src/internal.h

   1 /*
   2  * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_INTERNAL__
  28 #define __DISPATCH_INTERNAL__
  29
  30 #if __has_include(<config/config_ac.h>)
  31 #include <config/config_ac.h>
  32 #else
  33 #include <config/config.h>
  34 #endif
  35
  36 #define __DISPATCH_BUILDING_DISPATCH__
  37 #define __DISPATCH_INDIRECT__
  38
  39 #ifdef __APPLE__
  40 #include <Availability.h>
  41 #include <TargetConditionals.h>
  42
  43 #ifndef TARGET_OS_MAC_DESKTOP
  44 #define TARGET_OS_MAC_DESKTOP  (TARGET_OS_MAC && \
  45                 !TARGET_OS_SIMULATOR && !TARGET_OS_IPHONE && !TARGET_OS_EMBEDDED)
  46 #endif
  47
  48 #if TARGET_OS_MAC_DESKTOP
  49 #  define DISPATCH_HOST_SUPPORTS_OSX(x) \
  50                 (__MAC_OS_X_VERSION_MIN_REQUIRED >= (x))
  51 #  if !DISPATCH_HOST_SUPPORTS_OSX(101000)
  52 #    error "OS X hosts older than OS X 10.10 aren't supported anymore"
  53 #  endif // !DISPATCH_HOST_SUPPORTS_OSX(101000)
  54 #elif TARGET_OS_SIMULATOR
  55 #  define DISPATCH_HOST_SUPPORTS_OSX(x) \
  56                 (IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED >= (x))
  57 #  if !DISPATCH_HOST_SUPPORTS_OSX(101000)
  58 #    error "Simulator hosts older than OS X 10.10 aren't supported anymore"
  59 #  endif // !DISPATCH_HOST_SUPPORTS_OSX(101000)
  60 #else
  61 #  define DISPATCH_HOST_SUPPORTS_OSX(x) 1
  62 #  if __IPHONE_OS_VERSION_MIN_REQUIRED < 70000
  63 #    error "iOS hosts older than iOS 7.0 aren't supported anymore"
  64 #  endif
  65 #endif
  66
  67 #else // !__APPLE__
  68 #define DISPATCH_HOST_SUPPORTS_OSX(x) 0
  69 #endif // !__APPLE__
  70
  71
  72 #if !defined(DISPATCH_MACH_SPI) && TARGET_OS_MAC
  73 #define DISPATCH_MACH_SPI 1
  74 #endif
  75 #if !defined(OS_VOUCHER_CREATION_SPI) && TARGET_OS_MAC
  76 #define OS_VOUCHER_CREATION_SPI 1
  77 #endif
  78 #if !defined(OS_VOUCHER_ACTIVITY_SPI) && TARGET_OS_MAC
  79 #define OS_VOUCHER_ACTIVITY_SPI 1
  80 #endif
  81 #if !defined(OS_FIREHOSE_SPI) && TARGET_OS_MAC
  82 #define OS_FIREHOSE_SPI 1
  83 #endif
  84 #if !defined(DISPATCH_LAYOUT_SPI) && TARGET_OS_MAC
  85 #define DISPATCH_LAYOUT_SPI 1
  86 #endif
  87
  88 #if __has_include(<mach-o/dyld_priv.h>)
  89 #include <mach-o/dyld_priv.h>
  90 #if !defined(HAVE_DYLD_IS_MEMORY_IMMUTABLE)
  91 #if defined(DYLD_MACOSX_VERSION_10_12) || defined(DYLD_IOS_VERSION_10_0)
  92 #define HAVE_DYLD_IS_MEMORY_IMMUTABLE 1
  93 #else
  94 #define HAVE_DYLD_IS_MEMORY_IMMUTABLE 0
  95 #endif
  96 #endif // !defined(HAVE_DYLD_IS_MEMORY_IMMUTABLE)
  97 #endif // __has_include(<mach-o/dyld_priv.h>)
  98
  99 #if !defined(USE_OBJC) && HAVE_OBJC
 100 #define USE_OBJC 1
 101 #endif
 102
 103 #if USE_OBJC
 104 #define OS_OBJECT_HAVE_OBJC_SUPPORT 1
 105 #if defined(__OBJC__)
 106 #define OS_OBJECT_USE_OBJC 1
 107 // Force internal Objective-C sources to use class-visible headers
 108 // even when not compiling in Swift.
 109 #define OS_OBJECT_SWIFT3 1
 110 #else
 111 #define OS_OBJECT_USE_OBJC 0
 112 #endif // __OBJC__
 113 #else
 114 #define OS_OBJECT_HAVE_OBJC_SUPPORT 0
 115 #endif // USE_OBJC
 116
 117 #include <dispatch/dispatch.h>
 118 #include <dispatch/base.h>
 119
 120 #define __DISPATCH_HIDE_SYMBOL(sym, version) \
 121         __asm__(".section __TEXT,__const\n\t" \
 122                         ".globl $ld$hide$os" #version "$_" #sym "\n\t" \
 123                         "$ld$hide$os" #version "$_" #sym ":\n\t" \
 124                         "    .byte 0\n\t" \
 125                         ".previous")
 126
 127
 128 #ifndef DISPATCH_HIDE_SYMBOL
 129 #if TARGET_OS_MAC && !TARGET_OS_IPHONE
 130 #define DISPATCH_HIDE_SYMBOL(sym, osx, ios, tvos, watchos) \
 131                 __DISPATCH_HIDE_SYMBOL(sym, osx)
 132 #else
 133 #define DISPATCH_HIDE_SYMBOL(sym, osx, ios, tvos, watchos)
 134 #endif
 135 #endif
 136
 137 #include <os/object.h>
 138 #include <dispatch/time.h>
 139 #include <dispatch/object.h>
 140 #include <dispatch/queue.h>
 141 #include <dispatch/block.h>
 142 #include <dispatch/source.h>
 143 #include <dispatch/group.h>
 144 #include <dispatch/semaphore.h>
 145 #include <dispatch/once.h>
 146 #include <dispatch/data.h>
 147 #if !TARGET_OS_WIN32
 148 #include <dispatch/io.h>
 149 #endif
 150
 151 #if defined(__OBJC__) || defined(__cplusplus)
 152 #define DISPATCH_PURE_C 0
 153 #else
 154 #define DISPATCH_PURE_C 1
 155 #endif
 156
 157 /* private.h must be included last to avoid picking up installed headers. */
 158 #include "os/object_private.h"
 159 #include "queue_private.h"
 160 #include "source_private.h"
 161 #include "mach_private.h"
 162 #include "data_private.h"
 163 #include "os/voucher_private.h"
 164 #include "os/voucher_activity_private.h"
 165 #if !TARGET_OS_WIN32
 166 #include "io_private.h"
 167 #endif
 168 #include "layout_private.h"
 169 #include "benchmark.h"
 170 #include "private.h"
 171
 172 /* SPI for Libsystem-internal use */
 173 DISPATCH_EXPORT DISPATCH_NOTHROW void libdispatch_init(void);
 174 #if !TARGET_OS_WIN32
 175 DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_prepare(void);
 176 DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_parent(void);
 177 DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void);
 178 #endif
 179
 180 /* More #includes at EOF (dependent on the contents of internal.h) ... */
 181
 182 // Abort on uncaught exceptions thrown from client callouts rdar://8577499
 183 #if !defined(DISPATCH_USE_CLIENT_CALLOUT)
 184 #define DISPATCH_USE_CLIENT_CALLOUT 1
 185 #endif
 186
 187 /* The "_debug" library build */
 188 #ifndef DISPATCH_DEBUG
 189 #define DISPATCH_DEBUG 0
 190 #endif
 191
 192 #ifndef DISPATCH_PROFILE
 193 #define DISPATCH_PROFILE 0
 194 #endif
 195
 196 #if (!TARGET_OS_EMBEDDED || DISPATCH_DEBUG || DISPATCH_PROFILE) && \
 197                 !defined(DISPATCH_USE_DTRACE)
 198 #define DISPATCH_USE_DTRACE 1
 199 #endif
 200
 201 #if DISPATCH_USE_DTRACE && (DISPATCH_INTROSPECTION || DISPATCH_DEBUG || \
 202                 DISPATCH_PROFILE) && !defined(DISPATCH_USE_DTRACE_INTROSPECTION)
 203 #define DISPATCH_USE_DTRACE_INTROSPECTION 1
 204 #endif
 205
 206 #ifndef DISPATCH_DEBUG_QOS
 207 #define DISPATCH_DEBUG_QOS DISPATCH_DEBUG
 208 #endif
 209
 210 #if HAVE_LIBKERN_OSCROSSENDIAN_H
 211 #include <libkern/OSCrossEndian.h>
 212 #endif
 213 #if HAVE_LIBKERN_OSATOMIC_H
 214 #include <libkern/OSAtomic.h>
 215 #endif
 216 #if HAVE_MACH
 217 #include <mach/boolean.h>
 218 #include <mach/clock_types.h>
 219 #include <mach/clock.h>
 220 #include <mach/exception.h>
 221 #include <mach/mach.h>
 222 #include <mach/mach_error.h>
 223 #include <mach/mach_host.h>
 224 #include <mach/mach_interface.h>
 225 #include <mach/mach_time.h>
 226 #include <mach/mach_traps.h>
 227 #include <mach/message.h>
 228 #include <mach/mig_errors.h>
 229 #include <mach/host_special_ports.h>
 230 #include <mach/host_info.h>
 231 #include <mach/notify.h>
 232 #include <mach/mach_vm.h>
 233 #include <mach/vm_map.h>
 234 #endif /* HAVE_MACH */
 235 #if HAVE_MALLOC_MALLOC_H
 236 #include <malloc/malloc.h>
 237 #endif
 238 #if __has_include(<malloc_private.h>)
 239 #include <malloc_private.h>
 240 #endif // __has_include(<malloc_private.h)
 241
 242 #include <sys/stat.h>
 243
 244 #if !TARGET_OS_WIN32
 245 #include <sys/event.h>
 246 #include <sys/mount.h>
 247 #include <sys/queue.h>
 248 #include <sys/sysctl.h>
 249 #include <sys/socket.h>
 250 #include <sys/time.h>
 251 #include <sys/mman.h>
 252 #include <netinet/in.h>
 253 #endif
 254 #if defined(__linux__)
 255 #include <sys/eventfd.h>
 256 #endif
 257
 258 #ifdef __BLOCKS__
 259 #include <Block_private.h>
 260 #include <Block.h>
 261 #endif /* __BLOCKS__ */
 262
 263 #include <assert.h>
 264 #include <errno.h>
 265 #if HAVE_FCNTL_H
 266 #include <fcntl.h>
 267 #endif
 268 #include <limits.h>
 269 #include <search.h>
 270 #if USE_POSIX_SEM
 271 #include <semaphore.h>
 272 #endif
 273 #include <signal.h>
 274 #include <stdarg.h>
 275 #include <stdbool.h>
 276 #include <stdint.h>
 277 #include <stdio.h>
 278 #include <stdlib.h>
 279 #include <string.h>
 280 #if HAVE_UNISTD_H
 281 #include <unistd.h>
 282 #endif
 283
 284 #if __GNUC__
 285 #define DISPATCH_NOINLINE __attribute__((__noinline__))
 286 #define DISPATCH_USED __attribute__((__used__))
 287 #define DISPATCH_UNUSED __attribute__((__unused__))
 288 #define DISPATCH_WEAK __attribute__((__weak__))
 289 #define DISPATCH_OVERLOADABLE __attribute__((__overloadable__))
 290 #define DISPATCH_PACKED __attribute__((__packed__))
 291 #if DISPATCH_DEBUG
 292 #define DISPATCH_ALWAYS_INLINE_NDEBUG
 293 #else
 294 #define DISPATCH_ALWAYS_INLINE_NDEBUG __attribute__((__always_inline__))
 295 #endif
 296 #else   /* __GNUC__ */
 297 #define DISPATCH_NOINLINE
 298 #define DISPATCH_USED
 299 #define DISPATCH_UNUSED
 300 #define DISPATCH_WEAK
 301 #define DISPATCH_ALWAYS_INLINE_NDEBUG
 302 #endif  /* __GNUC__ */
 303
 304 #define DISPATCH_CONCAT(x,y) DISPATCH_CONCAT1(x,y)
 305 #define DISPATCH_CONCAT1(x,y) x ## y
 306
 307 // workaround 6368156
 308 #ifdef NSEC_PER_SEC
 309 #undef NSEC_PER_SEC
 310 #endif
 311 #ifdef USEC_PER_SEC
 312 #undef USEC_PER_SEC
 313 #endif
 314 #ifdef NSEC_PER_USEC
 315 #undef NSEC_PER_USEC
 316 #endif
 317 #define NSEC_PER_SEC 1000000000ull
 318 #define USEC_PER_SEC 1000000ull
 319 #define NSEC_PER_USEC 1000ull
 320
 321 /* I wish we had __builtin_expect_range() */
 322 #if __GNUC__
 323 #define _safe_cast_to_long(x) \
 324                 ({ _Static_assert(sizeof(typeof(x)) <= sizeof(long), \
 325                                 "__builtin_expect doesn't support types wider than long"); \
 326                                 (long)(x); })
 327 #define fastpath(x) ((typeof(x))__builtin_expect(_safe_cast_to_long(x), ~0l))
 328 #define slowpath(x) ((typeof(x))__builtin_expect(_safe_cast_to_long(x), 0l))
 329 #define likely(x) __builtin_expect(!!(x), 1)
 330 #define unlikely(x) __builtin_expect(!!(x), 0)
 331 #else
 332 #define fastpath(x) (x)
 333 #define slowpath(x) (x)
 334 #define likely(x) (!!(x))
 335 #define unlikely(x) (!!(x))
 336 #endif // __GNUC__
 337
 338 #if BYTE_ORDER == LITTLE_ENDIAN
 339 #define DISPATCH_STRUCT_LITTLE_ENDIAN_2(a, b)        struct { a; b; }
 340 #define DISPATCH_STRUCT_LITTLE_ENDIAN_3(a, b, c)     struct { a; b; c; }
 341 #define DISPATCH_STRUCT_LITTLE_ENDIAN_4(a, b, c, d)  struct { a; b; c; d; }
 342 #else
 343 #define DISPATCH_STRUCT_LITTLE_ENDIAN_2(a, b)        struct { b; a; }
 344 #define DISPATCH_STRUCT_LITTLE_ENDIAN_3(a, b, c)     struct { c; b; a; }
 345 #define DISPATCH_STRUCT_LITTLE_ENDIAN_4(a, b, c, d)  struct { d; c; b; a; }
 346 #endif
 347
 348 #define _TAILQ_IS_ENQUEUED(elm, field) \
 349                 ((elm)->field.tqe_prev != NULL)
 350 #define _TAILQ_MARK_NOT_ENQUEUED(elm, field) \
 351                 do { (elm)->field.tqe_prev = NULL; } while (0)
 352
 353 #if DISPATCH_DEBUG
 354 // sys/queue.h debugging
 355 #undef TRASHIT
 356 #define TRASHIT(x) do {(x) = (void *)-1;} while (0)
 357 #endif // DISPATCH_DEBUG
 358 #define _TAILQ_TRASH_ENTRY(elm, field) do { \
 359                         TRASHIT((elm)->field.tqe_next); \
 360                         TRASHIT((elm)->field.tqe_prev); \
 361                 } while (0)
 362 #define _TAILQ_TRASH_HEAD(head) do { \
 363                         TRASHIT((head)->tqh_first); \
 364                         TRASHIT((head)->tqh_last); \
 365                 } while (0)
 366
 367 DISPATCH_EXPORT DISPATCH_NOINLINE
 368 void _dispatch_bug(size_t line, long val);
 369
 370 #if HAVE_MACH
 371 DISPATCH_NOINLINE
 372 void _dispatch_bug_client(const char* msg);
 373 DISPATCH_NOINLINE
 374 void _dispatch_bug_mach_client(const char *msg, mach_msg_return_t kr);
 375 #endif // HAVE_MACH
 376 DISPATCH_NOINLINE
 377 void _dispatch_bug_kevent_client(const char* msg, const char* filter,
 378                 const char *operation, int err);
 379
 380 DISPATCH_NOINLINE
 381 void _dispatch_bug_deprecated(const char *msg);
 382
 383 DISPATCH_NOINLINE DISPATCH_NORETURN
 384 void _dispatch_abort(size_t line, long val);
 385
 386 #if !defined(DISPATCH_USE_OS_DEBUG_LOG) && DISPATCH_DEBUG
 387 #if __has_include(<os/debug_private.h>)
 388 #define DISPATCH_USE_OS_DEBUG_LOG 1
 389 #include <os/debug_private.h>
 390 #endif
 391 #endif // DISPATCH_USE_OS_DEBUG_LOG
 392
 393 #if !defined(DISPATCH_USE_SIMPLE_ASL) && !DISPATCH_USE_OS_DEBUG_LOG
 394 #if __has_include(<_simple.h>)
 395 #define DISPATCH_USE_SIMPLE_ASL 1
 396 #include <_simple.h>
 397 #endif
 398 #endif // DISPATCH_USE_SIMPLE_ASL
 399
 400 #if !DISPATCH_USE_SIMPLE_ASL && !DISPATCH_USE_OS_DEBUG_LOG && !TARGET_OS_WIN32
 401 #include <syslog.h>
 402 #endif
 403
 404 #if DISPATCH_USE_OS_DEBUG_LOG
 405 #define _dispatch_log(msg, ...) os_debug_log("libdispatch", msg, ## __VA_ARGS__)
 406 #else
 407 DISPATCH_EXPORT DISPATCH_NOINLINE __attribute__((__format__(__printf__,1,2)))
 408 void _dispatch_log(const char *msg, ...);
 409 #endif // DISPATCH_USE_OS_DEBUG_LOG
 410
 411 #define dsnprintf(buf, siz, ...) \
 412                 ({ size_t _siz = siz; int _r = snprintf(buf, _siz, __VA_ARGS__); \
 413                  _r < 0 ? 0u : ((size_t)_r > _siz ? _siz : (size_t)_r); })
 414
 415 #if __GNUC__
 416 #define dispatch_static_assert(e) ({ \
 417                 char __compile_time_assert__[(bool)(e) ? 1 : -1] DISPATCH_UNUSED; \
 418         })
 419 #else
 420 #define dispatch_static_assert(e)
 421 #endif
 422
 423 #define DISPATCH_BAD_INPUT              ((void *_Nonnull)0)
 424 #define DISPATCH_OUT_OF_MEMORY  ((void *_Nonnull)0)
 425
 426 /*
 427  * For reporting bugs within libdispatch when using the "_debug" version of the
 428  * library.
 429  */
 430 #if __GNUC__
 431 #define dispatch_assert(e) do { \
 432                 if (__builtin_constant_p(e)) { \
 433                         dispatch_static_assert(e); \
 434                 } else { \
 435                         typeof(e) _e = fastpath(e); /* always eval 'e' */ \
 436                         if (DISPATCH_DEBUG && !_e) { \
 437                                 _dispatch_abort(__LINE__, (long)_e); \
 438                         } \
 439                 } \
 440         } while (0)
 441 #else
 442 static inline void _dispatch_assert(long e, long line) {
 443         if (DISPATCH_DEBUG && !e) _dispatch_abort(line, e);
 444 }
 445 #define dispatch_assert(e) _dispatch_assert((long)(e), __LINE__)
 446 #endif  /* __GNUC__ */
 447
 448 #if __GNUC__
 449 /*
 450  * A lot of API return zero upon success and not-zero on fail. Let's capture
 451  * and log the non-zero value
 452  */
 453 #define dispatch_assert_zero(e) do { \
 454                 if (__builtin_constant_p(e)) { \
 455                         dispatch_static_assert(e); \
 456                 } else { \
 457                         typeof(e) _e = slowpath(e); /* always eval 'e' */ \
 458                         if (DISPATCH_DEBUG && _e) { \
 459                                 _dispatch_abort(__LINE__, (long)_e); \
 460                         } \
 461                 } \
 462         } while (0)
 463 #else
 464 static inline void _dispatch_assert_zero(long e, long line) {
 465         if (DISPATCH_DEBUG && e) _dispatch_abort(line, e);
 466 }
 467 #define dispatch_assert_zero(e) _dispatch_assert((long)(e), __LINE__)
 468 #endif  /* __GNUC__ */
 469
 470 /*
 471  * For reporting bugs or impedance mismatches between libdispatch and external
 472  * subsystems. These do NOT abort(), and are always compiled into the product.
 473  *
 474  * In particular, we wrap all system-calls with assume() macros.
 475  */
 476 #if __GNUC__
 477 #define dispatch_assume(e) ({ \
 478                 typeof(e) _e = fastpath(e); /* always eval 'e' */ \
 479                 if (!_e) { \
 480                         if (__builtin_constant_p(e)) { \
 481                                 dispatch_static_assert(e); \
 482                         } \
 483                         _dispatch_bug(__LINE__, (long)_e); \
 484                 } \
 485                 _e; \
 486         })
 487 #else
 488 static inline long _dispatch_assume(long e, long line) {
 489         if (!e) _dispatch_bug(line, e);
 490         return e;
 491 }
 492 #define dispatch_assume(e) _dispatch_assume((long)(e), __LINE__)
 493 #endif  /* __GNUC__ */
 494
 495 /*
 496  * A lot of API return zero upon success and not-zero on fail. Let's capture
 497  * and log the non-zero value
 498  */
 499 #if __GNUC__
 500 #define dispatch_assume_zero(e) ({ \
 501                 typeof(e) _e = slowpath(e); /* always eval 'e' */ \
 502                 if (_e) { \
 503                         if (__builtin_constant_p(e)) { \
 504                                 dispatch_static_assert(e); \
 505                         } \
 506                         _dispatch_bug(__LINE__, (long)_e); \
 507                 } \
 508                 _e; \
 509         })
 510 #else
 511 static inline long _dispatch_assume_zero(long e, long line) {
 512         if (e) _dispatch_bug(line, e);
 513         return e;
 514 }
 515 #define dispatch_assume_zero(e) _dispatch_assume_zero((long)(e), __LINE__)
 516 #endif  /* __GNUC__ */
 517
 518 /*
 519  * For reporting bugs in clients when using the "_debug" version of the library.
 520  */
 521 #if __GNUC__
 522 #define dispatch_debug_assert(e, msg, args...) do { \
 523                 if (__builtin_constant_p(e)) { \
 524                         dispatch_static_assert(e); \
 525                 } else { \
 526                         typeof(e) _e = fastpath(e); /* always eval 'e' */ \
 527                         if (DISPATCH_DEBUG && !_e) { \
 528                                 _dispatch_log("%s() 0x%lx: " msg, __func__, (long)_e, ##args); \
 529                                 abort(); \
 530                         } \
 531                 } \
 532         } while (0)
 533 #else
 534 #define dispatch_debug_assert(e, msg, args...) do { \
 535         long _e = (long)fastpath(e); /* always eval 'e' */ \
 536         if (DISPATCH_DEBUG && !_e) { \
 537                 _dispatch_log("%s() 0x%lx: " msg, __FUNCTION__, _e, ##args); \
 538                 abort(); \
 539         } \
 540 } while (0)
 541 #endif  /* __GNUC__ */
 542
 543 /* Make sure the debug statments don't get too stale */
 544 #define _dispatch_debug(x, args...) do { \
 545         if (DISPATCH_DEBUG) { \
 546                 _dispatch_log("%u\t%p\t" x, __LINE__, \
 547                                 (void *)_dispatch_thread_self(), ##args); \
 548         } \
 549 } while (0)
 550
 551 #if DISPATCH_DEBUG
 552 #if HAVE_MACH
 553 DISPATCH_NOINLINE DISPATCH_USED
 554 void dispatch_debug_machport(mach_port_t name, const char* str);
 555 #endif
 556 #endif
 557
 558 #if DISPATCH_DEBUG
 559 /* This is the private version of the deprecated dispatch_debug() */
 560 DISPATCH_NONNULL2 DISPATCH_NOTHROW
 561 __attribute__((__format__(printf,2,3)))
 562 void
 563 _dispatch_object_debug(dispatch_object_t object, const char *message, ...);
 564 #else
 565 #define _dispatch_object_debug(object, message, ...)
 566 #endif // DISPATCH_DEBUG
 567
 568 #ifdef __BLOCKS__
 569 #define _dispatch_Block_invoke(bb) \
 570                 ((dispatch_function_t)((struct Block_layout *)bb)->invoke)
 571 void *_dispatch_Block_copy(void *block);
 572 #if __GNUC__
 573 #define _dispatch_Block_copy(x) ((typeof(x))_dispatch_Block_copy(x))
 574 #endif
 575 void _dispatch_call_block_and_release(void *block);
 576 #endif /* __BLOCKS__ */
 577
 578 void _dispatch_temporary_resource_shortage(void);
 579 void *_dispatch_calloc(size_t num_items, size_t size);
 580 const char *_dispatch_strdup_if_mutable(const char *str);
 581 void _dispatch_vtable_init(void);
 582 char *_dispatch_get_build(void);
 583
 584 uint64_t _dispatch_timeout(dispatch_time_t when);
 585 uint64_t _dispatch_time_nanoseconds_since_epoch(dispatch_time_t when);
 586
 587 #define _DISPATCH_UNSAFE_FORK_MULTITHREADED  ((uint8_t)1)
 588 #define _DISPATCH_UNSAFE_FORK_PROHIBIT       ((uint8_t)2)
 589 extern uint8_t _dispatch_unsafe_fork;
 590 extern bool _dispatch_child_of_unsafe_fork;
 591 void _dispatch_fork_becomes_unsafe_slow(void);
 592
 593 #define _dispatch_is_multithreaded_inline() \
 594         ((_dispatch_unsafe_fork & _DISPATCH_UNSAFE_FORK_MULTITHREADED) != 0)
 595
 596 DISPATCH_ALWAYS_INLINE
 597 static inline void
 598 _dispatch_fork_becomes_unsafe(void)
 599 {
 600         if (!fastpath(_dispatch_is_multithreaded_inline())) {
 601                 _dispatch_fork_becomes_unsafe_slow();
 602                 DISPATCH_COMPILER_CAN_ASSUME(_dispatch_is_multithreaded_inline());
 603         }
 604 }
 605
 606 /* #includes dependent on internal.h */
 607 #include "shims.h"
 608
 609 // Older Mac OS X and iOS Simulator fallbacks
 610
 611 #if HAVE_PTHREAD_WORKQUEUES
 612 #ifndef WORKQ_ADDTHREADS_OPTION_OVERCOMMIT
 613 #define WORKQ_ADDTHREADS_OPTION_OVERCOMMIT 0x00000001
 614 #endif
 615 #endif // HAVE_PTHREAD_WORKQUEUES
 616 #if HAVE__PTHREAD_WORKQUEUE_INIT && PTHREAD_WORKQUEUE_SPI_VERSION >= 20140213 \
 617                 && !defined(HAVE_PTHREAD_WORKQUEUE_QOS)
 618 #define HAVE_PTHREAD_WORKQUEUE_QOS 1
 619 #endif
 620 #if HAVE__PTHREAD_WORKQUEUE_INIT && (PTHREAD_WORKQUEUE_SPI_VERSION >= 20150304 \
 621                 || (PTHREAD_WORKQUEUE_SPI_VERSION == 20140730 && \
 622                         defined(WORKQ_FEATURE_KEVENT))) \
 623                 && !defined(HAVE_PTHREAD_WORKQUEUE_KEVENT)
 624 #if PTHREAD_WORKQUEUE_SPI_VERSION == 20140730
 625 // rdar://problem/20609877
 626 typedef pthread_worqueue_function_kevent_t pthread_workqueue_function_kevent_t;
 627 #endif
 628 #define HAVE_PTHREAD_WORKQUEUE_KEVENT 1
 629 #endif
 630
 631 #ifndef PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK
 632 #if HAVE_PTHREAD_WORKQUEUE_QOS && DISPATCH_HOST_SUPPORTS_OSX(101200)
 633 #define PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK 1
 634 #else
 635 #define PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK 0
 636 #endif
 637 #endif // PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK
 638
 639 #if HAVE_MACH
 640 #if !defined(MACH_NOTIFY_SEND_POSSIBLE)
 641 #undef MACH_NOTIFY_SEND_POSSIBLE
 642 #define MACH_NOTIFY_SEND_POSSIBLE MACH_NOTIFY_DEAD_NAME
 643 #endif
 644 #endif // HAVE_MACH
 645
 646 #ifdef EVFILT_MEMORYSTATUS
 647 #ifndef DISPATCH_USE_MEMORYSTATUS
 648 #define DISPATCH_USE_MEMORYSTATUS 1
 649 #endif
 650 #endif // EVFILT_MEMORYSTATUS
 651
 652 #if defined(EVFILT_VM) && !DISPATCH_USE_MEMORYSTATUS
 653 #ifndef DISPATCH_USE_VM_PRESSURE
 654 #define DISPATCH_USE_VM_PRESSURE 1
 655 #endif
 656 #endif // EVFILT_VM
 657
 658 #if TARGET_OS_SIMULATOR
 659 #undef DISPATCH_USE_MEMORYPRESSURE_SOURCE
 660 #define DISPATCH_USE_MEMORYPRESSURE_SOURCE 0
 661 #undef DISPATCH_USE_VM_PRESSURE_SOURCE
 662 #define DISPATCH_USE_VM_PRESSURE_SOURCE 0
 663 #endif // TARGET_OS_SIMULATOR
 664 #if !defined(DISPATCH_USE_MEMORYPRESSURE_SOURCE) && DISPATCH_USE_MEMORYSTATUS
 665 #define DISPATCH_USE_MEMORYPRESSURE_SOURCE 1
 666 #elif !defined(DISPATCH_USE_VM_PRESSURE_SOURCE) && DISPATCH_USE_VM_PRESSURE
 667 #define DISPATCH_USE_VM_PRESSURE_SOURCE 1
 668 #endif
 669 #if DISPATCH_USE_MEMORYPRESSURE_SOURCE
 670 extern bool _dispatch_memory_warn;
 671 #endif
 672
 673 #if !defined(NOTE_LEEWAY)
 674 #undef NOTE_LEEWAY
 675 #define NOTE_LEEWAY 0
 676 #undef NOTE_CRITICAL
 677 #define NOTE_CRITICAL 0
 678 #undef NOTE_BACKGROUND
 679 #define NOTE_BACKGROUND 0
 680 #endif // NOTE_LEEWAY
 681
 682 #if !defined(NOTE_FUNLOCK)
 683 #define NOTE_FUNLOCK 0x00000100
 684 #endif
 685
 686 #if !defined(NOTE_MACH_CONTINUOUS_TIME)
 687 #define NOTE_MACH_CONTINUOUS_TIME 0
 688 #endif // NOTE_MACH_CONTINUOUS_TIME
 689
 690 #if !defined(HOST_NOTIFY_CALENDAR_SET)
 691 #define HOST_NOTIFY_CALENDAR_SET HOST_NOTIFY_CALENDAR_CHANGE
 692 #endif // HOST_NOTIFY_CALENDAR_SET
 693
 694 #if !defined(HOST_CALENDAR_SET_REPLYID)
 695 #define HOST_CALENDAR_SET_REPLYID 951
 696 #endif // HOST_CALENDAR_SET_REPLYID
 697
 698 #if HAVE_DECL_NOTE_REAP
 699 #if defined(NOTE_REAP) && defined(__APPLE__)
 700 #undef NOTE_REAP
 701 #define NOTE_REAP 0x10000000 // <rdar://problem/13338526>
 702 #endif
 703 #endif // HAVE_DECL_NOTE_REAP
 704
 705 #ifndef VQ_QUOTA
 706 #undef HAVE_DECL_VQ_QUOTA // rdar://problem/24160982
 707 #endif // VQ_QUOTA
 708
 709 #if !defined(NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) || \
 710                 !DISPATCH_HOST_SUPPORTS_OSX(101200)
 711 #undef NOTE_MEMORYSTATUS_PROC_LIMIT_WARN
 712 #define NOTE_MEMORYSTATUS_PROC_LIMIT_WARN 0
 713 #endif // NOTE_MEMORYSTATUS_PROC_LIMIT_WARN
 714
 715 #if !defined(NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) || \
 716                 !DISPATCH_HOST_SUPPORTS_OSX(101200)
 717 #undef NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL
 718 #define NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL 0
 719 #endif // NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL
 720
 721 #if !defined(EV_UDATA_SPECIFIC) || !DISPATCH_HOST_SUPPORTS_OSX(101100)
 722 #undef DISPATCH_USE_EV_UDATA_SPECIFIC
 723 #define DISPATCH_USE_EV_UDATA_SPECIFIC 0
 724 #elif !defined(DISPATCH_USE_EV_UDATA_SPECIFIC)
 725 #define DISPATCH_USE_EV_UDATA_SPECIFIC 1
 726 #endif // EV_UDATA_SPECIFIC
 727
 728 #if !DISPATCH_USE_EV_UDATA_SPECIFIC
 729 #undef EV_UDATA_SPECIFIC
 730 #define EV_UDATA_SPECIFIC 0
 731 #undef EV_VANISHED
 732 #define EV_VANISHED 0
 733 #endif // !DISPATCH_USE_EV_UDATA_SPECIFIC
 734
 735 #ifndef EV_VANISHED
 736 #define EV_VANISHED 0x0200
 737 #endif
 738
 739 #ifndef DISPATCH_KEVENT_TREAT_ENOENT_AS_EINPROGRESS
 740 #if TARGET_OS_MAC && !DISPATCH_HOST_SUPPORTS_OSX(101200)
 741 // deferred delete can return bogus ENOENTs on older kernels
 742 #define DISPATCH_KEVENT_TREAT_ENOENT_AS_EINPROGRESS 1
 743 #else
 744 #define DISPATCH_KEVENT_TREAT_ENOENT_AS_EINPROGRESS 0
 745 #endif
 746 #endif
 747
 748 #if !defined(EV_SET_QOS) || !DISPATCH_HOST_SUPPORTS_OSX(101100)
 749 #undef DISPATCH_USE_KEVENT_QOS
 750 #define DISPATCH_USE_KEVENT_QOS 0
 751 #elif !defined(DISPATCH_USE_KEVENT_QOS)
 752 #define DISPATCH_USE_KEVENT_QOS 1
 753 #endif // EV_SET_QOS
 754
 755 #if HAVE_PTHREAD_WORKQUEUE_KEVENT && defined(KEVENT_FLAG_WORKQ) && \
 756                 DISPATCH_USE_EV_UDATA_SPECIFIC && DISPATCH_USE_KEVENT_QOS && \
 757                 DISPATCH_HOST_SUPPORTS_OSX(101200) && \
 758                 !defined(DISPATCH_USE_KEVENT_WORKQUEUE)
 759 #define DISPATCH_USE_KEVENT_WORKQUEUE 1
 760 #endif
 761
 762
 763 #if (!DISPATCH_USE_KEVENT_WORKQUEUE || DISPATCH_DEBUG) && \
 764                 !defined(DISPATCH_USE_MGR_THREAD)
 765 #define DISPATCH_USE_MGR_THREAD 1
 766 #endif
 767
 768 #if DISPATCH_USE_KEVENT_WORKQUEUE && DISPATCH_USE_EV_UDATA_SPECIFIC && \
 769                 DISPATCH_HOST_SUPPORTS_OSX(101200) && \
 770                 !defined(DISPATCH_USE_EVFILT_MACHPORT_DIRECT)
 771 #define DISPATCH_USE_EVFILT_MACHPORT_DIRECT 1
 772 #endif
 773
 774 #ifndef MACH_SEND_OVERRIDE
 775 #define MACH_SEND_OVERRIDE 0x00000020
 776 typedef unsigned int mach_msg_priority_t;
 777 #define MACH_MSG_PRIORITY_UNSPECIFIED ((mach_msg_priority_t)0)
 778 #endif // MACH_SEND_OVERRIDE
 779
 780
 781 #if (!DISPATCH_USE_EVFILT_MACHPORT_DIRECT || DISPATCH_DEBUG) && \
 782                 !defined(DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK)
 783 #define DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK 1
 784 #endif
 785
 786 #if DISPATCH_USE_KEVENT_QOS
 787 typedef struct kevent_qos_s _dispatch_kevent_qos_s;
 788 typedef typeof(((struct kevent_qos_s*)NULL)->qos) _dispatch_kevent_priority_t;
 789 #else // DISPATCH_USE_KEVENT_QOS
 790 #ifndef KEVENT_FLAG_IMMEDIATE
 791 #define KEVENT_FLAG_NONE 0x00
 792 #define KEVENT_FLAG_IMMEDIATE 0x01
 793 #define KEVENT_FLAG_ERROR_EVENTS 0x02
 794 #endif // KEVENT_FLAG_IMMEDIATE
 795 typedef struct kevent64_s _dispatch_kevent_qos_s;
 796 #define kevent_qos(_kq, _changelist, _nchanges, _eventlist, _nevents, \
 797                 _data_out, _data_available, _flags) \
 798                 ({ unsigned int _f = (_flags); _dispatch_kevent_qos_s _kev_copy; \
 799                 const _dispatch_kevent_qos_s *_cl = (_changelist); \
 800                 int _n = (_nchanges); const struct timespec _timeout_immediately = {}; \
 801                 dispatch_static_assert(!(_data_out) && !(_data_available)); \
 802                 if (_f & KEVENT_FLAG_ERROR_EVENTS) { \
 803                         dispatch_static_assert(_n == 1); \
 804                         _kev_copy = *_cl; _kev_copy.flags |= EV_RECEIPT; } \
 805                 kevent64((_kq), _f & KEVENT_FLAG_ERROR_EVENTS ? &_kev_copy : _cl, _n, \
 806                         (_eventlist), (_nevents), 0, \
 807                         _f & KEVENT_FLAG_IMMEDIATE ? &_timeout_immediately : NULL); })
 808 #endif // DISPATCH_USE_KEVENT_QOS
 809
 810 #if defined(F_SETNOSIGPIPE) && defined(F_GETNOSIGPIPE)
 811 #ifndef DISPATCH_USE_SETNOSIGPIPE
 812 #define DISPATCH_USE_SETNOSIGPIPE 1
 813 #endif
 814 #endif // F_SETNOSIGPIPE
 815
 816 #if defined(MACH_SEND_NOIMPORTANCE)
 817 #ifndef DISPATCH_USE_CHECKIN_NOIMPORTANCE
 818 #define DISPATCH_USE_CHECKIN_NOIMPORTANCE 1 // rdar://problem/16996737
 819 #endif
 820 #ifndef DISPATCH_USE_NOIMPORTANCE_QOS
 821 #define DISPATCH_USE_NOIMPORTANCE_QOS 1 // rdar://problem/21414476
 822 #endif
 823 #endif // MACH_SEND_NOIMPORTANCE
 824
 825
 826 #if HAVE_LIBPROC_INTERNAL_H
 827 #include <libproc.h>
 828 #include <libproc_internal.h>
 829 #ifndef DISPATCH_USE_IMPORTANCE_ASSERTION
 830 #define DISPATCH_USE_IMPORTANCE_ASSERTION 1
 831 #endif
 832 #endif // HAVE_LIBPROC_INTERNAL_H
 833
 834 #if HAVE_SYS_GUARDED_H
 835 #include <sys/guarded.h>
 836 #ifndef DISPATCH_USE_GUARDED_FD
 837 #define DISPATCH_USE_GUARDED_FD 1
 838 #endif
 839 // change_fdguard_np() requires GUARD_DUP <rdar://problem/11814513>
 840 #if DISPATCH_USE_GUARDED_FD && RDAR_11814513
 841 #define DISPATCH_USE_GUARDED_FD_CHANGE_FDGUARD 1
 842 #endif
 843 #endif // HAVE_SYS_GUARDED_H
 844
 845
 846 #if __has_include(<sys/kdebug.h>)
 847 #include <sys/kdebug.h>
 848 #ifndef DBG_DISPATCH
 849 #define DBG_DISPATCH 46
 850 #endif
 851 #ifndef KDBG_CODE
 852 #define KDBG_CODE(...) 0
 853 #endif
 854 #define DISPATCH_CODE(subclass, code) \
 855                 KDBG_CODE(DBG_DISPATCH, DISPATCH_TRACE_SUBCLASS_##subclass, code)
 856 #ifdef ARIADNEDBG_CODE
 857 #define ARIADNE_ENTER_DISPATCH_MAIN_CODE ARIADNEDBG_CODE(220, 2)
 858 #else
 859 #define ARIADNE_ENTER_DISPATCH_MAIN_CODE 0
 860 #endif
 861 #if !defined(DISPATCH_USE_VOUCHER_KDEBUG_TRACE) && DISPATCH_INTROSPECTION
 862 #define DISPATCH_USE_VOUCHER_KDEBUG_TRACE 1
 863 #endif
 864
 865 #define DISPATCH_TRACE_SUBCLASS_DEFAULT 0
 866 #define DISPATCH_TRACE_SUBCLASS_VOUCHER 1
 867 #define DISPATCH_TRACE_SUBCLASS_PERF 2
 868 #define DISPATCH_TRACE_SUBCLASS_MACH_MSG 3
 869
 870 #define DISPATCH_PERF_non_leaf_retarget DISPATCH_CODE(PERF, 1)
 871 #define DISPATCH_PERF_post_activate_retarget DISPATCH_CODE(PERF, 2)
 872 #define DISPATCH_PERF_post_activate_mutation DISPATCH_CODE(PERF, 3)
 873 #define DISPATCH_PERF_delayed_registration DISPATCH_CODE(PERF, 4)
 874 #define DISPATCH_PERF_mutable_target DISPATCH_CODE(PERF, 5)
 875
 876 #define DISPATCH_MACH_MSG_hdr_move DISPATCH_CODE(MACH_MSG, 1)
 877
 878 DISPATCH_ALWAYS_INLINE
 879 static inline void
 880 _dispatch_ktrace_impl(uint32_t code, uint64_t a, uint64_t b,
 881                 uint64_t c, uint64_t d)
 882 {
 883         if (!code) return;
 884 #ifdef _COMM_PAGE_KDEBUG_ENABLE
 885         if (likely(*(volatile uint32_t *)_COMM_PAGE_KDEBUG_ENABLE == 0)) return;
 886 #endif
 887         kdebug_trace(code, a, b, c, d);
 888 }
 889 #define _dispatch_cast_to_uint64(e) \
 890                 __builtin_choose_expr(sizeof(e) > 4, \
 891                                 ((uint64_t)(e)), ((uint64_t)(uintptr_t)(e)))
 892 #define _dispatch_ktrace(code, a, b, c, d)  _dispatch_ktrace_impl(code, \
 893                 _dispatch_cast_to_uint64(a), _dispatch_cast_to_uint64(b), \
 894                 _dispatch_cast_to_uint64(c), _dispatch_cast_to_uint64(d))
 895
 896 #else // __has_include(<sys/kdebug.h>)
 897 #define DISPATCH_CODE(subclass, code) 0
 898 #define ARIADNE_ENTER_DISPATCH_MAIN_CODE 0
 899 #define DISPATCH_USE_VOUCHER_KDEBUG_TRACE 0
 900 #define _dispatch_ktrace(code, a, b, c, d)
 901 #endif // !__has_include(<sys/kdebug.h>)
 902 #define _dispatch_ktrace4(code, a, b, c, d) _dispatch_ktrace(code, a, b, c, d)
 903 #define _dispatch_ktrace3(code, a, b, c)    _dispatch_ktrace(code, a, b, c, 0)
 904 #define _dispatch_ktrace2(code, a, b)       _dispatch_ktrace(code, a, b, 0, 0)
 905 #define _dispatch_ktrace1(code, a)          _dispatch_ktrace(code, a, 0, 0, 0)
 906 #define _dispatch_ktrace0(code)             _dispatch_ktrace(code, 0, 0, 0, 0)
 907
 908 #ifndef MACH_MSGH_BITS_VOUCHER_MASK
 909 #define MACH_MSGH_BITS_VOUCHER_MASK     0x001f0000
 910 #define MACH_MSGH_BITS_SET_PORTS(remote, local, voucher)        \
 911         (((remote) & MACH_MSGH_BITS_REMOTE_MASK) |              \
 912          (((local) << 8) & MACH_MSGH_BITS_LOCAL_MASK) |         \
 913          (((voucher) << 16) & MACH_MSGH_BITS_VOUCHER_MASK))
 914 #define MACH_MSGH_BITS_VOUCHER(bits)                            \
 915                 (((bits) & MACH_MSGH_BITS_VOUCHER_MASK) >> 16)
 916 #define MACH_MSGH_BITS_HAS_VOUCHER(bits)                        \
 917         (MACH_MSGH_BITS_VOUCHER(bits) != MACH_MSGH_BITS_ZERO)
 918 #define msgh_voucher_port msgh_reserved
 919 #define mach_voucher_t mach_port_t
 920 #define MACH_VOUCHER_NULL MACH_PORT_NULL
 921 #define MACH_SEND_INVALID_VOUCHER 0x10000005
 922 #endif
 923
 924 #if TARGET_OS_SIMULATOR && IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101100
 925 #undef VOUCHER_USE_MACH_VOUCHER
 926 #define VOUCHER_USE_MACH_VOUCHER 0
 927 #endif
 928 #ifndef VOUCHER_USE_MACH_VOUCHER
 929 #if __has_include(<mach/mach_voucher.h>)
 930 #define VOUCHER_USE_MACH_VOUCHER 1
 931 #endif
 932 #endif
 933
 934 #if RDAR_24272659 // FIXME: <rdar://problem/24272659>
 935 #if !VOUCHER_USE_MACH_VOUCHER || !DISPATCH_HOST_SUPPORTS_OSX(101200)
 936 #undef VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
 937 #define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 0
 938 #elif !defined(VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER)
 939 #define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 1
 940 #endif
 941 #else // RDAR_24272659
 942 #undef VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
 943 #define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 0
 944 #endif // RDAR_24272659
 945
 946 #if !VOUCHER_USE_MACH_VOUCHER || !DISPATCH_HOST_SUPPORTS_OSX(101200)
 947 #undef VOUCHER_USE_BANK_AUTOREDEEM
 948 #define VOUCHER_USE_BANK_AUTOREDEEM 0
 949 #elif !defined(VOUCHER_USE_BANK_AUTOREDEEM)
 950 #define VOUCHER_USE_BANK_AUTOREDEEM 1
 951 #endif
 952
 953 #if !VOUCHER_USE_MACH_VOUCHER || \
 954                 !__has_include(<voucher/ipc_pthread_priority_types.h>) || \
 955                 !DISPATCH_HOST_SUPPORTS_OSX(101200)
 956 #undef VOUCHER_USE_MACH_VOUCHER_PRIORITY
 957 #define VOUCHER_USE_MACH_VOUCHER_PRIORITY 0
 958 #elif !defined(VOUCHER_USE_MACH_VOUCHER_PRIORITY)
 959 #define VOUCHER_USE_MACH_VOUCHER_PRIORITY 1
 960 #endif
 961
 962 #ifndef VOUCHER_USE_PERSONA
 963 #if VOUCHER_USE_MACH_VOUCHER && defined(BANK_PERSONA_TOKEN) && \
 964                 TARGET_OS_IOS && !TARGET_OS_SIMULATOR
 965 #define VOUCHER_USE_PERSONA 1
 966 #else
 967 #define VOUCHER_USE_PERSONA 0
 968 #endif
 969 #endif // VOUCHER_USE_PERSONA
 970
 971 #if VOUCHER_USE_MACH_VOUCHER
 972 #undef DISPATCH_USE_IMPORTANCE_ASSERTION
 973 #define DISPATCH_USE_IMPORTANCE_ASSERTION 0
 974 #else
 975 #undef MACH_RCV_VOUCHER
 976 #define MACH_RCV_VOUCHER 0
 977 #define VOUCHER_USE_PERSONA 0
 978 #endif // VOUCHER_USE_MACH_VOUCHER
 979
 980 #define _dispatch_hardware_crash() \
 981                 __asm__(""); __builtin_trap() // <rdar://problem/17464981>
 982
 983 #define _dispatch_set_crash_log_cause_and_message(ac, msg)
 984 #define _dispatch_set_crash_log_message(msg)
 985 #define _dispatch_set_crash_log_message_dynamic(msg)
 986
 987 #if HAVE_MACH
 988 // MIG_REPLY_MISMATCH means either:
 989 // 1) A signal handler is NOT using async-safe API. See the sigaction(2) man
 990 //    page for more info.
 991 // 2) A hand crafted call to mach_msg*() screwed up. Use MIG.
 992 #define DISPATCH_VERIFY_MIG(x) do { \
 993                 if ((x) == MIG_REPLY_MISMATCH) { \
 994                         _dispatch_set_crash_log_cause_and_message((x), \
 995                                         "MIG_REPLY_MISMATCH"); \
 996                         _dispatch_hardware_crash(); \
 997                 } \
 998         } while (0)
 999 #endif
1000
1001 #define DISPATCH_INTERNAL_CRASH(c, x) do { \
1002                 _dispatch_set_crash_log_cause_and_message((c), \
1003                                 "BUG IN LIBDISPATCH: " x); \
1004                 _dispatch_hardware_crash(); \
1005         } while (0)
1006
1007 #define DISPATCH_CLIENT_CRASH(c, x) do { \
1008                 _dispatch_set_crash_log_cause_and_message((c), \
1009                                 "BUG IN CLIENT OF LIBDISPATCH: " x); \
1010                 _dispatch_hardware_crash(); \
1011         } while (0)
1012
1013 #define _OS_OBJECT_CLIENT_CRASH(x) do { \
1014                 _dispatch_set_crash_log_message("API MISUSE: " x); \
1015                 _dispatch_hardware_crash(); \
1016         } while (0)
1017
1018 #define DISPATCH_ASSERTION_FAILED_MESSAGE \
1019                 "BUG IN CLIENT OF LIBDISPATCH: Assertion failed: "
1020
1021 #define _dispatch_assert_crash(msg)  do { \
1022                 const char *__msg = (msg); \
1023                 _dispatch_log("%s", __msg); \
1024                 _dispatch_set_crash_log_message_dynamic(__msg); \
1025                 _dispatch_hardware_crash(); \
1026         } while (0)
1027
1028 #define _dispatch_client_assert_fail(fmt, ...)  do { \
1029                 char *_msg = NULL; \
1030                 asprintf(&_msg, "%s" fmt, DISPATCH_ASSERTION_FAILED_MESSAGE, \
1031                                 ##__VA_ARGS__); \
1032                 _dispatch_assert_crash(_msg); \
1033                 free(_msg); \
1034         } while (0)
1035
1036 #define DISPATCH_NO_VOUCHER ((voucher_t)(void*)~0ul)
1037 #define DISPATCH_NO_PRIORITY ((pthread_priority_t)~0ul)
1038 DISPATCH_ENUM(_dispatch_thread_set_self, unsigned long,
1039         DISPATCH_PRIORITY_ENFORCE = 0x1,
1040         DISPATCH_VOUCHER_REPLACE = 0x2,
1041         DISPATCH_VOUCHER_CONSUME = 0x4,
1042         DISPATCH_THREAD_PARK = 0x8,
1043 );
1044 DISPATCH_WARN_RESULT
1045 static inline voucher_t _dispatch_adopt_priority_and_set_voucher(
1046                 pthread_priority_t priority, voucher_t voucher,
1047                 _dispatch_thread_set_self_t flags);
1048 #if HAVE_MACH
1049 mach_port_t _dispatch_get_mach_host_port(void);
1050 #endif
1051
1052 #if HAVE_PTHREAD_WORKQUEUE_QOS
1053 #if DISPATCH_DEBUG
1054 extern int _dispatch_set_qos_class_enabled;
1055 #else
1056 #define _dispatch_set_qos_class_enabled (1)
1057 #endif
1058 #endif // HAVE_PTHREAD_WORKQUEUE_QOS
1059 #if DISPATCH_USE_KEVENT_WORKQUEUE
1060 #if !HAVE_PTHREAD_WORKQUEUE_QOS || !DISPATCH_USE_KEVENT_QOS || \
1061                 !DISPATCH_USE_EV_UDATA_SPECIFIC
1062 #error Invalid build configuration
1063 #endif
1064 #if DISPATCH_USE_MGR_THREAD
1065 extern int _dispatch_kevent_workqueue_enabled;
1066 #else
1067 #define _dispatch_kevent_workqueue_enabled (1)
1068 #endif
1069 #endif // DISPATCH_USE_KEVENT_WORKQUEUE
1070
1071 #if DISPATCH_USE_EVFILT_MACHPORT_DIRECT
1072 #if !DISPATCH_USE_KEVENT_WORKQUEUE || !DISPATCH_USE_EV_UDATA_SPECIFIC
1073 #error Invalid build configuration
1074 #endif
1075 #if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
1076 extern int _dispatch_evfilt_machport_direct_enabled;
1077 #else
1078 #define _dispatch_evfilt_machport_direct_enabled (1)
1079 #endif
1080 #else
1081 #define _dispatch_evfilt_machport_direct_enabled (0)
1082 #endif // DISPATCH_USE_EVFILT_MACHPORT_DIRECT
1083
1084
1085 /* #includes dependent on internal.h */
1086 #include "object_internal.h"
1087 #include "semaphore_internal.h"
1088 #include "introspection_internal.h"
1089 #include "queue_internal.h"
1090 #include "source_internal.h"
1091 #include "voucher_internal.h"
1092 #include "data_internal.h"
1093 #if !TARGET_OS_WIN32
1094 #include "io_internal.h"
1095 #endif
1096 #include "inline_internal.h"
1097 #include "firehose/firehose_internal.h"
1098
1099 #endif /* __DISPATCH_INTERNAL__ */