src/internal.h

   1 /*
   2  * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_INTERNAL__
  28 #define __DISPATCH_INTERNAL__
  29
  30 #if __has_include(<config/config_ac.h>)
  31 #include <config/config_ac.h>
  32 #else
  33 #include <config/config.h>
  34 #endif
  35
  36 #define __DISPATCH_BUILDING_DISPATCH__
  37 #define __DISPATCH_INDIRECT__
  38
  39 #ifdef __APPLE__
  40 #include <Availability.h>
  41 #include <TargetConditionals.h>
  42
  43 #ifndef TARGET_OS_MAC_DESKTOP
  44 #define TARGET_OS_MAC_DESKTOP  (TARGET_OS_MAC && \
  45                 !TARGET_OS_SIMULATOR && !TARGET_OS_IPHONE && !TARGET_OS_EMBEDDED)
  46 #endif
  47
  48 #if TARGET_OS_MAC_DESKTOP
  49 #  define DISPATCH_HOST_SUPPORTS_OSX(x) \
  50                 (__MAC_OS_X_VERSION_MIN_REQUIRED >= (x))
  51 #  if !DISPATCH_HOST_SUPPORTS_OSX(101000)
  52 #    error "OS X hosts older than OS X 10.10 aren't supported anymore"
  53 #  endif // !DISPATCH_HOST_SUPPORTS_OSX(101000)
  54 #elif TARGET_OS_SIMULATOR
  55 #  define DISPATCH_HOST_SUPPORTS_OSX(x) \
  56                 (IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED >= (x))
  57 #  if !DISPATCH_HOST_SUPPORTS_OSX(101000)
  58 #    error "Simulator hosts older than OS X 10.10 aren't supported anymore"
  59 #  endif // !DISPATCH_HOST_SUPPORTS_OSX(101000)
  60 #else
  61 #  define DISPATCH_HOST_SUPPORTS_OSX(x) 1
  62 #  if __IPHONE_OS_VERSION_MIN_REQUIRED < 70000
  63 #    error "iOS hosts older than iOS 7.0 aren't supported anymore"
  64 #  endif
  65 #endif
  66
  67 #else // !__APPLE__
  68 #define DISPATCH_HOST_SUPPORTS_OSX(x) 0
  69 #endif // !__APPLE__
  70
  71
  72 #if !defined(DISPATCH_MACH_SPI) && TARGET_OS_MAC
  73 #define DISPATCH_MACH_SPI 1
  74 #endif
  75 #if !defined(OS_VOUCHER_CREATION_SPI) && TARGET_OS_MAC
  76 #define OS_VOUCHER_CREATION_SPI 1
  77 #endif
  78 #if !defined(OS_VOUCHER_ACTIVITY_SPI) && TARGET_OS_MAC
  79 #define OS_VOUCHER_ACTIVITY_SPI 1
  80 #endif
  81 #if !defined(OS_VOUCHER_ACTIVITY_GENERATE_SWAPS)
  82 #define OS_VOUCHER_ACTIVITY_GENERATE_SWAPS 0
  83 #endif
  84 #if !defined(OS_FIREHOSE_SPI) && TARGET_OS_MAC
  85 #define OS_FIREHOSE_SPI 1
  86 #endif
  87 #if !defined(DISPATCH_LAYOUT_SPI) && TARGET_OS_MAC
  88 #define DISPATCH_LAYOUT_SPI 1
  89 #endif
  90
  91 #if __has_include(<mach-o/dyld_priv.h>)
  92 #include <mach-o/dyld_priv.h>
  93 #if !defined(HAVE_DYLD_IS_MEMORY_IMMUTABLE)
  94 #if defined(DYLD_MACOSX_VERSION_10_12) || defined(DYLD_IOS_VERSION_10_0)
  95 #define HAVE_DYLD_IS_MEMORY_IMMUTABLE 1
  96 #else
  97 #define HAVE_DYLD_IS_MEMORY_IMMUTABLE 0
  98 #endif
  99 #endif // !defined(HAVE_DYLD_IS_MEMORY_IMMUTABLE)
 100 #endif // __has_include(<mach-o/dyld_priv.h>)
 101
 102 #if !defined(USE_OBJC) && HAVE_OBJC
 103 #define USE_OBJC 1
 104 #endif
 105
 106 #if USE_OBJC
 107 #define OS_OBJECT_HAVE_OBJC_SUPPORT 1
 108 #if defined(__OBJC__)
 109 #define OS_OBJECT_USE_OBJC 1
 110 // Force internal Objective-C sources to use class-visible headers
 111 // even when not compiling in Swift.
 112 #define OS_OBJECT_SWIFT3 1
 113 #else
 114 #define OS_OBJECT_USE_OBJC 0
 115 #endif // __OBJC__
 116 #else
 117 #define OS_OBJECT_HAVE_OBJC_SUPPORT 0
 118 #endif // USE_OBJC
 119
 120 #include <dispatch/dispatch.h>
 121 #include <dispatch/base.h>
 122
 123 #define __DISPATCH_HIDE_SYMBOL(sym, version) \
 124         __asm__(".section __TEXT,__const\n\t" \
 125                         ".globl $ld$hide$os" #version "$_" #sym "\n\t" \
 126                         "$ld$hide$os" #version "$_" #sym ":\n\t" \
 127                         "    .byte 0\n\t" \
 128                         ".previous")
 129
 130
 131 #ifndef DISPATCH_HIDE_SYMBOL
 132 #if TARGET_OS_MAC && !TARGET_OS_IPHONE
 133 #define DISPATCH_HIDE_SYMBOL(sym, osx, ios, tvos, watchos) \
 134                 __DISPATCH_HIDE_SYMBOL(sym, osx)
 135 #else
 136 #define DISPATCH_HIDE_SYMBOL(sym, osx, ios, tvos, watchos)
 137 #endif
 138 #endif
 139
 140 #include <os/object.h>
 141 #include <dispatch/time.h>
 142 #include <dispatch/object.h>
 143 #include <dispatch/queue.h>
 144 #include <dispatch/block.h>
 145 #include <dispatch/source.h>
 146 #include <dispatch/group.h>
 147 #include <dispatch/semaphore.h>
 148 #include <dispatch/once.h>
 149 #include <dispatch/data.h>
 150 #if !TARGET_OS_WIN32
 151 #include <dispatch/io.h>
 152 #endif
 153
 154 #if defined(__OBJC__) || defined(__cplusplus)
 155 #define DISPATCH_PURE_C 0
 156 #else
 157 #define DISPATCH_PURE_C 1
 158 #endif
 159
 160 /* private.h must be included last to avoid picking up installed headers. */
 161 #include "os/object_private.h"
 162 #include "queue_private.h"
 163 #include "source_private.h"
 164 #include "mach_private.h"
 165 #include "data_private.h"
 166 #include "os/voucher_private.h"
 167 #include "os/voucher_activity_private.h"
 168 #if !TARGET_OS_WIN32
 169 #include "io_private.h"
 170 #endif
 171 #include "layout_private.h"
 172 #include "benchmark.h"
 173 #include "private.h"
 174
 175 /* SPI for Libsystem-internal use */
 176 DISPATCH_EXPORT DISPATCH_NOTHROW void libdispatch_init(void);
 177 #if !TARGET_OS_WIN32
 178 DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_prepare(void);
 179 DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_parent(void);
 180 DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void);
 181 #endif
 182
 183 /* More #includes at EOF (dependent on the contents of internal.h) ... */
 184
 185 // Abort on uncaught exceptions thrown from client callouts rdar://8577499
 186 #if !defined(DISPATCH_USE_CLIENT_CALLOUT)
 187 #define DISPATCH_USE_CLIENT_CALLOUT 1
 188 #endif
 189
 190 /* The "_debug" library build */
 191 #ifndef DISPATCH_DEBUG
 192 #define DISPATCH_DEBUG 0
 193 #endif
 194
 195 #ifndef DISPATCH_PROFILE
 196 #define DISPATCH_PROFILE 0
 197 #endif
 198
 199 #if (!TARGET_OS_EMBEDDED || DISPATCH_DEBUG || DISPATCH_PROFILE) && \
 200                 !defined(DISPATCH_USE_DTRACE)
 201 #define DISPATCH_USE_DTRACE 1
 202 #endif
 203
 204 #if DISPATCH_USE_DTRACE && (DISPATCH_INTROSPECTION || DISPATCH_DEBUG || \
 205                 DISPATCH_PROFILE) && !defined(DISPATCH_USE_DTRACE_INTROSPECTION)
 206 #define DISPATCH_USE_DTRACE_INTROSPECTION 1
 207 #endif
 208
 209 #ifndef DISPATCH_DEBUG_QOS
 210 #define DISPATCH_DEBUG_QOS DISPATCH_DEBUG
 211 #endif
 212
 213 #if HAVE_LIBKERN_OSCROSSENDIAN_H
 214 #include <libkern/OSCrossEndian.h>
 215 #endif
 216 #if HAVE_LIBKERN_OSATOMIC_H
 217 #include <libkern/OSAtomic.h>
 218 #endif
 219 #if HAVE_MACH
 220 #include <mach/boolean.h>
 221 #include <mach/clock_types.h>
 222 #include <mach/clock.h>
 223 #include <mach/exception.h>
 224 #include <mach/mach.h>
 225 #include <mach/mach_error.h>
 226 #include <mach/mach_host.h>
 227 #include <mach/mach_interface.h>
 228 #include <mach/mach_time.h>
 229 #include <mach/mach_traps.h>
 230 #include <mach/message.h>
 231 #include <mach/mig_errors.h>
 232 #include <mach/host_special_ports.h>
 233 #include <mach/host_info.h>
 234 #include <mach/notify.h>
 235 #include <mach/mach_vm.h>
 236 #include <mach/vm_map.h>
 237 #endif /* HAVE_MACH */
 238 #if HAVE_MALLOC_MALLOC_H
 239 #include <malloc/malloc.h>
 240 #endif
 241 #if __has_include(<malloc_private.h>)
 242 #include <malloc_private.h>
 243 #endif // __has_include(<malloc_private.h)
 244
 245 #include <sys/stat.h>
 246
 247 #if !TARGET_OS_WIN32
 248 #include <sys/event.h>
 249 #include <sys/mount.h>
 250 #include <sys/queue.h>
 251 #include <sys/sysctl.h>
 252 #include <sys/socket.h>
 253 #include <sys/time.h>
 254 #include <sys/mman.h>
 255 #include <netinet/in.h>
 256 #endif
 257 #if defined(__linux__)
 258 #include <sys/eventfd.h>
 259 #endif
 260
 261 #ifdef __BLOCKS__
 262 #include <Block_private.h>
 263 #include <Block.h>
 264 #endif /* __BLOCKS__ */
 265
 266 #include <assert.h>
 267 #include <errno.h>
 268 #if HAVE_FCNTL_H
 269 #include <fcntl.h>
 270 #endif
 271 #include <limits.h>
 272 #include <search.h>
 273 #if USE_POSIX_SEM
 274 #include <semaphore.h>
 275 #endif
 276 #include <signal.h>
 277 #include <stdarg.h>
 278 #include <stdbool.h>
 279 #include <stdint.h>
 280 #include <stdio.h>
 281 #include <stdlib.h>
 282 #include <string.h>
 283 #if HAVE_UNISTD_H
 284 #include <unistd.h>
 285 #endif
 286
 287 #if __GNUC__
 288 #define DISPATCH_NOINLINE __attribute__((__noinline__))
 289 #define DISPATCH_USED __attribute__((__used__))
 290 #define DISPATCH_UNUSED __attribute__((__unused__))
 291 #define DISPATCH_WEAK __attribute__((__weak__))
 292 #define DISPATCH_OVERLOADABLE __attribute__((__overloadable__))
 293 #define DISPATCH_PACKED __attribute__((__packed__))
 294 #if DISPATCH_DEBUG
 295 #define DISPATCH_ALWAYS_INLINE_NDEBUG
 296 #else
 297 #define DISPATCH_ALWAYS_INLINE_NDEBUG __attribute__((__always_inline__))
 298 #endif
 299 #else   /* __GNUC__ */
 300 #define DISPATCH_NOINLINE
 301 #define DISPATCH_USED
 302 #define DISPATCH_UNUSED
 303 #define DISPATCH_WEAK
 304 #define DISPATCH_ALWAYS_INLINE_NDEBUG
 305 #endif  /* __GNUC__ */
 306
 307 #define DISPATCH_CONCAT(x,y) DISPATCH_CONCAT1(x,y)
 308 #define DISPATCH_CONCAT1(x,y) x ## y
 309
 310 // workaround 6368156
 311 #ifdef NSEC_PER_SEC
 312 #undef NSEC_PER_SEC
 313 #endif
 314 #ifdef USEC_PER_SEC
 315 #undef USEC_PER_SEC
 316 #endif
 317 #ifdef NSEC_PER_USEC
 318 #undef NSEC_PER_USEC
 319 #endif
 320 #define NSEC_PER_SEC 1000000000ull
 321 #define USEC_PER_SEC 1000000ull
 322 #define NSEC_PER_USEC 1000ull
 323
 324 /* I wish we had __builtin_expect_range() */
 325 #if __GNUC__
 326 #define _safe_cast_to_long(x) \
 327                 ({ _Static_assert(sizeof(typeof(x)) <= sizeof(long), \
 328                                 "__builtin_expect doesn't support types wider than long"); \
 329                                 (long)(x); })
 330 #define fastpath(x) ((typeof(x))__builtin_expect(_safe_cast_to_long(x), ~0l))
 331 #define slowpath(x) ((typeof(x))__builtin_expect(_safe_cast_to_long(x), 0l))
 332 #define likely(x) __builtin_expect(!!(x), 1)
 333 #define unlikely(x) __builtin_expect(!!(x), 0)
 334 #else
 335 #define fastpath(x) (x)
 336 #define slowpath(x) (x)
 337 #define likely(x) (!!(x))
 338 #define unlikely(x) (!!(x))
 339 #endif // __GNUC__
 340
 341 #if BYTE_ORDER == LITTLE_ENDIAN
 342 #define DISPATCH_STRUCT_LITTLE_ENDIAN_2(a, b)        struct { a; b; }
 343 #define DISPATCH_STRUCT_LITTLE_ENDIAN_3(a, b, c)     struct { a; b; c; }
 344 #define DISPATCH_STRUCT_LITTLE_ENDIAN_4(a, b, c, d)  struct { a; b; c; d; }
 345 #else
 346 #define DISPATCH_STRUCT_LITTLE_ENDIAN_2(a, b)        struct { b; a; }
 347 #define DISPATCH_STRUCT_LITTLE_ENDIAN_3(a, b, c)     struct { c; b; a; }
 348 #define DISPATCH_STRUCT_LITTLE_ENDIAN_4(a, b, c, d)  struct { d; c; b; a; }
 349 #endif
 350
 351 #define _TAILQ_IS_ENQUEUED(elm, field) \
 352                 ((elm)->field.tqe_prev != NULL)
 353 #define _TAILQ_MARK_NOT_ENQUEUED(elm, field) \
 354                 do { (elm)->field.tqe_prev = NULL; } while (0)
 355
 356 #if DISPATCH_DEBUG
 357 // sys/queue.h debugging
 358 #if defined(__linux__)
 359 #define QUEUE_MACRO_DEBUG 1
 360 #else
 361 #undef TRASHIT
 362 #define TRASHIT(x) do {(x) = (void *)-1;} while (0)
 363 #endif
 364 #endif // DISPATCH_DEBUG
 365 #define _TAILQ_TRASH_ENTRY(elm, field) do { \
 366                         TRASHIT((elm)->field.tqe_next); \
 367                         TRASHIT((elm)->field.tqe_prev); \
 368                 } while (0)
 369 #define _TAILQ_TRASH_HEAD(head) do { \
 370                         TRASHIT((head)->tqh_first); \
 371                         TRASHIT((head)->tqh_last); \
 372                 } while (0)
 373
 374 DISPATCH_EXPORT DISPATCH_NOINLINE
 375 void _dispatch_bug(size_t line, long val);
 376
 377 #if HAVE_MACH
 378 DISPATCH_NOINLINE
 379 void _dispatch_bug_client(const char* msg);
 380 DISPATCH_NOINLINE
 381 void _dispatch_bug_mach_client(const char *msg, mach_msg_return_t kr);
 382 #endif // HAVE_MACH
 383 DISPATCH_NOINLINE
 384 void _dispatch_bug_kevent_client(const char* msg, const char* filter,
 385                 const char *operation, int err);
 386
 387 DISPATCH_NOINLINE
 388 void _dispatch_bug_deprecated(const char *msg);
 389
 390 DISPATCH_NOINLINE DISPATCH_NORETURN
 391 void _dispatch_abort(size_t line, long val);
 392
 393 #if !defined(DISPATCH_USE_OS_DEBUG_LOG) && DISPATCH_DEBUG
 394 #if __has_include(<os/debug_private.h>)
 395 #define DISPATCH_USE_OS_DEBUG_LOG 1
 396 #include <os/debug_private.h>
 397 #endif
 398 #endif // DISPATCH_USE_OS_DEBUG_LOG
 399
 400 #if !defined(DISPATCH_USE_SIMPLE_ASL) && !DISPATCH_USE_OS_DEBUG_LOG
 401 #if __has_include(<_simple.h>)
 402 #define DISPATCH_USE_SIMPLE_ASL 1
 403 #include <_simple.h>
 404 #endif
 405 #endif // DISPATCH_USE_SIMPLE_ASL
 406
 407 #if !DISPATCH_USE_SIMPLE_ASL && !DISPATCH_USE_OS_DEBUG_LOG && !TARGET_OS_WIN32
 408 #include <syslog.h>
 409 #endif
 410
 411 #if DISPATCH_USE_OS_DEBUG_LOG
 412 #define _dispatch_log(msg, ...) os_debug_log("libdispatch", msg, ## __VA_ARGS__)
 413 #else
 414 DISPATCH_EXPORT DISPATCH_NOINLINE __attribute__((__format__(__printf__,1,2)))
 415 void _dispatch_log(const char *msg, ...);
 416 #endif // DISPATCH_USE_OS_DEBUG_LOG
 417
 418 #define dsnprintf(buf, siz, ...) \
 419                 ({ size_t _siz = siz; int _r = snprintf(buf, _siz, __VA_ARGS__); \
 420                  _r < 0 ? 0u : ((size_t)_r > _siz ? _siz : (size_t)_r); })
 421
 422 #if __GNUC__
 423 #define dispatch_static_assert(e) ({ \
 424                 char __compile_time_assert__[(bool)(e) ? 1 : -1] DISPATCH_UNUSED; \
 425         })
 426 #else
 427 #define dispatch_static_assert(e)
 428 #endif
 429
 430 #define DISPATCH_BAD_INPUT              ((void *_Nonnull)0)
 431 #define DISPATCH_OUT_OF_MEMORY  ((void *_Nonnull)0)
 432
 433 /*
 434  * For reporting bugs within libdispatch when using the "_debug" version of the
 435  * library.
 436  */
 437 #if __GNUC__
 438 #define dispatch_assert(e) do { \
 439                 if (__builtin_constant_p(e)) { \
 440                         dispatch_static_assert(e); \
 441                 } else { \
 442                         typeof(e) _e = fastpath(e); /* always eval 'e' */ \
 443                         if (DISPATCH_DEBUG && !_e) { \
 444                                 _dispatch_abort(__LINE__, (long)_e); \
 445                         } \
 446                 } \
 447         } while (0)
 448 #else
 449 static inline void _dispatch_assert(long e, long line) {
 450         if (DISPATCH_DEBUG && !e) _dispatch_abort(line, e);
 451 }
 452 #define dispatch_assert(e) _dispatch_assert((long)(e), __LINE__)
 453 #endif  /* __GNUC__ */
 454
 455 #if __GNUC__
 456 /*
 457  * A lot of API return zero upon success and not-zero on fail. Let's capture
 458  * and log the non-zero value
 459  */
 460 #define dispatch_assert_zero(e) do { \
 461                 if (__builtin_constant_p(e)) { \
 462                         dispatch_static_assert(e); \
 463                 } else { \
 464                         typeof(e) _e = slowpath(e); /* always eval 'e' */ \
 465                         if (DISPATCH_DEBUG && _e) { \
 466                                 _dispatch_abort(__LINE__, (long)_e); \
 467                         } \
 468                 } \
 469         } while (0)
 470 #else
 471 static inline void _dispatch_assert_zero(long e, long line) {
 472         if (DISPATCH_DEBUG && e) _dispatch_abort(line, e);
 473 }
 474 #define dispatch_assert_zero(e) _dispatch_assert((long)(e), __LINE__)
 475 #endif  /* __GNUC__ */
 476
 477 /*
 478  * For reporting bugs or impedance mismatches between libdispatch and external
 479  * subsystems. These do NOT abort(), and are always compiled into the product.
 480  *
 481  * In particular, we wrap all system-calls with assume() macros.
 482  */
 483 #if __GNUC__
 484 #define dispatch_assume(e) ({ \
 485                 typeof(e) _e = fastpath(e); /* always eval 'e' */ \
 486                 if (!_e) { \
 487                         if (__builtin_constant_p(e)) { \
 488                                 dispatch_static_assert(e); \
 489                         } \
 490                         _dispatch_bug(__LINE__, (long)_e); \
 491                 } \
 492                 _e; \
 493         })
 494 #else
 495 static inline long _dispatch_assume(long e, long line) {
 496         if (!e) _dispatch_bug(line, e);
 497         return e;
 498 }
 499 #define dispatch_assume(e) _dispatch_assume((long)(e), __LINE__)
 500 #endif  /* __GNUC__ */
 501
 502 /*
 503  * A lot of API return zero upon success and not-zero on fail. Let's capture
 504  * and log the non-zero value
 505  */
 506 #if __GNUC__
 507 #define dispatch_assume_zero(e) ({ \
 508                 typeof(e) _e = slowpath(e); /* always eval 'e' */ \
 509                 if (_e) { \
 510                         if (__builtin_constant_p(e)) { \
 511                                 dispatch_static_assert(e); \
 512                         } \
 513                         _dispatch_bug(__LINE__, (long)_e); \
 514                 } \
 515                 _e; \
 516         })
 517 #else
 518 static inline long _dispatch_assume_zero(long e, long line) {
 519         if (e) _dispatch_bug(line, e);
 520         return e;
 521 }
 522 #define dispatch_assume_zero(e) _dispatch_assume_zero((long)(e), __LINE__)
 523 #endif  /* __GNUC__ */
 524
 525 /*
 526  * For reporting bugs in clients when using the "_debug" version of the library.
 527  */
 528 #if __GNUC__
 529 #define dispatch_debug_assert(e, msg, args...) do { \
 530                 if (__builtin_constant_p(e)) { \
 531                         dispatch_static_assert(e); \
 532                 } else { \
 533                         typeof(e) _e = fastpath(e); /* always eval 'e' */ \
 534                         if (DISPATCH_DEBUG && !_e) { \
 535                                 _dispatch_log("%s() 0x%lx: " msg, __func__, (long)_e, ##args); \
 536                                 abort(); \
 537                         } \
 538                 } \
 539         } while (0)
 540 #else
 541 #define dispatch_debug_assert(e, msg, args...) do { \
 542         long _e = (long)fastpath(e); /* always eval 'e' */ \
 543         if (DISPATCH_DEBUG && !_e) { \
 544                 _dispatch_log("%s() 0x%lx: " msg, __FUNCTION__, _e, ##args); \
 545                 abort(); \
 546         } \
 547 } while (0)
 548 #endif  /* __GNUC__ */
 549
 550 /* Make sure the debug statments don't get too stale */
 551 #define _dispatch_debug(x, args...) do { \
 552         if (DISPATCH_DEBUG) { \
 553                 _dispatch_log("%u\t%p\t" x, __LINE__, \
 554                                 (void *)_dispatch_thread_self(), ##args); \
 555         } \
 556 } while (0)
 557
 558 #if DISPATCH_DEBUG
 559 #if HAVE_MACH
 560 DISPATCH_NOINLINE DISPATCH_USED
 561 void dispatch_debug_machport(mach_port_t name, const char* str);
 562 #endif
 563 #endif
 564
 565 #if DISPATCH_DEBUG
 566 /* This is the private version of the deprecated dispatch_debug() */
 567 DISPATCH_NONNULL2 DISPATCH_NOTHROW
 568 __attribute__((__format__(printf,2,3)))
 569 void
 570 _dispatch_object_debug(dispatch_object_t object, const char *message, ...);
 571 #else
 572 #define _dispatch_object_debug(object, message, ...)
 573 #endif // DISPATCH_DEBUG
 574
 575 #ifdef __BLOCKS__
 576 #define _dispatch_Block_invoke(bb) \
 577                 ((dispatch_function_t)((struct Block_layout *)bb)->invoke)
 578 void *_dispatch_Block_copy(void *block);
 579 #if __GNUC__
 580 #define _dispatch_Block_copy(x) ((typeof(x))_dispatch_Block_copy(x))
 581 #endif
 582 void _dispatch_call_block_and_release(void *block);
 583 #endif /* __BLOCKS__ */
 584
 585 void _dispatch_temporary_resource_shortage(void);
 586 void *_dispatch_calloc(size_t num_items, size_t size);
 587 const char *_dispatch_strdup_if_mutable(const char *str);
 588 void _dispatch_vtable_init(void);
 589 char *_dispatch_get_build(void);
 590
 591 uint64_t _dispatch_timeout(dispatch_time_t when);
 592 uint64_t _dispatch_time_nanoseconds_since_epoch(dispatch_time_t when);
 593
 594 #define _DISPATCH_UNSAFE_FORK_MULTITHREADED  ((uint8_t)1)
 595 #define _DISPATCH_UNSAFE_FORK_PROHIBIT       ((uint8_t)2)
 596 extern uint8_t _dispatch_unsafe_fork;
 597 extern bool _dispatch_child_of_unsafe_fork;
 598 void _dispatch_fork_becomes_unsafe_slow(void);
 599
 600 #define _dispatch_is_multithreaded_inline() \
 601         ((_dispatch_unsafe_fork & _DISPATCH_UNSAFE_FORK_MULTITHREADED) != 0)
 602
 603 DISPATCH_ALWAYS_INLINE
 604 static inline void
 605 _dispatch_fork_becomes_unsafe(void)
 606 {
 607         if (!fastpath(_dispatch_is_multithreaded_inline())) {
 608                 _dispatch_fork_becomes_unsafe_slow();
 609                 DISPATCH_COMPILER_CAN_ASSUME(_dispatch_is_multithreaded_inline());
 610         }
 611 }
 612
 613 /* #includes dependent on internal.h */
 614 #include "shims.h"
 615
 616 // Older Mac OS X and iOS Simulator fallbacks
 617
 618 #if HAVE_PTHREAD_WORKQUEUES
 619 #ifndef WORKQ_ADDTHREADS_OPTION_OVERCOMMIT
 620 #define WORKQ_ADDTHREADS_OPTION_OVERCOMMIT 0x00000001
 621 #endif
 622 #endif // HAVE_PTHREAD_WORKQUEUES
 623 #if HAVE__PTHREAD_WORKQUEUE_INIT && PTHREAD_WORKQUEUE_SPI_VERSION >= 20140213 \
 624                 && !defined(HAVE_PTHREAD_WORKQUEUE_QOS)
 625 #define HAVE_PTHREAD_WORKQUEUE_QOS 1
 626 #endif
 627 #if HAVE__PTHREAD_WORKQUEUE_INIT && (PTHREAD_WORKQUEUE_SPI_VERSION >= 20150304 \
 628                 || (PTHREAD_WORKQUEUE_SPI_VERSION == 20140730 && \
 629                         defined(WORKQ_FEATURE_KEVENT))) \
 630                 && !defined(HAVE_PTHREAD_WORKQUEUE_KEVENT)
 631 #if PTHREAD_WORKQUEUE_SPI_VERSION == 20140730
 632 // rdar://problem/20609877
 633 typedef pthread_worqueue_function_kevent_t pthread_workqueue_function_kevent_t;
 634 #endif
 635 #define HAVE_PTHREAD_WORKQUEUE_KEVENT 1
 636 #endif
 637
 638 #ifndef PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK
 639 #if HAVE_PTHREAD_WORKQUEUE_QOS && DISPATCH_HOST_SUPPORTS_OSX(101200)
 640 #define PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK 1
 641 #else
 642 #define PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK 0
 643 #endif
 644 #endif // PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK
 645
 646 #if HAVE_MACH
 647 #if !defined(MACH_NOTIFY_SEND_POSSIBLE)
 648 #undef MACH_NOTIFY_SEND_POSSIBLE
 649 #define MACH_NOTIFY_SEND_POSSIBLE MACH_NOTIFY_DEAD_NAME
 650 #endif
 651 #endif // HAVE_MACH
 652
 653 #ifdef EVFILT_MEMORYSTATUS
 654 #ifndef DISPATCH_USE_MEMORYSTATUS
 655 #define DISPATCH_USE_MEMORYSTATUS 1
 656 #endif
 657 #endif // EVFILT_MEMORYSTATUS
 658
 659 #if TARGET_OS_SIMULATOR
 660 #undef DISPATCH_USE_MEMORYPRESSURE_SOURCE
 661 #define DISPATCH_USE_MEMORYPRESSURE_SOURCE 0
 662 #endif // TARGET_OS_SIMULATOR
 663 #if !defined(DISPATCH_USE_MEMORYPRESSURE_SOURCE) && DISPATCH_USE_MEMORYSTATUS
 664 #define DISPATCH_USE_MEMORYPRESSURE_SOURCE 1
 665 #endif
 666 #if DISPATCH_USE_MEMORYPRESSURE_SOURCE
 667 extern bool _dispatch_memory_warn;
 668 #endif
 669
 670 #if !defined(NOTE_LEEWAY)
 671 #undef NOTE_LEEWAY
 672 #define NOTE_LEEWAY 0
 673 #undef NOTE_CRITICAL
 674 #define NOTE_CRITICAL 0
 675 #undef NOTE_BACKGROUND
 676 #define NOTE_BACKGROUND 0
 677 #endif // NOTE_LEEWAY
 678
 679 #if !defined(NOTE_FUNLOCK)
 680 #define NOTE_FUNLOCK 0x00000100
 681 #endif
 682
 683 #if !defined(NOTE_MACH_CONTINUOUS_TIME)
 684 #define NOTE_MACH_CONTINUOUS_TIME 0
 685 #endif // NOTE_MACH_CONTINUOUS_TIME
 686
 687 #if !defined(HOST_NOTIFY_CALENDAR_SET)
 688 #define HOST_NOTIFY_CALENDAR_SET HOST_NOTIFY_CALENDAR_CHANGE
 689 #endif // HOST_NOTIFY_CALENDAR_SET
 690
 691 #if !defined(HOST_CALENDAR_SET_REPLYID)
 692 #define HOST_CALENDAR_SET_REPLYID 951
 693 #endif // HOST_CALENDAR_SET_REPLYID
 694
 695 #if HAVE_DECL_NOTE_REAP
 696 #if defined(NOTE_REAP) && defined(__APPLE__)
 697 #undef NOTE_REAP
 698 #define NOTE_REAP 0x10000000 // <rdar://problem/13338526>
 699 #endif
 700 #endif // HAVE_DECL_NOTE_REAP
 701
 702 #ifndef VQ_QUOTA
 703 #undef HAVE_DECL_VQ_QUOTA // rdar://problem/24160982
 704 #endif // VQ_QUOTA
 705
 706 #ifndef VQ_NEARLOWDISK
 707 #undef HAVE_DECL_VQ_NEARLOWDISK
 708 #endif // VQ_NEARLOWDISK
 709
 710 #ifndef VQ_DESIRED_DISK
 711 #undef HAVE_DECL_VQ_DESIRED_DISK
 712 #endif // VQ_DESIRED_DISK
 713
 714 #if !defined(NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) || \
 715                 !DISPATCH_HOST_SUPPORTS_OSX(101200)
 716 #undef NOTE_MEMORYSTATUS_PROC_LIMIT_WARN
 717 #define NOTE_MEMORYSTATUS_PROC_LIMIT_WARN 0
 718 #endif // NOTE_MEMORYSTATUS_PROC_LIMIT_WARN
 719
 720 #if !defined(NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) || \
 721                 !DISPATCH_HOST_SUPPORTS_OSX(101200)
 722 #undef NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL
 723 #define NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL 0
 724 #endif // NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL
 725
 726 #if !defined(EV_UDATA_SPECIFIC) || !DISPATCH_HOST_SUPPORTS_OSX(101100)
 727 #undef DISPATCH_USE_EV_UDATA_SPECIFIC
 728 #define DISPATCH_USE_EV_UDATA_SPECIFIC 0
 729 #elif !defined(DISPATCH_USE_EV_UDATA_SPECIFIC)
 730 #define DISPATCH_USE_EV_UDATA_SPECIFIC 1
 731 #endif // EV_UDATA_SPECIFIC
 732
 733 #if !DISPATCH_USE_EV_UDATA_SPECIFIC
 734 #undef EV_UDATA_SPECIFIC
 735 #define EV_UDATA_SPECIFIC 0
 736 #undef EV_VANISHED
 737 #define EV_VANISHED 0
 738 #endif // !DISPATCH_USE_EV_UDATA_SPECIFIC
 739
 740 #ifndef EV_VANISHED
 741 #define EV_VANISHED 0x0200
 742 #endif
 743
 744 #ifndef DISPATCH_KEVENT_TREAT_ENOENT_AS_EINPROGRESS
 745 #if TARGET_OS_MAC && !DISPATCH_HOST_SUPPORTS_OSX(101200)
 746 // deferred delete can return bogus ENOENTs on older kernels
 747 #define DISPATCH_KEVENT_TREAT_ENOENT_AS_EINPROGRESS 1
 748 #else
 749 #define DISPATCH_KEVENT_TREAT_ENOENT_AS_EINPROGRESS 0
 750 #endif
 751 #endif
 752
 753 #if !defined(EV_SET_QOS) || !DISPATCH_HOST_SUPPORTS_OSX(101100)
 754 #undef DISPATCH_USE_KEVENT_QOS
 755 #define DISPATCH_USE_KEVENT_QOS 0
 756 #elif !defined(DISPATCH_USE_KEVENT_QOS)
 757 #define DISPATCH_USE_KEVENT_QOS 1
 758 #endif // EV_SET_QOS
 759
 760 #if HAVE_PTHREAD_WORKQUEUE_KEVENT && defined(KEVENT_FLAG_WORKQ) && \
 761                 DISPATCH_USE_EV_UDATA_SPECIFIC && DISPATCH_USE_KEVENT_QOS && \
 762                 DISPATCH_HOST_SUPPORTS_OSX(101200) && \
 763                 !defined(DISPATCH_USE_KEVENT_WORKQUEUE)
 764 #define DISPATCH_USE_KEVENT_WORKQUEUE 1
 765 #endif
 766
 767
 768 #if (!DISPATCH_USE_KEVENT_WORKQUEUE || DISPATCH_DEBUG) && \
 769                 !defined(DISPATCH_USE_MGR_THREAD)
 770 #define DISPATCH_USE_MGR_THREAD 1
 771 #endif
 772
 773 #if DISPATCH_USE_KEVENT_WORKQUEUE && DISPATCH_USE_EV_UDATA_SPECIFIC && \
 774                 DISPATCH_HOST_SUPPORTS_OSX(101200) && \
 775                 !defined(DISPATCH_USE_EVFILT_MACHPORT_DIRECT)
 776 #define DISPATCH_USE_EVFILT_MACHPORT_DIRECT 1
 777 #endif
 778
 779 #ifndef MACH_SEND_OVERRIDE
 780 #define MACH_SEND_OVERRIDE 0x00000020
 781 typedef unsigned int mach_msg_priority_t;
 782 #define MACH_MSG_PRIORITY_UNSPECIFIED ((mach_msg_priority_t)0)
 783 #endif // MACH_SEND_OVERRIDE
 784
 785
 786 #if (!DISPATCH_USE_EVFILT_MACHPORT_DIRECT || DISPATCH_DEBUG) && \
 787                 !defined(DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK)
 788 #define DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK 1
 789 #endif
 790
 791 #if DISPATCH_USE_KEVENT_QOS
 792 typedef struct kevent_qos_s _dispatch_kevent_qos_s;
 793 typedef typeof(((struct kevent_qos_s*)NULL)->qos) _dispatch_kevent_priority_t;
 794 #else // DISPATCH_USE_KEVENT_QOS
 795 #ifndef KEVENT_FLAG_IMMEDIATE
 796 #define KEVENT_FLAG_NONE 0x00
 797 #define KEVENT_FLAG_IMMEDIATE 0x01
 798 #define KEVENT_FLAG_ERROR_EVENTS 0x02
 799 #endif // KEVENT_FLAG_IMMEDIATE
 800 typedef struct kevent64_s _dispatch_kevent_qos_s;
 801 #define kevent_qos(_kq, _changelist, _nchanges, _eventlist, _nevents, \
 802                 _data_out, _data_available, _flags) \
 803                 ({ unsigned int _f = (_flags); _dispatch_kevent_qos_s _kev_copy; \
 804                 const _dispatch_kevent_qos_s *_cl = (_changelist); \
 805                 int _n = (_nchanges); const struct timespec _timeout_immediately = {}; \
 806                 dispatch_static_assert(!(_data_out) && !(_data_available)); \
 807                 if (_f & KEVENT_FLAG_ERROR_EVENTS) { \
 808                         dispatch_static_assert(_n == 1); \
 809                         _kev_copy = *_cl; _kev_copy.flags |= EV_RECEIPT; } \
 810                 kevent64((_kq), _f & KEVENT_FLAG_ERROR_EVENTS ? &_kev_copy : _cl, _n, \
 811                         (_eventlist), (_nevents), 0, \
 812                         _f & KEVENT_FLAG_IMMEDIATE ? &_timeout_immediately : NULL); })
 813 #endif // DISPATCH_USE_KEVENT_QOS
 814
 815 #if defined(F_SETNOSIGPIPE) && defined(F_GETNOSIGPIPE)
 816 #ifndef DISPATCH_USE_SETNOSIGPIPE
 817 #define DISPATCH_USE_SETNOSIGPIPE 1
 818 #endif
 819 #endif // F_SETNOSIGPIPE
 820
 821 #if defined(MACH_SEND_NOIMPORTANCE)
 822 #ifndef DISPATCH_USE_CHECKIN_NOIMPORTANCE
 823 #define DISPATCH_USE_CHECKIN_NOIMPORTANCE 1 // rdar://problem/16996737
 824 #endif
 825 #ifndef DISPATCH_USE_NOIMPORTANCE_QOS
 826 #define DISPATCH_USE_NOIMPORTANCE_QOS 1 // rdar://problem/21414476
 827 #endif
 828 #endif // MACH_SEND_NOIMPORTANCE
 829
 830
 831 #if HAVE_LIBPROC_INTERNAL_H
 832 #include <libproc.h>
 833 #include <libproc_internal.h>
 834 #ifndef DISPATCH_USE_IMPORTANCE_ASSERTION
 835 #define DISPATCH_USE_IMPORTANCE_ASSERTION 1
 836 #endif
 837 #endif // HAVE_LIBPROC_INTERNAL_H
 838
 839 #if HAVE_SYS_GUARDED_H
 840 #include <sys/guarded.h>
 841 #ifndef DISPATCH_USE_GUARDED_FD
 842 #define DISPATCH_USE_GUARDED_FD 1
 843 #endif
 844 #endif // HAVE_SYS_GUARDED_H
 845
 846
 847 #if __has_include(<sys/kdebug.h>)
 848 #include <sys/kdebug.h>
 849 #ifndef DBG_DISPATCH
 850 #define DBG_DISPATCH 46
 851 #endif
 852 #ifndef KDBG_CODE
 853 #define KDBG_CODE(...) 0
 854 #endif
 855 #define DISPATCH_CODE(subclass, code) \
 856                 KDBG_CODE(DBG_DISPATCH, DISPATCH_TRACE_SUBCLASS_##subclass, code)
 857 #ifdef ARIADNEDBG_CODE
 858 #define ARIADNE_ENTER_DISPATCH_MAIN_CODE ARIADNEDBG_CODE(220, 2)
 859 #else
 860 #define ARIADNE_ENTER_DISPATCH_MAIN_CODE 0
 861 #endif
 862 #if !defined(DISPATCH_USE_VOUCHER_KDEBUG_TRACE) && DISPATCH_INTROSPECTION
 863 #define DISPATCH_USE_VOUCHER_KDEBUG_TRACE 1
 864 #endif
 865
 866 #define DISPATCH_TRACE_SUBCLASS_DEFAULT 0
 867 #define DISPATCH_TRACE_SUBCLASS_VOUCHER 1
 868 #define DISPATCH_TRACE_SUBCLASS_PERF 2
 869 #define DISPATCH_TRACE_SUBCLASS_MACH_MSG 3
 870
 871 #define DISPATCH_PERF_non_leaf_retarget DISPATCH_CODE(PERF, 1)
 872 #define DISPATCH_PERF_post_activate_retarget DISPATCH_CODE(PERF, 2)
 873 #define DISPATCH_PERF_post_activate_mutation DISPATCH_CODE(PERF, 3)
 874 #define DISPATCH_PERF_delayed_registration DISPATCH_CODE(PERF, 4)
 875 #define DISPATCH_PERF_mutable_target DISPATCH_CODE(PERF, 5)
 876
 877 #define DISPATCH_MACH_MSG_hdr_move DISPATCH_CODE(MACH_MSG, 1)
 878
 879 DISPATCH_ALWAYS_INLINE
 880 static inline void
 881 _dispatch_ktrace_impl(uint32_t code, uint64_t a, uint64_t b,
 882                 uint64_t c, uint64_t d)
 883 {
 884         if (!code) return;
 885 #ifdef _COMM_PAGE_KDEBUG_ENABLE
 886         if (likely(*(volatile uint32_t *)_COMM_PAGE_KDEBUG_ENABLE == 0)) return;
 887 #endif
 888         kdebug_trace(code, a, b, c, d);
 889 }
 890 #define _dispatch_cast_to_uint64(e) \
 891                 __builtin_choose_expr(sizeof(e) > 4, \
 892                                 ((uint64_t)(e)), ((uint64_t)(uintptr_t)(e)))
 893 #define _dispatch_ktrace(code, a, b, c, d)  _dispatch_ktrace_impl(code, \
 894                 _dispatch_cast_to_uint64(a), _dispatch_cast_to_uint64(b), \
 895                 _dispatch_cast_to_uint64(c), _dispatch_cast_to_uint64(d))
 896
 897 #else // __has_include(<sys/kdebug.h>)
 898 #define DISPATCH_CODE(subclass, code) 0
 899 #define ARIADNE_ENTER_DISPATCH_MAIN_CODE 0
 900 #define DISPATCH_USE_VOUCHER_KDEBUG_TRACE 0
 901 #define _dispatch_ktrace(code, a, b, c, d)
 902 #endif // !__has_include(<sys/kdebug.h>)
 903 #define _dispatch_ktrace4(code, a, b, c, d) _dispatch_ktrace(code, a, b, c, d)
 904 #define _dispatch_ktrace3(code, a, b, c)    _dispatch_ktrace(code, a, b, c, 0)
 905 #define _dispatch_ktrace2(code, a, b)       _dispatch_ktrace(code, a, b, 0, 0)
 906 #define _dispatch_ktrace1(code, a)          _dispatch_ktrace(code, a, 0, 0, 0)
 907 #define _dispatch_ktrace0(code)             _dispatch_ktrace(code, 0, 0, 0, 0)
 908
 909 #ifndef MACH_MSGH_BITS_VOUCHER_MASK
 910 #define MACH_MSGH_BITS_VOUCHER_MASK     0x001f0000
 911 #define MACH_MSGH_BITS_SET_PORTS(remote, local, voucher)        \
 912         (((remote) & MACH_MSGH_BITS_REMOTE_MASK) |              \
 913          (((local) << 8) & MACH_MSGH_BITS_LOCAL_MASK) |         \
 914          (((voucher) << 16) & MACH_MSGH_BITS_VOUCHER_MASK))
 915 #define MACH_MSGH_BITS_VOUCHER(bits)                            \
 916                 (((bits) & MACH_MSGH_BITS_VOUCHER_MASK) >> 16)
 917 #define MACH_MSGH_BITS_HAS_VOUCHER(bits)                        \
 918         (MACH_MSGH_BITS_VOUCHER(bits) != MACH_MSGH_BITS_ZERO)
 919 #define msgh_voucher_port msgh_reserved
 920 #define mach_voucher_t mach_port_t
 921 #define MACH_VOUCHER_NULL MACH_PORT_NULL
 922 #define MACH_SEND_INVALID_VOUCHER 0x10000005
 923 #endif
 924
 925 #if TARGET_OS_SIMULATOR && IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101100
 926 #undef VOUCHER_USE_MACH_VOUCHER
 927 #define VOUCHER_USE_MACH_VOUCHER 0
 928 #endif
 929 #ifndef VOUCHER_USE_MACH_VOUCHER
 930 #if __has_include(<mach/mach_voucher.h>)
 931 #define VOUCHER_USE_MACH_VOUCHER 1
 932 #endif
 933 #endif
 934
 935 #if RDAR_24272659 // FIXME: <rdar://problem/24272659>
 936 #if !VOUCHER_USE_MACH_VOUCHER || !DISPATCH_HOST_SUPPORTS_OSX(101200)
 937 #undef VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
 938 #define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 0
 939 #elif !defined(VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER)
 940 #define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 1
 941 #endif
 942 #else // RDAR_24272659
 943 #undef VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
 944 #define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 0
 945 #endif // RDAR_24272659
 946
 947 #if !VOUCHER_USE_MACH_VOUCHER || !DISPATCH_HOST_SUPPORTS_OSX(101200)
 948 #undef VOUCHER_USE_BANK_AUTOREDEEM
 949 #define VOUCHER_USE_BANK_AUTOREDEEM 0
 950 #elif !defined(VOUCHER_USE_BANK_AUTOREDEEM)
 951 #define VOUCHER_USE_BANK_AUTOREDEEM 1
 952 #endif
 953
 954 #if !VOUCHER_USE_MACH_VOUCHER || \
 955                 !__has_include(<voucher/ipc_pthread_priority_types.h>) || \
 956                 !DISPATCH_HOST_SUPPORTS_OSX(101200)
 957 #undef VOUCHER_USE_MACH_VOUCHER_PRIORITY
 958 #define VOUCHER_USE_MACH_VOUCHER_PRIORITY 0
 959 #elif !defined(VOUCHER_USE_MACH_VOUCHER_PRIORITY)
 960 #define VOUCHER_USE_MACH_VOUCHER_PRIORITY 1
 961 #endif
 962
 963 #ifndef VOUCHER_USE_PERSONA
 964 #if VOUCHER_USE_MACH_VOUCHER && defined(BANK_PERSONA_TOKEN) && \
 965                 TARGET_OS_IOS && !TARGET_OS_SIMULATOR
 966 #define VOUCHER_USE_PERSONA 1
 967 #else
 968 #define VOUCHER_USE_PERSONA 0
 969 #endif
 970 #endif // VOUCHER_USE_PERSONA
 971
 972 #if VOUCHER_USE_MACH_VOUCHER
 973 #undef DISPATCH_USE_IMPORTANCE_ASSERTION
 974 #define DISPATCH_USE_IMPORTANCE_ASSERTION 0
 975 #else
 976 #undef MACH_RCV_VOUCHER
 977 #define MACH_RCV_VOUCHER 0
 978 #define VOUCHER_USE_PERSONA 0
 979 #endif // VOUCHER_USE_MACH_VOUCHER
 980
 981 #define _dispatch_hardware_crash() \
 982                 __asm__(""); __builtin_trap() // <rdar://problem/17464981>
 983
 984 #define _dispatch_set_crash_log_cause_and_message(ac, msg)
 985 #define _dispatch_set_crash_log_message(msg)
 986 #define _dispatch_set_crash_log_message_dynamic(msg)
 987
 988 #if HAVE_MACH
 989 // MIG_REPLY_MISMATCH means either:
 990 // 1) A signal handler is NOT using async-safe API. See the sigaction(2) man
 991 //    page for more info.
 992 // 2) A hand crafted call to mach_msg*() screwed up. Use MIG.
 993 #define DISPATCH_VERIFY_MIG(x) do { \
 994                 if ((x) == MIG_REPLY_MISMATCH) { \
 995                         _dispatch_set_crash_log_cause_and_message((x), \
 996                                         "MIG_REPLY_MISMATCH"); \
 997                         _dispatch_hardware_crash(); \
 998                 } \
 999         } while (0)
1000 #endif
1001
1002 #define DISPATCH_INTERNAL_CRASH(c, x) do { \
1003                 _dispatch_set_crash_log_cause_and_message((c), \
1004                                 "BUG IN LIBDISPATCH: " x); \
1005                 _dispatch_hardware_crash(); \
1006         } while (0)
1007
1008 #define DISPATCH_CLIENT_CRASH(c, x) do { \
1009                 _dispatch_set_crash_log_cause_and_message((c), \
1010                                 "BUG IN CLIENT OF LIBDISPATCH: " x); \
1011                 _dispatch_hardware_crash(); \
1012         } while (0)
1013
1014 #define _OS_OBJECT_CLIENT_CRASH(x) do { \
1015                 _dispatch_set_crash_log_message("API MISUSE: " x); \
1016                 _dispatch_hardware_crash(); \
1017         } while (0)
1018
1019 #define DISPATCH_ASSERTION_FAILED_MESSAGE \
1020                 "BUG IN CLIENT OF LIBDISPATCH: Assertion failed: "
1021
1022 #define _dispatch_assert_crash(msg)  do { \
1023                 const char *__msg = (msg); \
1024                 _dispatch_log("%s", __msg); \
1025                 _dispatch_set_crash_log_message_dynamic(__msg); \
1026                 _dispatch_hardware_crash(); \
1027         } while (0)
1028
1029 #define _dispatch_client_assert_fail(fmt, ...)  do { \
1030                 char *_msg = NULL; \
1031                 asprintf(&_msg, "%s" fmt, DISPATCH_ASSERTION_FAILED_MESSAGE, \
1032                                 ##__VA_ARGS__); \
1033                 _dispatch_assert_crash(_msg); \
1034                 free(_msg); \
1035         } while (0)
1036
1037 #define DISPATCH_NO_VOUCHER ((voucher_t)(void*)~0ul)
1038 #define DISPATCH_NO_PRIORITY ((pthread_priority_t)~0ul)
1039 DISPATCH_ENUM(_dispatch_thread_set_self, unsigned long,
1040         DISPATCH_PRIORITY_ENFORCE = 0x1,
1041         DISPATCH_VOUCHER_REPLACE = 0x2,
1042         DISPATCH_VOUCHER_CONSUME = 0x4,
1043         DISPATCH_THREAD_PARK = 0x8,
1044 );
1045 DISPATCH_WARN_RESULT
1046 static inline voucher_t _dispatch_adopt_priority_and_set_voucher(
1047                 pthread_priority_t priority, voucher_t voucher,
1048                 _dispatch_thread_set_self_t flags);
1049 #if HAVE_MACH
1050 mach_port_t _dispatch_get_mach_host_port(void);
1051 #endif
1052
1053 #if HAVE_PTHREAD_WORKQUEUE_QOS
1054 #if DISPATCH_DEBUG
1055 extern int _dispatch_set_qos_class_enabled;
1056 #else
1057 #define _dispatch_set_qos_class_enabled (1)
1058 #endif
1059 #endif // HAVE_PTHREAD_WORKQUEUE_QOS
1060 #if DISPATCH_USE_KEVENT_WORKQUEUE
1061 #if !HAVE_PTHREAD_WORKQUEUE_QOS || !DISPATCH_USE_KEVENT_QOS || \
1062                 !DISPATCH_USE_EV_UDATA_SPECIFIC
1063 #error Invalid build configuration
1064 #endif
1065 #if DISPATCH_USE_MGR_THREAD
1066 extern int _dispatch_kevent_workqueue_enabled;
1067 #else
1068 #define _dispatch_kevent_workqueue_enabled (1)
1069 #endif
1070 #endif // DISPATCH_USE_KEVENT_WORKQUEUE
1071
1072 #if DISPATCH_USE_EVFILT_MACHPORT_DIRECT
1073 #if !DISPATCH_USE_KEVENT_WORKQUEUE || !DISPATCH_USE_EV_UDATA_SPECIFIC
1074 #error Invalid build configuration
1075 #endif
1076 #if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
1077 extern int _dispatch_evfilt_machport_direct_enabled;
1078 #else
1079 #define _dispatch_evfilt_machport_direct_enabled (1)
1080 #endif
1081 #else
1082 #define _dispatch_evfilt_machport_direct_enabled (0)
1083 #endif // DISPATCH_USE_EVFILT_MACHPORT_DIRECT
1084
1085
1086 /* #includes dependent on internal.h */
1087 #include "object_internal.h"
1088 #include "semaphore_internal.h"
1089 #include "introspection_internal.h"
1090 #include "queue_internal.h"
1091 #include "source_internal.h"
1092 #include "voucher_internal.h"
1093 #include "data_internal.h"
1094 #if !TARGET_OS_WIN32
1095 #include "io_internal.h"
1096 #endif
1097 #include "inline_internal.h"
1098 #include "firehose/firehose_internal.h"
1099
1100 #endif /* __DISPATCH_INTERNAL__ */