src/internal.h

   1 /*
   2  * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_INTERNAL__
  28 #define __DISPATCH_INTERNAL__
  29
  30 #if __has_include(<config/config_ac.h>)
  31 #include <config/config_ac.h>
  32 #else
  33 #include <config/config.h>
  34 #endif
  35
  36 #define __DISPATCH_BUILDING_DISPATCH__
  37 #define __DISPATCH_INDIRECT__
  38
  39 #ifdef __APPLE__
  40 #include <Availability.h>
  41 #include <os/availability.h>
  42 #include <TargetConditionals.h>
  43
  44 #ifndef TARGET_OS_MAC_DESKTOP
  45 #define TARGET_OS_MAC_DESKTOP  (TARGET_OS_MAC && \
  46                 !TARGET_OS_SIMULATOR && !TARGET_OS_IPHONE && !TARGET_OS_EMBEDDED)
  47 #endif
  48
  49 #if TARGET_OS_MAC_DESKTOP
  50 #  define DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(x) \
  51                 (__MAC_OS_X_VERSION_MIN_REQUIRED >= (x))
  52 #  if !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
  53 #    error "OS X hosts older than OS X 10.12 aren't supported anymore"
  54 #  endif // !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
  55 #elif TARGET_OS_SIMULATOR
  56 #  define DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(x) \
  57                 (IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED >= (x))
  58 #  if !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
  59 #    error "Simulator hosts older than OS X 10.12 aren't supported anymore"
  60 #  endif // !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
  61 #else
  62 #  define DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(x) 1
  63 #  if __IPHONE_OS_VERSION_MIN_REQUIRED < 90000
  64 #    error "iOS hosts older than iOS 9.0 aren't supported anymore"
  65 #  endif
  66 #endif
  67
  68 #else // !__APPLE__
  69 #define DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(x) 0
  70 #endif // !__APPLE__
  71
  72
  73 #if !defined(DISPATCH_MACH_SPI) && TARGET_OS_MAC
  74 #define DISPATCH_MACH_SPI 1
  75 #endif
  76 #if !defined(OS_VOUCHER_CREATION_SPI) && TARGET_OS_MAC
  77 #define OS_VOUCHER_CREATION_SPI 1
  78 #endif
  79 #if !defined(OS_VOUCHER_ACTIVITY_SPI) && TARGET_OS_MAC
  80 #define OS_VOUCHER_ACTIVITY_SPI 1
  81 #endif
  82 #if !defined(OS_VOUCHER_ACTIVITY_GENERATE_SWAPS)
  83 #define OS_VOUCHER_ACTIVITY_GENERATE_SWAPS 0
  84 #endif
  85 #if !defined(OS_FIREHOSE_SPI) && TARGET_OS_MAC
  86 #define OS_FIREHOSE_SPI 1
  87 #endif
  88 #if !defined(DISPATCH_LAYOUT_SPI) && TARGET_OS_MAC
  89 #define DISPATCH_LAYOUT_SPI 1
  90 #endif
  91
  92 #if __has_include(<mach-o/dyld_priv.h>)
  93 #include <mach-o/dyld_priv.h>
  94 #if !defined(HAVE_DYLD_IS_MEMORY_IMMUTABLE)
  95 #if defined(DYLD_MACOSX_VERSION_10_12) || defined(DYLD_IOS_VERSION_10_0)
  96 #define HAVE_DYLD_IS_MEMORY_IMMUTABLE 1
  97 #else
  98 #define HAVE_DYLD_IS_MEMORY_IMMUTABLE 0
  99 #endif
 100 #endif // !defined(HAVE_DYLD_IS_MEMORY_IMMUTABLE)
 101 #endif // __has_include(<mach-o/dyld_priv.h>)
 102
 103 #if !defined(USE_OBJC) && HAVE_OBJC
 104 #define USE_OBJC 1
 105 #endif
 106
 107 #if USE_OBJC
 108 #define OS_OBJECT_HAVE_OBJC_SUPPORT 1
 109 #if defined(__OBJC__)
 110 #define OS_OBJECT_USE_OBJC 1
 111 // Force internal Objective-C sources to use class-visible headers
 112 // even when not compiling in Swift.
 113 #define OS_OBJECT_SWIFT3 1
 114 #else
 115 #define OS_OBJECT_USE_OBJC 0
 116 #endif // __OBJC__
 117 #else
 118 #define OS_OBJECT_HAVE_OBJC_SUPPORT 0
 119 #endif // USE_OBJC
 120
 121 #include <dispatch/dispatch.h>
 122 #include <dispatch/base.h>
 123
 124 #define __DISPATCH_HIDE_SYMBOL(sym, version) \
 125         __asm__(".section __TEXT,__const\n\t" \
 126                         ".globl $ld$hide$os" #version "$_" #sym "\n\t" \
 127                         "$ld$hide$os" #version "$_" #sym ":\n\t" \
 128                         "    .byte 0\n\t" \
 129                         ".previous")
 130
 131
 132 #ifndef DISPATCH_HIDE_SYMBOL
 133 #if TARGET_OS_MAC && !TARGET_OS_IPHONE
 134 #define DISPATCH_HIDE_SYMBOL(sym, osx, ios, tvos, watchos) \
 135                 __DISPATCH_HIDE_SYMBOL(sym, osx)
 136 #else
 137 #define DISPATCH_HIDE_SYMBOL(sym, osx, ios, tvos, watchos)
 138 #endif
 139 #endif
 140
 141 #include <os/object.h>
 142 #include <dispatch/time.h>
 143 #include <dispatch/object.h>
 144 #include <dispatch/queue.h>
 145 #include <dispatch/block.h>
 146 #include <dispatch/source.h>
 147 #include <dispatch/group.h>
 148 #include <dispatch/semaphore.h>
 149 #include <dispatch/once.h>
 150 #include <dispatch/data.h>
 151 #if !TARGET_OS_WIN32
 152 #include <dispatch/io.h>
 153 #endif
 154
 155 #if defined(__OBJC__) || defined(__cplusplus)
 156 #define DISPATCH_PURE_C 0
 157 #else
 158 #define DISPATCH_PURE_C 1
 159 #endif
 160
 161 /* private.h must be included last to avoid picking up installed headers. */
 162 #include <pthread.h>
 163 #include "os/object_private.h"
 164 #include "queue_private.h"
 165 #include "source_private.h"
 166 #include "mach_private.h"
 167 #include "data_private.h"
 168 #include "os/voucher_private.h"
 169 #include "os/voucher_activity_private.h"
 170 #if !TARGET_OS_WIN32
 171 #include "io_private.h"
 172 #endif
 173 #include "layout_private.h"
 174 #include "benchmark.h"
 175 #include "private.h"
 176
 177 /* SPI for Libsystem-internal use */
 178 DISPATCH_EXPORT DISPATCH_NOTHROW void libdispatch_init(void);
 179 #if !TARGET_OS_WIN32
 180 DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_prepare(void);
 181 DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_parent(void);
 182 DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void);
 183 #endif
 184
 185 /* More #includes at EOF (dependent on the contents of internal.h) ... */
 186
 187 // Abort on uncaught exceptions thrown from client callouts rdar://8577499
 188 #if !defined(DISPATCH_USE_CLIENT_CALLOUT)
 189 #define DISPATCH_USE_CLIENT_CALLOUT 1
 190 #endif
 191
 192 #define DISPATCH_ALLOW_NON_LEAF_RETARGET 1
 193
 194 /* The "_debug" library build */
 195 #ifndef DISPATCH_DEBUG
 196 #define DISPATCH_DEBUG 0
 197 #endif
 198
 199 #ifndef DISPATCH_PROFILE
 200 #define DISPATCH_PROFILE 0
 201 #endif
 202
 203 #if (!TARGET_OS_EMBEDDED || DISPATCH_DEBUG || DISPATCH_PROFILE) && \
 204                 !defined(DISPATCH_USE_DTRACE)
 205 #define DISPATCH_USE_DTRACE 1
 206 #endif
 207
 208 #if DISPATCH_USE_DTRACE && (DISPATCH_INTROSPECTION || DISPATCH_DEBUG || \
 209                 DISPATCH_PROFILE) && !defined(DISPATCH_USE_DTRACE_INTROSPECTION)
 210 #define DISPATCH_USE_DTRACE_INTROSPECTION 1
 211 #endif
 212
 213 #ifndef DISPATCH_DEBUG_QOS
 214 #define DISPATCH_DEBUG_QOS DISPATCH_DEBUG
 215 #endif
 216
 217 #if HAVE_LIBKERN_OSCROSSENDIAN_H
 218 #include <libkern/OSCrossEndian.h>
 219 #endif
 220 #if HAVE_LIBKERN_OSATOMIC_H
 221 #include <libkern/OSAtomic.h>
 222 #endif
 223 #if HAVE_MACH
 224 #include <mach/boolean.h>
 225 #include <mach/clock_types.h>
 226 #include <mach/clock.h>
 227 #include <mach/exception.h>
 228 #include <mach/mach.h>
 229 #include <mach/mach_error.h>
 230 #include <mach/mach_host.h>
 231 #include <mach/mach_interface.h>
 232 #include <mach/mach_time.h>
 233 #include <mach/mach_traps.h>
 234 #include <mach/message.h>
 235 #include <mach/mig_errors.h>
 236 #include <mach/host_special_ports.h>
 237 #include <mach/host_info.h>
 238 #include <mach/notify.h>
 239 #include <mach/mach_vm.h>
 240 #include <mach/vm_map.h>
 241 #endif /* HAVE_MACH */
 242 #if HAVE_MALLOC_MALLOC_H
 243 #include <malloc/malloc.h>
 244 #endif
 245
 246 #include <sys/stat.h>
 247
 248 #if !TARGET_OS_WIN32
 249 #include <sys/mount.h>
 250 #include <sys/queue.h>
 251 #ifdef __ANDROID__
 252 #include <linux/sysctl.h>
 253 #else
 254 #include <sys/sysctl.h>
 255 #endif /* __ANDROID__ */
 256 #include <sys/socket.h>
 257 #include <sys/time.h>
 258 #include <sys/mman.h>
 259 #include <netinet/in.h>
 260 #endif
 261
 262 #ifdef __BLOCKS__
 263 #if __has_include(<Block_private.h>)
 264 #include <Block_private.h>
 265 #else
 266 #include "BlocksRuntime/Block_private.h"
 267 #endif // __has_include(<Block_private.h>)
 268 #include <Block.h>
 269 #endif /* __BLOCKS__ */
 270
 271 #include <assert.h>
 272 #include <errno.h>
 273 #if HAVE_FCNTL_H
 274 #include <fcntl.h>
 275 #endif
 276 #include <limits.h>
 277 #include <search.h>
 278 #if USE_POSIX_SEM
 279 #include <semaphore.h>
 280 #endif
 281 #include <signal.h>
 282 #include <stdarg.h>
 283 #include <stdbool.h>
 284 #include <stdint.h>
 285 #include <stdio.h>
 286 #include <stdlib.h>
 287 #include <string.h>
 288 #if HAVE_UNISTD_H
 289 #include <unistd.h>
 290 #endif
 291
 292 #if __GNUC__
 293 #define DISPATCH_NOINLINE __attribute__((__noinline__))
 294 #define DISPATCH_USED __attribute__((__used__))
 295 #define DISPATCH_UNUSED __attribute__((__unused__))
 296 #define DISPATCH_WEAK __attribute__((__weak__))
 297 #define DISPATCH_OVERLOADABLE __attribute__((__overloadable__))
 298 #define DISPATCH_PACKED __attribute__((__packed__))
 299 #if DISPATCH_DEBUG
 300 #define DISPATCH_ALWAYS_INLINE_NDEBUG
 301 #else
 302 #define DISPATCH_ALWAYS_INLINE_NDEBUG __attribute__((__always_inline__))
 303 #endif
 304 #else   /* __GNUC__ */
 305 #define DISPATCH_NOINLINE
 306 #define DISPATCH_USED
 307 #define DISPATCH_UNUSED
 308 #define DISPATCH_WEAK
 309 #define DISPATCH_ALWAYS_INLINE_NDEBUG
 310 #endif  /* __GNUC__ */
 311
 312 #define DISPATCH_CONCAT(x,y) DISPATCH_CONCAT1(x,y)
 313 #define DISPATCH_CONCAT1(x,y) x ## y
 314
 315 #define DISPATCH_COUNT_ARGS(...) DISPATCH_COUNT_ARGS1(, ## __VA_ARGS__, \
 316                 _8, _7, _6, _5, _4, _3, _2, _1, _0)
 317 #define DISPATCH_COUNT_ARGS1(z, a, b, c, d, e, f, g, h, cnt, ...) cnt
 318
 319 #if BYTE_ORDER == LITTLE_ENDIAN
 320 #define DISPATCH_STRUCT_LE_2(a, b)        struct { a; b; }
 321 #define DISPATCH_STRUCT_LE_3(a, b, c)     struct { a; b; c; }
 322 #define DISPATCH_STRUCT_LE_4(a, b, c, d)  struct { a; b; c; d; }
 323 #else
 324 #define DISPATCH_STRUCT_LE_2(a, b)        struct { b; a; }
 325 #define DISPATCH_STRUCT_LE_3(a, b, c)     struct { c; b; a; }
 326 #define DISPATCH_STRUCT_LE_4(a, b, c, d)  struct { d; c; b; a; }
 327 #endif
 328 #if __has_feature(c_startic_assert)
 329 #define DISPATCH_UNION_ASSERT(alias, st) \
 330                 _Static_assert(sizeof(struct { alias; }) == sizeof(st), "bogus union");
 331 #else
 332 #define DISPATCH_UNION_ASSERT(alias, st)
 333 #endif
 334 #define DISPATCH_UNION_LE(alias, ...) \
 335                 DISPATCH_UNION_ASSERT(alias, DISPATCH_CONCAT(DISPATCH_STRUCT_LE, \
 336                                 DISPATCH_COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__)) \
 337                 union { alias; DISPATCH_CONCAT(DISPATCH_STRUCT_LE, \
 338                                 DISPATCH_COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__); }
 339
 340 // workaround 6368156
 341 #ifdef NSEC_PER_SEC
 342 #undef NSEC_PER_SEC
 343 #endif
 344 #ifdef USEC_PER_SEC
 345 #undef USEC_PER_SEC
 346 #endif
 347 #ifdef NSEC_PER_USEC
 348 #undef NSEC_PER_USEC
 349 #endif
 350 #define NSEC_PER_SEC 1000000000ull
 351 #define USEC_PER_SEC 1000000ull
 352 #define NSEC_PER_USEC 1000ull
 353
 354 /* I wish we had __builtin_expect_range() */
 355 #if __GNUC__
 356 #define _safe_cast_to_long(x) \
 357                 ({ _Static_assert(sizeof(typeof(x)) <= sizeof(long), \
 358                                 "__builtin_expect doesn't support types wider than long"); \
 359                                 (long)(x); })
 360 #define fastpath(x) ((typeof(x))__builtin_expect(_safe_cast_to_long(x), ~0l))
 361 #define slowpath(x) ((typeof(x))__builtin_expect(_safe_cast_to_long(x), 0l))
 362 #define likely(x) __builtin_expect(!!(x), 1)
 363 #define unlikely(x) __builtin_expect(!!(x), 0)
 364 #else
 365 #define fastpath(x) (x)
 366 #define slowpath(x) (x)
 367 #define likely(x) (!!(x))
 368 #define unlikely(x) (!!(x))
 369 #endif // __GNUC__
 370
 371 #define _TAILQ_IS_ENQUEUED(elm, field) \
 372                 ((elm)->field.tqe_prev != NULL)
 373 #define _TAILQ_MARK_NOT_ENQUEUED(elm, field) \
 374                 do { (elm)->field.tqe_prev = NULL; } while (0)
 375
 376 #if DISPATCH_DEBUG
 377 // sys/queue.h debugging
 378 #if defined(__linux__)
 379 #define QUEUE_MACRO_DEBUG 1
 380 #else
 381 #undef TRASHIT
 382 #define TRASHIT(x) do {(x) = (void *)-1;} while (0)
 383 #endif
 384 #endif // DISPATCH_DEBUG
 385 #define _TAILQ_TRASH_ENTRY(elm, field) do { \
 386                         TRASHIT((elm)->field.tqe_next); \
 387                         TRASHIT((elm)->field.tqe_prev); \
 388                 } while (0)
 389 #define _TAILQ_TRASH_HEAD(head) do { \
 390                         TRASHIT((head)->tqh_first); \
 391                         TRASHIT((head)->tqh_last); \
 392                 } while (0)
 393
 394 DISPATCH_EXPORT DISPATCH_NOINLINE
 395 void _dispatch_bug(size_t line, long val);
 396
 397 DISPATCH_NOINLINE
 398 void _dispatch_bug_client(const char* msg);
 399 #if HAVE_MACH
 400 DISPATCH_NOINLINE
 401 void _dispatch_bug_mach_client(const char *msg, mach_msg_return_t kr);
 402 #endif // HAVE_MACH
 403 DISPATCH_NOINLINE
 404 void _dispatch_bug_kevent_client(const char* msg, const char* filter,
 405                 const char *operation, int err);
 406
 407 DISPATCH_NOINLINE
 408 void _dispatch_bug_deprecated(const char *msg);
 409
 410 DISPATCH_NOINLINE DISPATCH_NORETURN
 411 void _dispatch_abort(size_t line, long val);
 412
 413 #if !defined(DISPATCH_USE_OS_DEBUG_LOG) && DISPATCH_DEBUG
 414 #if __has_include(<os/debug_private.h>)
 415 #define DISPATCH_USE_OS_DEBUG_LOG 1
 416 #include <os/debug_private.h>
 417 #endif
 418 #endif // DISPATCH_USE_OS_DEBUG_LOG
 419
 420 #if !defined(DISPATCH_USE_SIMPLE_ASL) && !DISPATCH_USE_OS_DEBUG_LOG
 421 #if __has_include(<_simple.h>)
 422 #define DISPATCH_USE_SIMPLE_ASL 1
 423 #include <_simple.h>
 424 #endif
 425 #endif // DISPATCH_USE_SIMPLE_ASL
 426
 427 #if !DISPATCH_USE_SIMPLE_ASL && !DISPATCH_USE_OS_DEBUG_LOG && !TARGET_OS_WIN32
 428 #include <syslog.h>
 429 #endif
 430
 431 #if DISPATCH_USE_OS_DEBUG_LOG
 432 #define _dispatch_log(msg, ...) os_debug_log("libdispatch", msg, ## __VA_ARGS__)
 433 #else
 434 DISPATCH_EXPORT DISPATCH_NOINLINE __attribute__((__format__(__printf__,1,2)))
 435 void _dispatch_log(const char *msg, ...);
 436 #endif // DISPATCH_USE_OS_DEBUG_LOG
 437
 438 #define dsnprintf(buf, siz, ...) \
 439                 ({ size_t _siz = siz; int _r = snprintf(buf, _siz, __VA_ARGS__); \
 440                  _r < 0 ? 0u : ((size_t)_r > _siz ? _siz : (size_t)_r); })
 441
 442 #if __GNUC__
 443 #define dispatch_static_assert(e) ({ \
 444                 char __compile_time_assert__[(bool)(e) ? 1 : -1] DISPATCH_UNUSED; \
 445         })
 446 #else
 447 #define dispatch_static_assert(e)
 448 #endif
 449
 450 #define DISPATCH_BAD_INPUT              ((void *_Nonnull)0)
 451 #define DISPATCH_OUT_OF_MEMORY  ((void *_Nonnull)0)
 452
 453 /*
 454  * For reporting bugs within libdispatch when using the "_debug" version of the
 455  * library.
 456  */
 457 #if __APPLE__
 458 #define dispatch_assert(e) do { \
 459                 if (__builtin_constant_p(e)) { \
 460                         dispatch_static_assert(e); \
 461                 } else { \
 462                         typeof(e) _e = (e); /* always eval 'e' */ \
 463                         if (unlikely(DISPATCH_DEBUG && !_e)) { \
 464                                 _dispatch_abort(__LINE__, (long)_e); \
 465                         } \
 466                 } \
 467         } while (0)
 468 #else
 469 static inline void
 470 _dispatch_assert(long e, size_t line)
 471 {
 472         if (DISPATCH_DEBUG && !e) _dispatch_abort(line, e);
 473 }
 474 #define dispatch_assert(e) _dispatch_assert((long)(e), __LINE__)
 475 #endif  /* __GNUC__ */
 476
 477 #if __APPLE__
 478 /*
 479  * A lot of API return zero upon success and not-zero on fail. Let's capture
 480  * and log the non-zero value
 481  */
 482 #define dispatch_assert_zero(e) do { \
 483                 if (__builtin_constant_p(e)) { \
 484                         dispatch_static_assert(e); \
 485                 } else { \
 486                         typeof(e) _e = (e); /* always eval 'e' */ \
 487                         if (unlikely(DISPATCH_DEBUG && _e)) { \
 488                                 _dispatch_abort(__LINE__, (long)_e); \
 489                         } \
 490                 } \
 491         } while (0)
 492 #else
 493 static inline void
 494 _dispatch_assert_zero(long e, size_t line)
 495 {
 496         if (DISPATCH_DEBUG && e) _dispatch_abort(line, e);
 497 }
 498 #define dispatch_assert_zero(e) _dispatch_assert_zero((long)(e), __LINE__)
 499 #endif  /* __GNUC__ */
 500
 501 /*
 502  * For reporting bugs or impedance mismatches between libdispatch and external
 503  * subsystems. These do NOT abort(), and are always compiled into the product.
 504  *
 505  * In particular, we wrap all system-calls with assume() macros.
 506  */
 507 #if __GNUC__
 508 #define dispatch_assume(e) ({ \
 509                 typeof(e) _e = (e); /* always eval 'e' */ \
 510                 if (unlikely(!_e)) { \
 511                         if (__builtin_constant_p(e)) { \
 512                                 dispatch_static_assert(e); \
 513                         } \
 514                         _dispatch_bug(__LINE__, (long)_e); \
 515                 } \
 516                 _e; \
 517         })
 518 #else
 519 static inline long
 520 _dispatch_assume(long e, long line)
 521 {
 522         if (!e) _dispatch_bug(line, e);
 523         return e;
 524 }
 525 #define dispatch_assume(e) _dispatch_assume((long)(e), __LINE__)
 526 #endif  /* __GNUC__ */
 527
 528 /*
 529  * A lot of API return zero upon success and not-zero on fail. Let's capture
 530  * and log the non-zero value
 531  */
 532 #if __GNUC__
 533 #define dispatch_assume_zero(e) ({ \
 534                 typeof(e) _e = (e); /* always eval 'e' */ \
 535                 if (unlikely(_e)) { \
 536                         if (__builtin_constant_p(e)) { \
 537                                 dispatch_static_assert(e); \
 538                         } \
 539                         _dispatch_bug(__LINE__, (long)_e); \
 540                 } \
 541                 _e; \
 542         })
 543 #else
 544 static inline long
 545 _dispatch_assume_zero(long e, long line)
 546 {
 547         if (e) _dispatch_bug(line, e);
 548         return e;
 549 }
 550 #define dispatch_assume_zero(e) _dispatch_assume_zero((long)(e), __LINE__)
 551 #endif  /* __GNUC__ */
 552
 553 /*
 554  * For reporting bugs in clients when using the "_debug" version of the library.
 555  */
 556 #if __GNUC__
 557 #define dispatch_debug_assert(e, msg, args...) do { \
 558                 if (__builtin_constant_p(e)) { \
 559                         dispatch_static_assert(e); \
 560                 } else { \
 561                         typeof(e) _e = (e); /* always eval 'e' */ \
 562                         if (unlikely(DISPATCH_DEBUG && !_e)) { \
 563                                 _dispatch_log("%s() 0x%lx: " msg, __func__, (long)_e, ##args); \
 564                                 abort(); \
 565                         } \
 566                 } \
 567         } while (0)
 568 #else
 569 #define dispatch_debug_assert(e, msg, args...) do { \
 570         typeof(e) _e = (e); /* always eval 'e' */ \
 571         if (unlikely(DISPATCH_DEBUG && !_e)) { \
 572                 _dispatch_log("%s() 0x%lx: " msg, __FUNCTION__, _e, ##args); \
 573                 abort(); \
 574         } \
 575 } while (0)
 576 #endif  /* __GNUC__ */
 577
 578 /* Make sure the debug statments don't get too stale */
 579 #define _dispatch_debug(x, args...) do { \
 580         if (DISPATCH_DEBUG) { \
 581                 _dispatch_log("%u\t%p\t" x, __LINE__, \
 582                                 (void *)_dispatch_thread_self(), ##args); \
 583         } \
 584 } while (0)
 585
 586 #if DISPATCH_DEBUG
 587 /* This is the private version of the deprecated dispatch_debug() */
 588 DISPATCH_NONNULL2 DISPATCH_NOTHROW
 589 __attribute__((__format__(printf,2,3)))
 590 void
 591 _dispatch_object_debug(dispatch_object_t object, const char *message, ...);
 592 #else
 593 #define _dispatch_object_debug(object, message, ...)
 594 #endif // DISPATCH_DEBUG
 595
 596 #ifdef __BLOCKS__
 597 #define _dispatch_Block_invoke(bb) \
 598                 ((dispatch_function_t)((struct Block_layout *)bb)->invoke)
 599 void *_dispatch_Block_copy(void *block);
 600 #if __GNUC__
 601 #define _dispatch_Block_copy(x) ((typeof(x))_dispatch_Block_copy(x))
 602 #endif
 603 void _dispatch_call_block_and_release(void *block);
 604 #endif /* __BLOCKS__ */
 605
 606 void _dispatch_temporary_resource_shortage(void);
 607 void *_dispatch_calloc(size_t num_items, size_t size);
 608 const char *_dispatch_strdup_if_mutable(const char *str);
 609 void _dispatch_vtable_init(void);
 610 char *_dispatch_get_build(void);
 611 int _dispatch_sigmask(void);
 612
 613 uint64_t _dispatch_timeout(dispatch_time_t when);
 614 uint64_t _dispatch_time_nanoseconds_since_epoch(dispatch_time_t when);
 615
 616 #define _DISPATCH_UNSAFE_FORK_MULTITHREADED  ((uint8_t)1)
 617 #define _DISPATCH_UNSAFE_FORK_PROHIBIT       ((uint8_t)2)
 618 extern uint8_t _dispatch_unsafe_fork;
 619 extern bool _dispatch_child_of_unsafe_fork;
 620 void _dispatch_fork_becomes_unsafe_slow(void);
 621
 622 #define _dispatch_is_multithreaded_inline() \
 623         ((_dispatch_unsafe_fork & _DISPATCH_UNSAFE_FORK_MULTITHREADED) != 0)
 624
 625 DISPATCH_ALWAYS_INLINE
 626 static inline void
 627 _dispatch_fork_becomes_unsafe(void)
 628 {
 629         if (unlikely(!_dispatch_is_multithreaded_inline())) {
 630                 _dispatch_fork_becomes_unsafe_slow();
 631                 DISPATCH_COMPILER_CAN_ASSUME(_dispatch_is_multithreaded_inline());
 632         }
 633 }
 634
 635 #if DISPATCH_INTROSPECTION
 636 #undef DISPATCH_PERF_MON
 637 #define DISPATCH_PERF_MON 0
 638 #endif
 639
 640 /* #includes dependent on internal.h */
 641 #include "shims.h"
 642 #include "event/event_internal.h"
 643
 644 // Older Mac OS X and iOS Simulator fallbacks
 645
 646 #if HAVE__PTHREAD_WORKQUEUE_INIT && PTHREAD_WORKQUEUE_SPI_VERSION >= 20140213 \
 647                 && !defined(HAVE_PTHREAD_WORKQUEUE_QOS)
 648 #define HAVE_PTHREAD_WORKQUEUE_QOS 1
 649 #endif
 650 #if HAVE__PTHREAD_WORKQUEUE_INIT && PTHREAD_WORKQUEUE_SPI_VERSION >= 20150304 \
 651                 && !defined(HAVE_PTHREAD_WORKQUEUE_KEVENT)
 652 #define HAVE_PTHREAD_WORKQUEUE_KEVENT 1
 653 #endif
 654
 655
 656 #ifndef HAVE_PTHREAD_WORKQUEUE_NARROWING
 657 #if !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(109900)
 658 #define HAVE_PTHREAD_WORKQUEUE_NARROWING 0
 659 #else
 660 #define HAVE_PTHREAD_WORKQUEUE_NARROWING 1
 661 #endif
 662 #endif
 663
 664 #ifdef EVFILT_MEMORYSTATUS
 665 #ifndef DISPATCH_USE_MEMORYSTATUS
 666 #define DISPATCH_USE_MEMORYSTATUS 1
 667 #endif
 668 #endif // EVFILT_MEMORYSTATUS
 669
 670 #if TARGET_OS_SIMULATOR
 671 #undef DISPATCH_USE_MEMORYPRESSURE_SOURCE
 672 #define DISPATCH_USE_MEMORYPRESSURE_SOURCE 0
 673 #endif // TARGET_OS_SIMULATOR
 674 #if !defined(DISPATCH_USE_MEMORYPRESSURE_SOURCE) && DISPATCH_USE_MEMORYSTATUS
 675 #define DISPATCH_USE_MEMORYPRESSURE_SOURCE 1
 676 #endif
 677 #if DISPATCH_USE_MEMORYPRESSURE_SOURCE
 678 #if __has_include(<malloc_private.h>)
 679 #include <malloc_private.h>
 680 #else
 681 extern void malloc_memory_event_handler(unsigned long);
 682 #endif // __has_include(<malloc_private.h)
 683 extern bool _dispatch_memory_warn;
 684 #endif
 685
 686 #if HAVE_PTHREAD_WORKQUEUE_KEVENT && defined(KEVENT_FLAG_WORKQ) && \
 687                 !defined(DISPATCH_USE_KEVENT_WORKQUEUE)
 688 #define DISPATCH_USE_KEVENT_WORKQUEUE 1
 689 #endif
 690
 691 #if (!DISPATCH_USE_KEVENT_WORKQUEUE || DISPATCH_DEBUG || DISPATCH_PROFILE) && \
 692                 !defined(DISPATCH_USE_MGR_THREAD)
 693 #define DISPATCH_USE_MGR_THREAD 1
 694 #endif
 695
 696
 697 #if defined(MACH_SEND_SYNC_OVERRIDE) && defined(MACH_RCV_SYNC_WAIT) && \
 698                 DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(109900) && \
 699                 !defined(DISPATCH_USE_MACH_SEND_SYNC_OVERRIDE)
 700 #define DISPATCH_USE_MACH_SEND_SYNC_OVERRIDE 1
 701 #endif
 702
 703 #if defined(F_SETNOSIGPIPE) && defined(F_GETNOSIGPIPE)
 704 #ifndef DISPATCH_USE_SETNOSIGPIPE
 705 #define DISPATCH_USE_SETNOSIGPIPE 1
 706 #endif
 707 #endif // F_SETNOSIGPIPE
 708
 709 #if defined(MACH_SEND_NOIMPORTANCE)
 710 #ifndef DISPATCH_USE_CHECKIN_NOIMPORTANCE
 711 #define DISPATCH_USE_CHECKIN_NOIMPORTANCE 1 // rdar://problem/16996737
 712 #endif
 713 #ifndef DISPATCH_USE_NOIMPORTANCE_QOS
 714 #define DISPATCH_USE_NOIMPORTANCE_QOS 1 // rdar://problem/21414476
 715 #endif
 716 #endif // MACH_SEND_NOIMPORTANCE
 717
 718
 719 #if HAVE_LIBPROC_INTERNAL_H
 720 #include <libproc.h>
 721 #include <libproc_internal.h>
 722 #ifndef DISPATCH_USE_IMPORTANCE_ASSERTION
 723 #define DISPATCH_USE_IMPORTANCE_ASSERTION 1
 724 #endif
 725 #endif // HAVE_LIBPROC_INTERNAL_H
 726
 727 #if HAVE_SYS_GUARDED_H
 728 #include <sys/guarded.h>
 729 #ifndef DISPATCH_USE_GUARDED_FD
 730 #define DISPATCH_USE_GUARDED_FD 1
 731 #endif
 732 #endif // HAVE_SYS_GUARDED_H
 733
 734
 735 #if DISPATCH_USE_DTRACE || DISPATCH_USE_DTRACE_INTROSPECTION
 736 typedef struct dispatch_trace_timer_params_s {
 737         int64_t deadline, interval, leeway;
 738 } *dispatch_trace_timer_params_t;
 739
 740 #include "provider.h"
 741 #endif // DISPATCH_USE_DTRACE || DISPATCH_USE_DTRACE_INTROSPECTION
 742
 743 #if __has_include(<sys/kdebug.h>)
 744 #include <sys/kdebug.h>
 745 #ifndef DBG_DISPATCH
 746 #define DBG_DISPATCH 46
 747 #endif
 748 #ifndef KDBG_CODE
 749 #define KDBG_CODE(...) 0
 750 #define DBG_FUNC_START 0
 751 #define DBG_FUNC_END 0
 752 #endif
 753 #define DISPATCH_CODE(subclass, code) \
 754                 KDBG_CODE(DBG_DISPATCH, DISPATCH_TRACE_SUBCLASS_##subclass, code)
 755 #define DISPATCH_CODE_START(subclass, code) \
 756                 (DISPATCH_CODE(subclass, code) | DBG_FUNC_START)
 757 #define DISPATCH_CODE_END(subclass, code) \
 758                 (DISPATCH_CODE(subclass, code) | DBG_FUNC_END)
 759 #ifdef ARIADNEDBG_CODE
 760 #define ARIADNE_ENTER_DISPATCH_MAIN_CODE ARIADNEDBG_CODE(220, 2)
 761 #else
 762 #define ARIADNE_ENTER_DISPATCH_MAIN_CODE 0
 763 #endif
 764 #if !defined(DISPATCH_USE_VOUCHER_KDEBUG_TRACE) && \
 765                 (DISPATCH_INTROSPECTION || DISPATCH_PROFILE || DISPATCH_DEBUG)
 766 #define DISPATCH_USE_VOUCHER_KDEBUG_TRACE 1
 767 #endif
 768
 769 #define DISPATCH_TRACE_SUBCLASS_DEFAULT 0
 770 #define DISPATCH_TRACE_SUBCLASS_VOUCHER 1
 771 #define DISPATCH_TRACE_SUBCLASS_PERF 2
 772 #define DISPATCH_TRACE_SUBCLASS_MACH_MSG 3
 773 #define DISPATCH_TRACE_SUBCLASS_PERF_MON 4
 774
 775 #define DISPATCH_PERF_non_leaf_retarget DISPATCH_CODE(PERF, 1)
 776 #define DISPATCH_PERF_post_activate_retarget DISPATCH_CODE(PERF, 2)
 777 #define DISPATCH_PERF_post_activate_mutation DISPATCH_CODE(PERF, 3)
 778 #define DISPATCH_PERF_delayed_registration DISPATCH_CODE(PERF, 4)
 779 #define DISPATCH_PERF_mutable_target DISPATCH_CODE(PERF, 5)
 780 #define DISPATCH_PERF_strict_bg_timer DISPATCH_CODE(PERF, 6)
 781
 782 #define DISPATCH_MACH_MSG_hdr_move DISPATCH_CODE(MACH_MSG, 1)
 783
 784 #define DISPATCH_PERF_MON_worker_thread_start DISPATCH_CODE_START(PERF_MON, 1)
 785 #define DISPATCH_PERF_MON_worker_thread_end DISPATCH_CODE_END(PERF_MON, 1)
 786 #define DISPATCH_PERF_MON_worker_useless DISPATCH_CODE(PERF_MON, 2)
 787
 788 DISPATCH_ALWAYS_INLINE
 789 static inline void
 790 _dispatch_ktrace_impl(uint32_t code, uint64_t a, uint64_t b,
 791                 uint64_t c, uint64_t d)
 792 {
 793         if (!code) return;
 794 #ifdef _COMM_PAGE_KDEBUG_ENABLE
 795         if (likely(*(volatile uint32_t *)_COMM_PAGE_KDEBUG_ENABLE == 0)) return;
 796 #endif
 797         kdebug_trace(code, a, b, c, d);
 798 }
 799 #define _dispatch_cast_to_uint64(e) \
 800                 __builtin_choose_expr(sizeof(e) > 4, \
 801                                 ((uint64_t)(e)), ((uint64_t)(uintptr_t)(e)))
 802 #define _dispatch_ktrace(code, a, b, c, d)  _dispatch_ktrace_impl(code, \
 803                 _dispatch_cast_to_uint64(a), _dispatch_cast_to_uint64(b), \
 804                 _dispatch_cast_to_uint64(c), _dispatch_cast_to_uint64(d))
 805
 806 #else // __has_include(<sys/kdebug.h>)
 807 #define DISPATCH_CODE(subclass, code) 0
 808 #define ARIADNE_ENTER_DISPATCH_MAIN_CODE 0
 809 #define DISPATCH_USE_VOUCHER_KDEBUG_TRACE 0
 810 #define _dispatch_ktrace(code, a, b, c, d)
 811 #endif // !__has_include(<sys/kdebug.h>)
 812 #define _dispatch_ktrace4(code, a, b, c, d) _dispatch_ktrace(code, a, b, c, d)
 813 #define _dispatch_ktrace3(code, a, b, c)    _dispatch_ktrace(code, a, b, c, 0)
 814 #define _dispatch_ktrace2(code, a, b)       _dispatch_ktrace(code, a, b, 0, 0)
 815 #define _dispatch_ktrace1(code, a)          _dispatch_ktrace(code, a, 0, 0, 0)
 816 #define _dispatch_ktrace0(code)             _dispatch_ktrace(code, 0, 0, 0, 0)
 817
 818 #ifndef MACH_MSGH_BITS_VOUCHER_MASK
 819 #define MACH_MSGH_BITS_VOUCHER_MASK     0x001f0000
 820 #define MACH_MSGH_BITS_SET_PORTS(remote, local, voucher)        \
 821         (((remote) & MACH_MSGH_BITS_REMOTE_MASK) |              \
 822          (((local) << 8) & MACH_MSGH_BITS_LOCAL_MASK) |         \
 823          (((voucher) << 16) & MACH_MSGH_BITS_VOUCHER_MASK))
 824 #define MACH_MSGH_BITS_VOUCHER(bits)                            \
 825                 (((bits) & MACH_MSGH_BITS_VOUCHER_MASK) >> 16)
 826 #define MACH_MSGH_BITS_HAS_VOUCHER(bits)                        \
 827         (MACH_MSGH_BITS_VOUCHER(bits) != MACH_MSGH_BITS_ZERO)
 828 #define msgh_voucher_port msgh_reserved
 829 #define mach_voucher_t mach_port_t
 830 #define MACH_VOUCHER_NULL MACH_PORT_NULL
 831 #define MACH_SEND_INVALID_VOUCHER 0x10000005
 832 #endif
 833
 834 #ifndef VOUCHER_USE_MACH_VOUCHER
 835 #if __has_include(<mach/mach_voucher.h>)
 836 #define VOUCHER_USE_MACH_VOUCHER 1
 837 #endif
 838 #endif // VOUCHER_USE_MACH_VOUCHER
 839
 840 #ifndef VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
 841 #if RDAR_24272659 // FIXME: <rdar://problem/24272659>
 842 #define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 1
 843 #else // RDAR_24272659
 844 #define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 0
 845 #endif // RDAR_24272659
 846 #endif
 847
 848 #ifndef VOUCHER_USE_PERSONA
 849 #if VOUCHER_USE_MACH_VOUCHER && defined(BANK_PERSONA_TOKEN) && \
 850                 TARGET_OS_IOS && !TARGET_OS_SIMULATOR
 851 #define VOUCHER_USE_PERSONA 1
 852 #else
 853 #define VOUCHER_USE_PERSONA 0
 854 #endif
 855 #endif // VOUCHER_USE_PERSONA
 856
 857 #if VOUCHER_USE_MACH_VOUCHER
 858 #undef DISPATCH_USE_IMPORTANCE_ASSERTION
 859 #define DISPATCH_USE_IMPORTANCE_ASSERTION 0
 860 #else
 861 #undef MACH_RCV_VOUCHER
 862 #define MACH_RCV_VOUCHER 0
 863 #define VOUCHER_USE_PERSONA 0
 864 #endif // VOUCHER_USE_MACH_VOUCHER
 865
 866 #define _dispatch_hardware_crash() \
 867                 __asm__(""); __builtin_trap() // <rdar://problem/17464981>
 868
 869 #define _dispatch_set_crash_log_cause_and_message(ac, msg) ((void)(ac))
 870 #define _dispatch_set_crash_log_message(msg)
 871 #define _dispatch_set_crash_log_message_dynamic(msg)
 872
 873 #if HAVE_MACH
 874 // MIG_REPLY_MISMATCH means either:
 875 // 1) A signal handler is NOT using async-safe API. See the sigaction(2) man
 876 //    page for more info.
 877 // 2) A hand crafted call to mach_msg*() screwed up. Use MIG.
 878 #define DISPATCH_VERIFY_MIG(x) do { \
 879                 if ((x) == MIG_REPLY_MISMATCH) { \
 880                         _dispatch_set_crash_log_cause_and_message((x), \
 881                                         "MIG_REPLY_MISMATCH"); \
 882                         _dispatch_hardware_crash(); \
 883                 } \
 884         } while (0)
 885 #endif
 886
 887 #define DISPATCH_INTERNAL_CRASH(c, x) do { \
 888                 _dispatch_set_crash_log_cause_and_message((c), \
 889                                 "BUG IN LIBDISPATCH: " x); \
 890                 _dispatch_hardware_crash(); \
 891         } while (0)
 892
 893 #define DISPATCH_CLIENT_CRASH(c, x) do { \
 894                 _dispatch_set_crash_log_cause_and_message((c), \
 895                                 "BUG IN CLIENT OF LIBDISPATCH: " x); \
 896                 _dispatch_hardware_crash(); \
 897         } while (0)
 898
 899 #define _OS_OBJECT_CLIENT_CRASH(x) do { \
 900                 _dispatch_set_crash_log_message("API MISUSE: " x); \
 901                 _dispatch_hardware_crash(); \
 902         } while (0)
 903
 904 #define DISPATCH_ASSERTION_FAILED_MESSAGE \
 905                 "BUG IN CLIENT OF LIBDISPATCH: Assertion failed: "
 906
 907 #define _dispatch_assert_crash(msg)  do { \
 908                 const char *__msg = (msg); \
 909                 _dispatch_log("%s", __msg); \
 910                 _dispatch_set_crash_log_message_dynamic(__msg); \
 911                 _dispatch_hardware_crash(); \
 912         } while (0)
 913
 914 #define _dispatch_client_assert_fail(fmt, ...)  do { \
 915                 char *_msg = NULL; \
 916                 asprintf(&_msg, "%s" fmt, DISPATCH_ASSERTION_FAILED_MESSAGE, \
 917                                 ##__VA_ARGS__); \
 918                 _dispatch_assert_crash(_msg); \
 919                 free(_msg); \
 920         } while (0)
 921
 922 #define DISPATCH_NO_VOUCHER ((voucher_t)(void*)~0ul)
 923 #define DISPATCH_NO_PRIORITY ((pthread_priority_t)~0ul)
 924 DISPATCH_ENUM(dispatch_thread_set_self, unsigned long,
 925         DISPATCH_PRIORITY_ENFORCE = 0x1,
 926         DISPATCH_VOUCHER_REPLACE = 0x2,
 927         DISPATCH_VOUCHER_CONSUME = 0x4,
 928         DISPATCH_THREAD_PARK = 0x8,
 929 );
 930 DISPATCH_WARN_RESULT
 931 static inline voucher_t _dispatch_adopt_priority_and_set_voucher(
 932                 pthread_priority_t priority, voucher_t voucher,
 933                 dispatch_thread_set_self_t flags);
 934 #if HAVE_MACH
 935 mach_port_t _dispatch_get_mach_host_port(void);
 936 #endif
 937
 938 #if HAVE_PTHREAD_WORKQUEUE_QOS
 939 #if DISPATCH_DEBUG
 940 extern int _dispatch_set_qos_class_enabled;
 941 #else
 942 #define _dispatch_set_qos_class_enabled (1)
 943 #endif
 944 #endif // HAVE_PTHREAD_WORKQUEUE_QOS
 945 #if DISPATCH_USE_KEVENT_WORKQUEUE
 946 #if !HAVE_PTHREAD_WORKQUEUE_QOS || !EV_UDATA_SPECIFIC
 947 #error Invalid build configuration
 948 #endif
 949 #if DISPATCH_USE_MGR_THREAD
 950 extern int _dispatch_kevent_workqueue_enabled;
 951 #else
 952 #define _dispatch_kevent_workqueue_enabled (1)
 953 #endif
 954 #else
 955 #define _dispatch_kevent_workqueue_enabled (0)
 956 #endif // DISPATCH_USE_KEVENT_WORKQUEUE
 957
 958
 959 /* #includes dependent on internal.h */
 960 #include "object_internal.h"
 961 #include "semaphore_internal.h"
 962 #include "introspection_internal.h"
 963 #include "queue_internal.h"
 964 #include "source_internal.h"
 965 #include "mach_internal.h"
 966 #include "voucher_internal.h"
 967 #include "data_internal.h"
 968 #if !TARGET_OS_WIN32
 969 #include "io_internal.h"
 970 #endif
 971 #include "inline_internal.h"
 972 #include "firehose/firehose_internal.h"
 973
 974 #endif /* __DISPATCH_INTERNAL__ */