src/init.c

   1 /*
   2  * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 // Contains exported global data and initialization & other routines that must
  22 // only exist once in the shared library even when resolvers are used.
  23
  24 #include "internal.h"
  25
  26 #if HAVE_MACH
  27 #include "protocolServer.h"
  28 #endif
  29
  30 #pragma mark -
  31 #pragma mark dispatch_init
  32
  33 #if USE_LIBDISPATCH_INIT_CONSTRUCTOR
  34 DISPATCH_NOTHROW __attribute__((constructor))
  35 void
  36 _libdispatch_init(void);
  37
  38 DISPATCH_EXPORT DISPATCH_NOTHROW
  39 void
  40 _libdispatch_init(void)
  41 {
  42         libdispatch_init();
  43 }
  44 #endif
  45
  46 DISPATCH_EXPORT DISPATCH_NOTHROW
  47 void
  48 dispatch_atfork_prepare(void)
  49 {
  50 }
  51
  52 DISPATCH_EXPORT DISPATCH_NOTHROW
  53 void
  54 dispatch_atfork_parent(void)
  55 {
  56 }
  57
  58 #pragma mark -
  59 #pragma mark dispatch_globals
  60
  61 #if DISPATCH_COCOA_COMPAT
  62 void (*dispatch_begin_thread_4GC)(void);
  63 void (*dispatch_end_thread_4GC)(void);
  64 void *(*_dispatch_begin_NSAutoReleasePool)(void);
  65 void (*_dispatch_end_NSAutoReleasePool)(void *);
  66 #endif
  67
  68 #if !DISPATCH_USE_DIRECT_TSD
  69 pthread_key_t dispatch_queue_key;
  70 pthread_key_t dispatch_sema4_key;
  71 pthread_key_t dispatch_cache_key;
  72 pthread_key_t dispatch_io_key;
  73 pthread_key_t dispatch_apply_key;
  74 #if DISPATCH_INTROSPECTION
  75 pthread_key_t dispatch_introspection_key;
  76 #elif DISPATCH_PERF_MON
  77 pthread_key_t dispatch_bcounter_key;
  78 #endif
  79 #endif // !DISPATCH_USE_DIRECT_TSD
  80
  81 struct _dispatch_hw_config_s _dispatch_hw_config;
  82 bool _dispatch_safe_fork = true, _dispatch_child_of_unsafe_fork;
  83
  84 DISPATCH_NOINLINE
  85 bool
  86 _dispatch_is_multithreaded(void)
  87 {
  88         return !_dispatch_safe_fork;
  89 }
  90
  91
  92 DISPATCH_NOINLINE
  93 bool
  94 _dispatch_is_fork_of_multithreaded_parent(void)
  95 {
  96         return _dispatch_child_of_unsafe_fork;
  97 }
  98
  99 const struct dispatch_queue_offsets_s dispatch_queue_offsets = {
 100         .dqo_version = 4,
 101         .dqo_label = offsetof(struct dispatch_queue_s, dq_label),
 102         .dqo_label_size = sizeof(((dispatch_queue_t)NULL)->dq_label),
 103         .dqo_flags = 0,
 104         .dqo_flags_size = 0,
 105         .dqo_width = offsetof(struct dispatch_queue_s, dq_width),
 106         .dqo_width_size = sizeof(((dispatch_queue_t)NULL)->dq_width),
 107         .dqo_serialnum = offsetof(struct dispatch_queue_s, dq_serialnum),
 108         .dqo_serialnum_size = sizeof(((dispatch_queue_t)NULL)->dq_serialnum),
 109         .dqo_running = offsetof(struct dispatch_queue_s, dq_running),
 110         .dqo_running_size = sizeof(((dispatch_queue_t)NULL)->dq_running),
 111 };
 112
 113 // 6618342 Contact the team that owns the Instrument DTrace probe before
 114 //         renaming this symbol
 115 DISPATCH_CACHELINE_ALIGN
 116 struct dispatch_queue_s _dispatch_main_q = {
 117         .do_vtable = DISPATCH_VTABLE(queue),
 118 #if !DISPATCH_USE_RESOLVERS
 119         .do_targetq = &_dispatch_root_queues[
 120                         DISPATCH_ROOT_QUEUE_IDX_DEFAULT_OVERCOMMIT_PRIORITY],
 121 #endif
 122         .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT,
 123         .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT,
 124         .do_suspend_cnt = DISPATCH_OBJECT_SUSPEND_LOCK,
 125         .dq_label = "com.apple.main-thread",
 126         .dq_running = 1,
 127         .dq_width = 1,
 128         .dq_is_thread_bound = 1,
 129         .dq_serialnum = 1,
 130 };
 131
 132 struct dispatch_queue_attr_s _dispatch_queue_attr_concurrent = {
 133         .do_vtable = DISPATCH_VTABLE(queue_attr),
 134         .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT,
 135         .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT,
 136         .do_next = DISPATCH_OBJECT_LISTLESS,
 137 };
 138
 139 #pragma mark -
 140 #pragma mark dispatch_vtables
 141
 142 DISPATCH_VTABLE_INSTANCE(semaphore,
 143         .do_type = DISPATCH_SEMAPHORE_TYPE,
 144         .do_kind = "semaphore",
 145         .do_dispose = _dispatch_semaphore_dispose,
 146         .do_debug = _dispatch_semaphore_debug,
 147 );
 148
 149 DISPATCH_VTABLE_INSTANCE(group,
 150         .do_type = DISPATCH_GROUP_TYPE,
 151         .do_kind = "group",
 152         .do_dispose = _dispatch_semaphore_dispose,
 153         .do_debug = _dispatch_semaphore_debug,
 154 );
 155
 156 DISPATCH_VTABLE_INSTANCE(queue,
 157         .do_type = DISPATCH_QUEUE_TYPE,
 158         .do_kind = "queue",
 159         .do_dispose = _dispatch_queue_dispose,
 160         .do_invoke = _dispatch_queue_invoke,
 161         .do_probe = _dispatch_queue_probe,
 162         .do_debug = dispatch_queue_debug,
 163 );
 164
 165 DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_root, queue,
 166         .do_type = DISPATCH_QUEUE_ROOT_TYPE,
 167         .do_kind = "global-queue",
 168         .do_dispose = _dispatch_pthread_root_queue_dispose,
 169         .do_probe = _dispatch_root_queue_probe,
 170         .do_debug = dispatch_queue_debug,
 171 );
 172
 173 DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_runloop, queue,
 174         .do_type = DISPATCH_QUEUE_ROOT_TYPE,
 175         .do_kind = "runloop-queue",
 176         .do_dispose = _dispatch_runloop_queue_dispose,
 177         .do_invoke = _dispatch_queue_invoke,
 178         .do_probe = _dispatch_runloop_queue_probe,
 179         .do_debug = dispatch_queue_debug,
 180 );
 181
 182 DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_mgr, queue,
 183         .do_type = DISPATCH_QUEUE_MGR_TYPE,
 184         .do_kind = "mgr-queue",
 185         .do_invoke = _dispatch_mgr_thread,
 186         .do_probe = _dispatch_mgr_queue_probe,
 187         .do_debug = dispatch_queue_debug,
 188 );
 189
 190 DISPATCH_VTABLE_INSTANCE(queue_specific_queue,
 191         .do_type = DISPATCH_QUEUE_SPECIFIC_TYPE,
 192         .do_kind = "queue-context",
 193         .do_dispose = _dispatch_queue_specific_queue_dispose,
 194         .do_invoke = (void*)_dispatch_queue_invoke,
 195         .do_probe = (void *)_dispatch_queue_probe,
 196         .do_debug = (void *)dispatch_queue_debug,
 197 );
 198
 199 DISPATCH_VTABLE_INSTANCE(queue_attr,
 200         .do_type = DISPATCH_QUEUE_ATTR_TYPE,
 201         .do_kind = "queue-attr",
 202 );
 203
 204 DISPATCH_VTABLE_INSTANCE(source,
 205         .do_type = DISPATCH_SOURCE_KEVENT_TYPE,
 206         .do_kind = "kevent-source",
 207         .do_dispose = _dispatch_source_dispose,
 208         .do_invoke = _dispatch_source_invoke,
 209         .do_probe = _dispatch_source_probe,
 210         .do_debug = _dispatch_source_debug,
 211 );
 212
 213 DISPATCH_VTABLE_INSTANCE(mach,
 214         .do_type = DISPATCH_MACH_CHANNEL_TYPE,
 215         .do_kind = "mach-channel",
 216         .do_dispose = _dispatch_mach_dispose,
 217         .do_invoke = _dispatch_mach_invoke,
 218         .do_probe = _dispatch_mach_probe,
 219         .do_debug = _dispatch_mach_debug,
 220 );
 221
 222 DISPATCH_VTABLE_INSTANCE(mach_msg,
 223         .do_type = DISPATCH_MACH_MSG_TYPE,
 224         .do_kind = "mach-msg",
 225         .do_dispose = _dispatch_mach_msg_dispose,
 226         .do_invoke = _dispatch_mach_msg_invoke,
 227         .do_debug = _dispatch_mach_msg_debug,
 228 );
 229
 230 #if !USE_OBJC
 231 DISPATCH_VTABLE_INSTANCE(data,
 232         .do_type = DISPATCH_DATA_TYPE,
 233         .do_kind = "data",
 234         .do_dispose = _dispatch_data_dispose,
 235         .do_debug = _dispatch_data_debug,
 236 );
 237 #endif
 238
 239 DISPATCH_VTABLE_INSTANCE(io,
 240         .do_type = DISPATCH_IO_TYPE,
 241         .do_kind = "channel",
 242         .do_dispose = _dispatch_io_dispose,
 243         .do_debug = _dispatch_io_debug,
 244 );
 245
 246 DISPATCH_VTABLE_INSTANCE(operation,
 247         .do_type = DISPATCH_OPERATION_TYPE,
 248         .do_kind = "operation",
 249         .do_dispose = _dispatch_operation_dispose,
 250         .do_debug = _dispatch_operation_debug,
 251 );
 252
 253 DISPATCH_VTABLE_INSTANCE(disk,
 254         .do_type = DISPATCH_DISK_TYPE,
 255         .do_kind = "disk",
 256         .do_dispose = _dispatch_disk_dispose,
 257 );
 258
 259 void
 260 _dispatch_vtable_init(void)
 261 {
 262 #if USE_OBJC
 263         // ObjC classes and dispatch vtables are co-located via linker order and
 264         // alias files, verify correct layout during initialization rdar://10640168
 265         DISPATCH_OBJC_CLASS_DECL(semaphore);
 266         dispatch_assert((char*)DISPATCH_VTABLE(semaphore) -
 267                         (char*)DISPATCH_OBJC_CLASS(semaphore) == 0);
 268         dispatch_assert((char*)&DISPATCH_CONCAT(_,DISPATCH_CLASS(semaphore_vtable))
 269                         - (char*)DISPATCH_OBJC_CLASS(semaphore) ==
 270                         sizeof(_os_object_class_s));
 271 #endif // USE_OBJC
 272 }
 273
 274 #pragma mark -
 275 #pragma mark dispatch_bug
 276
 277 static char _dispatch_build[16];
 278
 279 static void
 280 _dispatch_build_init(void *context DISPATCH_UNUSED)
 281 {
 282 #ifdef __APPLE__
 283         int mib[] = { CTL_KERN, KERN_OSVERSION };
 284         size_t bufsz = sizeof(_dispatch_build);
 285
 286         sysctl(mib, 2, _dispatch_build, &bufsz, NULL, 0);
 287 #else
 288         /*
 289          * XXXRW: What to do here for !Mac OS X?
 290          */
 291         memset(_dispatch_build, 0, sizeof(_dispatch_build));
 292 #endif
 293 }
 294
 295 static dispatch_once_t _dispatch_build_pred;
 296
 297 char*
 298 _dispatch_get_build(void)
 299 {
 300         dispatch_once_f(&_dispatch_build_pred, NULL, _dispatch_build_init);
 301         return _dispatch_build;
 302 }
 303
 304 #define _dispatch_bug_log(msg, ...) do { \
 305         static void *last_seen; \
 306         void *ra = __builtin_return_address(0); \
 307         if (last_seen != ra) { \
 308                 last_seen = ra; \
 309                 _dispatch_log(msg, ##__VA_ARGS__); \
 310         } \
 311 } while(0)
 312
 313 void
 314 _dispatch_bug(size_t line, long val)
 315 {
 316         dispatch_once_f(&_dispatch_build_pred, NULL, _dispatch_build_init);
 317         _dispatch_bug_log("BUG in libdispatch: %s - %lu - 0x%lx",
 318                         _dispatch_build, (unsigned long)line, val);
 319 }
 320
 321 void
 322 _dispatch_bug_client(const char* msg)
 323 {
 324         _dispatch_bug_log("BUG in libdispatch client: %s", msg);
 325 }
 326
 327 void
 328 _dispatch_bug_mach_client(const char* msg, mach_msg_return_t kr)
 329 {
 330         _dispatch_bug_log("BUG in libdispatch client: %s %s - 0x%x", msg,
 331                         mach_error_string(kr), kr);
 332 }
 333
 334 void
 335 _dispatch_bug_kevent_client(const char* msg, const char* filter,
 336                 const char *operation, int err)
 337 {
 338         _dispatch_bug_log("BUG in libdispatch client: %s[%s] %s: \"%s\" - 0x%x",
 339                         msg, filter, operation, strerror(err), err);
 340 }
 341
 342 void
 343 _dispatch_abort(size_t line, long val)
 344 {
 345         _dispatch_bug(line, val);
 346         abort();
 347 }
 348
 349 #if !DISPATCH_USE_OS_TRACE
 350
 351 #pragma mark -
 352 #pragma mark dispatch_log
 353
 354 static int dispatch_logfile = -1;
 355 static bool dispatch_log_disabled;
 356 static dispatch_once_t _dispatch_logv_pred;
 357
 358 static void
 359 _dispatch_logv_init(void *context DISPATCH_UNUSED)
 360 {
 361 #if DISPATCH_DEBUG
 362         bool log_to_file = true;
 363 #else
 364         bool log_to_file = false;
 365 #endif
 366         char *e = getenv("LIBDISPATCH_LOG");
 367         if (e) {
 368                 if (strcmp(e, "YES") == 0) {
 369                         // default
 370                 } else if (strcmp(e, "NO") == 0) {
 371                         dispatch_log_disabled = true;
 372                 } else if (strcmp(e, "syslog") == 0) {
 373                         log_to_file = false;
 374                 } else if (strcmp(e, "file") == 0) {
 375                         log_to_file = true;
 376                 } else if (strcmp(e, "stderr") == 0) {
 377                         log_to_file = true;
 378                         dispatch_logfile = STDERR_FILENO;
 379                 }
 380         }
 381         if (!dispatch_log_disabled) {
 382                 if (log_to_file && dispatch_logfile == -1) {
 383                         char path[PATH_MAX];
 384                         snprintf(path, sizeof(path), "/var/tmp/libdispatch.%d.log",
 385                                         getpid());
 386                         dispatch_logfile = open(path, O_WRONLY | O_APPEND | O_CREAT |
 387                                         O_NOFOLLOW | O_CLOEXEC, 0666);
 388                 }
 389                 if (dispatch_logfile != -1) {
 390                         struct timeval tv;
 391                         gettimeofday(&tv, NULL);
 392                         dprintf(dispatch_logfile, "=== log file opened for %s[%u] at "
 393                                         "%ld.%06u ===\n", getprogname() ?: "", getpid(),
 394                                         tv.tv_sec, tv.tv_usec);
 395                 }
 396         }
 397 }
 398
 399 static inline void
 400 _dispatch_log_file(char *buf, size_t len)
 401 {
 402         ssize_t r;
 403
 404         buf[len++] = '\n';
 405 retry:
 406         r = write(dispatch_logfile, buf, len);
 407         if (slowpath(r == -1) && errno == EINTR) {
 408                 goto retry;
 409         }
 410 }
 411
 412 DISPATCH_NOINLINE
 413 static void
 414 _dispatch_logv_file(const char *msg, va_list ap)
 415 {
 416         char buf[2048];
 417         int r = vsnprintf(buf, sizeof(buf), msg, ap);
 418         if (r < 0) return;
 419         size_t len = (size_t)r;
 420         if (len > sizeof(buf) - 1) {
 421                 len = sizeof(buf) - 1;
 422         }
 423         _dispatch_log_file(buf, len);
 424 }
 425
 426 DISPATCH_ALWAYS_INLINE
 427 static inline void
 428 _dispatch_logv(const char *msg, size_t len, va_list ap)
 429 {
 430         dispatch_once_f(&_dispatch_logv_pred, NULL, _dispatch_logv_init);
 431         if (slowpath(dispatch_log_disabled)) {
 432                 return;
 433         }
 434         if (slowpath(dispatch_logfile != -1)) {
 435                 if (!ap) {
 436                         return _dispatch_log_file((char*)msg, len);
 437                 }
 438                 return _dispatch_logv_file(msg, ap);
 439         }
 440         if (!ap) {
 441                 return syslog(LOG_NOTICE, "%s", msg);
 442         }
 443         return vsyslog(LOG_NOTICE, msg, ap);
 444 }
 445
 446 DISPATCH_NOINLINE
 447 void
 448 _dispatch_log(const char *msg, ...)
 449 {
 450         va_list ap;
 451
 452         va_start(ap, msg);
 453         _dispatch_logv(msg, 0, ap);
 454         va_end(ap);
 455 }
 456
 457 #endif // DISPATCH_USE_OS_TRACE
 458
 459 #pragma mark -
 460 #pragma mark dispatch_debug
 461
 462 static size_t
 463 _dispatch_object_debug2(dispatch_object_t dou, char* buf, size_t bufsiz)
 464 {
 465         DISPATCH_OBJECT_TFB(_dispatch_objc_debug, dou, buf, bufsiz);
 466         if (dou._do->do_vtable->do_debug) {
 467                 return dx_debug(dou._do, buf, bufsiz);
 468         }
 469         return strlcpy(buf, "NULL vtable slot: ", bufsiz);
 470 }
 471
 472 DISPATCH_NOINLINE
 473 static void
 474 _dispatch_debugv(dispatch_object_t dou, const char *msg, va_list ap)
 475 {
 476         char buf[2048];
 477         int r;
 478         size_t offs;
 479         if (dou._do) {
 480                 offs = _dispatch_object_debug2(dou, buf, sizeof(buf));
 481                 dispatch_assert(offs + 2 < sizeof(buf));
 482                 buf[offs++] = ':';
 483                 buf[offs++] = ' ';
 484                 buf[offs]   = '\0';
 485         } else {
 486                 offs = strlcpy(buf, "NULL: ", sizeof(buf));
 487         }
 488         r = vsnprintf(buf + offs, sizeof(buf) - offs, msg, ap);
 489 #if !DISPATCH_USE_OS_TRACE
 490         size_t len = offs + (r < 0 ? 0 : (size_t)r);
 491         if (len > sizeof(buf) - 1) {
 492                 len = sizeof(buf) - 1;
 493         }
 494         _dispatch_logv(buf, len, NULL);
 495 #else
 496         _dispatch_log("%s", buf);
 497 #endif
 498 }
 499
 500 DISPATCH_NOINLINE
 501 void
 502 dispatch_debugv(dispatch_object_t dou, const char *msg, va_list ap)
 503 {
 504         _dispatch_debugv(dou, msg, ap);
 505 }
 506
 507 DISPATCH_NOINLINE
 508 void
 509 dispatch_debug(dispatch_object_t dou, const char *msg, ...)
 510 {
 511         va_list ap;
 512
 513         va_start(ap, msg);
 514         _dispatch_debugv(dou, msg, ap);
 515         va_end(ap);
 516 }
 517
 518 #if DISPATCH_DEBUG
 519 DISPATCH_NOINLINE
 520 void
 521 _dispatch_object_debug(dispatch_object_t dou, const char *msg, ...)
 522 {
 523         va_list ap;
 524
 525         va_start(ap, msg);
 526         _dispatch_debugv(dou._do, msg, ap);
 527         va_end(ap);
 528 }
 529 #endif
 530
 531 #pragma mark -
 532 #pragma mark dispatch_calloc
 533
 534 DISPATCH_NOINLINE
 535 void
 536 _dispatch_temporary_resource_shortage(void)
 537 {
 538         sleep(1);
 539 }
 540
 541 void *
 542 _dispatch_calloc(size_t num_items, size_t size)
 543 {
 544         void *buf;
 545         while (!fastpath(buf = calloc(num_items, size))) {
 546                 _dispatch_temporary_resource_shortage();
 547         }
 548         return buf;
 549 }
 550
 551 #pragma mark -
 552 #pragma mark dispatch_block_t
 553
 554 #ifdef __BLOCKS__
 555
 556 #undef _dispatch_Block_copy
 557 dispatch_block_t
 558 _dispatch_Block_copy(dispatch_block_t db)
 559 {
 560         dispatch_block_t rval;
 561
 562         if (fastpath(db)) {
 563                 while (!fastpath(rval = Block_copy(db))) {
 564                         _dispatch_temporary_resource_shortage();
 565                 }
 566                 return rval;
 567         }
 568         DISPATCH_CLIENT_CRASH("NULL was passed where a block should have been");
 569 }
 570
 571 void
 572 _dispatch_call_block_and_release(void *block)
 573 {
 574         void (^b)(void) = block;
 575         b();
 576         Block_release(b);
 577 }
 578
 579 #endif // __BLOCKS__
 580
 581 #pragma mark -
 582 #pragma mark dispatch_client_callout
 583
 584 // Abort on uncaught exceptions thrown from client callouts rdar://8577499
 585 #if DISPATCH_USE_CLIENT_CALLOUT && (__arm__ || !USE_OBJC)
 586 // On platforms with SjLj exceptions, avoid the SjLj overhead on every callout
 587 // by clearing the unwinder's TSD pointer to the handler stack around callouts
 588
 589 #define _dispatch_get_tsd_base()
 590 #define _dispatch_get_unwind_tsd() (NULL)
 591 #define _dispatch_set_unwind_tsd(u) do {(void)(u);} while (0)
 592 #define _dispatch_free_unwind_tsd()
 593
 594 #undef _dispatch_client_callout
 595 DISPATCH_NOINLINE
 596 void
 597 _dispatch_client_callout(void *ctxt, dispatch_function_t f)
 598 {
 599         _dispatch_get_tsd_base();
 600         void *u = _dispatch_get_unwind_tsd();
 601         if (fastpath(!u)) return f(ctxt);
 602         _dispatch_set_unwind_tsd(NULL);
 603         f(ctxt);
 604         _dispatch_free_unwind_tsd();
 605         _dispatch_set_unwind_tsd(u);
 606 }
 607
 608 #undef _dispatch_client_callout2
 609 DISPATCH_NOINLINE
 610 void
 611 _dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t))
 612 {
 613         _dispatch_get_tsd_base();
 614         void *u = _dispatch_get_unwind_tsd();
 615         if (fastpath(!u)) return f(ctxt, i);
 616         _dispatch_set_unwind_tsd(NULL);
 617         f(ctxt, i);
 618         _dispatch_free_unwind_tsd();
 619         _dispatch_set_unwind_tsd(u);
 620 }
 621
 622 #undef _dispatch_client_callout3
 623 bool
 624 _dispatch_client_callout3(void *ctxt, dispatch_data_t region, size_t offset,
 625                 const void *buffer, size_t size, dispatch_data_applier_function_t f)
 626 {
 627         _dispatch_get_tsd_base();
 628         void *u = _dispatch_get_unwind_tsd();
 629         if (fastpath(!u)) return f(ctxt, region, offset, buffer, size);
 630         _dispatch_set_unwind_tsd(NULL);
 631         bool res = f(ctxt, region, offset, buffer, size);
 632         _dispatch_free_unwind_tsd();
 633         _dispatch_set_unwind_tsd(u);
 634         return res;
 635 }
 636
 637 #undef _dispatch_client_callout4
 638 void
 639 _dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason,
 640                 dispatch_mach_msg_t dmsg, mach_error_t error,
 641                 dispatch_mach_handler_function_t f)
 642 {
 643         _dispatch_get_tsd_base();
 644         void *u = _dispatch_get_unwind_tsd();
 645         if (fastpath(!u)) return f(ctxt, reason, dmsg, error);
 646         _dispatch_set_unwind_tsd(NULL);
 647         f(ctxt, reason, dmsg, error);
 648         _dispatch_free_unwind_tsd();
 649         _dispatch_set_unwind_tsd(u);
 650 }
 651
 652 #endif // DISPATCH_USE_CLIENT_CALLOUT
 653
 654 #pragma mark -
 655 #pragma mark _os_object_t no_objc
 656
 657 #if !USE_OBJC
 658
 659 static const _os_object_class_s _os_object_class;
 660
 661 void
 662 _os_object_init(void)
 663 {
 664         return;
 665 }
 666
 667 inline _os_object_t
 668 _os_object_alloc_realized(const void *cls, size_t size)
 669 {
 670         _os_object_t obj;
 671         dispatch_assert(size >= sizeof(struct _os_object_s));
 672         while (!fastpath(obj = calloc(1u, size))) {
 673                 _dispatch_temporary_resource_shortage();
 674         }
 675         obj->os_obj_isa = cls;
 676         return obj;
 677 }
 678
 679 _os_object_t
 680 _os_object_alloc(const void *cls, size_t size)
 681 {
 682         if (!cls) cls = &_os_object_class;
 683         return _os_object_alloc_realized(cls, size);
 684 }
 685
 686 void
 687 _os_object_dealloc(_os_object_t obj)
 688 {
 689         *((void *volatile*)&obj->os_obj_isa) = (void *)0x200;
 690         return free(obj);
 691 }
 692
 693 void
 694 _os_object_xref_dispose(_os_object_t obj)
 695 {
 696         if (fastpath(obj->os_obj_isa->_os_obj_xref_dispose)) {
 697                 return obj->os_obj_isa->_os_obj_xref_dispose(obj);
 698         }
 699         return _os_object_release_internal(obj);
 700 }
 701
 702 void
 703 _os_object_dispose(_os_object_t obj)
 704 {
 705         if (fastpath(obj->os_obj_isa->_os_obj_dispose)) {
 706                 return obj->os_obj_isa->_os_obj_dispose(obj);
 707         }
 708         return _os_object_dealloc(obj);
 709 }
 710
 711 #pragma mark -
 712 #pragma mark dispatch_autorelease_pool no_objc
 713
 714 #if DISPATCH_COCOA_COMPAT
 715
 716 void *_dispatch_autorelease_pool_push(void) {
 717         void *pool = NULL;
 718         if (_dispatch_begin_NSAutoReleasePool) {
 719                 pool = _dispatch_begin_NSAutoReleasePool();
 720         }
 721         return pool;
 722 }
 723
 724 void _dispatch_autorelease_pool_pop(void *pool) {
 725         if (_dispatch_end_NSAutoReleasePool) {
 726                 _dispatch_end_NSAutoReleasePool(pool);
 727         }
 728 }
 729
 730 #endif // DISPATCH_COCOA_COMPAT
 731 #endif // !USE_OBJC
 732
 733 #pragma mark -
 734 #pragma mark dispatch_source_types
 735
 736 static void
 737 dispatch_source_type_timer_init(dispatch_source_t ds,
 738         dispatch_source_type_t type DISPATCH_UNUSED,
 739         uintptr_t handle DISPATCH_UNUSED,
 740         unsigned long mask,
 741         dispatch_queue_t q)
 742 {
 743         if (fastpath(!ds->ds_refs)) {
 744                 ds->ds_refs = _dispatch_calloc(1ul,
 745                                 sizeof(struct dispatch_timer_source_refs_s));
 746         }
 747         ds->ds_needs_rearm = true;
 748         ds->ds_is_timer = true;
 749         if (q == dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0)
 750                         || q == dispatch_get_global_queue(
 751                         DISPATCH_QUEUE_PRIORITY_BACKGROUND, DISPATCH_QUEUE_OVERCOMMIT)){
 752                 mask |= DISPATCH_TIMER_BACKGROUND; // <rdar://problem/12200216>
 753         }
 754         ds_timer(ds->ds_refs).flags = mask;
 755 }
 756
 757 const struct dispatch_source_type_s _dispatch_source_type_timer = {
 758         .ke = {
 759                 .filter = DISPATCH_EVFILT_TIMER,
 760         },
 761         .mask = DISPATCH_TIMER_STRICT|DISPATCH_TIMER_BACKGROUND|
 762                         DISPATCH_TIMER_WALL_CLOCK,
 763         .init = dispatch_source_type_timer_init,
 764 };
 765
 766 static void
 767 dispatch_source_type_timer_with_aggregate_init(dispatch_source_t ds,
 768         dispatch_source_type_t type, uintptr_t handle, unsigned long mask,
 769         dispatch_queue_t q)
 770 {
 771         ds->ds_refs = _dispatch_calloc(1ul,
 772                         sizeof(struct dispatch_timer_source_aggregate_refs_s));
 773         dispatch_source_type_timer_init(ds, type, handle, mask, q);
 774         ds_timer(ds->ds_refs).flags |= DISPATCH_TIMER_WITH_AGGREGATE;
 775         ds->dq_specific_q = (void*)handle;
 776         _dispatch_retain(ds->dq_specific_q);
 777 }
 778
 779 const struct dispatch_source_type_s _dispatch_source_type_timer_with_aggregate={
 780         .ke = {
 781                 .filter = DISPATCH_EVFILT_TIMER,
 782                 .ident = ~0ull,
 783         },
 784         .mask = DISPATCH_TIMER_STRICT|DISPATCH_TIMER_BACKGROUND,
 785         .init = dispatch_source_type_timer_with_aggregate_init,
 786 };
 787
 788 static void
 789 dispatch_source_type_interval_init(dispatch_source_t ds,
 790         dispatch_source_type_t type, uintptr_t handle, unsigned long mask,
 791         dispatch_queue_t q)
 792 {
 793         dispatch_source_type_timer_init(ds, type, handle, mask, q);
 794         ds_timer(ds->ds_refs).flags |= DISPATCH_TIMER_INTERVAL;
 795         unsigned long ident = _dispatch_source_timer_idx(ds->ds_refs);
 796         ds->ds_dkev->dk_kevent.ident = ds->ds_ident_hack = ident;
 797         _dispatch_source_set_interval(ds, handle);
 798 }
 799
 800 const struct dispatch_source_type_s _dispatch_source_type_interval = {
 801         .ke = {
 802                 .filter = DISPATCH_EVFILT_TIMER,
 803                 .ident = ~0ull,
 804         },
 805         .mask = DISPATCH_TIMER_STRICT|DISPATCH_TIMER_BACKGROUND|
 806                         DISPATCH_INTERVAL_UI_ANIMATION,
 807         .init = dispatch_source_type_interval_init,
 808 };
 809
 810 const struct dispatch_source_type_s _dispatch_source_type_read = {
 811         .ke = {
 812                 .filter = EVFILT_READ,
 813                 .flags = EV_DISPATCH,
 814         },
 815 };
 816
 817 const struct dispatch_source_type_s _dispatch_source_type_write = {
 818         .ke = {
 819                 .filter = EVFILT_WRITE,
 820                 .flags = EV_DISPATCH,
 821         },
 822 };
 823
 824 #if DISPATCH_USE_VM_PRESSURE
 825 #if TARGET_IPHONE_SIMULATOR // rdar://problem/9219483
 826 static int _dispatch_ios_simulator_memory_warnings_fd = -1;
 827 static void
 828 _dispatch_ios_simulator_vm_source_init(void *context DISPATCH_UNUSED)
 829 {
 830         char *e = getenv("IPHONE_SIMULATOR_MEMORY_WARNINGS");
 831         if (!e) return;
 832         _dispatch_ios_simulator_memory_warnings_fd = open(e, O_EVTONLY);
 833         if (_dispatch_ios_simulator_memory_warnings_fd == -1) {
 834                 (void)dispatch_assume_zero(errno);
 835         }
 836 }
 837 static void
 838 dispatch_source_type_vm_init(dispatch_source_t ds,
 839         dispatch_source_type_t type DISPATCH_UNUSED,
 840         uintptr_t handle DISPATCH_UNUSED,
 841         unsigned long mask,
 842         dispatch_queue_t q DISPATCH_UNUSED)
 843 {
 844         static dispatch_once_t pred;
 845         dispatch_once_f(&pred, NULL, _dispatch_ios_simulator_vm_source_init);
 846         ds->ds_dkev->dk_kevent.ident = (uint64_t)(mask & DISPATCH_VM_PRESSURE ?
 847                         _dispatch_ios_simulator_memory_warnings_fd : -1);
 848 }
 849
 850 const struct dispatch_source_type_s _dispatch_source_type_vm = {
 851         .ke = {
 852                 .filter = EVFILT_VNODE,
 853                 .flags = EV_CLEAR,
 854         },
 855         .mask = NOTE_ATTRIB,
 856         .init = dispatch_source_type_vm_init,
 857 };
 858 #else
 859 static void
 860 dispatch_source_type_vm_init(dispatch_source_t ds,
 861         dispatch_source_type_t type DISPATCH_UNUSED,
 862         uintptr_t handle DISPATCH_UNUSED,
 863         unsigned long mask DISPATCH_UNUSED,
 864         dispatch_queue_t q DISPATCH_UNUSED)
 865 {
 866         ds->ds_is_level = false;
 867 }
 868
 869 const struct dispatch_source_type_s _dispatch_source_type_vm = {
 870         .ke = {
 871                 .filter = EVFILT_VM,
 872                 .flags = EV_DISPATCH,
 873         },
 874         .mask = NOTE_VM_PRESSURE,
 875         .init = dispatch_source_type_vm_init,
 876 };
 877 #endif
 878 #endif
 879
 880 #ifdef DISPATCH_USE_MEMORYSTATUS
 881 static void
 882 dispatch_source_type_memorystatus_init(dispatch_source_t ds,
 883         dispatch_source_type_t type DISPATCH_UNUSED,
 884         uintptr_t handle DISPATCH_UNUSED,
 885         unsigned long mask DISPATCH_UNUSED,
 886         dispatch_queue_t q DISPATCH_UNUSED)
 887 {
 888         ds->ds_is_level = false;
 889 }
 890
 891 const struct dispatch_source_type_s _dispatch_source_type_memorystatus = {
 892         .ke = {
 893                 .filter = EVFILT_MEMORYSTATUS,
 894                 .flags = EV_DISPATCH,
 895         },
 896         .mask = NOTE_MEMORYSTATUS_PRESSURE_NORMAL|NOTE_MEMORYSTATUS_PRESSURE_WARN
 897 #ifdef NOTE_MEMORYSTATUS_PRESSURE_CRITICAL
 898                         |NOTE_MEMORYSTATUS_PRESSURE_CRITICAL
 899 #endif
 900                         ,
 901         .init = dispatch_source_type_memorystatus_init,
 902 };
 903 #endif
 904
 905 const struct dispatch_source_type_s _dispatch_source_type_proc = {
 906         .ke = {
 907                 .filter = EVFILT_PROC,
 908                 .flags = EV_CLEAR,
 909         },
 910         .mask = NOTE_EXIT|NOTE_FORK|NOTE_EXEC
 911 #if HAVE_DECL_NOTE_SIGNAL
 912                         |NOTE_SIGNAL
 913 #endif
 914 #if HAVE_DECL_NOTE_REAP
 915                         |NOTE_REAP
 916 #endif
 917                         ,
 918 };
 919
 920 const struct dispatch_source_type_s _dispatch_source_type_signal = {
 921         .ke = {
 922                 .filter = EVFILT_SIGNAL,
 923         },
 924 };
 925
 926 const struct dispatch_source_type_s _dispatch_source_type_vnode = {
 927         .ke = {
 928                 .filter = EVFILT_VNODE,
 929                 .flags = EV_CLEAR,
 930         },
 931         .mask = NOTE_DELETE|NOTE_WRITE|NOTE_EXTEND|NOTE_ATTRIB|NOTE_LINK|
 932                         NOTE_RENAME|NOTE_REVOKE
 933 #if HAVE_DECL_NOTE_NONE
 934                         |NOTE_NONE
 935 #endif
 936                         ,
 937 };
 938
 939 const struct dispatch_source_type_s _dispatch_source_type_vfs = {
 940         .ke = {
 941                 .filter = EVFILT_FS,
 942                 .flags = EV_CLEAR,
 943         },
 944         .mask = VQ_NOTRESP|VQ_NEEDAUTH|VQ_LOWDISK|VQ_MOUNT|VQ_UNMOUNT|VQ_DEAD|
 945                         VQ_ASSIST|VQ_NOTRESPLOCK
 946 #if HAVE_DECL_VQ_UPDATE
 947                         |VQ_UPDATE
 948 #endif
 949 #if HAVE_DECL_VQ_VERYLOWDISK
 950                         |VQ_VERYLOWDISK
 951 #endif
 952                         ,
 953 };
 954
 955 const struct dispatch_source_type_s _dispatch_source_type_sock = {
 956 #ifdef EVFILT_SOCK
 957         .ke = {
 958                 .filter = EVFILT_SOCK,
 959                 .flags = EV_CLEAR,
 960         },
 961         .mask = NOTE_CONNRESET |  NOTE_READCLOSED | NOTE_WRITECLOSED |
 962                 NOTE_TIMEOUT | NOTE_NOSRCADDR |  NOTE_IFDENIED | NOTE_SUSPEND |
 963                 NOTE_RESUME | NOTE_KEEPALIVE
 964 #ifdef NOTE_ADAPTIVE_WTIMO
 965                 | NOTE_ADAPTIVE_WTIMO | NOTE_ADAPTIVE_RTIMO
 966 #endif
 967 #ifdef NOTE_CONNECTED
 968                 | NOTE_CONNECTED | NOTE_DISCONNECTED | NOTE_CONNINFO_UPDATED
 969 #endif
 970                 ,
 971 #endif // EVFILT_SOCK
 972 };
 973
 974 const struct dispatch_source_type_s _dispatch_source_type_data_add = {
 975         .ke = {
 976                 .filter = DISPATCH_EVFILT_CUSTOM_ADD,
 977         },
 978 };
 979
 980 const struct dispatch_source_type_s _dispatch_source_type_data_or = {
 981         .ke = {
 982                 .filter = DISPATCH_EVFILT_CUSTOM_OR,
 983                 .flags = EV_CLEAR,
 984                 .fflags = ~0u,
 985         },
 986 };
 987
 988 #if HAVE_MACH
 989
 990 static void
 991 dispatch_source_type_mach_send_init(dispatch_source_t ds,
 992         dispatch_source_type_t type DISPATCH_UNUSED,
 993         uintptr_t handle DISPATCH_UNUSED, unsigned long mask,
 994         dispatch_queue_t q DISPATCH_UNUSED)
 995 {
 996         if (!mask) {
 997                 // Preserve legacy behavior that (mask == 0) => DISPATCH_MACH_SEND_DEAD
 998                 ds->ds_dkev->dk_kevent.fflags = DISPATCH_MACH_SEND_DEAD;
 999                 ds->ds_pending_data_mask = DISPATCH_MACH_SEND_DEAD;
1000         }
1001 }
1002
1003 const struct dispatch_source_type_s _dispatch_source_type_mach_send = {
1004         .ke = {
1005                 .filter = DISPATCH_EVFILT_MACH_NOTIFICATION,
1006                 .flags = EV_CLEAR,
1007         },
1008         .mask = DISPATCH_MACH_SEND_DEAD|DISPATCH_MACH_SEND_POSSIBLE,
1009         .init = dispatch_source_type_mach_send_init,
1010 };
1011
1012 static void
1013 dispatch_source_type_mach_recv_init(dispatch_source_t ds,
1014         dispatch_source_type_t type DISPATCH_UNUSED,
1015         uintptr_t handle DISPATCH_UNUSED,
1016         unsigned long mask DISPATCH_UNUSED,
1017         dispatch_queue_t q DISPATCH_UNUSED)
1018 {
1019         ds->ds_is_level = false;
1020 }
1021
1022 const struct dispatch_source_type_s _dispatch_source_type_mach_recv = {
1023         .ke = {
1024                 .filter = EVFILT_MACHPORT,
1025                 .flags = EV_DISPATCH,
1026                 .fflags = DISPATCH_MACH_RECV_MESSAGE,
1027         },
1028         .init = dispatch_source_type_mach_recv_init,
1029 };
1030
1031 #pragma mark -
1032 #pragma mark dispatch_mig
1033
1034 void *
1035 dispatch_mach_msg_get_context(mach_msg_header_t *msg)
1036 {
1037         mach_msg_context_trailer_t *tp;
1038         void *context = NULL;
1039
1040         tp = (mach_msg_context_trailer_t *)((uint8_t *)msg +
1041                         round_msg(msg->msgh_size));
1042         if (tp->msgh_trailer_size >=
1043                         (mach_msg_size_t)sizeof(mach_msg_context_trailer_t)) {
1044                 context = (void *)(uintptr_t)tp->msgh_context;
1045         }
1046         return context;
1047 }
1048
1049 kern_return_t
1050 _dispatch_wakeup_runloop_thread(mach_port_t mp DISPATCH_UNUSED)
1051 {
1052         // dummy function just to pop a runloop thread out of mach_msg()
1053         return 0;
1054 }
1055
1056 kern_return_t
1057 _dispatch_consume_send_once_right(mach_port_t mp DISPATCH_UNUSED)
1058 {
1059         // dummy function to consume a send-once right
1060         return 0;
1061 }
1062
1063 kern_return_t
1064 _dispatch_mach_notify_port_destroyed(mach_port_t notify DISPATCH_UNUSED,
1065                 mach_port_t name)
1066 {
1067         kern_return_t kr;
1068         // this function should never be called
1069         (void)dispatch_assume_zero(name);
1070         kr = mach_port_mod_refs(mach_task_self(), name, MACH_PORT_RIGHT_RECEIVE,-1);
1071         DISPATCH_VERIFY_MIG(kr);
1072         (void)dispatch_assume_zero(kr);
1073         return KERN_SUCCESS;
1074 }
1075
1076 kern_return_t
1077 _dispatch_mach_notify_no_senders(mach_port_t notify,
1078                 mach_port_mscount_t mscnt DISPATCH_UNUSED)
1079 {
1080         // this function should never be called
1081         (void)dispatch_assume_zero(notify);
1082         return KERN_SUCCESS;
1083 }
1084
1085 kern_return_t
1086 _dispatch_mach_notify_send_once(mach_port_t notify DISPATCH_UNUSED)
1087 {
1088         // we only register for dead-name notifications
1089         // some code deallocated our send-once right without consuming it
1090 #if DISPATCH_DEBUG
1091         _dispatch_log("Corruption: An app/library deleted a libdispatch "
1092                         "dead-name notification");
1093 #endif
1094         return KERN_SUCCESS;
1095 }
1096
1097 #endif // HAVE_MACH