src/init.c

   1 /*
   2  * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 // Contains exported global data and initialization & other routines that must
  22 // only exist once in the shared library even when resolvers are used.
  23
  24 #include "internal.h"
  25
  26 #if HAVE_MACH
  27 #include "protocolServer.h"
  28 #endif
  29
  30 #pragma mark -
  31 #pragma mark dispatch_init
  32
  33 #if USE_LIBDISPATCH_INIT_CONSTRUCTOR
  34 DISPATCH_NOTHROW __attribute__((constructor))
  35 void
  36 _libdispatch_init(void);
  37
  38 DISPATCH_EXPORT DISPATCH_NOTHROW
  39 void
  40 _libdispatch_init(void)
  41 {
  42         libdispatch_init();
  43 }
  44 #endif
  45
  46 DISPATCH_EXPORT DISPATCH_NOTHROW
  47 void
  48 dispatch_atfork_prepare(void)
  49 {
  50         _os_object_atfork_prepare();
  51 }
  52
  53 DISPATCH_EXPORT DISPATCH_NOTHROW
  54 void
  55 dispatch_atfork_parent(void)
  56 {
  57         _os_object_atfork_parent();
  58 }
  59
  60 DISPATCH_EXPORT DISPATCH_NOTHROW
  61 void
  62 dispatch_atfork_child(void)
  63 {
  64         _os_object_atfork_child();
  65         _voucher_atfork_child();
  66         if (_dispatch_is_multithreaded_inline()) {
  67                 _dispatch_child_of_unsafe_fork = true;
  68         }
  69         _dispatch_queue_atfork_child();
  70         // clear the _PROHIBIT and _MULTITHREADED bits if set
  71         _dispatch_unsafe_fork = 0;
  72 }
  73
  74 #pragma mark -
  75 #pragma mark dispatch_globals
  76
  77 DISPATCH_HIDE_SYMBOL(dispatch_assert_queue, 10.12, 10.0, 10.0, 3.0);
  78 DISPATCH_HIDE_SYMBOL(dispatch_assert_queue_not, 10.12, 10.0, 10.0, 3.0);
  79 DISPATCH_HIDE_SYMBOL(dispatch_queue_create_with_target, 10.12, 10.0, 10.0, 3.0);
  80
  81 #if DISPATCH_COCOA_COMPAT
  82 void *(*_dispatch_begin_NSAutoReleasePool)(void);
  83 void (*_dispatch_end_NSAutoReleasePool)(void *);
  84 #endif
  85
  86 #if DISPATCH_USE_THREAD_LOCAL_STORAGE
  87 __thread struct dispatch_tsd __dispatch_tsd;
  88 pthread_key_t __dispatch_tsd_key;
  89 #elif !DISPATCH_USE_DIRECT_TSD
  90 pthread_key_t dispatch_queue_key;
  91 pthread_key_t dispatch_frame_key;
  92 pthread_key_t dispatch_cache_key;
  93 pthread_key_t dispatch_context_key;
  94 pthread_key_t dispatch_pthread_root_queue_observer_hooks_key;
  95 pthread_key_t dispatch_defaultpriority_key;
  96 #if DISPATCH_INTROSPECTION
  97 pthread_key_t dispatch_introspection_key;
  98 #elif DISPATCH_PERF_MON
  99 pthread_key_t dispatch_bcounter_key;
 100 #endif
 101 pthread_key_t dispatch_sema4_key;
 102 pthread_key_t dispatch_voucher_key;
 103 pthread_key_t dispatch_deferred_items_key;
 104 #endif // !DISPATCH_USE_DIRECT_TSD && !DISPATCH_USE_THREAD_LOCAL_STORAGE
 105
 106 #if VOUCHER_USE_MACH_VOUCHER
 107 dispatch_once_t _voucher_task_mach_voucher_pred;
 108 mach_voucher_t _voucher_task_mach_voucher;
 109 #if !VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
 110 mach_voucher_t _voucher_default_task_mach_voucher;
 111 #endif
 112 dispatch_once_t _firehose_task_buffer_pred;
 113 firehose_buffer_t _firehose_task_buffer;
 114 const uint32_t _firehose_spi_version = OS_FIREHOSE_SPI_VERSION;
 115 uint64_t _voucher_unique_pid;
 116 voucher_activity_hooks_t _voucher_libtrace_hooks;
 117 dispatch_mach_t _voucher_activity_debug_channel;
 118 #endif
 119 #if HAVE_PTHREAD_WORKQUEUE_QOS && DISPATCH_DEBUG
 120 int _dispatch_set_qos_class_enabled;
 121 #endif
 122 #if DISPATCH_USE_KEVENT_WORKQUEUE && DISPATCH_USE_MGR_THREAD
 123 int _dispatch_kevent_workqueue_enabled;
 124 #endif
 125 #if DISPATCH_USE_EVFILT_MACHPORT_DIRECT && \
 126                 DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
 127 int _dispatch_evfilt_machport_direct_enabled;
 128 #endif
 129
 130 DISPATCH_HW_CONFIG();
 131 uint8_t _dispatch_unsafe_fork;
 132 bool _dispatch_child_of_unsafe_fork;
 133 #if DISPATCH_USE_MEMORYPRESSURE_SOURCE
 134 bool _dispatch_memory_warn;
 135 int _dispatch_continuation_cache_limit = DISPATCH_CONTINUATION_CACHE_LIMIT;
 136 #endif
 137
 138 DISPATCH_NOINLINE
 139 bool
 140 _dispatch_is_multithreaded(void)
 141 {
 142         return _dispatch_is_multithreaded_inline();
 143 }
 144
 145 DISPATCH_NOINLINE
 146 bool
 147 _dispatch_is_fork_of_multithreaded_parent(void)
 148 {
 149         return _dispatch_child_of_unsafe_fork;
 150 }
 151
 152 DISPATCH_NOINLINE
 153 void
 154 _dispatch_fork_becomes_unsafe_slow(void)
 155 {
 156         uint8_t value = os_atomic_or(&_dispatch_unsafe_fork,
 157                         _DISPATCH_UNSAFE_FORK_MULTITHREADED, relaxed);
 158         if (value & _DISPATCH_UNSAFE_FORK_PROHIBIT) {
 159                 DISPATCH_CLIENT_CRASH(0, "Transition to multithreaded is prohibited");
 160         }
 161 }
 162
 163 DISPATCH_NOINLINE
 164 void
 165 _dispatch_prohibit_transition_to_multithreaded(bool prohibit)
 166 {
 167         if (prohibit) {
 168                 uint8_t value = os_atomic_or(&_dispatch_unsafe_fork,
 169                                 _DISPATCH_UNSAFE_FORK_PROHIBIT, relaxed);
 170                 if (value & _DISPATCH_UNSAFE_FORK_MULTITHREADED) {
 171                         DISPATCH_CLIENT_CRASH(0, "The executable is already multithreaded");
 172                 }
 173         } else {
 174                 os_atomic_and(&_dispatch_unsafe_fork,
 175                                 (uint8_t)~_DISPATCH_UNSAFE_FORK_PROHIBIT, relaxed);
 176         }
 177 }
 178
 179 const struct dispatch_queue_offsets_s dispatch_queue_offsets = {
 180         .dqo_version = 6,
 181         .dqo_label = offsetof(struct dispatch_queue_s, dq_label),
 182         .dqo_label_size = sizeof(((dispatch_queue_t)NULL)->dq_label),
 183         .dqo_flags = 0,
 184         .dqo_flags_size = 0,
 185         .dqo_serialnum = offsetof(struct dispatch_queue_s, dq_serialnum),
 186         .dqo_serialnum_size = sizeof(((dispatch_queue_t)NULL)->dq_serialnum),
 187         .dqo_width = offsetof(struct dispatch_queue_s, dq_width),
 188         .dqo_width_size = sizeof(((dispatch_queue_t)NULL)->dq_width),
 189         .dqo_running = 0,
 190         .dqo_running_size = 0,
 191         .dqo_suspend_cnt = 0,
 192         .dqo_suspend_cnt_size = 0,
 193         .dqo_target_queue = offsetof(struct dispatch_queue_s, do_targetq),
 194         .dqo_target_queue_size = sizeof(((dispatch_queue_t)NULL)->do_targetq),
 195         .dqo_priority = offsetof(struct dispatch_queue_s, dq_priority),
 196         .dqo_priority_size = sizeof(((dispatch_queue_t)NULL)->dq_priority),
 197 };
 198
 199 #if DISPATCH_USE_DIRECT_TSD
 200 const struct dispatch_tsd_indexes_s dispatch_tsd_indexes = {
 201         .dti_version = 2,
 202         .dti_queue_index = dispatch_queue_key,
 203         .dti_voucher_index = dispatch_voucher_key,
 204         .dti_qos_class_index = dispatch_priority_key,
 205 };
 206 #endif // DISPATCH_USE_DIRECT_TSD
 207
 208 // 6618342 Contact the team that owns the Instrument DTrace probe before
 209 //         renaming this symbol
 210 DISPATCH_CACHELINE_ALIGN
 211 struct dispatch_queue_s _dispatch_main_q = {
 212         DISPATCH_GLOBAL_OBJECT_HEADER(queue_main),
 213 #if !DISPATCH_USE_RESOLVERS
 214         .do_targetq = &_dispatch_root_queues[
 215                         DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT],
 216 #endif
 217         .dq_state = DISPATCH_QUEUE_STATE_INIT_VALUE(1),
 218         .dq_label = "com.apple.main-thread",
 219         .dq_width = 1,
 220         .dq_atomic_bits = DQF_THREAD_BOUND | DQF_CANNOT_TRYSYNC,
 221         .dq_override_voucher = DISPATCH_NO_VOUCHER,
 222         .dq_serialnum = 1,
 223 };
 224
 225 #pragma mark -
 226 #pragma mark dispatch_queue_attr_t
 227
 228 #define DISPATCH_QUEUE_ATTR_INIT(qos, prio, overcommit, freq, concurrent, inactive) \
 229                 { \
 230                         DISPATCH_GLOBAL_OBJECT_HEADER(queue_attr), \
 231                         .dqa_qos_class = (qos), \
 232                         .dqa_relative_priority = (qos) ? (prio) : 0, \
 233                         .dqa_overcommit = _dispatch_queue_attr_overcommit_##overcommit, \
 234                         .dqa_autorelease_frequency = DISPATCH_AUTORELEASE_FREQUENCY_##freq, \
 235                         .dqa_concurrent = (concurrent), \
 236                         .dqa_inactive = (inactive), \
 237                 }
 238
 239 #define DISPATCH_QUEUE_ATTR_ACTIVE_INIT(qos, prio, overcommit, freq, concurrent) \
 240                 { \
 241                         [DQA_INDEX_ACTIVE] = DISPATCH_QUEUE_ATTR_INIT(\
 242                                         qos, prio, overcommit, freq, concurrent, false), \
 243                         [DQA_INDEX_INACTIVE] = DISPATCH_QUEUE_ATTR_INIT(\
 244                                         qos, prio, overcommit, freq, concurrent, true), \
 245                 }
 246
 247 #define DISPATCH_QUEUE_ATTR_OVERCOMMIT_INIT(qos, prio, overcommit) \
 248                 { \
 249                         [DQA_INDEX_AUTORELEASE_FREQUENCY_INHERIT][DQA_INDEX_CONCURRENT] = \
 250                                         DISPATCH_QUEUE_ATTR_ACTIVE_INIT(qos, prio, overcommit, INHERIT, 1), \
 251                         [DQA_INDEX_AUTORELEASE_FREQUENCY_INHERIT][DQA_INDEX_SERIAL] = \
 252                                         DISPATCH_QUEUE_ATTR_ACTIVE_INIT(qos, prio, overcommit, INHERIT, 0), \
 253                         [DQA_INDEX_AUTORELEASE_FREQUENCY_WORK_ITEM][DQA_INDEX_CONCURRENT] = \
 254                                         DISPATCH_QUEUE_ATTR_ACTIVE_INIT(qos, prio, overcommit, WORK_ITEM, 1), \
 255                         [DQA_INDEX_AUTORELEASE_FREQUENCY_WORK_ITEM][DQA_INDEX_SERIAL] = \
 256                                         DISPATCH_QUEUE_ATTR_ACTIVE_INIT(qos, prio, overcommit, WORK_ITEM, 0), \
 257                         [DQA_INDEX_AUTORELEASE_FREQUENCY_NEVER][DQA_INDEX_CONCURRENT] = \
 258                                         DISPATCH_QUEUE_ATTR_ACTIVE_INIT(qos, prio, overcommit, NEVER, 1), \
 259                         [DQA_INDEX_AUTORELEASE_FREQUENCY_NEVER][DQA_INDEX_SERIAL] = \
 260                                         DISPATCH_QUEUE_ATTR_ACTIVE_INIT(qos, prio, overcommit, NEVER, 0), \
 261                 }
 262
 263 #define DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, prio) \
 264                 [prio] = { \
 265                         [DQA_INDEX_UNSPECIFIED_OVERCOMMIT] = \
 266                                         DISPATCH_QUEUE_ATTR_OVERCOMMIT_INIT(qos, -(prio), unspecified), \
 267                         [DQA_INDEX_NON_OVERCOMMIT] = \
 268                                         DISPATCH_QUEUE_ATTR_OVERCOMMIT_INIT(qos, -(prio), disabled), \
 269                         [DQA_INDEX_OVERCOMMIT] = \
 270                                         DISPATCH_QUEUE_ATTR_OVERCOMMIT_INIT(qos, -(prio), enabled), \
 271                 }
 272
 273 #define DISPATCH_QUEUE_ATTR_PRIO_INIT(qos) \
 274                 { \
 275                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 0), \
 276                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 1), \
 277                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 2), \
 278                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 3), \
 279                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 4), \
 280                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 5), \
 281                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 6), \
 282                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 7), \
 283                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 8), \
 284                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 9), \
 285                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 10), \
 286                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 11), \
 287                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 12), \
 288                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 13), \
 289                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 14), \
 290                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 15), \
 291                 }
 292
 293 #define DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(qos) \
 294                 [DQA_INDEX_QOS_CLASS_##qos] = \
 295                                 DISPATCH_QUEUE_ATTR_PRIO_INIT(_DISPATCH_QOS_CLASS_##qos)
 296
 297 // DISPATCH_QUEUE_CONCURRENT resp. _dispatch_queue_attr_concurrent is aliased
 298 // to array member [0][0][0][0][0][0] and their properties must match!
 299 const struct dispatch_queue_attr_s _dispatch_queue_attrs[]
 300                 [DISPATCH_QUEUE_ATTR_PRIO_COUNT]
 301                 [DISPATCH_QUEUE_ATTR_OVERCOMMIT_COUNT]
 302                 [DISPATCH_QUEUE_ATTR_AUTORELEASE_FREQUENCY_COUNT]
 303                 [DISPATCH_QUEUE_ATTR_CONCURRENCY_COUNT]
 304                 [DISPATCH_QUEUE_ATTR_INACTIVE_COUNT] = {
 305         DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(UNSPECIFIED),
 306         DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(MAINTENANCE),
 307         DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(BACKGROUND),
 308         DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(UTILITY),
 309         DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(DEFAULT),
 310         DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(USER_INITIATED),
 311         DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(USER_INTERACTIVE),
 312 };
 313
 314 #if DISPATCH_VARIANT_STATIC
 315 // <rdar://problem/16778703>
 316 struct dispatch_queue_attr_s _dispatch_queue_attr_concurrent =
 317         DISPATCH_QUEUE_ATTR_INIT(_DISPATCH_QOS_CLASS_UNSPECIFIED, 0,
 318                         unspecified, INHERIT, 1, false);
 319 #endif // DISPATCH_VARIANT_STATIC
 320
 321 // _dispatch_queue_attr_concurrent is aliased using libdispatch.aliases
 322 // and the -alias_list linker option on Darwin but needs to be done manually
 323 // for other platforms.
 324 #ifndef __APPLE__
 325 extern struct dispatch_queue_attr_s _dispatch_queue_attr_concurrent
 326         __attribute__((__alias__("_dispatch_queue_attrs")));
 327 #endif
 328
 329 #pragma mark -
 330 #pragma mark dispatch_vtables
 331
 332 DISPATCH_VTABLE_INSTANCE(semaphore,
 333         .do_type = DISPATCH_SEMAPHORE_TYPE,
 334         .do_kind = "semaphore",
 335         .do_dispose = _dispatch_semaphore_dispose,
 336         .do_debug = _dispatch_semaphore_debug,
 337 );
 338
 339 DISPATCH_VTABLE_INSTANCE(group,
 340         .do_type = DISPATCH_GROUP_TYPE,
 341         .do_kind = "group",
 342         .do_dispose = _dispatch_group_dispose,
 343         .do_debug = _dispatch_group_debug,
 344 );
 345
 346 DISPATCH_VTABLE_INSTANCE(queue,
 347         .do_type = DISPATCH_QUEUE_LEGACY_TYPE,
 348         .do_kind = "queue",
 349         .do_dispose = _dispatch_queue_dispose,
 350         .do_suspend = _dispatch_queue_suspend,
 351         .do_resume = _dispatch_queue_resume,
 352         .do_invoke = _dispatch_queue_invoke,
 353         .do_wakeup = _dispatch_queue_wakeup,
 354         .do_debug = dispatch_queue_debug,
 355         .do_set_targetq = _dispatch_queue_set_target_queue,
 356 );
 357
 358 DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_serial, queue,
 359         .do_type = DISPATCH_QUEUE_SERIAL_TYPE,
 360         .do_kind = "serial-queue",
 361         .do_dispose = _dispatch_queue_dispose,
 362         .do_suspend = _dispatch_queue_suspend,
 363         .do_resume = _dispatch_queue_resume,
 364         .do_finalize_activation = _dispatch_queue_finalize_activation,
 365         .do_invoke = _dispatch_queue_invoke,
 366         .do_wakeup = _dispatch_queue_wakeup,
 367         .do_debug = dispatch_queue_debug,
 368         .do_set_targetq = _dispatch_queue_set_target_queue,
 369 );
 370
 371 DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_concurrent, queue,
 372         .do_type = DISPATCH_QUEUE_CONCURRENT_TYPE,
 373         .do_kind = "concurrent-queue",
 374         .do_dispose = _dispatch_queue_dispose,
 375         .do_suspend = _dispatch_queue_suspend,
 376         .do_resume = _dispatch_queue_resume,
 377         .do_finalize_activation = _dispatch_queue_finalize_activation,
 378         .do_invoke = _dispatch_queue_invoke,
 379         .do_wakeup = _dispatch_queue_wakeup,
 380         .do_debug = dispatch_queue_debug,
 381         .do_set_targetq = _dispatch_queue_set_target_queue,
 382 );
 383
 384
 385 DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_root, queue,
 386         .do_type = DISPATCH_QUEUE_GLOBAL_ROOT_TYPE,
 387         .do_kind = "global-queue",
 388         .do_dispose = _dispatch_pthread_root_queue_dispose,
 389         .do_wakeup = _dispatch_root_queue_wakeup,
 390         .do_debug = dispatch_queue_debug,
 391 );
 392
 393 DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_main, queue,
 394         .do_type = DISPATCH_QUEUE_SERIAL_TYPE,
 395         .do_kind = "main-queue",
 396         .do_dispose = _dispatch_queue_dispose,
 397         .do_invoke = _dispatch_queue_invoke,
 398         .do_wakeup = _dispatch_main_queue_wakeup,
 399         .do_debug = dispatch_queue_debug,
 400 );
 401
 402 DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_runloop, queue,
 403         .do_type = DISPATCH_QUEUE_RUNLOOP_TYPE,
 404         .do_kind = "runloop-queue",
 405         .do_dispose = _dispatch_runloop_queue_dispose,
 406         .do_invoke = _dispatch_queue_invoke,
 407         .do_wakeup = _dispatch_runloop_queue_wakeup,
 408         .do_debug = dispatch_queue_debug,
 409 );
 410
 411 DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_mgr, queue,
 412         .do_type = DISPATCH_QUEUE_MGR_TYPE,
 413         .do_kind = "mgr-queue",
 414         .do_invoke = _dispatch_mgr_thread,
 415         .do_wakeup = _dispatch_mgr_queue_wakeup,
 416         .do_debug = dispatch_queue_debug,
 417 );
 418
 419 DISPATCH_VTABLE_INSTANCE(queue_specific_queue,
 420         .do_type = DISPATCH_QUEUE_SPECIFIC_TYPE,
 421         .do_kind = "queue-context",
 422         .do_dispose = _dispatch_queue_specific_queue_dispose,
 423         .do_invoke = (void *)_dispatch_queue_invoke,
 424         .do_wakeup = (void *)_dispatch_queue_wakeup,
 425         .do_debug = (void *)dispatch_queue_debug,
 426 );
 427
 428 DISPATCH_VTABLE_INSTANCE(queue_attr,
 429         .do_type = DISPATCH_QUEUE_ATTR_TYPE,
 430         .do_kind = "queue-attr",
 431 );
 432
 433 DISPATCH_VTABLE_INSTANCE(source,
 434         .do_type = DISPATCH_SOURCE_KEVENT_TYPE,
 435         .do_kind = "kevent-source",
 436         .do_dispose = _dispatch_source_dispose,
 437         .do_suspend = (void *)_dispatch_queue_suspend,
 438         .do_resume = (void *)_dispatch_queue_resume,
 439         .do_finalize_activation = _dispatch_source_finalize_activation,
 440         .do_invoke = _dispatch_source_invoke,
 441         .do_wakeup = _dispatch_source_wakeup,
 442         .do_debug = _dispatch_source_debug,
 443         .do_set_targetq = (void *)_dispatch_queue_set_target_queue,
 444 );
 445
 446 #if HAVE_MACH
 447 DISPATCH_VTABLE_INSTANCE(mach,
 448         .do_type = DISPATCH_MACH_CHANNEL_TYPE,
 449         .do_kind = "mach-channel",
 450         .do_dispose = _dispatch_mach_dispose,
 451         .do_suspend = (void *)_dispatch_queue_suspend,
 452         .do_resume = (void *)_dispatch_queue_resume,
 453         .do_finalize_activation = _dispatch_mach_finalize_activation,
 454         .do_invoke = _dispatch_mach_invoke,
 455         .do_wakeup = _dispatch_mach_wakeup,
 456         .do_debug = _dispatch_mach_debug,
 457         .do_set_targetq = (void *)_dispatch_queue_set_target_queue,
 458 );
 459
 460 DISPATCH_VTABLE_INSTANCE(mach_msg,
 461         .do_type = DISPATCH_MACH_MSG_TYPE,
 462         .do_kind = "mach-msg",
 463         .do_dispose = _dispatch_mach_msg_dispose,
 464         .do_invoke = _dispatch_mach_msg_invoke,
 465         .do_debug = _dispatch_mach_msg_debug,
 466 );
 467 #endif // HAVE_MACH
 468
 469 #if !DISPATCH_DATA_IS_BRIDGED_TO_NSDATA
 470 DISPATCH_VTABLE_INSTANCE(data,
 471         .do_type = DISPATCH_DATA_TYPE,
 472         .do_kind = "data",
 473         .do_dispose = _dispatch_data_dispose,
 474         .do_debug = _dispatch_data_debug,
 475 );
 476 #endif
 477
 478 DISPATCH_VTABLE_INSTANCE(io,
 479         .do_type = DISPATCH_IO_TYPE,
 480         .do_kind = "channel",
 481         .do_dispose = _dispatch_io_dispose,
 482         .do_debug = _dispatch_io_debug,
 483         .do_set_targetq = _dispatch_io_set_target_queue,
 484 );
 485
 486 DISPATCH_VTABLE_INSTANCE(operation,
 487         .do_type = DISPATCH_OPERATION_TYPE,
 488         .do_kind = "operation",
 489         .do_dispose = _dispatch_operation_dispose,
 490         .do_debug = _dispatch_operation_debug,
 491 );
 492
 493 DISPATCH_VTABLE_INSTANCE(disk,
 494         .do_type = DISPATCH_DISK_TYPE,
 495         .do_kind = "disk",
 496         .do_dispose = _dispatch_disk_dispose,
 497 );
 498
 499
 500 const struct dispatch_continuation_vtable_s _dispatch_continuation_vtables[] = {
 501         DC_VTABLE_ENTRY(ASYNC_REDIRECT,
 502                 .do_kind = "dc-redirect",
 503                 .do_invoke = _dispatch_async_redirect_invoke),
 504 #if HAVE_MACH
 505         DC_VTABLE_ENTRY(MACH_SEND_BARRRIER_DRAIN,
 506                 .do_kind = "dc-mach-send-drain",
 507                 .do_invoke = _dispatch_mach_send_barrier_drain_invoke),
 508         DC_VTABLE_ENTRY(MACH_SEND_BARRIER,
 509                 .do_kind = "dc-mach-send-barrier",
 510                 .do_invoke = _dispatch_mach_barrier_invoke),
 511         DC_VTABLE_ENTRY(MACH_RECV_BARRIER,
 512                 .do_kind = "dc-mach-recv-barrier",
 513                 .do_invoke = _dispatch_mach_barrier_invoke),
 514 #endif
 515 #if HAVE_PTHREAD_WORKQUEUE_QOS
 516         DC_VTABLE_ENTRY(OVERRIDE_STEALING,
 517                 .do_kind = "dc-override-stealing",
 518                 .do_invoke = _dispatch_queue_override_invoke),
 519         DC_VTABLE_ENTRY(OVERRIDE_OWNING,
 520                 .do_kind = "dc-override-owning",
 521                 .do_invoke = _dispatch_queue_override_invoke),
 522 #endif
 523 };
 524
 525 void
 526 _dispatch_vtable_init(void)
 527 {
 528 #if OS_OBJECT_HAVE_OBJC2
 529         // ObjC classes and dispatch vtables are co-located via linker order and
 530         // alias files, verify correct layout during initialization rdar://10640168
 531         dispatch_assert((char*)&DISPATCH_CONCAT(_,DISPATCH_CLASS(semaphore_vtable))
 532                         - (char*)DISPATCH_VTABLE(semaphore) ==
 533                         offsetof(struct dispatch_semaphore_vtable_s, _os_obj_vtable));
 534 #endif // USE_OBJC
 535 }
 536
 537 #pragma mark -
 538 #pragma mark dispatch_bug
 539
 540 static char _dispatch_build[16];
 541
 542 static void
 543 _dispatch_build_init(void *context DISPATCH_UNUSED)
 544 {
 545 #ifdef __APPLE__
 546         int mib[] = { CTL_KERN, KERN_OSVERSION };
 547         size_t bufsz = sizeof(_dispatch_build);
 548
 549         sysctl(mib, 2, _dispatch_build, &bufsz, NULL, 0);
 550 #if TARGET_IPHONE_SIMULATOR
 551         char *sim_version = getenv("SIMULATOR_RUNTIME_BUILD_VERSION");
 552         if (sim_version) {
 553                 (void)strlcat(_dispatch_build, " ", sizeof(_dispatch_build));
 554                 (void)strlcat(_dispatch_build, sim_version, sizeof(_dispatch_build));
 555         }
 556 #endif // TARGET_IPHONE_SIMULATOR
 557
 558 #else
 559         /*
 560          * XXXRW: What to do here for !Mac OS X?
 561          */
 562         memset(_dispatch_build, 0, sizeof(_dispatch_build));
 563 #endif // __APPLE__
 564 }
 565
 566 static dispatch_once_t _dispatch_build_pred;
 567
 568 char*
 569 _dispatch_get_build(void)
 570 {
 571         dispatch_once_f(&_dispatch_build_pred, NULL, _dispatch_build_init);
 572         return _dispatch_build;
 573 }
 574
 575 #define _dispatch_bug_log(msg, ...) do { \
 576         static void *last_seen; \
 577         void *ra = __builtin_return_address(0); \
 578         if (last_seen != ra) { \
 579                 last_seen = ra; \
 580                 _dispatch_log(msg, ##__VA_ARGS__); \
 581         } \
 582 } while(0)
 583
 584 void
 585 _dispatch_bug(size_t line, long val)
 586 {
 587         dispatch_once_f(&_dispatch_build_pred, NULL, _dispatch_build_init);
 588         _dispatch_bug_log("BUG in libdispatch: %s - %lu - 0x%lx",
 589                         _dispatch_build, (unsigned long)line, val);
 590 }
 591
 592 void
 593 _dispatch_bug_client(const char* msg)
 594 {
 595         _dispatch_bug_log("BUG in libdispatch client: %s", msg);
 596 }
 597
 598 #if HAVE_MACH
 599 void
 600 _dispatch_bug_mach_client(const char* msg, mach_msg_return_t kr)
 601 {
 602         _dispatch_bug_log("BUG in libdispatch client: %s %s - 0x%x", msg,
 603                         mach_error_string(kr), kr);
 604 }
 605 #endif
 606
 607 void
 608 _dispatch_bug_kevent_client(const char* msg, const char* filter,
 609                 const char *operation, int err)
 610 {
 611         if (operation && err) {
 612                 _dispatch_bug_log("BUG in libdispatch client: %s[%s] %s: \"%s\" - 0x%x",
 613                                 msg, filter, operation, strerror(err), err);
 614         } else if (operation) {
 615                 _dispatch_bug_log("BUG in libdispatch client: %s[%s] %s",
 616                                 msg, filter, operation);
 617         } else {
 618                 _dispatch_bug_log("BUG in libdispatch: %s[%s]: \"%s\" - 0x%x",
 619                                 msg, filter, strerror(err), err);
 620         }
 621 }
 622
 623 void
 624 _dispatch_bug_deprecated(const char *msg)
 625 {
 626         _dispatch_bug_log("DEPRECATED USE in libdispatch client: %s", msg);
 627 }
 628
 629 void
 630 _dispatch_abort(size_t line, long val)
 631 {
 632         _dispatch_bug(line, val);
 633         abort();
 634 }
 635
 636 #if !DISPATCH_USE_OS_DEBUG_LOG
 637
 638 #pragma mark -
 639 #pragma mark dispatch_log
 640
 641 static int dispatch_logfile = -1;
 642 static bool dispatch_log_disabled;
 643 #if DISPATCH_DEBUG
 644 static uint64_t dispatch_log_basetime;
 645 #endif
 646 static dispatch_once_t _dispatch_logv_pred;
 647
 648 static void
 649 _dispatch_logv_init(void *context DISPATCH_UNUSED)
 650 {
 651 #if DISPATCH_DEBUG
 652         bool log_to_file = true;
 653 #else
 654         bool log_to_file = false;
 655 #endif
 656         char *e = getenv("LIBDISPATCH_LOG");
 657         if (e) {
 658                 if (strcmp(e, "YES") == 0) {
 659                         // default
 660                 } else if (strcmp(e, "NO") == 0) {
 661                         dispatch_log_disabled = true;
 662                 } else if (strcmp(e, "syslog") == 0) {
 663                         log_to_file = false;
 664                 } else if (strcmp(e, "file") == 0) {
 665                         log_to_file = true;
 666                 } else if (strcmp(e, "stderr") == 0) {
 667                         log_to_file = true;
 668                         dispatch_logfile = STDERR_FILENO;
 669                 }
 670         }
 671         if (!dispatch_log_disabled) {
 672                 if (log_to_file && dispatch_logfile == -1) {
 673                         char path[PATH_MAX];
 674                         snprintf(path, sizeof(path), "/var/tmp/libdispatch.%d.log",
 675                                         getpid());
 676                         dispatch_logfile = open(path, O_WRONLY | O_APPEND | O_CREAT |
 677                                         O_NOFOLLOW | O_CLOEXEC, 0666);
 678                 }
 679                 if (dispatch_logfile != -1) {
 680                         struct timeval tv;
 681                         gettimeofday(&tv, NULL);
 682 #if DISPATCH_DEBUG
 683                         dispatch_log_basetime = _dispatch_absolute_time();
 684 #endif
 685                         dprintf(dispatch_logfile, "=== log file opened for %s[%u] at "
 686                                         "%ld.%06u ===\n", getprogname() ?: "", getpid(),
 687                                         tv.tv_sec, (int)tv.tv_usec);
 688                 }
 689         }
 690 }
 691
 692 static inline void
 693 _dispatch_log_file(char *buf, size_t len)
 694 {
 695         ssize_t r;
 696
 697         buf[len++] = '\n';
 698 retry:
 699         r = write(dispatch_logfile, buf, len);
 700         if (slowpath(r == -1) && errno == EINTR) {
 701                 goto retry;
 702         }
 703 }
 704
 705 DISPATCH_NOINLINE
 706 static void
 707 _dispatch_logv_file(const char *msg, va_list ap)
 708 {
 709         char buf[2048];
 710         size_t bufsiz = sizeof(buf), offset = 0;
 711         int r;
 712
 713 #if DISPATCH_DEBUG
 714         offset += dsnprintf(&buf[offset], bufsiz - offset, "%llu\t",
 715                         _dispatch_absolute_time() - dispatch_log_basetime);
 716 #endif
 717         r = vsnprintf(&buf[offset], bufsiz - offset, msg, ap);
 718         if (r < 0) return;
 719         offset += (size_t)r;
 720         if (offset > bufsiz - 1) {
 721                 offset = bufsiz - 1;
 722         }
 723         _dispatch_log_file(buf, offset);
 724 }
 725
 726 #if DISPATCH_USE_SIMPLE_ASL
 727 static inline void
 728 _dispatch_syslog(const char *msg)
 729 {
 730         _simple_asl_log(ASL_LEVEL_NOTICE, "com.apple.libsystem.libdispatch", msg);
 731 }
 732
 733 static inline void
 734 _dispatch_vsyslog(const char *msg, va_list ap)
 735 {
 736         char *str;
 737     vasprintf(&str, msg, ap);
 738         if (str) {
 739                 _dispatch_syslog(str);
 740                 free(str);
 741         }
 742 }
 743 #else // DISPATCH_USE_SIMPLE_ASL
 744 static inline void
 745 _dispatch_syslog(const char *msg)
 746 {
 747         syslog(LOG_NOTICE, "%s", msg);
 748 }
 749
 750 static inline void
 751 _dispatch_vsyslog(const char *msg, va_list ap)
 752 {
 753         vsyslog(LOG_NOTICE, msg, ap);
 754 }
 755 #endif // DISPATCH_USE_SIMPLE_ASL
 756
 757 DISPATCH_ALWAYS_INLINE
 758 static inline void
 759 _dispatch_logv(const char *msg, size_t len, va_list *ap_ptr)
 760 {
 761         dispatch_once_f(&_dispatch_logv_pred, NULL, _dispatch_logv_init);
 762         if (slowpath(dispatch_log_disabled)) {
 763                 return;
 764         }
 765         if (slowpath(dispatch_logfile != -1)) {
 766                 if (!ap_ptr) {
 767                         return _dispatch_log_file((char*)msg, len);
 768                 }
 769                 return _dispatch_logv_file(msg, *ap_ptr);
 770         }
 771         if (!ap_ptr) {
 772                 return _dispatch_syslog(msg);
 773         }
 774         return _dispatch_vsyslog(msg, *ap_ptr);
 775 }
 776
 777 DISPATCH_NOINLINE
 778 void
 779 _dispatch_log(const char *msg, ...)
 780 {
 781         va_list ap;
 782
 783         va_start(ap, msg);
 784         _dispatch_logv(msg, 0, &ap);
 785         va_end(ap);
 786 }
 787
 788 #endif // DISPATCH_USE_OS_DEBUG_LOG
 789
 790 #pragma mark -
 791 #pragma mark dispatch_debug
 792
 793 static size_t
 794 _dispatch_object_debug2(dispatch_object_t dou, char* buf, size_t bufsiz)
 795 {
 796         DISPATCH_OBJECT_TFB(_dispatch_objc_debug, dou, buf, bufsiz);
 797         if (dx_vtable(dou._do)->do_debug) {
 798                 return dx_debug(dou._do, buf, bufsiz);
 799         }
 800         return strlcpy(buf, "NULL vtable slot: ", bufsiz);
 801 }
 802
 803 DISPATCH_NOINLINE
 804 static void
 805 _dispatch_debugv(dispatch_object_t dou, const char *msg, va_list ap)
 806 {
 807         char buf[2048];
 808         size_t bufsiz = sizeof(buf), offset = 0;
 809         int r;
 810 #if DISPATCH_DEBUG && !DISPATCH_USE_OS_DEBUG_LOG
 811         offset += dsnprintf(&buf[offset], bufsiz - offset, "%llu\t\t%p\t",
 812                         _dispatch_absolute_time() - dispatch_log_basetime,
 813                         (void *)_dispatch_thread_self());
 814 #endif
 815         if (dou._do) {
 816                 offset += _dispatch_object_debug2(dou, &buf[offset], bufsiz - offset);
 817                 dispatch_assert(offset + 2 < bufsiz);
 818                 buf[offset++] = ':';
 819                 buf[offset++] = ' ';
 820                 buf[offset]   = '\0';
 821         } else {
 822                 offset += strlcpy(&buf[offset], "NULL: ", bufsiz - offset);
 823         }
 824         r = vsnprintf(&buf[offset], bufsiz - offset, msg, ap);
 825 #if !DISPATCH_USE_OS_DEBUG_LOG
 826         size_t len = offset + (r < 0 ? 0 : (size_t)r);
 827         if (len > bufsiz - 1) {
 828                 len = bufsiz - 1;
 829         }
 830         _dispatch_logv(buf, len, NULL);
 831 #else
 832         _dispatch_log("%s", buf);
 833 #endif
 834 }
 835
 836 DISPATCH_NOINLINE
 837 void
 838 dispatch_debugv(dispatch_object_t dou, const char *msg, va_list ap)
 839 {
 840         _dispatch_debugv(dou, msg, ap);
 841 }
 842
 843 DISPATCH_NOINLINE
 844 void
 845 dispatch_debug(dispatch_object_t dou, const char *msg, ...)
 846 {
 847         va_list ap;
 848
 849         va_start(ap, msg);
 850         _dispatch_debugv(dou, msg, ap);
 851         va_end(ap);
 852 }
 853
 854 #if DISPATCH_DEBUG
 855 DISPATCH_NOINLINE
 856 void
 857 _dispatch_object_debug(dispatch_object_t dou, const char *msg, ...)
 858 {
 859         va_list ap;
 860
 861         va_start(ap, msg);
 862         _dispatch_debugv(dou._do, msg, ap);
 863         va_end(ap);
 864 }
 865 #endif
 866
 867 #pragma mark -
 868 #pragma mark dispatch_calloc
 869
 870 DISPATCH_NOINLINE
 871 void
 872 _dispatch_temporary_resource_shortage(void)
 873 {
 874         sleep(1);
 875 }
 876
 877 void *
 878 _dispatch_calloc(size_t num_items, size_t size)
 879 {
 880         void *buf;
 881         while (!fastpath(buf = calloc(num_items, size))) {
 882                 _dispatch_temporary_resource_shortage();
 883         }
 884         return buf;
 885 }
 886
 887 /**
 888  * If the source string is mutable, allocates memory and copies the contents.
 889  * Otherwise returns the source string.
 890  */
 891 const char *
 892 _dispatch_strdup_if_mutable(const char *str)
 893 {
 894 #if HAVE_DYLD_IS_MEMORY_IMMUTABLE
 895         size_t size = strlen(str) + 1;
 896         if (slowpath(!_dyld_is_memory_immutable(str, size))) {
 897                 char *clone = (char *) malloc(size);
 898                 if (dispatch_assume(clone)) {
 899                         memcpy(clone, str, size);
 900                 }
 901                 return clone;
 902         }
 903         return str;
 904 #else
 905         return strdup(str);
 906 #endif
 907 }
 908
 909 #pragma mark -
 910 #pragma mark dispatch_block_t
 911
 912 #ifdef __BLOCKS__
 913
 914 void *
 915 (_dispatch_Block_copy)(void *db)
 916 {
 917         dispatch_block_t rval;
 918
 919         if (fastpath(db)) {
 920                 while (!fastpath(rval = Block_copy(db))) {
 921                         _dispatch_temporary_resource_shortage();
 922                 }
 923                 return rval;
 924         }
 925         DISPATCH_CLIENT_CRASH(0, "NULL was passed where a block should have been");
 926 }
 927
 928 void
 929 _dispatch_call_block_and_release(void *block)
 930 {
 931         void (^b)(void) = block;
 932         b();
 933         Block_release(b);
 934 }
 935
 936 #endif // __BLOCKS__
 937
 938 #pragma mark -
 939 #pragma mark dispatch_client_callout
 940
 941 // Abort on uncaught exceptions thrown from client callouts rdar://8577499
 942 #if DISPATCH_USE_CLIENT_CALLOUT && (__USING_SJLJ_EXCEPTIONS__ || !USE_OBJC || \
 943                 OS_OBJECT_HAVE_OBJC1)
 944 // On platforms with SjLj exceptions, avoid the SjLj overhead on every callout
 945 // by clearing the unwinder's TSD pointer to the handler stack around callouts
 946
 947 #define _dispatch_get_tsd_base()
 948 #define _dispatch_get_unwind_tsd() (NULL)
 949 #define _dispatch_set_unwind_tsd(u) do {(void)(u);} while (0)
 950 #define _dispatch_free_unwind_tsd()
 951
 952 #undef _dispatch_client_callout
 953 DISPATCH_NOINLINE
 954 void
 955 _dispatch_client_callout(void *ctxt, dispatch_function_t f)
 956 {
 957         _dispatch_get_tsd_base();
 958         void *u = _dispatch_get_unwind_tsd();
 959         if (fastpath(!u)) return f(ctxt);
 960         _dispatch_set_unwind_tsd(NULL);
 961         f(ctxt);
 962         _dispatch_free_unwind_tsd();
 963         _dispatch_set_unwind_tsd(u);
 964 }
 965
 966 #undef _dispatch_client_callout2
 967 DISPATCH_NOINLINE
 968 void
 969 _dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t))
 970 {
 971         _dispatch_get_tsd_base();
 972         void *u = _dispatch_get_unwind_tsd();
 973         if (fastpath(!u)) return f(ctxt, i);
 974         _dispatch_set_unwind_tsd(NULL);
 975         f(ctxt, i);
 976         _dispatch_free_unwind_tsd();
 977         _dispatch_set_unwind_tsd(u);
 978 }
 979
 980 #if HAVE_MACH
 981 #undef _dispatch_client_callout4
 982 void
 983 _dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason,
 984                 dispatch_mach_msg_t dmsg, mach_error_t error,
 985                 dispatch_mach_handler_function_t f)
 986 {
 987         _dispatch_get_tsd_base();
 988         void *u = _dispatch_get_unwind_tsd();
 989         if (fastpath(!u)) return f(ctxt, reason, dmsg, error);
 990         _dispatch_set_unwind_tsd(NULL);
 991         f(ctxt, reason, dmsg, error);
 992         _dispatch_free_unwind_tsd();
 993         _dispatch_set_unwind_tsd(u);
 994 }
 995 #endif // HAVE_MACH
 996
 997 #endif // DISPATCH_USE_CLIENT_CALLOUT
 998
 999 #pragma mark -
1000 #pragma mark _os_object_t no_objc
1001
1002 #if !USE_OBJC
1003
1004 static const _os_object_vtable_s _os_object_vtable;
1005
1006 void
1007 _os_object_init(void)
1008 {
1009         return;
1010 }
1011
1012 inline _os_object_t
1013 _os_object_alloc_realized(const void *cls, size_t size)
1014 {
1015         _os_object_t obj;
1016         dispatch_assert(size >= sizeof(struct _os_object_s));
1017         while (!fastpath(obj = calloc(1u, size))) {
1018                 _dispatch_temporary_resource_shortage();
1019         }
1020         obj->os_obj_isa = cls;
1021         return obj;
1022 }
1023
1024 _os_object_t
1025 _os_object_alloc(const void *cls, size_t size)
1026 {
1027         if (!cls) cls = &_os_object_vtable;
1028         return _os_object_alloc_realized(cls, size);
1029 }
1030
1031 void
1032 _os_object_dealloc(_os_object_t obj)
1033 {
1034         *((void *volatile*)&obj->os_obj_isa) = (void *)0x200;
1035         return free(obj);
1036 }
1037
1038 void
1039 _os_object_xref_dispose(_os_object_t obj)
1040 {
1041         _os_object_xrefcnt_dispose_barrier(obj);
1042         if (fastpath(obj->os_obj_isa->_os_obj_xref_dispose)) {
1043                 return obj->os_obj_isa->_os_obj_xref_dispose(obj);
1044         }
1045         return _os_object_release_internal(obj);
1046 }
1047
1048 void
1049 _os_object_dispose(_os_object_t obj)
1050 {
1051         _os_object_refcnt_dispose_barrier(obj);
1052         if (fastpath(obj->os_obj_isa->_os_obj_dispose)) {
1053                 return obj->os_obj_isa->_os_obj_dispose(obj);
1054         }
1055         return _os_object_dealloc(obj);
1056 }
1057
1058 void*
1059 os_retain(void *obj)
1060 {
1061         if (fastpath(obj)) {
1062                 return _os_object_retain(obj);
1063         }
1064         return obj;
1065 }
1066
1067 #undef os_release
1068 void
1069 os_release(void *obj)
1070 {
1071         if (fastpath(obj)) {
1072                 return _os_object_release(obj);
1073         }
1074 }
1075
1076 void
1077 _os_object_atfork_prepare(void)
1078 {
1079         return;
1080 }
1081
1082 void
1083 _os_object_atfork_parent(void)
1084 {
1085         return;
1086 }
1087
1088 void
1089 _os_object_atfork_child(void)
1090 {
1091         return;
1092 }
1093
1094 #pragma mark -
1095 #pragma mark dispatch_autorelease_pool no_objc
1096
1097 #if DISPATCH_COCOA_COMPAT
1098
1099 void*
1100 _dispatch_autorelease_pool_push(void)
1101 {
1102         void *pool = NULL;
1103         if (_dispatch_begin_NSAutoReleasePool) {
1104                 pool = _dispatch_begin_NSAutoReleasePool();
1105         }
1106         return pool;
1107 }
1108
1109 void
1110 _dispatch_autorelease_pool_pop(void *pool)
1111 {
1112         if (_dispatch_end_NSAutoReleasePool) {
1113                 _dispatch_end_NSAutoReleasePool(pool);
1114         }
1115 }
1116
1117 void*
1118 _dispatch_last_resort_autorelease_pool_push(void)
1119 {
1120         return _dispatch_autorelease_pool_push();
1121 }
1122
1123 void
1124 _dispatch_last_resort_autorelease_pool_pop(void *pool)
1125 {
1126         _dispatch_autorelease_pool_pop(pool);
1127 }
1128
1129 #endif // DISPATCH_COCOA_COMPAT
1130 #endif // !USE_OBJC
1131
1132 #pragma mark -
1133 #pragma mark dispatch_source_types
1134
1135 static void
1136 dispatch_source_type_timer_init(dispatch_source_t ds,
1137         dispatch_source_type_t type DISPATCH_UNUSED,
1138         uintptr_t handle DISPATCH_UNUSED,
1139         unsigned long mask)
1140 {
1141         if (fastpath(!ds->ds_refs)) {
1142                 ds->ds_refs = _dispatch_calloc(1ul,
1143                                 sizeof(struct dispatch_timer_source_refs_s));
1144         }
1145         ds->ds_needs_rearm = true;
1146         ds->ds_is_timer = true;
1147         ds_timer(ds->ds_refs).flags = mask;
1148 }
1149
1150 const struct dispatch_source_type_s _dispatch_source_type_timer = {
1151         .ke = {
1152                 .filter = DISPATCH_EVFILT_TIMER,
1153         },
1154         .mask = DISPATCH_TIMER_STRICT|DISPATCH_TIMER_BACKGROUND|
1155                         DISPATCH_TIMER_WALL_CLOCK,
1156         .init = dispatch_source_type_timer_init,
1157 };
1158
1159 static void
1160 dispatch_source_type_after_init(dispatch_source_t ds,
1161         dispatch_source_type_t type, uintptr_t handle, unsigned long mask)
1162 {
1163         dispatch_source_type_timer_init(ds, type, handle, mask);
1164         ds->ds_needs_rearm = false;
1165         ds_timer(ds->ds_refs).flags |= DISPATCH_TIMER_AFTER;
1166 }
1167
1168 const struct dispatch_source_type_s _dispatch_source_type_after = {
1169         .ke = {
1170                 .filter = DISPATCH_EVFILT_TIMER,
1171         },
1172         .init = dispatch_source_type_after_init,
1173 };
1174
1175 static void
1176 dispatch_source_type_timer_with_aggregate_init(dispatch_source_t ds,
1177         dispatch_source_type_t type, uintptr_t handle, unsigned long mask)
1178 {
1179         ds->ds_refs = _dispatch_calloc(1ul,
1180                         sizeof(struct dispatch_timer_source_aggregate_refs_s));
1181         dispatch_source_type_timer_init(ds, type, handle, mask);
1182         ds_timer(ds->ds_refs).flags |= DISPATCH_TIMER_WITH_AGGREGATE;
1183         ds->dq_specific_q = (void*)handle;
1184         _dispatch_retain(ds->dq_specific_q);
1185 }
1186
1187 const struct dispatch_source_type_s _dispatch_source_type_timer_with_aggregate={
1188         .ke = {
1189                 .filter = DISPATCH_EVFILT_TIMER,
1190                 .ident = ~0ull,
1191         },
1192         .mask = DISPATCH_TIMER_STRICT|DISPATCH_TIMER_BACKGROUND,
1193         .init = dispatch_source_type_timer_with_aggregate_init,
1194 };
1195
1196 static void
1197 dispatch_source_type_interval_init(dispatch_source_t ds,
1198         dispatch_source_type_t type, uintptr_t handle, unsigned long mask)
1199 {
1200         dispatch_source_type_timer_init(ds, type, handle, mask);
1201         ds_timer(ds->ds_refs).flags |= DISPATCH_TIMER_INTERVAL;
1202         unsigned long ident = _dispatch_source_timer_idx(ds->ds_refs);
1203         ds->ds_dkev->dk_kevent.ident = ds->ds_ident_hack = ident;
1204         _dispatch_source_set_interval(ds, handle);
1205 }
1206
1207 const struct dispatch_source_type_s _dispatch_source_type_interval = {
1208         .ke = {
1209                 .filter = DISPATCH_EVFILT_TIMER,
1210                 .ident = ~0ull,
1211         },
1212         .mask = DISPATCH_TIMER_STRICT|DISPATCH_TIMER_BACKGROUND|
1213                         DISPATCH_INTERVAL_UI_ANIMATION,
1214         .init = dispatch_source_type_interval_init,
1215 };
1216
1217 static void
1218 dispatch_source_type_readwrite_init(dispatch_source_t ds,
1219         dispatch_source_type_t type DISPATCH_UNUSED,
1220         uintptr_t handle DISPATCH_UNUSED,
1221         unsigned long mask DISPATCH_UNUSED)
1222 {
1223         ds->ds_is_level = true;
1224 #if HAVE_DECL_NOTE_LOWAT
1225         // bypass kernel check for device kqueue support rdar://19004921
1226         ds->ds_dkev->dk_kevent.fflags = NOTE_LOWAT;
1227 #endif
1228         ds->ds_dkev->dk_kevent.data = 1;
1229 }
1230
1231 const struct dispatch_source_type_s _dispatch_source_type_read = {
1232         .ke = {
1233                 .filter = EVFILT_READ,
1234                 .flags = EV_VANISHED|EV_DISPATCH|EV_UDATA_SPECIFIC,
1235         },
1236         .init = dispatch_source_type_readwrite_init,
1237 };
1238
1239 const struct dispatch_source_type_s _dispatch_source_type_write = {
1240         .ke = {
1241                 .filter = EVFILT_WRITE,
1242                 .flags = EV_VANISHED|EV_DISPATCH|EV_UDATA_SPECIFIC,
1243         },
1244         .init = dispatch_source_type_readwrite_init,
1245 };
1246
1247 #if DISPATCH_USE_MEMORYSTATUS
1248
1249 #if TARGET_IPHONE_SIMULATOR // rdar://problem/9219483
1250 static int _dispatch_ios_simulator_memory_warnings_fd = -1;
1251 static void
1252 _dispatch_ios_simulator_memorypressure_init(void *context DISPATCH_UNUSED)
1253 {
1254         char *e = getenv("SIMULATOR_MEMORY_WARNINGS");
1255         if (!e) return;
1256         _dispatch_ios_simulator_memory_warnings_fd = open(e, O_EVTONLY);
1257         if (_dispatch_ios_simulator_memory_warnings_fd == -1) {
1258                 (void)dispatch_assume_zero(errno);
1259         }
1260 }
1261 #endif
1262
1263 #if TARGET_IPHONE_SIMULATOR
1264 static void
1265 dispatch_source_type_memorypressure_init(dispatch_source_t ds,
1266         dispatch_source_type_t type DISPATCH_UNUSED,
1267         uintptr_t handle DISPATCH_UNUSED,
1268         unsigned long mask DISPATCH_UNUSED)
1269 {
1270         static dispatch_once_t pred;
1271         dispatch_once_f(&pred, NULL, _dispatch_ios_simulator_memorypressure_init);
1272         handle = (uintptr_t)_dispatch_ios_simulator_memory_warnings_fd;
1273         mask = NOTE_ATTRIB;
1274         ds->ds_dkev->dk_kevent.filter = EVFILT_VNODE;
1275         ds->ds_dkev->dk_kevent.ident = handle;
1276         ds->ds_dkev->dk_kevent.flags |= EV_CLEAR;
1277         ds->ds_dkev->dk_kevent.fflags = (uint32_t)mask;
1278         ds->ds_ident_hack = handle;
1279         ds->ds_pending_data_mask = mask;
1280         ds->ds_memorypressure_override = 1;
1281 }
1282 #else
1283 #define dispatch_source_type_memorypressure_init NULL
1284 #endif
1285
1286 #ifndef NOTE_MEMORYSTATUS_LOW_SWAP
1287 #define NOTE_MEMORYSTATUS_LOW_SWAP 0x8
1288 #endif
1289
1290 const struct dispatch_source_type_s _dispatch_source_type_memorypressure = {
1291         .ke = {
1292                 .filter = EVFILT_MEMORYSTATUS,
1293                 .flags = EV_DISPATCH|EV_UDATA_SPECIFIC,
1294         },
1295         .mask = NOTE_MEMORYSTATUS_PRESSURE_NORMAL|NOTE_MEMORYSTATUS_PRESSURE_WARN
1296                         |NOTE_MEMORYSTATUS_PRESSURE_CRITICAL|NOTE_MEMORYSTATUS_LOW_SWAP
1297                         |NOTE_MEMORYSTATUS_PROC_LIMIT_WARN|NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL,
1298         .init = dispatch_source_type_memorypressure_init,
1299 };
1300
1301 static void
1302 dispatch_source_type_vm_init(dispatch_source_t ds,
1303         dispatch_source_type_t type DISPATCH_UNUSED,
1304         uintptr_t handle DISPATCH_UNUSED,
1305         unsigned long mask DISPATCH_UNUSED)
1306 {
1307         // Map legacy vm pressure to memorypressure warning rdar://problem/15907505
1308         mask = NOTE_MEMORYSTATUS_PRESSURE_WARN;
1309         ds->ds_dkev->dk_kevent.fflags = (uint32_t)mask;
1310         ds->ds_pending_data_mask = mask;
1311         ds->ds_vmpressure_override = 1;
1312 #if TARGET_IPHONE_SIMULATOR
1313         dispatch_source_type_memorypressure_init(ds, type, handle, mask);
1314 #endif
1315 }
1316
1317 const struct dispatch_source_type_s _dispatch_source_type_vm = {
1318         .ke = {
1319                 .filter = EVFILT_MEMORYSTATUS,
1320                 .flags = EV_DISPATCH|EV_UDATA_SPECIFIC,
1321         },
1322         .mask = NOTE_VM_PRESSURE,
1323         .init = dispatch_source_type_vm_init,
1324 };
1325
1326 #endif // DISPATCH_USE_MEMORYSTATUS
1327
1328 const struct dispatch_source_type_s _dispatch_source_type_signal = {
1329         .ke = {
1330                 .filter = EVFILT_SIGNAL,
1331                 .flags = EV_UDATA_SPECIFIC,
1332         },
1333 };
1334
1335 #if !defined(__linux__)
1336 static void
1337 dispatch_source_type_proc_init(dispatch_source_t ds,
1338         dispatch_source_type_t type DISPATCH_UNUSED,
1339         uintptr_t handle DISPATCH_UNUSED,
1340         unsigned long mask DISPATCH_UNUSED)
1341 {
1342         ds->ds_dkev->dk_kevent.fflags |= NOTE_EXIT; // rdar://16655831
1343 }
1344
1345 const struct dispatch_source_type_s _dispatch_source_type_proc = {
1346         .ke = {
1347                 .filter = EVFILT_PROC,
1348                 .flags = EV_CLEAR|EV_UDATA_SPECIFIC,
1349         },
1350         .mask = NOTE_EXIT|NOTE_FORK|NOTE_EXEC
1351 #if HAVE_DECL_NOTE_SIGNAL
1352                         |NOTE_SIGNAL
1353 #endif
1354 #if HAVE_DECL_NOTE_REAP
1355                         |NOTE_REAP
1356 #endif
1357                         ,
1358         .init = dispatch_source_type_proc_init,
1359 };
1360
1361 const struct dispatch_source_type_s _dispatch_source_type_vnode = {
1362         .ke = {
1363                 .filter = EVFILT_VNODE,
1364                 .flags = EV_CLEAR|EV_VANISHED|EV_UDATA_SPECIFIC,
1365         },
1366         .mask = NOTE_DELETE|NOTE_WRITE|NOTE_EXTEND|NOTE_ATTRIB|NOTE_LINK|
1367                         NOTE_RENAME|NOTE_FUNLOCK
1368 #if HAVE_DECL_NOTE_REVOKE
1369                         |NOTE_REVOKE
1370 #endif
1371 #if HAVE_DECL_NOTE_NONE
1372                         |NOTE_NONE
1373 #endif
1374                         ,
1375 };
1376
1377 const struct dispatch_source_type_s _dispatch_source_type_vfs = {
1378         .ke = {
1379                 .filter = EVFILT_FS,
1380                 .flags = EV_CLEAR|EV_UDATA_SPECIFIC,
1381         },
1382         .mask = VQ_NOTRESP|VQ_NEEDAUTH|VQ_LOWDISK|VQ_MOUNT|VQ_UNMOUNT|VQ_DEAD|
1383                         VQ_ASSIST|VQ_NOTRESPLOCK
1384 #if HAVE_DECL_VQ_UPDATE
1385                         |VQ_UPDATE
1386 #endif
1387 #if HAVE_DECL_VQ_VERYLOWDISK
1388                         |VQ_VERYLOWDISK
1389 #endif
1390 #if HAVE_DECL_VQ_QUOTA
1391                         |VQ_QUOTA
1392 #endif
1393 #if HAVE_DECL_VQ_NEARLOWDISK
1394                         |VQ_NEARLOWDISK
1395 #endif
1396 #if HAVE_DECL_VQ_DESIRED_DISK
1397                         |VQ_DESIRED_DISK
1398 #endif
1399                         ,
1400 };
1401
1402 const struct dispatch_source_type_s _dispatch_source_type_sock = {
1403 #ifdef EVFILT_SOCK
1404         .ke = {
1405                 .filter = EVFILT_SOCK,
1406                 .flags = EV_CLEAR|EV_VANISHED|EV_UDATA_SPECIFIC,
1407         },
1408         .mask = NOTE_CONNRESET |  NOTE_READCLOSED | NOTE_WRITECLOSED |
1409                 NOTE_TIMEOUT | NOTE_NOSRCADDR |  NOTE_IFDENIED | NOTE_SUSPEND |
1410                 NOTE_RESUME | NOTE_KEEPALIVE
1411 #ifdef NOTE_ADAPTIVE_WTIMO
1412                 | NOTE_ADAPTIVE_WTIMO | NOTE_ADAPTIVE_RTIMO
1413 #endif
1414 #ifdef NOTE_CONNECTED
1415                 | NOTE_CONNECTED | NOTE_DISCONNECTED | NOTE_CONNINFO_UPDATED
1416 #endif
1417 #ifdef NOTE_NOTIFY_ACK
1418                 | NOTE_NOTIFY_ACK
1419 #endif
1420                 ,
1421 #endif // EVFILT_SOCK
1422 };
1423 #endif // !defined(__linux__)
1424
1425 static void
1426 dispatch_source_type_data_init(dispatch_source_t ds,
1427         dispatch_source_type_t type DISPATCH_UNUSED,
1428         uintptr_t handle DISPATCH_UNUSED,
1429         unsigned long mask DISPATCH_UNUSED)
1430 {
1431         ds->ds_is_installed = true;
1432         ds->ds_is_custom_source = true;
1433         ds->ds_is_direct_kevent = true;
1434         ds->ds_pending_data_mask = ~0ul;
1435         ds->ds_needs_rearm = false; // not registered with kevent
1436 }
1437
1438 const struct dispatch_source_type_s _dispatch_source_type_data_add = {
1439         .ke = {
1440                 .filter = DISPATCH_EVFILT_CUSTOM_ADD,
1441                 .flags = EV_UDATA_SPECIFIC,
1442         },
1443         .init = dispatch_source_type_data_init,
1444 };
1445
1446 const struct dispatch_source_type_s _dispatch_source_type_data_or = {
1447         .ke = {
1448                 .filter = DISPATCH_EVFILT_CUSTOM_OR,
1449                 .flags = EV_CLEAR|EV_UDATA_SPECIFIC,
1450                 .fflags = ~0u,
1451         },
1452         .init = dispatch_source_type_data_init,
1453 };
1454
1455 #if HAVE_MACH
1456
1457 static void
1458 dispatch_source_type_mach_send_init(dispatch_source_t ds,
1459         dispatch_source_type_t type DISPATCH_UNUSED,
1460         uintptr_t handle DISPATCH_UNUSED, unsigned long mask)
1461 {
1462         if (!mask) {
1463                 // Preserve legacy behavior that (mask == 0) => DISPATCH_MACH_SEND_DEAD
1464                 ds->ds_dkev->dk_kevent.fflags = DISPATCH_MACH_SEND_DEAD;
1465                 ds->ds_pending_data_mask = DISPATCH_MACH_SEND_DEAD;
1466         }
1467 }
1468
1469 const struct dispatch_source_type_s _dispatch_source_type_mach_send = {
1470         .ke = {
1471                 .filter = DISPATCH_EVFILT_MACH_NOTIFICATION,
1472                 .flags = EV_CLEAR,
1473         },
1474         .mask = DISPATCH_MACH_SEND_DEAD|DISPATCH_MACH_SEND_POSSIBLE,
1475         .init = dispatch_source_type_mach_send_init,
1476 };
1477
1478 static void
1479 dispatch_source_type_mach_recv_init(dispatch_source_t ds,
1480         dispatch_source_type_t type DISPATCH_UNUSED,
1481         uintptr_t handle DISPATCH_UNUSED,
1482         unsigned long mask DISPATCH_UNUSED)
1483 {
1484         ds->ds_pending_data_mask = DISPATCH_MACH_RECV_MESSAGE;
1485 #if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
1486         if (_dispatch_evfilt_machport_direct_enabled) return;
1487         ds->ds_dkev->dk_kevent.fflags = DISPATCH_MACH_RECV_MESSAGE;
1488         ds->ds_dkev->dk_kevent.flags &= ~(EV_UDATA_SPECIFIC|EV_VANISHED);
1489         ds->ds_is_direct_kevent = false;
1490 #endif
1491 }
1492
1493 const struct dispatch_source_type_s _dispatch_source_type_mach_recv = {
1494         .ke = {
1495                 .filter = EVFILT_MACHPORT,
1496                 .flags = EV_VANISHED|EV_DISPATCH|EV_UDATA_SPECIFIC,
1497         },
1498         .init = dispatch_source_type_mach_recv_init,
1499 };
1500
1501 #pragma mark -
1502 #pragma mark dispatch_mig
1503
1504 void *
1505 dispatch_mach_msg_get_context(mach_msg_header_t *msg)
1506 {
1507         mach_msg_context_trailer_t *tp;
1508         void *context = NULL;
1509
1510         tp = (mach_msg_context_trailer_t *)((uint8_t *)msg +
1511                         round_msg(msg->msgh_size));
1512         if (tp->msgh_trailer_size >=
1513                         (mach_msg_size_t)sizeof(mach_msg_context_trailer_t)) {
1514                 context = (void *)(uintptr_t)tp->msgh_context;
1515         }
1516         return context;
1517 }
1518
1519 kern_return_t
1520 _dispatch_wakeup_runloop_thread(mach_port_t mp DISPATCH_UNUSED)
1521 {
1522         // dummy function just to pop a runloop thread out of mach_msg()
1523         return 0;
1524 }
1525
1526 kern_return_t
1527 _dispatch_consume_send_once_right(mach_port_t mp DISPATCH_UNUSED)
1528 {
1529         // dummy function to consume a send-once right
1530         return 0;
1531 }
1532
1533 kern_return_t
1534 _dispatch_mach_notify_port_destroyed(mach_port_t notify DISPATCH_UNUSED,
1535                 mach_port_t name)
1536 {
1537         kern_return_t kr;
1538         // this function should never be called
1539         (void)dispatch_assume_zero(name);
1540         kr = mach_port_mod_refs(mach_task_self(), name, MACH_PORT_RIGHT_RECEIVE,-1);
1541         DISPATCH_VERIFY_MIG(kr);
1542         (void)dispatch_assume_zero(kr);
1543         return KERN_SUCCESS;
1544 }
1545
1546 kern_return_t
1547 _dispatch_mach_notify_no_senders(mach_port_t notify,
1548                 mach_port_mscount_t mscnt DISPATCH_UNUSED)
1549 {
1550         // this function should never be called
1551         (void)dispatch_assume_zero(notify);
1552         return KERN_SUCCESS;
1553 }
1554
1555 kern_return_t
1556 _dispatch_mach_notify_send_once(mach_port_t notify DISPATCH_UNUSED)
1557 {
1558         // we only register for dead-name notifications
1559         // some code deallocated our send-once right without consuming it
1560 #if DISPATCH_DEBUG
1561         _dispatch_log("Corruption: An app/library deleted a libdispatch "
1562                         "dead-name notification");
1563 #endif
1564         return KERN_SUCCESS;
1565 }
1566
1567 #endif // HAVE_MACH