src/init.c

   1 /*
   2  * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 // Contains exported global data and initialization & other routines that must
  22 // only exist once in the shared library even when resolvers are used.
  23
  24 #include "internal.h"
  25
  26 #if HAVE_MACH
  27 #include "protocolServer.h"
  28 #endif
  29
  30 #pragma mark -
  31 #pragma mark dispatch_init
  32
  33 #if USE_LIBDISPATCH_INIT_CONSTRUCTOR
  34 DISPATCH_NOTHROW __attribute__((constructor))
  35 void
  36 _libdispatch_init(void);
  37
  38 DISPATCH_EXPORT DISPATCH_NOTHROW
  39 void
  40 _libdispatch_init(void)
  41 {
  42         libdispatch_init();
  43 }
  44 #endif
  45
  46 DISPATCH_EXPORT DISPATCH_NOTHROW
  47 void
  48 dispatch_atfork_prepare(void)
  49 {
  50 }
  51
  52 DISPATCH_EXPORT DISPATCH_NOTHROW
  53 void
  54 dispatch_atfork_parent(void)
  55 {
  56 }
  57
  58 #pragma mark -
  59 #pragma mark dispatch_globals
  60
  61 DISPATCH_HIDE_SYMBOL(dispatch_assert_queue, 10.12, 10.0, 10.0, 3.0);
  62 DISPATCH_HIDE_SYMBOL(dispatch_assert_queue_not, 10.12, 10.0, 10.0, 3.0);
  63 DISPATCH_HIDE_SYMBOL(dispatch_queue_create_with_target, 10.12, 10.0, 10.0, 3.0);
  64
  65 #if DISPATCH_COCOA_COMPAT
  66 void *(*_dispatch_begin_NSAutoReleasePool)(void);
  67 void (*_dispatch_end_NSAutoReleasePool)(void *);
  68 #endif
  69
  70 #if DISPATCH_USE_THREAD_LOCAL_STORAGE
  71 __thread struct dispatch_tsd __dispatch_tsd;
  72 pthread_key_t __dispatch_tsd_key;
  73 #elif !DISPATCH_USE_DIRECT_TSD
  74 pthread_key_t dispatch_queue_key;
  75 pthread_key_t dispatch_frame_key;
  76 pthread_key_t dispatch_cache_key;
  77 pthread_key_t dispatch_context_key;
  78 pthread_key_t dispatch_pthread_root_queue_observer_hooks_key;
  79 pthread_key_t dispatch_defaultpriority_key;
  80 #if DISPATCH_INTROSPECTION
  81 pthread_key_t dispatch_introspection_key;
  82 #elif DISPATCH_PERF_MON
  83 pthread_key_t dispatch_bcounter_key;
  84 #endif
  85 pthread_key_t dispatch_sema4_key;
  86 pthread_key_t dispatch_voucher_key;
  87 pthread_key_t dispatch_deferred_items_key;
  88 #endif // !DISPATCH_USE_DIRECT_TSD && !DISPATCH_USE_THREAD_LOCAL_STORAGE
  89
  90 #if VOUCHER_USE_MACH_VOUCHER
  91 dispatch_once_t _voucher_task_mach_voucher_pred;
  92 mach_voucher_t _voucher_task_mach_voucher;
  93 #if !VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
  94 mach_voucher_t _voucher_default_task_mach_voucher;
  95 #endif
  96 dispatch_once_t _firehose_task_buffer_pred;
  97 firehose_buffer_t _firehose_task_buffer;
  98 const uint32_t _firehose_spi_version = OS_FIREHOSE_SPI_VERSION;
  99 uint64_t _voucher_unique_pid;
 100 voucher_activity_hooks_t _voucher_libtrace_hooks;
 101 dispatch_mach_t _voucher_activity_debug_channel;
 102 #endif
 103 #if HAVE_PTHREAD_WORKQUEUE_QOS && DISPATCH_DEBUG
 104 int _dispatch_set_qos_class_enabled;
 105 #endif
 106 #if DISPATCH_USE_KEVENT_WORKQUEUE && DISPATCH_USE_MGR_THREAD
 107 int _dispatch_kevent_workqueue_enabled;
 108 #endif
 109 #if DISPATCH_USE_EVFILT_MACHPORT_DIRECT && \
 110                 DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
 111 int _dispatch_evfilt_machport_direct_enabled;
 112 #endif
 113
 114 DISPATCH_HW_CONFIG();
 115 uint8_t _dispatch_unsafe_fork;
 116 bool _dispatch_child_of_unsafe_fork;
 117 #if DISPATCH_USE_MEMORYPRESSURE_SOURCE
 118 bool _dispatch_memory_warn;
 119 int _dispatch_continuation_cache_limit = DISPATCH_CONTINUATION_CACHE_LIMIT;
 120 #endif
 121
 122 DISPATCH_NOINLINE
 123 bool
 124 _dispatch_is_multithreaded(void)
 125 {
 126         return _dispatch_is_multithreaded_inline();
 127 }
 128
 129 DISPATCH_NOINLINE
 130 bool
 131 _dispatch_is_fork_of_multithreaded_parent(void)
 132 {
 133         return _dispatch_child_of_unsafe_fork;
 134 }
 135
 136 DISPATCH_NOINLINE
 137 void
 138 _dispatch_fork_becomes_unsafe_slow(void)
 139 {
 140         uint8_t value = os_atomic_or(&_dispatch_unsafe_fork,
 141                         _DISPATCH_UNSAFE_FORK_MULTITHREADED, relaxed);
 142         if (value & _DISPATCH_UNSAFE_FORK_PROHIBIT) {
 143                 DISPATCH_CLIENT_CRASH(0, "Transition to multithreaded is prohibited");
 144         }
 145 }
 146
 147 DISPATCH_NOINLINE
 148 void
 149 _dispatch_prohibit_transition_to_multithreaded(bool prohibit)
 150 {
 151         if (prohibit) {
 152                 uint8_t value = os_atomic_or(&_dispatch_unsafe_fork,
 153                                 _DISPATCH_UNSAFE_FORK_PROHIBIT, relaxed);
 154                 if (value & _DISPATCH_UNSAFE_FORK_MULTITHREADED) {
 155                         DISPATCH_CLIENT_CRASH(0, "The executable is already multithreaded");
 156                 }
 157         } else {
 158                 os_atomic_and(&_dispatch_unsafe_fork,
 159                                 (uint8_t)~_DISPATCH_UNSAFE_FORK_PROHIBIT, relaxed);
 160         }
 161 }
 162
 163 const struct dispatch_queue_offsets_s dispatch_queue_offsets = {
 164         .dqo_version = 6,
 165         .dqo_label = offsetof(struct dispatch_queue_s, dq_label),
 166         .dqo_label_size = sizeof(((dispatch_queue_t)NULL)->dq_label),
 167         .dqo_flags = 0,
 168         .dqo_flags_size = 0,
 169         .dqo_serialnum = offsetof(struct dispatch_queue_s, dq_serialnum),
 170         .dqo_serialnum_size = sizeof(((dispatch_queue_t)NULL)->dq_serialnum),
 171         .dqo_width = offsetof(struct dispatch_queue_s, dq_width),
 172         .dqo_width_size = sizeof(((dispatch_queue_t)NULL)->dq_width),
 173         .dqo_running = 0,
 174         .dqo_running_size = 0,
 175         .dqo_suspend_cnt = 0,
 176         .dqo_suspend_cnt_size = 0,
 177         .dqo_target_queue = offsetof(struct dispatch_queue_s, do_targetq),
 178         .dqo_target_queue_size = sizeof(((dispatch_queue_t)NULL)->do_targetq),
 179         .dqo_priority = offsetof(struct dispatch_queue_s, dq_priority),
 180         .dqo_priority_size = sizeof(((dispatch_queue_t)NULL)->dq_priority),
 181 };
 182
 183 #if DISPATCH_USE_DIRECT_TSD
 184 const struct dispatch_tsd_indexes_s dispatch_tsd_indexes = {
 185         .dti_version = 2,
 186         .dti_queue_index = dispatch_queue_key,
 187         .dti_voucher_index = dispatch_voucher_key,
 188         .dti_qos_class_index = dispatch_priority_key,
 189 };
 190 #endif // DISPATCH_USE_DIRECT_TSD
 191
 192 // 6618342 Contact the team that owns the Instrument DTrace probe before
 193 //         renaming this symbol
 194 DISPATCH_CACHELINE_ALIGN
 195 struct dispatch_queue_s _dispatch_main_q = {
 196         DISPATCH_GLOBAL_OBJECT_HEADER(queue_main),
 197 #if !DISPATCH_USE_RESOLVERS
 198         .do_targetq = &_dispatch_root_queues[
 199                         DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT],
 200 #endif
 201         .dq_state = DISPATCH_QUEUE_STATE_INIT_VALUE(1),
 202         .dq_label = "com.apple.main-thread",
 203         .dq_width = 1,
 204         .dq_atomic_bits = DQF_THREAD_BOUND | DQF_CANNOT_TRYSYNC,
 205         .dq_override_voucher = DISPATCH_NO_VOUCHER,
 206         .dq_serialnum = 1,
 207 };
 208
 209 #pragma mark -
 210 #pragma mark dispatch_queue_attr_t
 211
 212 #define DISPATCH_QUEUE_ATTR_INIT(qos, prio, overcommit, freq, concurrent, inactive) \
 213                 { \
 214                         DISPATCH_GLOBAL_OBJECT_HEADER(queue_attr), \
 215                         .dqa_qos_class = (qos), \
 216                         .dqa_relative_priority = (qos) ? (prio) : 0, \
 217                         .dqa_overcommit = _dispatch_queue_attr_overcommit_##overcommit, \
 218                         .dqa_autorelease_frequency = DISPATCH_AUTORELEASE_FREQUENCY_##freq, \
 219                         .dqa_concurrent = (concurrent), \
 220                         .dqa_inactive = (inactive), \
 221                 }
 222
 223 #define DISPATCH_QUEUE_ATTR_ACTIVE_INIT(qos, prio, overcommit, freq, concurrent) \
 224                 { \
 225                         [DQA_INDEX_ACTIVE] = DISPATCH_QUEUE_ATTR_INIT(\
 226                                         qos, prio, overcommit, freq, concurrent, false), \
 227                         [DQA_INDEX_INACTIVE] = DISPATCH_QUEUE_ATTR_INIT(\
 228                                         qos, prio, overcommit, freq, concurrent, true), \
 229                 }
 230
 231 #define DISPATCH_QUEUE_ATTR_OVERCOMMIT_INIT(qos, prio, overcommit) \
 232                 { \
 233                         [DQA_INDEX_AUTORELEASE_FREQUENCY_INHERIT][DQA_INDEX_CONCURRENT] = \
 234                                         DISPATCH_QUEUE_ATTR_ACTIVE_INIT(qos, prio, overcommit, INHERIT, 1), \
 235                         [DQA_INDEX_AUTORELEASE_FREQUENCY_INHERIT][DQA_INDEX_SERIAL] = \
 236                                         DISPATCH_QUEUE_ATTR_ACTIVE_INIT(qos, prio, overcommit, INHERIT, 0), \
 237                         [DQA_INDEX_AUTORELEASE_FREQUENCY_WORK_ITEM][DQA_INDEX_CONCURRENT] = \
 238                                         DISPATCH_QUEUE_ATTR_ACTIVE_INIT(qos, prio, overcommit, WORK_ITEM, 1), \
 239                         [DQA_INDEX_AUTORELEASE_FREQUENCY_WORK_ITEM][DQA_INDEX_SERIAL] = \
 240                                         DISPATCH_QUEUE_ATTR_ACTIVE_INIT(qos, prio, overcommit, WORK_ITEM, 0), \
 241                         [DQA_INDEX_AUTORELEASE_FREQUENCY_NEVER][DQA_INDEX_CONCURRENT] = \
 242                                         DISPATCH_QUEUE_ATTR_ACTIVE_INIT(qos, prio, overcommit, NEVER, 1), \
 243                         [DQA_INDEX_AUTORELEASE_FREQUENCY_NEVER][DQA_INDEX_SERIAL] = \
 244                                         DISPATCH_QUEUE_ATTR_ACTIVE_INIT(qos, prio, overcommit, NEVER, 0), \
 245                 }
 246
 247 #define DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, prio) \
 248                 [prio] = { \
 249                         [DQA_INDEX_UNSPECIFIED_OVERCOMMIT] = \
 250                                         DISPATCH_QUEUE_ATTR_OVERCOMMIT_INIT(qos, -(prio), unspecified), \
 251                         [DQA_INDEX_NON_OVERCOMMIT] = \
 252                                         DISPATCH_QUEUE_ATTR_OVERCOMMIT_INIT(qos, -(prio), disabled), \
 253                         [DQA_INDEX_OVERCOMMIT] = \
 254                                         DISPATCH_QUEUE_ATTR_OVERCOMMIT_INIT(qos, -(prio), enabled), \
 255                 }
 256
 257 #define DISPATCH_QUEUE_ATTR_PRIO_INIT(qos) \
 258                 { \
 259                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 0), \
 260                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 1), \
 261                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 2), \
 262                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 3), \
 263                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 4), \
 264                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 5), \
 265                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 6), \
 266                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 7), \
 267                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 8), \
 268                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 9), \
 269                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 10), \
 270                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 11), \
 271                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 12), \
 272                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 13), \
 273                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 14), \
 274                         DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 15), \
 275                 }
 276
 277 #define DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(qos) \
 278                 [DQA_INDEX_QOS_CLASS_##qos] = \
 279                                 DISPATCH_QUEUE_ATTR_PRIO_INIT(_DISPATCH_QOS_CLASS_##qos)
 280
 281 // DISPATCH_QUEUE_CONCURRENT resp. _dispatch_queue_attr_concurrent is aliased
 282 // to array member [0][0][0][0][0][0] and their properties must match!
 283 const struct dispatch_queue_attr_s _dispatch_queue_attrs[]
 284                 [DISPATCH_QUEUE_ATTR_PRIO_COUNT]
 285                 [DISPATCH_QUEUE_ATTR_OVERCOMMIT_COUNT]
 286                 [DISPATCH_QUEUE_ATTR_AUTORELEASE_FREQUENCY_COUNT]
 287                 [DISPATCH_QUEUE_ATTR_CONCURRENCY_COUNT]
 288                 [DISPATCH_QUEUE_ATTR_INACTIVE_COUNT] = {
 289         DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(UNSPECIFIED),
 290         DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(MAINTENANCE),
 291         DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(BACKGROUND),
 292         DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(UTILITY),
 293         DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(DEFAULT),
 294         DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(USER_INITIATED),
 295         DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(USER_INTERACTIVE),
 296 };
 297
 298 #if DISPATCH_VARIANT_STATIC
 299 // <rdar://problem/16778703>
 300 struct dispatch_queue_attr_s _dispatch_queue_attr_concurrent =
 301         DISPATCH_QUEUE_ATTR_INIT(_DISPATCH_QOS_CLASS_UNSPECIFIED, 0,
 302                         unspecified, INHERIT, 1, false);
 303 #endif // DISPATCH_VARIANT_STATIC
 304
 305 // _dispatch_queue_attr_concurrent is aliased using libdispatch.aliases
 306 // and the -alias_list linker option on Darwin but needs to be done manually
 307 // for other platforms.
 308 #ifndef __APPLE__
 309 extern struct dispatch_queue_attr_s _dispatch_queue_attr_concurrent
 310         __attribute__((__alias__("_dispatch_queue_attrs")));
 311 #endif
 312
 313 #pragma mark -
 314 #pragma mark dispatch_vtables
 315
 316 DISPATCH_VTABLE_INSTANCE(semaphore,
 317         .do_type = DISPATCH_SEMAPHORE_TYPE,
 318         .do_kind = "semaphore",
 319         .do_dispose = _dispatch_semaphore_dispose,
 320         .do_debug = _dispatch_semaphore_debug,
 321 );
 322
 323 DISPATCH_VTABLE_INSTANCE(group,
 324         .do_type = DISPATCH_GROUP_TYPE,
 325         .do_kind = "group",
 326         .do_dispose = _dispatch_group_dispose,
 327         .do_debug = _dispatch_group_debug,
 328 );
 329
 330 DISPATCH_VTABLE_INSTANCE(queue,
 331         .do_type = DISPATCH_QUEUE_LEGACY_TYPE,
 332         .do_kind = "queue",
 333         .do_dispose = _dispatch_queue_dispose,
 334         .do_suspend = _dispatch_queue_suspend,
 335         .do_resume = _dispatch_queue_resume,
 336         .do_invoke = _dispatch_queue_invoke,
 337         .do_wakeup = _dispatch_queue_wakeup,
 338         .do_debug = dispatch_queue_debug,
 339         .do_set_targetq = _dispatch_queue_set_target_queue,
 340 );
 341
 342 DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_serial, queue,
 343         .do_type = DISPATCH_QUEUE_SERIAL_TYPE,
 344         .do_kind = "serial-queue",
 345         .do_dispose = _dispatch_queue_dispose,
 346         .do_suspend = _dispatch_queue_suspend,
 347         .do_resume = _dispatch_queue_resume,
 348         .do_finalize_activation = _dispatch_queue_finalize_activation,
 349         .do_invoke = _dispatch_queue_invoke,
 350         .do_wakeup = _dispatch_queue_wakeup,
 351         .do_debug = dispatch_queue_debug,
 352         .do_set_targetq = _dispatch_queue_set_target_queue,
 353 );
 354
 355 DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_concurrent, queue,
 356         .do_type = DISPATCH_QUEUE_CONCURRENT_TYPE,
 357         .do_kind = "concurrent-queue",
 358         .do_dispose = _dispatch_queue_dispose,
 359         .do_suspend = _dispatch_queue_suspend,
 360         .do_resume = _dispatch_queue_resume,
 361         .do_finalize_activation = _dispatch_queue_finalize_activation,
 362         .do_invoke = _dispatch_queue_invoke,
 363         .do_wakeup = _dispatch_queue_wakeup,
 364         .do_debug = dispatch_queue_debug,
 365         .do_set_targetq = _dispatch_queue_set_target_queue,
 366 );
 367
 368
 369 DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_root, queue,
 370         .do_type = DISPATCH_QUEUE_GLOBAL_ROOT_TYPE,
 371         .do_kind = "global-queue",
 372         .do_dispose = _dispatch_pthread_root_queue_dispose,
 373         .do_wakeup = _dispatch_root_queue_wakeup,
 374         .do_debug = dispatch_queue_debug,
 375 );
 376
 377 DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_main, queue,
 378         .do_type = DISPATCH_QUEUE_SERIAL_TYPE,
 379         .do_kind = "main-queue",
 380         .do_dispose = _dispatch_queue_dispose,
 381         .do_invoke = _dispatch_queue_invoke,
 382         .do_wakeup = _dispatch_main_queue_wakeup,
 383         .do_debug = dispatch_queue_debug,
 384 );
 385
 386 DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_runloop, queue,
 387         .do_type = DISPATCH_QUEUE_RUNLOOP_TYPE,
 388         .do_kind = "runloop-queue",
 389         .do_dispose = _dispatch_runloop_queue_dispose,
 390         .do_invoke = _dispatch_queue_invoke,
 391         .do_wakeup = _dispatch_runloop_queue_wakeup,
 392         .do_debug = dispatch_queue_debug,
 393 );
 394
 395 DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_mgr, queue,
 396         .do_type = DISPATCH_QUEUE_MGR_TYPE,
 397         .do_kind = "mgr-queue",
 398         .do_invoke = _dispatch_mgr_thread,
 399         .do_wakeup = _dispatch_mgr_queue_wakeup,
 400         .do_debug = dispatch_queue_debug,
 401 );
 402
 403 DISPATCH_VTABLE_INSTANCE(queue_specific_queue,
 404         .do_type = DISPATCH_QUEUE_SPECIFIC_TYPE,
 405         .do_kind = "queue-context",
 406         .do_dispose = _dispatch_queue_specific_queue_dispose,
 407         .do_invoke = (void *)_dispatch_queue_invoke,
 408         .do_wakeup = (void *)_dispatch_queue_wakeup,
 409         .do_debug = (void *)dispatch_queue_debug,
 410 );
 411
 412 DISPATCH_VTABLE_INSTANCE(queue_attr,
 413         .do_type = DISPATCH_QUEUE_ATTR_TYPE,
 414         .do_kind = "queue-attr",
 415 );
 416
 417 DISPATCH_VTABLE_INSTANCE(source,
 418         .do_type = DISPATCH_SOURCE_KEVENT_TYPE,
 419         .do_kind = "kevent-source",
 420         .do_dispose = _dispatch_source_dispose,
 421         .do_suspend = (void *)_dispatch_queue_suspend,
 422         .do_resume = (void *)_dispatch_queue_resume,
 423         .do_finalize_activation = _dispatch_source_finalize_activation,
 424         .do_invoke = _dispatch_source_invoke,
 425         .do_wakeup = _dispatch_source_wakeup,
 426         .do_debug = _dispatch_source_debug,
 427         .do_set_targetq = (void *)_dispatch_queue_set_target_queue,
 428 );
 429
 430 #if HAVE_MACH
 431 DISPATCH_VTABLE_INSTANCE(mach,
 432         .do_type = DISPATCH_MACH_CHANNEL_TYPE,
 433         .do_kind = "mach-channel",
 434         .do_dispose = _dispatch_mach_dispose,
 435         .do_suspend = (void *)_dispatch_queue_suspend,
 436         .do_resume = (void *)_dispatch_queue_resume,
 437         .do_finalize_activation = _dispatch_mach_finalize_activation,
 438         .do_invoke = _dispatch_mach_invoke,
 439         .do_wakeup = _dispatch_mach_wakeup,
 440         .do_debug = _dispatch_mach_debug,
 441         .do_set_targetq = (void *)_dispatch_queue_set_target_queue,
 442 );
 443
 444 DISPATCH_VTABLE_INSTANCE(mach_msg,
 445         .do_type = DISPATCH_MACH_MSG_TYPE,
 446         .do_kind = "mach-msg",
 447         .do_dispose = _dispatch_mach_msg_dispose,
 448         .do_invoke = _dispatch_mach_msg_invoke,
 449         .do_debug = _dispatch_mach_msg_debug,
 450 );
 451 #endif // HAVE_MACH
 452
 453 #if !DISPATCH_DATA_IS_BRIDGED_TO_NSDATA
 454 DISPATCH_VTABLE_INSTANCE(data,
 455         .do_type = DISPATCH_DATA_TYPE,
 456         .do_kind = "data",
 457         .do_dispose = _dispatch_data_dispose,
 458         .do_debug = _dispatch_data_debug,
 459 );
 460 #endif
 461
 462 DISPATCH_VTABLE_INSTANCE(io,
 463         .do_type = DISPATCH_IO_TYPE,
 464         .do_kind = "channel",
 465         .do_dispose = _dispatch_io_dispose,
 466         .do_debug = _dispatch_io_debug,
 467         .do_set_targetq = _dispatch_io_set_target_queue,
 468 );
 469
 470 DISPATCH_VTABLE_INSTANCE(operation,
 471         .do_type = DISPATCH_OPERATION_TYPE,
 472         .do_kind = "operation",
 473         .do_dispose = _dispatch_operation_dispose,
 474         .do_debug = _dispatch_operation_debug,
 475 );
 476
 477 DISPATCH_VTABLE_INSTANCE(disk,
 478         .do_type = DISPATCH_DISK_TYPE,
 479         .do_kind = "disk",
 480         .do_dispose = _dispatch_disk_dispose,
 481 );
 482
 483
 484 const struct dispatch_continuation_vtable_s _dispatch_continuation_vtables[] = {
 485         DC_VTABLE_ENTRY(ASYNC_REDIRECT,
 486                 .do_kind = "dc-redirect",
 487                 .do_invoke = _dispatch_async_redirect_invoke),
 488 #if HAVE_MACH
 489         DC_VTABLE_ENTRY(MACH_SEND_BARRRIER_DRAIN,
 490                 .do_kind = "dc-mach-send-drain",
 491                 .do_invoke = _dispatch_mach_send_barrier_drain_invoke),
 492         DC_VTABLE_ENTRY(MACH_SEND_BARRIER,
 493                 .do_kind = "dc-mach-send-barrier",
 494                 .do_invoke = _dispatch_mach_barrier_invoke),
 495         DC_VTABLE_ENTRY(MACH_RECV_BARRIER,
 496                 .do_kind = "dc-mach-recv-barrier",
 497                 .do_invoke = _dispatch_mach_barrier_invoke),
 498 #endif
 499 #if HAVE_PTHREAD_WORKQUEUE_QOS
 500         DC_VTABLE_ENTRY(OVERRIDE_STEALING,
 501                 .do_kind = "dc-override-stealing",
 502                 .do_invoke = _dispatch_queue_override_invoke),
 503         DC_VTABLE_ENTRY(OVERRIDE_OWNING,
 504                 .do_kind = "dc-override-owning",
 505                 .do_invoke = _dispatch_queue_override_invoke),
 506 #endif
 507 };
 508
 509 void
 510 _dispatch_vtable_init(void)
 511 {
 512 #if OS_OBJECT_HAVE_OBJC2
 513         // ObjC classes and dispatch vtables are co-located via linker order and
 514         // alias files, verify correct layout during initialization rdar://10640168
 515         dispatch_assert((char*)&DISPATCH_CONCAT(_,DISPATCH_CLASS(semaphore_vtable))
 516                         - (char*)DISPATCH_VTABLE(semaphore) ==
 517                         offsetof(struct dispatch_semaphore_vtable_s, _os_obj_vtable));
 518 #endif // USE_OBJC
 519 }
 520
 521 #pragma mark -
 522 #pragma mark dispatch_bug
 523
 524 static char _dispatch_build[16];
 525
 526 static void
 527 _dispatch_build_init(void *context DISPATCH_UNUSED)
 528 {
 529 #ifdef __APPLE__
 530         int mib[] = { CTL_KERN, KERN_OSVERSION };
 531         size_t bufsz = sizeof(_dispatch_build);
 532
 533         sysctl(mib, 2, _dispatch_build, &bufsz, NULL, 0);
 534 #if TARGET_IPHONE_SIMULATOR
 535         char *sim_version = getenv("SIMULATOR_RUNTIME_BUILD_VERSION");
 536         if (sim_version) {
 537                 (void)strlcat(_dispatch_build, " ", sizeof(_dispatch_build));
 538                 (void)strlcat(_dispatch_build, sim_version, sizeof(_dispatch_build));
 539         }
 540 #endif // TARGET_IPHONE_SIMULATOR
 541
 542 #else
 543         /*
 544          * XXXRW: What to do here for !Mac OS X?
 545          */
 546         memset(_dispatch_build, 0, sizeof(_dispatch_build));
 547 #endif // __APPLE__
 548 }
 549
 550 static dispatch_once_t _dispatch_build_pred;
 551
 552 char*
 553 _dispatch_get_build(void)
 554 {
 555         dispatch_once_f(&_dispatch_build_pred, NULL, _dispatch_build_init);
 556         return _dispatch_build;
 557 }
 558
 559 #define _dispatch_bug_log(msg, ...) do { \
 560         static void *last_seen; \
 561         void *ra = __builtin_return_address(0); \
 562         if (last_seen != ra) { \
 563                 last_seen = ra; \
 564                 _dispatch_log(msg, ##__VA_ARGS__); \
 565         } \
 566 } while(0)
 567
 568 void
 569 _dispatch_bug(size_t line, long val)
 570 {
 571         dispatch_once_f(&_dispatch_build_pred, NULL, _dispatch_build_init);
 572         _dispatch_bug_log("BUG in libdispatch: %s - %lu - 0x%lx",
 573                         _dispatch_build, (unsigned long)line, val);
 574 }
 575
 576 void
 577 _dispatch_bug_client(const char* msg)
 578 {
 579         _dispatch_bug_log("BUG in libdispatch client: %s", msg);
 580 }
 581
 582 #if HAVE_MACH
 583 void
 584 _dispatch_bug_mach_client(const char* msg, mach_msg_return_t kr)
 585 {
 586         _dispatch_bug_log("BUG in libdispatch client: %s %s - 0x%x", msg,
 587                         mach_error_string(kr), kr);
 588 }
 589 #endif
 590
 591 void
 592 _dispatch_bug_kevent_client(const char* msg, const char* filter,
 593                 const char *operation, int err)
 594 {
 595         if (operation && err) {
 596                 _dispatch_bug_log("BUG in libdispatch client: %s[%s] %s: \"%s\" - 0x%x",
 597                                 msg, filter, operation, strerror(err), err);
 598         } else if (operation) {
 599                 _dispatch_bug_log("BUG in libdispatch client: %s[%s] %s",
 600                                 msg, filter, operation);
 601         } else {
 602                 _dispatch_bug_log("BUG in libdispatch: %s[%s]: \"%s\" - 0x%x",
 603                                 msg, filter, strerror(err), err);
 604         }
 605 }
 606
 607 void
 608 _dispatch_bug_deprecated(const char *msg)
 609 {
 610         _dispatch_bug_log("DEPRECATED USE in libdispatch client: %s", msg);
 611 }
 612
 613 void
 614 _dispatch_abort(size_t line, long val)
 615 {
 616         _dispatch_bug(line, val);
 617         abort();
 618 }
 619
 620 #if !DISPATCH_USE_OS_DEBUG_LOG
 621
 622 #pragma mark -
 623 #pragma mark dispatch_log
 624
 625 static int dispatch_logfile = -1;
 626 static bool dispatch_log_disabled;
 627 #if DISPATCH_DEBUG
 628 static uint64_t dispatch_log_basetime;
 629 #endif
 630 static dispatch_once_t _dispatch_logv_pred;
 631
 632 static void
 633 _dispatch_logv_init(void *context DISPATCH_UNUSED)
 634 {
 635 #if DISPATCH_DEBUG
 636         bool log_to_file = true;
 637 #else
 638         bool log_to_file = false;
 639 #endif
 640         char *e = getenv("LIBDISPATCH_LOG");
 641         if (e) {
 642                 if (strcmp(e, "YES") == 0) {
 643                         // default
 644                 } else if (strcmp(e, "NO") == 0) {
 645                         dispatch_log_disabled = true;
 646                 } else if (strcmp(e, "syslog") == 0) {
 647                         log_to_file = false;
 648                 } else if (strcmp(e, "file") == 0) {
 649                         log_to_file = true;
 650                 } else if (strcmp(e, "stderr") == 0) {
 651                         log_to_file = true;
 652                         dispatch_logfile = STDERR_FILENO;
 653                 }
 654         }
 655         if (!dispatch_log_disabled) {
 656                 if (log_to_file && dispatch_logfile == -1) {
 657                         char path[PATH_MAX];
 658                         snprintf(path, sizeof(path), "/var/tmp/libdispatch.%d.log",
 659                                         getpid());
 660                         dispatch_logfile = open(path, O_WRONLY | O_APPEND | O_CREAT |
 661                                         O_NOFOLLOW | O_CLOEXEC, 0666);
 662                 }
 663                 if (dispatch_logfile != -1) {
 664                         struct timeval tv;
 665                         gettimeofday(&tv, NULL);
 666 #if DISPATCH_DEBUG
 667                         dispatch_log_basetime = _dispatch_absolute_time();
 668 #endif
 669                         dprintf(dispatch_logfile, "=== log file opened for %s[%u] at "
 670                                         "%ld.%06u ===\n", getprogname() ?: "", getpid(),
 671                                         tv.tv_sec, (int)tv.tv_usec);
 672                 }
 673         }
 674 }
 675
 676 static inline void
 677 _dispatch_log_file(char *buf, size_t len)
 678 {
 679         ssize_t r;
 680
 681         buf[len++] = '\n';
 682 retry:
 683         r = write(dispatch_logfile, buf, len);
 684         if (slowpath(r == -1) && errno == EINTR) {
 685                 goto retry;
 686         }
 687 }
 688
 689 DISPATCH_NOINLINE
 690 static void
 691 _dispatch_logv_file(const char *msg, va_list ap)
 692 {
 693         char buf[2048];
 694         size_t bufsiz = sizeof(buf), offset = 0;
 695         int r;
 696
 697 #if DISPATCH_DEBUG
 698         offset += dsnprintf(&buf[offset], bufsiz - offset, "%llu\t",
 699                         _dispatch_absolute_time() - dispatch_log_basetime);
 700 #endif
 701         r = vsnprintf(&buf[offset], bufsiz - offset, msg, ap);
 702         if (r < 0) return;
 703         offset += (size_t)r;
 704         if (offset > bufsiz - 1) {
 705                 offset = bufsiz - 1;
 706         }
 707         _dispatch_log_file(buf, offset);
 708 }
 709
 710 #if DISPATCH_USE_SIMPLE_ASL
 711 static inline void
 712 _dispatch_syslog(const char *msg)
 713 {
 714         _simple_asl_log(ASL_LEVEL_NOTICE, "com.apple.libsystem.libdispatch", msg);
 715 }
 716
 717 static inline void
 718 _dispatch_vsyslog(const char *msg, va_list ap)
 719 {
 720         char *str;
 721     vasprintf(&str, msg, ap);
 722         if (str) {
 723                 _dispatch_syslog(str);
 724                 free(str);
 725         }
 726 }
 727 #else // DISPATCH_USE_SIMPLE_ASL
 728 static inline void
 729 _dispatch_syslog(const char *msg)
 730 {
 731         syslog(LOG_NOTICE, "%s", msg);
 732 }
 733
 734 static inline void
 735 _dispatch_vsyslog(const char *msg, va_list ap)
 736 {
 737         vsyslog(LOG_NOTICE, msg, ap);
 738 }
 739 #endif // DISPATCH_USE_SIMPLE_ASL
 740
 741 DISPATCH_ALWAYS_INLINE
 742 static inline void
 743 _dispatch_logv(const char *msg, size_t len, va_list *ap_ptr)
 744 {
 745         dispatch_once_f(&_dispatch_logv_pred, NULL, _dispatch_logv_init);
 746         if (slowpath(dispatch_log_disabled)) {
 747                 return;
 748         }
 749         if (slowpath(dispatch_logfile != -1)) {
 750                 if (!ap_ptr) {
 751                         return _dispatch_log_file((char*)msg, len);
 752                 }
 753                 return _dispatch_logv_file(msg, *ap_ptr);
 754         }
 755         if (!ap_ptr) {
 756                 return _dispatch_syslog(msg);
 757         }
 758         return _dispatch_vsyslog(msg, *ap_ptr);
 759 }
 760
 761 DISPATCH_NOINLINE
 762 void
 763 _dispatch_log(const char *msg, ...)
 764 {
 765         va_list ap;
 766
 767         va_start(ap, msg);
 768         _dispatch_logv(msg, 0, &ap);
 769         va_end(ap);
 770 }
 771
 772 #endif // DISPATCH_USE_OS_DEBUG_LOG
 773
 774 #pragma mark -
 775 #pragma mark dispatch_debug
 776
 777 static size_t
 778 _dispatch_object_debug2(dispatch_object_t dou, char* buf, size_t bufsiz)
 779 {
 780         DISPATCH_OBJECT_TFB(_dispatch_objc_debug, dou, buf, bufsiz);
 781         if (dx_vtable(dou._do)->do_debug) {
 782                 return dx_debug(dou._do, buf, bufsiz);
 783         }
 784         return strlcpy(buf, "NULL vtable slot: ", bufsiz);
 785 }
 786
 787 DISPATCH_NOINLINE
 788 static void
 789 _dispatch_debugv(dispatch_object_t dou, const char *msg, va_list ap)
 790 {
 791         char buf[2048];
 792         size_t bufsiz = sizeof(buf), offset = 0;
 793         int r;
 794 #if DISPATCH_DEBUG && !DISPATCH_USE_OS_DEBUG_LOG
 795         offset += dsnprintf(&buf[offset], bufsiz - offset, "%llu\t\t%p\t",
 796                         _dispatch_absolute_time() - dispatch_log_basetime,
 797                         (void *)_dispatch_thread_self());
 798 #endif
 799         if (dou._do) {
 800                 offset += _dispatch_object_debug2(dou, &buf[offset], bufsiz - offset);
 801                 dispatch_assert(offset + 2 < bufsiz);
 802                 buf[offset++] = ':';
 803                 buf[offset++] = ' ';
 804                 buf[offset]   = '\0';
 805         } else {
 806                 offset += strlcpy(&buf[offset], "NULL: ", bufsiz - offset);
 807         }
 808         r = vsnprintf(&buf[offset], bufsiz - offset, msg, ap);
 809 #if !DISPATCH_USE_OS_DEBUG_LOG
 810         size_t len = offset + (r < 0 ? 0 : (size_t)r);
 811         if (len > bufsiz - 1) {
 812                 len = bufsiz - 1;
 813         }
 814         _dispatch_logv(buf, len, NULL);
 815 #else
 816         _dispatch_log("%s", buf);
 817 #endif
 818 }
 819
 820 DISPATCH_NOINLINE
 821 void
 822 dispatch_debugv(dispatch_object_t dou, const char *msg, va_list ap)
 823 {
 824         _dispatch_debugv(dou, msg, ap);
 825 }
 826
 827 DISPATCH_NOINLINE
 828 void
 829 dispatch_debug(dispatch_object_t dou, const char *msg, ...)
 830 {
 831         va_list ap;
 832
 833         va_start(ap, msg);
 834         _dispatch_debugv(dou, msg, ap);
 835         va_end(ap);
 836 }
 837
 838 #if DISPATCH_DEBUG
 839 DISPATCH_NOINLINE
 840 void
 841 _dispatch_object_debug(dispatch_object_t dou, const char *msg, ...)
 842 {
 843         va_list ap;
 844
 845         va_start(ap, msg);
 846         _dispatch_debugv(dou._do, msg, ap);
 847         va_end(ap);
 848 }
 849 #endif
 850
 851 #pragma mark -
 852 #pragma mark dispatch_calloc
 853
 854 DISPATCH_NOINLINE
 855 void
 856 _dispatch_temporary_resource_shortage(void)
 857 {
 858         sleep(1);
 859 }
 860
 861 void *
 862 _dispatch_calloc(size_t num_items, size_t size)
 863 {
 864         void *buf;
 865         while (!fastpath(buf = calloc(num_items, size))) {
 866                 _dispatch_temporary_resource_shortage();
 867         }
 868         return buf;
 869 }
 870
 871 /**
 872  * If the source string is mutable, allocates memory and copies the contents.
 873  * Otherwise returns the source string.
 874  */
 875 const char *
 876 _dispatch_strdup_if_mutable(const char *str)
 877 {
 878 #if HAVE_DYLD_IS_MEMORY_IMMUTABLE
 879         size_t size = strlen(str) + 1;
 880         if (slowpath(!_dyld_is_memory_immutable(str, size))) {
 881                 char *clone = (char *) malloc(size);
 882                 if (dispatch_assume(clone)) {
 883                         memcpy(clone, str, size);
 884                 }
 885                 return clone;
 886         }
 887         return str;
 888 #else
 889         return strdup(str);
 890 #endif
 891 }
 892
 893 #pragma mark -
 894 #pragma mark dispatch_block_t
 895
 896 #ifdef __BLOCKS__
 897
 898 void *
 899 (_dispatch_Block_copy)(void *db)
 900 {
 901         dispatch_block_t rval;
 902
 903         if (fastpath(db)) {
 904                 while (!fastpath(rval = Block_copy(db))) {
 905                         _dispatch_temporary_resource_shortage();
 906                 }
 907                 return rval;
 908         }
 909         DISPATCH_CLIENT_CRASH(0, "NULL was passed where a block should have been");
 910 }
 911
 912 void
 913 _dispatch_call_block_and_release(void *block)
 914 {
 915         void (^b)(void) = block;
 916         b();
 917         Block_release(b);
 918 }
 919
 920 #endif // __BLOCKS__
 921
 922 #pragma mark -
 923 #pragma mark dispatch_client_callout
 924
 925 // Abort on uncaught exceptions thrown from client callouts rdar://8577499
 926 #if DISPATCH_USE_CLIENT_CALLOUT && (__USING_SJLJ_EXCEPTIONS__ || !USE_OBJC || \
 927                 OS_OBJECT_HAVE_OBJC1)
 928 // On platforms with SjLj exceptions, avoid the SjLj overhead on every callout
 929 // by clearing the unwinder's TSD pointer to the handler stack around callouts
 930
 931 #define _dispatch_get_tsd_base()
 932 #define _dispatch_get_unwind_tsd() (NULL)
 933 #define _dispatch_set_unwind_tsd(u) do {(void)(u);} while (0)
 934 #define _dispatch_free_unwind_tsd()
 935
 936 #undef _dispatch_client_callout
 937 DISPATCH_NOINLINE
 938 void
 939 _dispatch_client_callout(void *ctxt, dispatch_function_t f)
 940 {
 941         _dispatch_get_tsd_base();
 942         void *u = _dispatch_get_unwind_tsd();
 943         if (fastpath(!u)) return f(ctxt);
 944         _dispatch_set_unwind_tsd(NULL);
 945         f(ctxt);
 946         _dispatch_free_unwind_tsd();
 947         _dispatch_set_unwind_tsd(u);
 948 }
 949
 950 #undef _dispatch_client_callout2
 951 DISPATCH_NOINLINE
 952 void
 953 _dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t))
 954 {
 955         _dispatch_get_tsd_base();
 956         void *u = _dispatch_get_unwind_tsd();
 957         if (fastpath(!u)) return f(ctxt, i);
 958         _dispatch_set_unwind_tsd(NULL);
 959         f(ctxt, i);
 960         _dispatch_free_unwind_tsd();
 961         _dispatch_set_unwind_tsd(u);
 962 }
 963
 964 #if HAVE_MACH
 965 #undef _dispatch_client_callout4
 966 void
 967 _dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason,
 968                 dispatch_mach_msg_t dmsg, mach_error_t error,
 969                 dispatch_mach_handler_function_t f)
 970 {
 971         _dispatch_get_tsd_base();
 972         void *u = _dispatch_get_unwind_tsd();
 973         if (fastpath(!u)) return f(ctxt, reason, dmsg, error);
 974         _dispatch_set_unwind_tsd(NULL);
 975         f(ctxt, reason, dmsg, error);
 976         _dispatch_free_unwind_tsd();
 977         _dispatch_set_unwind_tsd(u);
 978 }
 979 #endif // HAVE_MACH
 980
 981 #endif // DISPATCH_USE_CLIENT_CALLOUT
 982
 983 #pragma mark -
 984 #pragma mark _os_object_t no_objc
 985
 986 #if !USE_OBJC
 987
 988 static const _os_object_vtable_s _os_object_vtable;
 989
 990 void
 991 _os_object_init(void)
 992 {
 993         return;
 994 }
 995
 996 inline _os_object_t
 997 _os_object_alloc_realized(const void *cls, size_t size)
 998 {
 999         _os_object_t obj;
1000         dispatch_assert(size >= sizeof(struct _os_object_s));
1001         while (!fastpath(obj = calloc(1u, size))) {
1002                 _dispatch_temporary_resource_shortage();
1003         }
1004         obj->os_obj_isa = cls;
1005         return obj;
1006 }
1007
1008 _os_object_t
1009 _os_object_alloc(const void *cls, size_t size)
1010 {
1011         if (!cls) cls = &_os_object_vtable;
1012         return _os_object_alloc_realized(cls, size);
1013 }
1014
1015 void
1016 _os_object_dealloc(_os_object_t obj)
1017 {
1018         *((void *volatile*)&obj->os_obj_isa) = (void *)0x200;
1019         return free(obj);
1020 }
1021
1022 void
1023 _os_object_xref_dispose(_os_object_t obj)
1024 {
1025         _os_object_xrefcnt_dispose_barrier(obj);
1026         if (fastpath(obj->os_obj_isa->_os_obj_xref_dispose)) {
1027                 return obj->os_obj_isa->_os_obj_xref_dispose(obj);
1028         }
1029         return _os_object_release_internal(obj);
1030 }
1031
1032 void
1033 _os_object_dispose(_os_object_t obj)
1034 {
1035         _os_object_refcnt_dispose_barrier(obj);
1036         if (fastpath(obj->os_obj_isa->_os_obj_dispose)) {
1037                 return obj->os_obj_isa->_os_obj_dispose(obj);
1038         }
1039         return _os_object_dealloc(obj);
1040 }
1041
1042 void*
1043 os_retain(void *obj)
1044 {
1045         if (fastpath(obj)) {
1046                 return _os_object_retain(obj);
1047         }
1048         return obj;
1049 }
1050
1051 #undef os_release
1052 void
1053 os_release(void *obj)
1054 {
1055         if (fastpath(obj)) {
1056                 return _os_object_release(obj);
1057         }
1058 }
1059
1060 #pragma mark -
1061 #pragma mark dispatch_autorelease_pool no_objc
1062
1063 #if DISPATCH_COCOA_COMPAT
1064
1065 void*
1066 _dispatch_autorelease_pool_push(void)
1067 {
1068         void *pool = NULL;
1069         if (_dispatch_begin_NSAutoReleasePool) {
1070                 pool = _dispatch_begin_NSAutoReleasePool();
1071         }
1072         return pool;
1073 }
1074
1075 void
1076 _dispatch_autorelease_pool_pop(void *pool)
1077 {
1078         if (_dispatch_end_NSAutoReleasePool) {
1079                 _dispatch_end_NSAutoReleasePool(pool);
1080         }
1081 }
1082
1083 void*
1084 _dispatch_last_resort_autorelease_pool_push(void)
1085 {
1086         return _dispatch_autorelease_pool_push();
1087 }
1088
1089 void
1090 _dispatch_last_resort_autorelease_pool_pop(void *pool)
1091 {
1092         _dispatch_autorelease_pool_pop(pool);
1093 }
1094
1095 #endif // DISPATCH_COCOA_COMPAT
1096 #endif // !USE_OBJC
1097
1098 #pragma mark -
1099 #pragma mark dispatch_source_types
1100
1101 static void
1102 dispatch_source_type_timer_init(dispatch_source_t ds,
1103         dispatch_source_type_t type DISPATCH_UNUSED,
1104         uintptr_t handle DISPATCH_UNUSED,
1105         unsigned long mask,
1106         dispatch_queue_t q)
1107 {
1108         if (fastpath(!ds->ds_refs)) {
1109                 ds->ds_refs = _dispatch_calloc(1ul,
1110                                 sizeof(struct dispatch_timer_source_refs_s));
1111         }
1112         ds->ds_needs_rearm = true;
1113         ds->ds_is_timer = true;
1114         if (q == dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0)
1115                         || q == dispatch_get_global_queue(
1116                         DISPATCH_QUEUE_PRIORITY_BACKGROUND, DISPATCH_QUEUE_OVERCOMMIT)){
1117                 mask |= DISPATCH_TIMER_BACKGROUND; // <rdar://problem/12200216>
1118         }
1119         ds_timer(ds->ds_refs).flags = mask;
1120 }
1121
1122 const struct dispatch_source_type_s _dispatch_source_type_timer = {
1123         .ke = {
1124                 .filter = DISPATCH_EVFILT_TIMER,
1125         },
1126         .mask = DISPATCH_TIMER_STRICT|DISPATCH_TIMER_BACKGROUND|
1127                         DISPATCH_TIMER_WALL_CLOCK,
1128         .init = dispatch_source_type_timer_init,
1129 };
1130
1131 static void
1132 dispatch_source_type_after_init(dispatch_source_t ds,
1133         dispatch_source_type_t type, uintptr_t handle, unsigned long mask,
1134         dispatch_queue_t q)
1135 {
1136         dispatch_source_type_timer_init(ds, type, handle, mask, q);
1137         ds->ds_needs_rearm = false;
1138         ds_timer(ds->ds_refs).flags |= DISPATCH_TIMER_AFTER;
1139 }
1140
1141 const struct dispatch_source_type_s _dispatch_source_type_after = {
1142         .ke = {
1143                 .filter = DISPATCH_EVFILT_TIMER,
1144         },
1145         .init = dispatch_source_type_after_init,
1146 };
1147
1148 static void
1149 dispatch_source_type_timer_with_aggregate_init(dispatch_source_t ds,
1150         dispatch_source_type_t type, uintptr_t handle, unsigned long mask,
1151         dispatch_queue_t q)
1152 {
1153         ds->ds_refs = _dispatch_calloc(1ul,
1154                         sizeof(struct dispatch_timer_source_aggregate_refs_s));
1155         dispatch_source_type_timer_init(ds, type, handle, mask, q);
1156         ds_timer(ds->ds_refs).flags |= DISPATCH_TIMER_WITH_AGGREGATE;
1157         ds->dq_specific_q = (void*)handle;
1158         _dispatch_retain(ds->dq_specific_q);
1159 }
1160
1161 const struct dispatch_source_type_s _dispatch_source_type_timer_with_aggregate={
1162         .ke = {
1163                 .filter = DISPATCH_EVFILT_TIMER,
1164                 .ident = ~0ull,
1165         },
1166         .mask = DISPATCH_TIMER_STRICT|DISPATCH_TIMER_BACKGROUND,
1167         .init = dispatch_source_type_timer_with_aggregate_init,
1168 };
1169
1170 static void
1171 dispatch_source_type_interval_init(dispatch_source_t ds,
1172         dispatch_source_type_t type, uintptr_t handle, unsigned long mask,
1173         dispatch_queue_t q)
1174 {
1175         dispatch_source_type_timer_init(ds, type, handle, mask, q);
1176         ds_timer(ds->ds_refs).flags |= DISPATCH_TIMER_INTERVAL;
1177         unsigned long ident = _dispatch_source_timer_idx(ds->ds_refs);
1178         ds->ds_dkev->dk_kevent.ident = ds->ds_ident_hack = ident;
1179         _dispatch_source_set_interval(ds, handle);
1180 }
1181
1182 const struct dispatch_source_type_s _dispatch_source_type_interval = {
1183         .ke = {
1184                 .filter = DISPATCH_EVFILT_TIMER,
1185                 .ident = ~0ull,
1186         },
1187         .mask = DISPATCH_TIMER_STRICT|DISPATCH_TIMER_BACKGROUND|
1188                         DISPATCH_INTERVAL_UI_ANIMATION,
1189         .init = dispatch_source_type_interval_init,
1190 };
1191
1192 static void
1193 dispatch_source_type_readwrite_init(dispatch_source_t ds,
1194         dispatch_source_type_t type DISPATCH_UNUSED,
1195         uintptr_t handle DISPATCH_UNUSED,
1196         unsigned long mask DISPATCH_UNUSED,
1197         dispatch_queue_t q DISPATCH_UNUSED)
1198 {
1199         ds->ds_is_level = true;
1200 #ifdef HAVE_DECL_NOTE_LOWAT
1201         // bypass kernel check for device kqueue support rdar://19004921
1202         ds->ds_dkev->dk_kevent.fflags = NOTE_LOWAT;
1203 #endif
1204         ds->ds_dkev->dk_kevent.data = 1;
1205 }
1206
1207 const struct dispatch_source_type_s _dispatch_source_type_read = {
1208         .ke = {
1209                 .filter = EVFILT_READ,
1210                 .flags = EV_VANISHED|EV_DISPATCH|EV_UDATA_SPECIFIC,
1211         },
1212         .init = dispatch_source_type_readwrite_init,
1213 };
1214
1215 const struct dispatch_source_type_s _dispatch_source_type_write = {
1216         .ke = {
1217                 .filter = EVFILT_WRITE,
1218                 .flags = EV_VANISHED|EV_DISPATCH|EV_UDATA_SPECIFIC,
1219         },
1220         .init = dispatch_source_type_readwrite_init,
1221 };
1222
1223 #if DISPATCH_USE_MEMORYSTATUS
1224
1225 #if TARGET_IPHONE_SIMULATOR // rdar://problem/9219483
1226 static int _dispatch_ios_simulator_memory_warnings_fd = -1;
1227 static void
1228 _dispatch_ios_simulator_memorypressure_init(void *context DISPATCH_UNUSED)
1229 {
1230         char *e = getenv("SIMULATOR_MEMORY_WARNINGS");
1231         if (!e) return;
1232         _dispatch_ios_simulator_memory_warnings_fd = open(e, O_EVTONLY);
1233         if (_dispatch_ios_simulator_memory_warnings_fd == -1) {
1234                 (void)dispatch_assume_zero(errno);
1235         }
1236 }
1237 #endif
1238
1239 #if TARGET_IPHONE_SIMULATOR
1240 static void
1241 dispatch_source_type_memorypressure_init(dispatch_source_t ds,
1242         dispatch_source_type_t type DISPATCH_UNUSED,
1243         uintptr_t handle DISPATCH_UNUSED,
1244         unsigned long mask DISPATCH_UNUSED,
1245         dispatch_queue_t q DISPATCH_UNUSED)
1246 {
1247         static dispatch_once_t pred;
1248         dispatch_once_f(&pred, NULL, _dispatch_ios_simulator_memorypressure_init);
1249         handle = (uintptr_t)_dispatch_ios_simulator_memory_warnings_fd;
1250         mask = NOTE_ATTRIB;
1251         ds->ds_dkev->dk_kevent.filter = EVFILT_VNODE;
1252         ds->ds_dkev->dk_kevent.ident = handle;
1253         ds->ds_dkev->dk_kevent.flags |= EV_CLEAR;
1254         ds->ds_dkev->dk_kevent.fflags = (uint32_t)mask;
1255         ds->ds_ident_hack = handle;
1256         ds->ds_pending_data_mask = mask;
1257         ds->ds_memorypressure_override = 1;
1258 }
1259 #else
1260 #define dispatch_source_type_memorypressure_init NULL
1261 #endif
1262
1263 #ifndef NOTE_MEMORYSTATUS_LOW_SWAP
1264 #define NOTE_MEMORYSTATUS_LOW_SWAP 0x8
1265 #endif
1266
1267 const struct dispatch_source_type_s _dispatch_source_type_memorypressure = {
1268         .ke = {
1269                 .filter = EVFILT_MEMORYSTATUS,
1270                 .flags = EV_DISPATCH|EV_UDATA_SPECIFIC,
1271         },
1272         .mask = NOTE_MEMORYSTATUS_PRESSURE_NORMAL|NOTE_MEMORYSTATUS_PRESSURE_WARN
1273                         |NOTE_MEMORYSTATUS_PRESSURE_CRITICAL|NOTE_MEMORYSTATUS_LOW_SWAP
1274                         |NOTE_MEMORYSTATUS_PROC_LIMIT_WARN|NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL,
1275         .init = dispatch_source_type_memorypressure_init,
1276 };
1277
1278 static void
1279 dispatch_source_type_vm_init(dispatch_source_t ds,
1280         dispatch_source_type_t type DISPATCH_UNUSED,
1281         uintptr_t handle DISPATCH_UNUSED,
1282         unsigned long mask DISPATCH_UNUSED,
1283         dispatch_queue_t q DISPATCH_UNUSED)
1284 {
1285         // Map legacy vm pressure to memorypressure warning rdar://problem/15907505
1286         mask = NOTE_MEMORYSTATUS_PRESSURE_WARN;
1287         ds->ds_dkev->dk_kevent.fflags = (uint32_t)mask;
1288         ds->ds_pending_data_mask = mask;
1289         ds->ds_vmpressure_override = 1;
1290 #if TARGET_IPHONE_SIMULATOR
1291         dispatch_source_type_memorypressure_init(ds, type, handle, mask, q);
1292 #endif
1293 }
1294
1295 const struct dispatch_source_type_s _dispatch_source_type_vm = {
1296         .ke = {
1297                 .filter = EVFILT_MEMORYSTATUS,
1298                 .flags = EV_DISPATCH|EV_UDATA_SPECIFIC,
1299         },
1300         .mask = NOTE_VM_PRESSURE,
1301         .init = dispatch_source_type_vm_init,
1302 };
1303
1304 #elif DISPATCH_USE_VM_PRESSURE
1305
1306 const struct dispatch_source_type_s _dispatch_source_type_vm = {
1307         .ke = {
1308                 .filter = EVFILT_VM,
1309                 .flags = EV_DISPATCH|EV_UDATA_SPECIFIC,
1310         },
1311         .mask = NOTE_VM_PRESSURE,
1312 };
1313
1314 #endif // DISPATCH_USE_VM_PRESSURE
1315
1316 const struct dispatch_source_type_s _dispatch_source_type_signal = {
1317         .ke = {
1318                 .filter = EVFILT_SIGNAL,
1319                 .flags = EV_UDATA_SPECIFIC,
1320         },
1321 };
1322
1323 #if !defined(__linux__)
1324 static void
1325 dispatch_source_type_proc_init(dispatch_source_t ds,
1326         dispatch_source_type_t type DISPATCH_UNUSED,
1327         uintptr_t handle DISPATCH_UNUSED,
1328         unsigned long mask DISPATCH_UNUSED,
1329         dispatch_queue_t q DISPATCH_UNUSED)
1330 {
1331         ds->ds_dkev->dk_kevent.fflags |= NOTE_EXIT; // rdar://16655831
1332 }
1333
1334 const struct dispatch_source_type_s _dispatch_source_type_proc = {
1335         .ke = {
1336                 .filter = EVFILT_PROC,
1337                 .flags = EV_CLEAR|EV_UDATA_SPECIFIC,
1338         },
1339         .mask = NOTE_EXIT|NOTE_FORK|NOTE_EXEC
1340 #if HAVE_DECL_NOTE_SIGNAL
1341                         |NOTE_SIGNAL
1342 #endif
1343 #if HAVE_DECL_NOTE_REAP
1344                         |NOTE_REAP
1345 #endif
1346                         ,
1347         .init = dispatch_source_type_proc_init,
1348 };
1349
1350 const struct dispatch_source_type_s _dispatch_source_type_vnode = {
1351         .ke = {
1352                 .filter = EVFILT_VNODE,
1353                 .flags = EV_CLEAR|EV_VANISHED|EV_UDATA_SPECIFIC,
1354         },
1355         .mask = NOTE_DELETE|NOTE_WRITE|NOTE_EXTEND|NOTE_ATTRIB|NOTE_LINK|
1356                         NOTE_RENAME|NOTE_FUNLOCK
1357 #if HAVE_DECL_NOTE_REVOKE
1358                         |NOTE_REVOKE
1359 #endif
1360 #if HAVE_DECL_NOTE_NONE
1361                         |NOTE_NONE
1362 #endif
1363                         ,
1364 };
1365
1366 const struct dispatch_source_type_s _dispatch_source_type_vfs = {
1367         .ke = {
1368                 .filter = EVFILT_FS,
1369                 .flags = EV_CLEAR|EV_UDATA_SPECIFIC,
1370         },
1371         .mask = VQ_NOTRESP|VQ_NEEDAUTH|VQ_LOWDISK|VQ_MOUNT|VQ_UNMOUNT|VQ_DEAD|
1372                         VQ_ASSIST|VQ_NOTRESPLOCK
1373 #if HAVE_DECL_VQ_UPDATE
1374                         |VQ_UPDATE
1375 #endif
1376 #if HAVE_DECL_VQ_VERYLOWDISK
1377                         |VQ_VERYLOWDISK
1378 #endif
1379 #if HAVE_DECL_VQ_QUOTA
1380                         |VQ_QUOTA
1381 #endif
1382                         ,
1383 };
1384
1385 const struct dispatch_source_type_s _dispatch_source_type_sock = {
1386 #ifdef EVFILT_SOCK
1387         .ke = {
1388                 .filter = EVFILT_SOCK,
1389                 .flags = EV_CLEAR|EV_VANISHED|EV_UDATA_SPECIFIC,
1390         },
1391         .mask = NOTE_CONNRESET |  NOTE_READCLOSED | NOTE_WRITECLOSED |
1392                 NOTE_TIMEOUT | NOTE_NOSRCADDR |  NOTE_IFDENIED | NOTE_SUSPEND |
1393                 NOTE_RESUME | NOTE_KEEPALIVE
1394 #ifdef NOTE_ADAPTIVE_WTIMO
1395                 | NOTE_ADAPTIVE_WTIMO | NOTE_ADAPTIVE_RTIMO
1396 #endif
1397 #ifdef NOTE_CONNECTED
1398                 | NOTE_CONNECTED | NOTE_DISCONNECTED | NOTE_CONNINFO_UPDATED
1399 #endif
1400 #ifdef NOTE_NOTIFY_ACK
1401                 | NOTE_NOTIFY_ACK
1402 #endif
1403                 ,
1404 #endif // EVFILT_SOCK
1405 };
1406 #endif // !defined(__linux__)
1407
1408 static void
1409 dispatch_source_type_data_init(dispatch_source_t ds,
1410         dispatch_source_type_t type DISPATCH_UNUSED,
1411         uintptr_t handle DISPATCH_UNUSED,
1412         unsigned long mask DISPATCH_UNUSED,
1413         dispatch_queue_t q DISPATCH_UNUSED)
1414 {
1415         ds->ds_is_installed = true;
1416         ds->ds_is_custom_source = true;
1417         ds->ds_is_direct_kevent = true;
1418         ds->ds_pending_data_mask = ~0ul;
1419         ds->ds_needs_rearm = false; // not registered with kevent
1420 }
1421
1422 const struct dispatch_source_type_s _dispatch_source_type_data_add = {
1423         .ke = {
1424                 .filter = DISPATCH_EVFILT_CUSTOM_ADD,
1425                 .flags = EV_UDATA_SPECIFIC,
1426         },
1427         .init = dispatch_source_type_data_init,
1428 };
1429
1430 const struct dispatch_source_type_s _dispatch_source_type_data_or = {
1431         .ke = {
1432                 .filter = DISPATCH_EVFILT_CUSTOM_OR,
1433                 .flags = EV_CLEAR|EV_UDATA_SPECIFIC,
1434                 .fflags = ~0u,
1435         },
1436         .init = dispatch_source_type_data_init,
1437 };
1438
1439 #if HAVE_MACH
1440
1441 static void
1442 dispatch_source_type_mach_send_init(dispatch_source_t ds,
1443         dispatch_source_type_t type DISPATCH_UNUSED,
1444         uintptr_t handle DISPATCH_UNUSED, unsigned long mask,
1445         dispatch_queue_t q DISPATCH_UNUSED)
1446 {
1447         if (!mask) {
1448                 // Preserve legacy behavior that (mask == 0) => DISPATCH_MACH_SEND_DEAD
1449                 ds->ds_dkev->dk_kevent.fflags = DISPATCH_MACH_SEND_DEAD;
1450                 ds->ds_pending_data_mask = DISPATCH_MACH_SEND_DEAD;
1451         }
1452 }
1453
1454 const struct dispatch_source_type_s _dispatch_source_type_mach_send = {
1455         .ke = {
1456                 .filter = DISPATCH_EVFILT_MACH_NOTIFICATION,
1457                 .flags = EV_CLEAR,
1458         },
1459         .mask = DISPATCH_MACH_SEND_DEAD|DISPATCH_MACH_SEND_POSSIBLE,
1460         .init = dispatch_source_type_mach_send_init,
1461 };
1462
1463 static void
1464 dispatch_source_type_mach_recv_init(dispatch_source_t ds,
1465         dispatch_source_type_t type DISPATCH_UNUSED,
1466         uintptr_t handle DISPATCH_UNUSED,
1467         unsigned long mask DISPATCH_UNUSED,
1468         dispatch_queue_t q DISPATCH_UNUSED)
1469 {
1470         ds->ds_pending_data_mask = DISPATCH_MACH_RECV_MESSAGE;
1471 #if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
1472         if (_dispatch_evfilt_machport_direct_enabled) return;
1473         ds->ds_dkev->dk_kevent.fflags = DISPATCH_MACH_RECV_MESSAGE;
1474         ds->ds_dkev->dk_kevent.flags &= ~(EV_UDATA_SPECIFIC|EV_VANISHED);
1475         ds->ds_is_direct_kevent = false;
1476 #endif
1477 }
1478
1479 const struct dispatch_source_type_s _dispatch_source_type_mach_recv = {
1480         .ke = {
1481                 .filter = EVFILT_MACHPORT,
1482                 .flags = EV_VANISHED|EV_DISPATCH|EV_UDATA_SPECIFIC,
1483         },
1484         .init = dispatch_source_type_mach_recv_init,
1485 };
1486
1487 #pragma mark -
1488 #pragma mark dispatch_mig
1489
1490 void *
1491 dispatch_mach_msg_get_context(mach_msg_header_t *msg)
1492 {
1493         mach_msg_context_trailer_t *tp;
1494         void *context = NULL;
1495
1496         tp = (mach_msg_context_trailer_t *)((uint8_t *)msg +
1497                         round_msg(msg->msgh_size));
1498         if (tp->msgh_trailer_size >=
1499                         (mach_msg_size_t)sizeof(mach_msg_context_trailer_t)) {
1500                 context = (void *)(uintptr_t)tp->msgh_context;
1501         }
1502         return context;
1503 }
1504
1505 kern_return_t
1506 _dispatch_wakeup_runloop_thread(mach_port_t mp DISPATCH_UNUSED)
1507 {
1508         // dummy function just to pop a runloop thread out of mach_msg()
1509         return 0;
1510 }
1511
1512 kern_return_t
1513 _dispatch_consume_send_once_right(mach_port_t mp DISPATCH_UNUSED)
1514 {
1515         // dummy function to consume a send-once right
1516         return 0;
1517 }
1518
1519 kern_return_t
1520 _dispatch_mach_notify_port_destroyed(mach_port_t notify DISPATCH_UNUSED,
1521                 mach_port_t name)
1522 {
1523         kern_return_t kr;
1524         // this function should never be called
1525         (void)dispatch_assume_zero(name);
1526         kr = mach_port_mod_refs(mach_task_self(), name, MACH_PORT_RIGHT_RECEIVE,-1);
1527         DISPATCH_VERIFY_MIG(kr);
1528         (void)dispatch_assume_zero(kr);
1529         return KERN_SUCCESS;
1530 }
1531
1532 kern_return_t
1533 _dispatch_mach_notify_no_senders(mach_port_t notify,
1534                 mach_port_mscount_t mscnt DISPATCH_UNUSED)
1535 {
1536         // this function should never be called
1537         (void)dispatch_assume_zero(notify);
1538         return KERN_SUCCESS;
1539 }
1540
1541 kern_return_t
1542 _dispatch_mach_notify_send_once(mach_port_t notify DISPATCH_UNUSED)
1543 {
1544         // we only register for dead-name notifications
1545         // some code deallocated our send-once right without consuming it
1546 #if DISPATCH_DEBUG
1547         _dispatch_log("Corruption: An app/library deleted a libdispatch "
1548                         "dead-name notification");
1549 #endif
1550         return KERN_SUCCESS;
1551 }
1552
1553 #endif // HAVE_MACH