2 * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
4 * @APPLE_APACHE_LICENSE_HEADER_START@
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
10 * http://www.apache.org/licenses/LICENSE-2.0
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
18 * @APPLE_APACHE_LICENSE_HEADER_END@
22 * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
23 * which are subject to change in future releases of Mac OS X. Any applications
24 * relying on these interfaces WILL break.
27 #ifndef __DISPATCH_INTERNAL__
28 #define __DISPATCH_INTERNAL__
30 #if __has_include(<config/config_ac.h>)
31 #include <config/config_ac.h>
33 #include <config/config.h>
36 #define __DISPATCH_BUILDING_DISPATCH__
37 #define __DISPATCH_INDIRECT__
40 #include <Availability.h>
41 #include <TargetConditionals.h>
43 #ifndef TARGET_OS_MAC_DESKTOP
44 #define TARGET_OS_MAC_DESKTOP (TARGET_OS_MAC && \
45 !TARGET_OS_SIMULATOR && !TARGET_OS_IPHONE && !TARGET_OS_EMBEDDED)
48 #if TARGET_OS_MAC_DESKTOP
49 # define DISPATCH_HOST_SUPPORTS_OSX(x) \
50 (__MAC_OS_X_VERSION_MIN_REQUIRED >= (x))
51 # if !DISPATCH_HOST_SUPPORTS_OSX(101000)
52 # error "OS X hosts older than OS X 10.10 aren't supported anymore"
53 # endif // !DISPATCH_HOST_SUPPORTS_OSX(101000)
54 #elif TARGET_OS_SIMULATOR
55 # define DISPATCH_HOST_SUPPORTS_OSX(x) \
56 (IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED >= (x))
57 # if !DISPATCH_HOST_SUPPORTS_OSX(101000)
58 # error "Simulator hosts older than OS X 10.10 aren't supported anymore"
59 # endif // !DISPATCH_HOST_SUPPORTS_OSX(101000)
61 # define DISPATCH_HOST_SUPPORTS_OSX(x) 1
62 # if __IPHONE_OS_VERSION_MIN_REQUIRED < 70000
63 # error "iOS hosts older than iOS 7.0 aren't supported anymore"
68 #define DISPATCH_HOST_SUPPORTS_OSX(x) 0
72 #if !defined(DISPATCH_MACH_SPI) && TARGET_OS_MAC
73 #define DISPATCH_MACH_SPI 1
75 #if !defined(OS_VOUCHER_CREATION_SPI) && TARGET_OS_MAC
76 #define OS_VOUCHER_CREATION_SPI 1
78 #if !defined(OS_VOUCHER_ACTIVITY_SPI) && TARGET_OS_MAC
79 #define OS_VOUCHER_ACTIVITY_SPI 1
81 #if !defined(OS_FIREHOSE_SPI) && TARGET_OS_MAC
82 #define OS_FIREHOSE_SPI 1
84 #if !defined(DISPATCH_LAYOUT_SPI) && TARGET_OS_MAC
85 #define DISPATCH_LAYOUT_SPI 1
88 #if __has_include(<mach-o/dyld_priv.h>)
89 #include <mach-o/dyld_priv.h>
90 #if !defined(HAVE_DYLD_IS_MEMORY_IMMUTABLE)
91 #if defined(DYLD_MACOSX_VERSION_10_12) || defined(DYLD_IOS_VERSION_10_0)
92 #define HAVE_DYLD_IS_MEMORY_IMMUTABLE 1
94 #define HAVE_DYLD_IS_MEMORY_IMMUTABLE 0
96 #endif // !defined(HAVE_DYLD_IS_MEMORY_IMMUTABLE)
97 #endif // __has_include(<mach-o/dyld_priv.h>)
99 #if !defined(USE_OBJC) && HAVE_OBJC
104 #define OS_OBJECT_HAVE_OBJC_SUPPORT 1
105 #if defined(__OBJC__)
106 #define OS_OBJECT_USE_OBJC 1
107 // Force internal Objective-C sources to use class-visible headers
108 // even when not compiling in Swift.
109 #define OS_OBJECT_SWIFT3 1
111 #define OS_OBJECT_USE_OBJC 0
114 #define OS_OBJECT_HAVE_OBJC_SUPPORT 0
117 #include <dispatch/dispatch.h>
118 #include <dispatch/base.h>
120 #define __DISPATCH_HIDE_SYMBOL(sym, version) \
121 __asm__(".section __TEXT,__const\n\t" \
122 ".globl $ld$hide$os" #version "$_" #sym "\n\t" \
123 "$ld$hide$os" #version "$_" #sym ":\n\t" \
128 #ifndef DISPATCH_HIDE_SYMBOL
129 #if TARGET_OS_MAC && !TARGET_OS_IPHONE
130 #define DISPATCH_HIDE_SYMBOL(sym, osx, ios, tvos, watchos) \
131 __DISPATCH_HIDE_SYMBOL(sym, osx)
133 #define DISPATCH_HIDE_SYMBOL(sym, osx, ios, tvos, watchos)
137 #include <os/object.h>
138 #include <dispatch/time.h>
139 #include <dispatch/object.h>
140 #include <dispatch/queue.h>
141 #include <dispatch/block.h>
142 #include <dispatch/source.h>
143 #include <dispatch/group.h>
144 #include <dispatch/semaphore.h>
145 #include <dispatch/once.h>
146 #include <dispatch/data.h>
148 #include <dispatch/io.h>
151 #if defined(__OBJC__) || defined(__cplusplus)
152 #define DISPATCH_PURE_C 0
154 #define DISPATCH_PURE_C 1
157 /* private.h must be included last to avoid picking up installed headers. */
158 #include "os/object_private.h"
159 #include "queue_private.h"
160 #include "source_private.h"
161 #include "mach_private.h"
162 #include "data_private.h"
163 #include "os/voucher_private.h"
164 #include "os/voucher_activity_private.h"
166 #include "io_private.h"
168 #include "layout_private.h"
169 #include "benchmark.h"
172 /* SPI for Libsystem-internal use */
173 DISPATCH_EXPORT DISPATCH_NOTHROW
void libdispatch_init(void);
175 DISPATCH_EXPORT DISPATCH_NOTHROW
void dispatch_atfork_prepare(void);
176 DISPATCH_EXPORT DISPATCH_NOTHROW
void dispatch_atfork_parent(void);
177 DISPATCH_EXPORT DISPATCH_NOTHROW
void dispatch_atfork_child(void);
180 /* More #includes at EOF (dependent on the contents of internal.h) ... */
182 // Abort on uncaught exceptions thrown from client callouts rdar://8577499
183 #if !defined(DISPATCH_USE_CLIENT_CALLOUT)
184 #define DISPATCH_USE_CLIENT_CALLOUT 1
187 /* The "_debug" library build */
188 #ifndef DISPATCH_DEBUG
189 #define DISPATCH_DEBUG 0
192 #ifndef DISPATCH_PROFILE
193 #define DISPATCH_PROFILE 0
196 #if (!TARGET_OS_EMBEDDED || DISPATCH_DEBUG || DISPATCH_PROFILE) && \
197 !defined(DISPATCH_USE_DTRACE)
198 #define DISPATCH_USE_DTRACE 1
201 #if DISPATCH_USE_DTRACE && (DISPATCH_INTROSPECTION || DISPATCH_DEBUG || \
202 DISPATCH_PROFILE) && !defined(DISPATCH_USE_DTRACE_INTROSPECTION)
203 #define DISPATCH_USE_DTRACE_INTROSPECTION 1
206 #ifndef DISPATCH_DEBUG_QOS
207 #define DISPATCH_DEBUG_QOS DISPATCH_DEBUG
210 #if HAVE_LIBKERN_OSCROSSENDIAN_H
211 #include <libkern/OSCrossEndian.h>
213 #if HAVE_LIBKERN_OSATOMIC_H
214 #include <libkern/OSAtomic.h>
217 #include <mach/boolean.h>
218 #include <mach/clock_types.h>
219 #include <mach/clock.h>
220 #include <mach/exception.h>
221 #include <mach/mach.h>
222 #include <mach/mach_error.h>
223 #include <mach/mach_host.h>
224 #include <mach/mach_interface.h>
225 #include <mach/mach_time.h>
226 #include <mach/mach_traps.h>
227 #include <mach/message.h>
228 #include <mach/mig_errors.h>
229 #include <mach/host_special_ports.h>
230 #include <mach/host_info.h>
231 #include <mach/notify.h>
232 #include <mach/mach_vm.h>
233 #include <mach/vm_map.h>
234 #endif /* HAVE_MACH */
235 #if HAVE_MALLOC_MALLOC_H
236 #include <malloc/malloc.h>
238 #if __has_include(<malloc_private.h>)
239 #include <malloc_private.h>
240 #endif // __has_include(<malloc_private.h)
242 #include <sys/stat.h>
245 #include <sys/event.h>
246 #include <sys/mount.h>
247 #include <sys/queue.h>
248 #include <sys/sysctl.h>
249 #include <sys/socket.h>
250 #include <sys/time.h>
251 #include <sys/mman.h>
252 #include <netinet/in.h>
254 #if defined(__linux__)
255 #include <sys/eventfd.h>
259 #include <Block_private.h>
261 #endif /* __BLOCKS__ */
271 #include <semaphore.h>
285 #define DISPATCH_NOINLINE __attribute__((__noinline__))
286 #define DISPATCH_USED __attribute__((__used__))
287 #define DISPATCH_UNUSED __attribute__((__unused__))
288 #define DISPATCH_WEAK __attribute__((__weak__))
289 #define DISPATCH_OVERLOADABLE __attribute__((__overloadable__))
290 #define DISPATCH_PACKED __attribute__((__packed__))
292 #define DISPATCH_ALWAYS_INLINE_NDEBUG
294 #define DISPATCH_ALWAYS_INLINE_NDEBUG __attribute__((__always_inline__))
297 #define DISPATCH_NOINLINE
298 #define DISPATCH_USED
299 #define DISPATCH_UNUSED
300 #define DISPATCH_WEAK
301 #define DISPATCH_ALWAYS_INLINE_NDEBUG
302 #endif /* __GNUC__ */
304 #define DISPATCH_CONCAT(x,y) DISPATCH_CONCAT1(x,y)
305 #define DISPATCH_CONCAT1(x,y) x ## y
307 // workaround 6368156
317 #define NSEC_PER_SEC 1000000000ull
318 #define USEC_PER_SEC 1000000ull
319 #define NSEC_PER_USEC 1000ull
321 /* I wish we had __builtin_expect_range() */
323 #define _safe_cast_to_long(x) \
324 ({ _Static_assert(sizeof(typeof(x)) <= sizeof(long), \
325 "__builtin_expect doesn't support types wider than long"); \
327 #define fastpath(x) ((typeof(x))__builtin_expect(_safe_cast_to_long(x), ~0l))
328 #define slowpath(x) ((typeof(x))__builtin_expect(_safe_cast_to_long(x), 0l))
329 #define likely(x) __builtin_expect(!!(x), 1)
330 #define unlikely(x) __builtin_expect(!!(x), 0)
332 #define fastpath(x) (x)
333 #define slowpath(x) (x)
334 #define likely(x) (!!(x))
335 #define unlikely(x) (!!(x))
338 #if BYTE_ORDER == LITTLE_ENDIAN
339 #define DISPATCH_STRUCT_LITTLE_ENDIAN_2(a, b) struct { a; b; }
340 #define DISPATCH_STRUCT_LITTLE_ENDIAN_3(a, b, c) struct { a; b; c; }
341 #define DISPATCH_STRUCT_LITTLE_ENDIAN_4(a, b, c, d) struct { a; b; c; d; }
343 #define DISPATCH_STRUCT_LITTLE_ENDIAN_2(a, b) struct { b; a; }
344 #define DISPATCH_STRUCT_LITTLE_ENDIAN_3(a, b, c) struct { c; b; a; }
345 #define DISPATCH_STRUCT_LITTLE_ENDIAN_4(a, b, c, d) struct { d; c; b; a; }
348 #define _TAILQ_IS_ENQUEUED(elm, field) \
349 ((elm)->field.tqe_prev != NULL)
350 #define _TAILQ_MARK_NOT_ENQUEUED(elm, field) \
351 do { (elm)->field.tqe_prev = NULL; } while (0)
354 // sys/queue.h debugging
356 #define TRASHIT(x) do {(x) = (void *)-1;} while (0)
357 #endif // DISPATCH_DEBUG
358 #define _TAILQ_TRASH_ENTRY(elm, field) do { \
359 TRASHIT((elm)->field.tqe_next); \
360 TRASHIT((elm)->field.tqe_prev); \
362 #define _TAILQ_TRASH_HEAD(head) do { \
363 TRASHIT((head)->tqh_first); \
364 TRASHIT((head)->tqh_last); \
367 DISPATCH_EXPORT DISPATCH_NOINLINE
368 void _dispatch_bug(size_t line
, long val
);
372 void _dispatch_bug_client(const char* msg
);
374 void _dispatch_bug_mach_client(const char *msg
, mach_msg_return_t kr
);
377 void _dispatch_bug_kevent_client(const char* msg
, const char* filter
,
378 const char *operation
, int err
);
381 void _dispatch_bug_deprecated(const char *msg
);
383 DISPATCH_NOINLINE DISPATCH_NORETURN
384 void _dispatch_abort(size_t line
, long val
);
386 #if !defined(DISPATCH_USE_OS_DEBUG_LOG) && DISPATCH_DEBUG
387 #if __has_include(<os/debug_private.h>)
388 #define DISPATCH_USE_OS_DEBUG_LOG 1
389 #include <os/debug_private.h>
391 #endif // DISPATCH_USE_OS_DEBUG_LOG
393 #if !defined(DISPATCH_USE_SIMPLE_ASL) && !DISPATCH_USE_OS_DEBUG_LOG
394 #if __has_include(<_simple.h>)
395 #define DISPATCH_USE_SIMPLE_ASL 1
398 #endif // DISPATCH_USE_SIMPLE_ASL
400 #if !DISPATCH_USE_SIMPLE_ASL && !DISPATCH_USE_OS_DEBUG_LOG && !TARGET_OS_WIN32
404 #if DISPATCH_USE_OS_DEBUG_LOG
405 #define _dispatch_log(msg, ...) os_debug_log("libdispatch", msg, ## __VA_ARGS__)
407 DISPATCH_EXPORT DISPATCH_NOINLINE
__attribute__((__format__(__printf__
,1,2)))
408 void _dispatch_log(const char *msg
, ...);
409 #endif // DISPATCH_USE_OS_DEBUG_LOG
411 #define dsnprintf(buf, siz, ...) \
412 ({ size_t _siz = siz; int _r = snprintf(buf, _siz, __VA_ARGS__); \
413 _r < 0 ? 0u : ((size_t)_r > _siz ? _siz : (size_t)_r); })
416 #define dispatch_static_assert(e) ({ \
417 char __compile_time_assert__[(bool)(e) ? 1 : -1] DISPATCH_UNUSED; \
420 #define dispatch_static_assert(e)
423 #define DISPATCH_BAD_INPUT ((void *_Nonnull)0)
424 #define DISPATCH_OUT_OF_MEMORY ((void *_Nonnull)0)
427 * For reporting bugs within libdispatch when using the "_debug" version of the
431 #define dispatch_assert(e) do { \
432 if (__builtin_constant_p(e)) { \
433 dispatch_static_assert(e); \
435 typeof(e) _e = fastpath(e); /* always eval 'e' */ \
436 if (DISPATCH_DEBUG && !_e) { \
437 _dispatch_abort(__LINE__, (long)_e); \
442 static inline void _dispatch_assert(long e
, long line
) {
443 if (DISPATCH_DEBUG
&& !e
) _dispatch_abort(line
, e
);
445 #define dispatch_assert(e) _dispatch_assert((long)(e), __LINE__)
446 #endif /* __GNUC__ */
450 * A lot of API return zero upon success and not-zero on fail. Let's capture
451 * and log the non-zero value
453 #define dispatch_assert_zero(e) do { \
454 if (__builtin_constant_p(e)) { \
455 dispatch_static_assert(e); \
457 typeof(e) _e = slowpath(e); /* always eval 'e' */ \
458 if (DISPATCH_DEBUG && _e) { \
459 _dispatch_abort(__LINE__, (long)_e); \
464 static inline void _dispatch_assert_zero(long e
, long line
) {
465 if (DISPATCH_DEBUG
&& e
) _dispatch_abort(line
, e
);
467 #define dispatch_assert_zero(e) _dispatch_assert((long)(e), __LINE__)
468 #endif /* __GNUC__ */
471 * For reporting bugs or impedance mismatches between libdispatch and external
472 * subsystems. These do NOT abort(), and are always compiled into the product.
474 * In particular, we wrap all system-calls with assume() macros.
477 #define dispatch_assume(e) ({ \
478 typeof(e) _e = fastpath(e); /* always eval 'e' */ \
480 if (__builtin_constant_p(e)) { \
481 dispatch_static_assert(e); \
483 _dispatch_bug(__LINE__, (long)_e); \
488 static inline long _dispatch_assume(long e
, long line
) {
489 if (!e
) _dispatch_bug(line
, e
);
492 #define dispatch_assume(e) _dispatch_assume((long)(e), __LINE__)
493 #endif /* __GNUC__ */
496 * A lot of API return zero upon success and not-zero on fail. Let's capture
497 * and log the non-zero value
500 #define dispatch_assume_zero(e) ({ \
501 typeof(e) _e = slowpath(e); /* always eval 'e' */ \
503 if (__builtin_constant_p(e)) { \
504 dispatch_static_assert(e); \
506 _dispatch_bug(__LINE__, (long)_e); \
511 static inline long _dispatch_assume_zero(long e
, long line
) {
512 if (e
) _dispatch_bug(line
, e
);
515 #define dispatch_assume_zero(e) _dispatch_assume_zero((long)(e), __LINE__)
516 #endif /* __GNUC__ */
519 * For reporting bugs in clients when using the "_debug" version of the library.
522 #define dispatch_debug_assert(e, msg, args...) do { \
523 if (__builtin_constant_p(e)) { \
524 dispatch_static_assert(e); \
526 typeof(e) _e = fastpath(e); /* always eval 'e' */ \
527 if (DISPATCH_DEBUG && !_e) { \
528 _dispatch_log("%s() 0x%lx: " msg, __func__, (long)_e, ##args); \
534 #define dispatch_debug_assert(e, msg, args...) do { \
535 long _e = (long)fastpath(e); /* always eval 'e' */ \
536 if (DISPATCH_DEBUG && !_e) { \
537 _dispatch_log("%s() 0x%lx: " msg, __FUNCTION__, _e, ##args); \
541 #endif /* __GNUC__ */
543 /* Make sure the debug statments don't get too stale */
544 #define _dispatch_debug(x, args...) do { \
545 if (DISPATCH_DEBUG) { \
546 _dispatch_log("%u\t%p\t" x, __LINE__, \
547 (void *)_dispatch_thread_self(), ##args); \
553 DISPATCH_NOINLINE DISPATCH_USED
554 void dispatch_debug_machport(mach_port_t name
, const char* str
);
559 /* This is the private version of the deprecated dispatch_debug() */
560 DISPATCH_NONNULL2 DISPATCH_NOTHROW
561 __attribute__((__format__(printf
,2,3)))
563 _dispatch_object_debug(dispatch_object_t object
, const char *message
, ...);
565 #define _dispatch_object_debug(object, message, ...)
566 #endif // DISPATCH_DEBUG
569 #define _dispatch_Block_invoke(bb) \
570 ((dispatch_function_t)((struct Block_layout *)bb)->invoke)
571 void *_dispatch_Block_copy(void *block
);
573 #define _dispatch_Block_copy(x) ((typeof(x))_dispatch_Block_copy(x))
575 void _dispatch_call_block_and_release(void *block
);
576 #endif /* __BLOCKS__ */
578 void _dispatch_temporary_resource_shortage(void);
579 void *_dispatch_calloc(size_t num_items
, size_t size
);
580 const char *_dispatch_strdup_if_mutable(const char *str
);
581 void _dispatch_vtable_init(void);
582 char *_dispatch_get_build(void);
584 uint64_t _dispatch_timeout(dispatch_time_t when
);
585 uint64_t _dispatch_time_nanoseconds_since_epoch(dispatch_time_t when
);
587 #define _DISPATCH_UNSAFE_FORK_MULTITHREADED ((uint8_t)1)
588 #define _DISPATCH_UNSAFE_FORK_PROHIBIT ((uint8_t)2)
589 extern uint8_t _dispatch_unsafe_fork
;
590 extern bool _dispatch_child_of_unsafe_fork
;
591 void _dispatch_fork_becomes_unsafe_slow(void);
593 #define _dispatch_is_multithreaded_inline() \
594 ((_dispatch_unsafe_fork & _DISPATCH_UNSAFE_FORK_MULTITHREADED) != 0)
596 DISPATCH_ALWAYS_INLINE
598 _dispatch_fork_becomes_unsafe(void)
600 if (!fastpath(_dispatch_is_multithreaded_inline())) {
601 _dispatch_fork_becomes_unsafe_slow();
602 DISPATCH_COMPILER_CAN_ASSUME(_dispatch_is_multithreaded_inline());
606 /* #includes dependent on internal.h */
609 // Older Mac OS X and iOS Simulator fallbacks
611 #if HAVE_PTHREAD_WORKQUEUES
612 #ifndef WORKQ_ADDTHREADS_OPTION_OVERCOMMIT
613 #define WORKQ_ADDTHREADS_OPTION_OVERCOMMIT 0x00000001
615 #endif // HAVE_PTHREAD_WORKQUEUES
616 #if HAVE__PTHREAD_WORKQUEUE_INIT && PTHREAD_WORKQUEUE_SPI_VERSION >= 20140213 \
617 && !defined(HAVE_PTHREAD_WORKQUEUE_QOS)
618 #define HAVE_PTHREAD_WORKQUEUE_QOS 1
620 #if HAVE__PTHREAD_WORKQUEUE_INIT && (PTHREAD_WORKQUEUE_SPI_VERSION >= 20150304 \
621 || (PTHREAD_WORKQUEUE_SPI_VERSION == 20140730 && \
622 defined(WORKQ_FEATURE_KEVENT))) \
623 && !defined(HAVE_PTHREAD_WORKQUEUE_KEVENT)
624 #if PTHREAD_WORKQUEUE_SPI_VERSION == 20140730
625 // rdar://problem/20609877
626 typedef pthread_worqueue_function_kevent_t pthread_workqueue_function_kevent_t
;
628 #define HAVE_PTHREAD_WORKQUEUE_KEVENT 1
631 #ifndef PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK
632 #if HAVE_PTHREAD_WORKQUEUE_QOS && DISPATCH_HOST_SUPPORTS_OSX(101200)
633 #define PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK 1
635 #define PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK 0
637 #endif // PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK
640 #if !defined(MACH_NOTIFY_SEND_POSSIBLE)
641 #undef MACH_NOTIFY_SEND_POSSIBLE
642 #define MACH_NOTIFY_SEND_POSSIBLE MACH_NOTIFY_DEAD_NAME
646 #ifdef EVFILT_MEMORYSTATUS
647 #ifndef DISPATCH_USE_MEMORYSTATUS
648 #define DISPATCH_USE_MEMORYSTATUS 1
650 #endif // EVFILT_MEMORYSTATUS
652 #if defined(EVFILT_VM) && !DISPATCH_USE_MEMORYSTATUS
653 #ifndef DISPATCH_USE_VM_PRESSURE
654 #define DISPATCH_USE_VM_PRESSURE 1
658 #if TARGET_OS_SIMULATOR
659 #undef DISPATCH_USE_MEMORYPRESSURE_SOURCE
660 #define DISPATCH_USE_MEMORYPRESSURE_SOURCE 0
661 #undef DISPATCH_USE_VM_PRESSURE_SOURCE
662 #define DISPATCH_USE_VM_PRESSURE_SOURCE 0
663 #endif // TARGET_OS_SIMULATOR
664 #if !defined(DISPATCH_USE_MEMORYPRESSURE_SOURCE) && DISPATCH_USE_MEMORYSTATUS
665 #define DISPATCH_USE_MEMORYPRESSURE_SOURCE 1
666 #elif !defined(DISPATCH_USE_VM_PRESSURE_SOURCE) && DISPATCH_USE_VM_PRESSURE
667 #define DISPATCH_USE_VM_PRESSURE_SOURCE 1
669 #if DISPATCH_USE_MEMORYPRESSURE_SOURCE
670 extern bool _dispatch_memory_warn
;
673 #if !defined(NOTE_LEEWAY)
675 #define NOTE_LEEWAY 0
677 #define NOTE_CRITICAL 0
678 #undef NOTE_BACKGROUND
679 #define NOTE_BACKGROUND 0
680 #endif // NOTE_LEEWAY
682 #if !defined(NOTE_FUNLOCK)
683 #define NOTE_FUNLOCK 0x00000100
686 #if !defined(NOTE_MACH_CONTINUOUS_TIME)
687 #define NOTE_MACH_CONTINUOUS_TIME 0
688 #endif // NOTE_MACH_CONTINUOUS_TIME
690 #if !defined(HOST_NOTIFY_CALENDAR_SET)
691 #define HOST_NOTIFY_CALENDAR_SET HOST_NOTIFY_CALENDAR_CHANGE
692 #endif // HOST_NOTIFY_CALENDAR_SET
694 #if !defined(HOST_CALENDAR_SET_REPLYID)
695 #define HOST_CALENDAR_SET_REPLYID 951
696 #endif // HOST_CALENDAR_SET_REPLYID
698 #if HAVE_DECL_NOTE_REAP
699 #if defined(NOTE_REAP) && defined(__APPLE__)
701 #define NOTE_REAP 0x10000000 // <rdar://problem/13338526>
703 #endif // HAVE_DECL_NOTE_REAP
706 #undef HAVE_DECL_VQ_QUOTA // rdar://problem/24160982
709 #if !defined(NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) || \
710 !DISPATCH_HOST_SUPPORTS_OSX(101200)
711 #undef NOTE_MEMORYSTATUS_PROC_LIMIT_WARN
712 #define NOTE_MEMORYSTATUS_PROC_LIMIT_WARN 0
713 #endif // NOTE_MEMORYSTATUS_PROC_LIMIT_WARN
715 #if !defined(NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) || \
716 !DISPATCH_HOST_SUPPORTS_OSX(101200)
717 #undef NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL
718 #define NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL 0
719 #endif // NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL
721 #if !defined(EV_UDATA_SPECIFIC) || !DISPATCH_HOST_SUPPORTS_OSX(101100)
722 #undef DISPATCH_USE_EV_UDATA_SPECIFIC
723 #define DISPATCH_USE_EV_UDATA_SPECIFIC 0
724 #elif !defined(DISPATCH_USE_EV_UDATA_SPECIFIC)
725 #define DISPATCH_USE_EV_UDATA_SPECIFIC 1
726 #endif // EV_UDATA_SPECIFIC
728 #if !DISPATCH_USE_EV_UDATA_SPECIFIC
729 #undef EV_UDATA_SPECIFIC
730 #define EV_UDATA_SPECIFIC 0
732 #define EV_VANISHED 0
733 #endif // !DISPATCH_USE_EV_UDATA_SPECIFIC
736 #define EV_VANISHED 0x0200
739 #ifndef DISPATCH_KEVENT_TREAT_ENOENT_AS_EINPROGRESS
740 #if TARGET_OS_MAC && !DISPATCH_HOST_SUPPORTS_OSX(101200)
741 // deferred delete can return bogus ENOENTs on older kernels
742 #define DISPATCH_KEVENT_TREAT_ENOENT_AS_EINPROGRESS 1
744 #define DISPATCH_KEVENT_TREAT_ENOENT_AS_EINPROGRESS 0
748 #if !defined(EV_SET_QOS) || !DISPATCH_HOST_SUPPORTS_OSX(101100)
749 #undef DISPATCH_USE_KEVENT_QOS
750 #define DISPATCH_USE_KEVENT_QOS 0
751 #elif !defined(DISPATCH_USE_KEVENT_QOS)
752 #define DISPATCH_USE_KEVENT_QOS 1
755 #if HAVE_PTHREAD_WORKQUEUE_KEVENT && defined(KEVENT_FLAG_WORKQ) && \
756 DISPATCH_USE_EV_UDATA_SPECIFIC && DISPATCH_USE_KEVENT_QOS && \
757 DISPATCH_HOST_SUPPORTS_OSX(101200) && \
758 !defined(DISPATCH_USE_KEVENT_WORKQUEUE)
759 #define DISPATCH_USE_KEVENT_WORKQUEUE 1
763 #if (!DISPATCH_USE_KEVENT_WORKQUEUE || DISPATCH_DEBUG) && \
764 !defined(DISPATCH_USE_MGR_THREAD)
765 #define DISPATCH_USE_MGR_THREAD 1
768 #if DISPATCH_USE_KEVENT_WORKQUEUE && DISPATCH_USE_EV_UDATA_SPECIFIC && \
769 DISPATCH_HOST_SUPPORTS_OSX(101200) && \
770 !defined(DISPATCH_USE_EVFILT_MACHPORT_DIRECT)
771 #define DISPATCH_USE_EVFILT_MACHPORT_DIRECT 1
774 #ifndef MACH_SEND_OVERRIDE
775 #define MACH_SEND_OVERRIDE 0x00000020
776 typedef unsigned int mach_msg_priority_t
;
777 #define MACH_MSG_PRIORITY_UNSPECIFIED ((mach_msg_priority_t)0)
778 #endif // MACH_SEND_OVERRIDE
781 #if (!DISPATCH_USE_EVFILT_MACHPORT_DIRECT || DISPATCH_DEBUG) && \
782 !defined(DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK)
783 #define DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK 1
786 #if DISPATCH_USE_KEVENT_QOS
787 typedef struct kevent_qos_s _dispatch_kevent_qos_s
;
788 typedef typeof(((struct kevent_qos_s
*)NULL
)->qos
) _dispatch_kevent_priority_t
;
789 #else // DISPATCH_USE_KEVENT_QOS
790 #ifndef KEVENT_FLAG_IMMEDIATE
791 #define KEVENT_FLAG_NONE 0x00
792 #define KEVENT_FLAG_IMMEDIATE 0x01
793 #define KEVENT_FLAG_ERROR_EVENTS 0x02
794 #endif // KEVENT_FLAG_IMMEDIATE
795 typedef struct kevent64_s _dispatch_kevent_qos_s
;
796 #define kevent_qos(_kq, _changelist, _nchanges, _eventlist, _nevents, \
797 _data_out, _data_available, _flags) \
798 ({ unsigned int _f = (_flags); _dispatch_kevent_qos_s _kev_copy; \
799 const _dispatch_kevent_qos_s *_cl = (_changelist); \
800 int _n = (_nchanges); const struct timespec _timeout_immediately = {}; \
801 dispatch_static_assert(!(_data_out) && !(_data_available)); \
802 if (_f & KEVENT_FLAG_ERROR_EVENTS) { \
803 dispatch_static_assert(_n == 1); \
804 _kev_copy = *_cl; _kev_copy.flags |= EV_RECEIPT; } \
805 kevent64((_kq), _f & KEVENT_FLAG_ERROR_EVENTS ? &_kev_copy : _cl, _n, \
806 (_eventlist), (_nevents), 0, \
807 _f & KEVENT_FLAG_IMMEDIATE ? &_timeout_immediately : NULL); })
808 #endif // DISPATCH_USE_KEVENT_QOS
810 #if defined(F_SETNOSIGPIPE) && defined(F_GETNOSIGPIPE)
811 #ifndef DISPATCH_USE_SETNOSIGPIPE
812 #define DISPATCH_USE_SETNOSIGPIPE 1
814 #endif // F_SETNOSIGPIPE
816 #if defined(MACH_SEND_NOIMPORTANCE)
817 #ifndef DISPATCH_USE_CHECKIN_NOIMPORTANCE
818 #define DISPATCH_USE_CHECKIN_NOIMPORTANCE 1 // rdar://problem/16996737
820 #ifndef DISPATCH_USE_NOIMPORTANCE_QOS
821 #define DISPATCH_USE_NOIMPORTANCE_QOS 1 // rdar://problem/21414476
823 #endif // MACH_SEND_NOIMPORTANCE
826 #if HAVE_LIBPROC_INTERNAL_H
828 #include <libproc_internal.h>
829 #ifndef DISPATCH_USE_IMPORTANCE_ASSERTION
830 #define DISPATCH_USE_IMPORTANCE_ASSERTION 1
832 #endif // HAVE_LIBPROC_INTERNAL_H
834 #if HAVE_SYS_GUARDED_H
835 #include <sys/guarded.h>
836 #ifndef DISPATCH_USE_GUARDED_FD
837 #define DISPATCH_USE_GUARDED_FD 1
839 // change_fdguard_np() requires GUARD_DUP <rdar://problem/11814513>
840 #if DISPATCH_USE_GUARDED_FD && RDAR_11814513
841 #define DISPATCH_USE_GUARDED_FD_CHANGE_FDGUARD 1
843 #endif // HAVE_SYS_GUARDED_H
846 #if __has_include(<sys/kdebug.h>)
847 #include <sys/kdebug.h>
849 #define DBG_DISPATCH 46
852 #define KDBG_CODE(...) 0
854 #define DISPATCH_CODE(subclass, code) \
855 KDBG_CODE(DBG_DISPATCH, DISPATCH_TRACE_SUBCLASS_##subclass, code)
856 #ifdef ARIADNEDBG_CODE
857 #define ARIADNE_ENTER_DISPATCH_MAIN_CODE ARIADNEDBG_CODE(220, 2)
859 #define ARIADNE_ENTER_DISPATCH_MAIN_CODE 0
861 #if !defined(DISPATCH_USE_VOUCHER_KDEBUG_TRACE) && DISPATCH_INTROSPECTION
862 #define DISPATCH_USE_VOUCHER_KDEBUG_TRACE 1
865 #define DISPATCH_TRACE_SUBCLASS_DEFAULT 0
866 #define DISPATCH_TRACE_SUBCLASS_VOUCHER 1
867 #define DISPATCH_TRACE_SUBCLASS_PERF 2
868 #define DISPATCH_TRACE_SUBCLASS_MACH_MSG 3
870 #define DISPATCH_PERF_non_leaf_retarget DISPATCH_CODE(PERF, 1)
871 #define DISPATCH_PERF_post_activate_retarget DISPATCH_CODE(PERF, 2)
872 #define DISPATCH_PERF_post_activate_mutation DISPATCH_CODE(PERF, 3)
873 #define DISPATCH_PERF_delayed_registration DISPATCH_CODE(PERF, 4)
874 #define DISPATCH_PERF_mutable_target DISPATCH_CODE(PERF, 5)
876 #define DISPATCH_MACH_MSG_hdr_move DISPATCH_CODE(MACH_MSG, 1)
878 DISPATCH_ALWAYS_INLINE
880 _dispatch_ktrace_impl(uint32_t code
, uint64_t a
, uint64_t b
,
881 uint64_t c
, uint64_t d
)
884 #ifdef _COMM_PAGE_KDEBUG_ENABLE
885 if (likely(*(volatile uint32_t *)_COMM_PAGE_KDEBUG_ENABLE
== 0)) return;
887 kdebug_trace(code
, a
, b
, c
, d
);
889 #define _dispatch_cast_to_uint64(e) \
890 __builtin_choose_expr(sizeof(e) > 4, \
891 ((uint64_t)(e)), ((uint64_t)(uintptr_t)(e)))
892 #define _dispatch_ktrace(code, a, b, c, d) _dispatch_ktrace_impl(code, \
893 _dispatch_cast_to_uint64(a), _dispatch_cast_to_uint64(b), \
894 _dispatch_cast_to_uint64(c), _dispatch_cast_to_uint64(d))
896 #else // __has_include(<sys/kdebug.h>)
897 #define DISPATCH_CODE(subclass, code) 0
898 #define ARIADNE_ENTER_DISPATCH_MAIN_CODE 0
899 #define DISPATCH_USE_VOUCHER_KDEBUG_TRACE 0
900 #define _dispatch_ktrace(code, a, b, c, d)
901 #endif // !__has_include(<sys/kdebug.h>)
902 #define _dispatch_ktrace4(code, a, b, c, d) _dispatch_ktrace(code, a, b, c, d)
903 #define _dispatch_ktrace3(code, a, b, c) _dispatch_ktrace(code, a, b, c, 0)
904 #define _dispatch_ktrace2(code, a, b) _dispatch_ktrace(code, a, b, 0, 0)
905 #define _dispatch_ktrace1(code, a) _dispatch_ktrace(code, a, 0, 0, 0)
906 #define _dispatch_ktrace0(code) _dispatch_ktrace(code, 0, 0, 0, 0)
908 #ifndef MACH_MSGH_BITS_VOUCHER_MASK
909 #define MACH_MSGH_BITS_VOUCHER_MASK 0x001f0000
910 #define MACH_MSGH_BITS_SET_PORTS(remote, local, voucher) \
911 (((remote) & MACH_MSGH_BITS_REMOTE_MASK) | \
912 (((local) << 8) & MACH_MSGH_BITS_LOCAL_MASK) | \
913 (((voucher) << 16) & MACH_MSGH_BITS_VOUCHER_MASK))
914 #define MACH_MSGH_BITS_VOUCHER(bits) \
915 (((bits) & MACH_MSGH_BITS_VOUCHER_MASK) >> 16)
916 #define MACH_MSGH_BITS_HAS_VOUCHER(bits) \
917 (MACH_MSGH_BITS_VOUCHER(bits) != MACH_MSGH_BITS_ZERO)
918 #define msgh_voucher_port msgh_reserved
919 #define mach_voucher_t mach_port_t
920 #define MACH_VOUCHER_NULL MACH_PORT_NULL
921 #define MACH_SEND_INVALID_VOUCHER 0x10000005
924 #if TARGET_OS_SIMULATOR && IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101100
925 #undef VOUCHER_USE_MACH_VOUCHER
926 #define VOUCHER_USE_MACH_VOUCHER 0
928 #ifndef VOUCHER_USE_MACH_VOUCHER
929 #if __has_include(<mach/mach_voucher.h>)
930 #define VOUCHER_USE_MACH_VOUCHER 1
934 #if RDAR_24272659 // FIXME: <rdar://problem/24272659>
935 #if !VOUCHER_USE_MACH_VOUCHER || !DISPATCH_HOST_SUPPORTS_OSX(101200)
936 #undef VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
937 #define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 0
938 #elif !defined(VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER)
939 #define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 1
941 #else // RDAR_24272659
942 #undef VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
943 #define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 0
944 #endif // RDAR_24272659
946 #if !VOUCHER_USE_MACH_VOUCHER || !DISPATCH_HOST_SUPPORTS_OSX(101200)
947 #undef VOUCHER_USE_BANK_AUTOREDEEM
948 #define VOUCHER_USE_BANK_AUTOREDEEM 0
949 #elif !defined(VOUCHER_USE_BANK_AUTOREDEEM)
950 #define VOUCHER_USE_BANK_AUTOREDEEM 1
953 #if !VOUCHER_USE_MACH_VOUCHER || \
954 !__has_include(<voucher/ipc_pthread_priority_types.h>) || \
955 !DISPATCH_HOST_SUPPORTS_OSX(101200)
956 #undef VOUCHER_USE_MACH_VOUCHER_PRIORITY
957 #define VOUCHER_USE_MACH_VOUCHER_PRIORITY 0
958 #elif !defined(VOUCHER_USE_MACH_VOUCHER_PRIORITY)
959 #define VOUCHER_USE_MACH_VOUCHER_PRIORITY 1
962 #ifndef VOUCHER_USE_PERSONA
963 #if VOUCHER_USE_MACH_VOUCHER && defined(BANK_PERSONA_TOKEN) && \
964 TARGET_OS_IOS && !TARGET_OS_SIMULATOR
965 #define VOUCHER_USE_PERSONA 1
967 #define VOUCHER_USE_PERSONA 0
969 #endif // VOUCHER_USE_PERSONA
971 #if VOUCHER_USE_MACH_VOUCHER
972 #undef DISPATCH_USE_IMPORTANCE_ASSERTION
973 #define DISPATCH_USE_IMPORTANCE_ASSERTION 0
975 #undef MACH_RCV_VOUCHER
976 #define MACH_RCV_VOUCHER 0
977 #define VOUCHER_USE_PERSONA 0
978 #endif // VOUCHER_USE_MACH_VOUCHER
980 #define _dispatch_hardware_crash() \
981 __asm__(""); __builtin_trap() // <rdar://problem/17464981>
983 #define _dispatch_set_crash_log_cause_and_message(ac, msg)
984 #define _dispatch_set_crash_log_message(msg)
985 #define _dispatch_set_crash_log_message_dynamic(msg)
988 // MIG_REPLY_MISMATCH means either:
989 // 1) A signal handler is NOT using async-safe API. See the sigaction(2) man
990 // page for more info.
991 // 2) A hand crafted call to mach_msg*() screwed up. Use MIG.
992 #define DISPATCH_VERIFY_MIG(x) do { \
993 if ((x) == MIG_REPLY_MISMATCH) { \
994 _dispatch_set_crash_log_cause_and_message((x), \
995 "MIG_REPLY_MISMATCH"); \
996 _dispatch_hardware_crash(); \
1001 #define DISPATCH_INTERNAL_CRASH(c, x) do { \
1002 _dispatch_set_crash_log_cause_and_message((c), \
1003 "BUG IN LIBDISPATCH: " x); \
1004 _dispatch_hardware_crash(); \
1007 #define DISPATCH_CLIENT_CRASH(c, x) do { \
1008 _dispatch_set_crash_log_cause_and_message((c), \
1009 "BUG IN CLIENT OF LIBDISPATCH: " x); \
1010 _dispatch_hardware_crash(); \
1013 #define _OS_OBJECT_CLIENT_CRASH(x) do { \
1014 _dispatch_set_crash_log_message("API MISUSE: " x); \
1015 _dispatch_hardware_crash(); \
1018 #define DISPATCH_ASSERTION_FAILED_MESSAGE \
1019 "BUG IN CLIENT OF LIBDISPATCH: Assertion failed: "
1021 #define _dispatch_assert_crash(msg) do { \
1022 const char *__msg = (msg); \
1023 _dispatch_log("%s", __msg); \
1024 _dispatch_set_crash_log_message_dynamic(__msg); \
1025 _dispatch_hardware_crash(); \
1028 #define _dispatch_client_assert_fail(fmt, ...) do { \
1029 char *_msg = NULL; \
1030 asprintf(&_msg, "%s" fmt, DISPATCH_ASSERTION_FAILED_MESSAGE, \
1032 _dispatch_assert_crash(_msg); \
1036 #define DISPATCH_NO_VOUCHER ((voucher_t)(void*)~0ul)
1037 #define DISPATCH_NO_PRIORITY ((pthread_priority_t)~0ul)
1038 DISPATCH_ENUM(_dispatch_thread_set_self
, unsigned long,
1039 DISPATCH_PRIORITY_ENFORCE
= 0x1,
1040 DISPATCH_VOUCHER_REPLACE
= 0x2,
1041 DISPATCH_VOUCHER_CONSUME
= 0x4,
1042 DISPATCH_THREAD_PARK
= 0x8,
1044 DISPATCH_WARN_RESULT
1045 static inline voucher_t
_dispatch_adopt_priority_and_set_voucher(
1046 pthread_priority_t priority
, voucher_t voucher
,
1047 _dispatch_thread_set_self_t flags
);
1049 mach_port_t
_dispatch_get_mach_host_port(void);
1052 #if HAVE_PTHREAD_WORKQUEUE_QOS
1054 extern int _dispatch_set_qos_class_enabled
;
1056 #define _dispatch_set_qos_class_enabled (1)
1058 #endif // HAVE_PTHREAD_WORKQUEUE_QOS
1059 #if DISPATCH_USE_KEVENT_WORKQUEUE
1060 #if !HAVE_PTHREAD_WORKQUEUE_QOS || !DISPATCH_USE_KEVENT_QOS || \
1061 !DISPATCH_USE_EV_UDATA_SPECIFIC
1062 #error Invalid build configuration
1064 #if DISPATCH_USE_MGR_THREAD
1065 extern int _dispatch_kevent_workqueue_enabled
;
1067 #define _dispatch_kevent_workqueue_enabled (1)
1069 #endif // DISPATCH_USE_KEVENT_WORKQUEUE
1071 #if DISPATCH_USE_EVFILT_MACHPORT_DIRECT
1072 #if !DISPATCH_USE_KEVENT_WORKQUEUE || !DISPATCH_USE_EV_UDATA_SPECIFIC
1073 #error Invalid build configuration
1075 #if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
1076 extern int _dispatch_evfilt_machport_direct_enabled
;
1078 #define _dispatch_evfilt_machport_direct_enabled (1)
1081 #define _dispatch_evfilt_machport_direct_enabled (0)
1082 #endif // DISPATCH_USE_EVFILT_MACHPORT_DIRECT
1085 /* #includes dependent on internal.h */
1086 #include "object_internal.h"
1087 #include "semaphore_internal.h"
1088 #include "introspection_internal.h"
1089 #include "queue_internal.h"
1090 #include "source_internal.h"
1091 #include "voucher_internal.h"
1092 #include "data_internal.h"
1093 #if !TARGET_OS_WIN32
1094 #include "io_internal.h"
1096 #include "inline_internal.h"
1097 #include "firehose/firehose_internal.h"
1099 #endif /* __DISPATCH_INTERNAL__ */