X-Git-Url: https://git.saurik.com/apple/libdispatch.git/blobdiff_plain/517da941284910bcce6aed25a1e923708f0ed33f..refs/heads/master:/src/internal.h diff --git a/src/internal.h b/src/internal.h index ed1a9c7..286e534 100644 --- a/src/internal.h +++ b/src/internal.h @@ -27,36 +27,90 @@ #ifndef __DISPATCH_INTERNAL__ #define __DISPATCH_INTERNAL__ +#if __has_include() +#include +#else #include +#endif #define __DISPATCH_BUILDING_DISPATCH__ #define __DISPATCH_INDIRECT__ #ifdef __APPLE__ #include +#include #include + +#ifndef TARGET_OS_MAC_DESKTOP +#define TARGET_OS_MAC_DESKTOP (TARGET_OS_MAC && \ + !TARGET_OS_SIMULATOR && !TARGET_OS_IPHONE && !TARGET_OS_EMBEDDED) +#endif + +#if TARGET_OS_MAC_DESKTOP +# define DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(x) \ + (__MAC_OS_X_VERSION_MIN_REQUIRED >= (x)) +# if !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200) +# error "OS X hosts older than OS X 10.12 aren't supported anymore" +# endif // !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200) +#elif TARGET_OS_SIMULATOR +# define DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(x) \ + (IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED >= (x)) +# if !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200) +# error "Simulator hosts older than OS X 10.12 aren't supported anymore" +# endif // !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200) +#else +# define DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(x) 1 +# if __IPHONE_OS_VERSION_MIN_REQUIRED < 90000 +# error "iOS hosts older than iOS 9.0 aren't supported anymore" +# endif #endif +#else // !__APPLE__ +#define DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(x) 0 +#endif // !__APPLE__ + #if !defined(DISPATCH_MACH_SPI) && TARGET_OS_MAC #define DISPATCH_MACH_SPI 1 #endif +#if !defined(OS_VOUCHER_CREATION_SPI) && TARGET_OS_MAC +#define OS_VOUCHER_CREATION_SPI 1 +#endif +#if !defined(OS_VOUCHER_ACTIVITY_SPI) && TARGET_OS_MAC +#define OS_VOUCHER_ACTIVITY_SPI 1 +#endif +#if !defined(OS_VOUCHER_ACTIVITY_GENERATE_SWAPS) +#define OS_VOUCHER_ACTIVITY_GENERATE_SWAPS 0 +#endif +#if !defined(OS_FIREHOSE_SPI) && TARGET_OS_MAC +#define OS_FIREHOSE_SPI 1 +#endif +#if !defined(DISPATCH_LAYOUT_SPI) && TARGET_OS_MAC +#define DISPATCH_LAYOUT_SPI 1 +#endif -#if !defined(USE_OBJC) && HAVE_OBJC -#define USE_OBJC 1 +#if __has_include() +#include +#if !defined(HAVE_DYLD_IS_MEMORY_IMMUTABLE) +#if defined(DYLD_MACOSX_VERSION_10_12) || defined(DYLD_IOS_VERSION_10_0) +#define HAVE_DYLD_IS_MEMORY_IMMUTABLE 1 +#else +#define HAVE_DYLD_IS_MEMORY_IMMUTABLE 0 #endif +#endif // !defined(HAVE_DYLD_IS_MEMORY_IMMUTABLE) +#endif // __has_include() -#if USE_OBJC && ((!TARGET_IPHONE_SIMULATOR && defined(__i386__)) || \ - (!TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED < 1080)) -// Disable Objective-C support on platforms with legacy objc runtime -#undef USE_OBJC -#define USE_OBJC 0 +#if !defined(USE_OBJC) && HAVE_OBJC +#define USE_OBJC 1 #endif #if USE_OBJC #define OS_OBJECT_HAVE_OBJC_SUPPORT 1 -#if __OBJC__ +#if defined(__OBJC__) #define OS_OBJECT_USE_OBJC 1 +// Force internal Objective-C sources to use class-visible headers +// even when not compiling in Swift. +#define OS_OBJECT_SWIFT3 1 #else #define OS_OBJECT_USE_OBJC 0 #endif // __OBJC__ @@ -67,11 +121,28 @@ #include #include +#define __DISPATCH_HIDE_SYMBOL(sym, version) \ + __asm__(".section __TEXT,__const\n\t" \ + ".globl $ld$hide$os" #version "$_" #sym "\n\t" \ + "$ld$hide$os" #version "$_" #sym ":\n\t" \ + " .byte 0\n\t" \ + ".previous") + + +#ifndef DISPATCH_HIDE_SYMBOL +#if TARGET_OS_MAC && !TARGET_OS_IPHONE +#define DISPATCH_HIDE_SYMBOL(sym, osx, ios, tvos, watchos) \ + __DISPATCH_HIDE_SYMBOL(sym, osx) +#else +#define DISPATCH_HIDE_SYMBOL(sym, osx, ios, tvos, watchos) +#endif +#endif #include -#include #include +#include #include +#include #include #include #include @@ -81,45 +152,25 @@ #include #endif -#define DISPATCH_STRUCT_DECL(type, name, ...) \ - struct type __VA_ARGS__ name - -// Visual Studio C++ does not support C99 designated initializers. -// This means that static declarations should be zero initialized and cannot -// be const since we must fill in the values during DLL initialization. -#if !TARGET_OS_WIN32 -#define DISPATCH_STRUCT_INSTANCE(type, name, ...) \ -struct type name = { \ -__VA_ARGS__ \ -} -#else -#define DISPATCH_STRUCT_INSTANCE(type, name, ...) \ -struct type name = { 0 } -#endif - -#if !TARGET_OS_WIN32 -#define DISPATCH_CONST_STRUCT_DECL(type, name, ...) \ - const DISPATCH_STRUCT_DECL(type, name, __VA_ARGS__) - -#define DISPATCH_CONST_STRUCT_INSTANCE(type, name, ...) \ - const DISPATCH_STRUCT_INSTANCE(type, name, __VA_ARGS__) +#if defined(__OBJC__) || defined(__cplusplus) +#define DISPATCH_PURE_C 0 #else -#define DISPATCH_CONST_STRUCT_DECL(type, name, ...) \ - DISPATCH_STRUCT_DECL(type, name, __VA_ARGS__) - -#define DISPATCH_CONST_STRUCT_INSTANCE(type, name, ...) \ - DISPATCH_STRUCT_INSTANCE(type, name, __VA_ARGS__) +#define DISPATCH_PURE_C 1 #endif /* private.h must be included last to avoid picking up installed headers. */ -#include "object_private.h" +#include +#include "os/object_private.h" #include "queue_private.h" #include "source_private.h" #include "mach_private.h" #include "data_private.h" +#include "os/voucher_private.h" +#include "os/voucher_activity_private.h" #if !TARGET_OS_WIN32 #include "io_private.h" #endif +#include "layout_private.h" #include "benchmark.h" #include "private.h" @@ -138,6 +189,8 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void); #define DISPATCH_USE_CLIENT_CALLOUT 1 #endif +#define DISPATCH_ALLOW_NON_LEAF_RETARGET 1 + /* The "_debug" library build */ #ifndef DISPATCH_DEBUG #define DISPATCH_DEBUG 0 @@ -152,11 +205,15 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void); #define DISPATCH_USE_DTRACE 1 #endif -#if ((!TARGET_OS_EMBEDDED && DISPATCH_INTROSPECTION) || DISPATCH_DEBUG || \ +#if DISPATCH_USE_DTRACE && (DISPATCH_INTROSPECTION || DISPATCH_DEBUG || \ DISPATCH_PROFILE) && !defined(DISPATCH_USE_DTRACE_INTROSPECTION) #define DISPATCH_USE_DTRACE_INTROSPECTION 1 #endif +#ifndef DISPATCH_DEBUG_QOS +#define DISPATCH_DEBUG_QOS DISPATCH_DEBUG +#endif + #if HAVE_LIBKERN_OSCROSSENDIAN_H #include #endif @@ -176,6 +233,7 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void); #include #include #include +#include #include #include #include @@ -187,21 +245,26 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void); #include -#if !TARGET_OS_WIN32 -#include +#if !TARGET_OS_WIN32 #include #include +#ifdef __ANDROID__ +#include +#else #include +#endif /* __ANDROID__ */ #include #include #include #include -#else -#include "sys_queue.h" #endif #ifdef __BLOCKS__ +#if __has_include() #include +#else +#include "BlocksRuntime/Block_private.h" +#endif // __has_include() #include #endif /* __BLOCKS__ */ @@ -222,32 +285,17 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void); #include #include #include -#if !TARGET_OS_WIN32 -#include -#endif #if HAVE_UNISTD_H #include #endif -#ifndef __has_builtin -#define __has_builtin(x) 0 -#endif -#ifndef __has_include -#define __has_include(x) 0 -#endif -#ifndef __has_feature -#define __has_feature(x) 0 -#endif -#ifndef __has_attribute -#define __has_attribute(x) 0 -#endif - #if __GNUC__ #define DISPATCH_NOINLINE __attribute__((__noinline__)) #define DISPATCH_USED __attribute__((__used__)) #define DISPATCH_UNUSED __attribute__((__unused__)) #define DISPATCH_WEAK __attribute__((__weak__)) #define DISPATCH_OVERLOADABLE __attribute__((__overloadable__)) +#define DISPATCH_PACKED __attribute__((__packed__)) #if DISPATCH_DEBUG #define DISPATCH_ALWAYS_INLINE_NDEBUG #else @@ -264,6 +312,31 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void); #define DISPATCH_CONCAT(x,y) DISPATCH_CONCAT1(x,y) #define DISPATCH_CONCAT1(x,y) x ## y +#define DISPATCH_COUNT_ARGS(...) DISPATCH_COUNT_ARGS1(, ## __VA_ARGS__, \ + _8, _7, _6, _5, _4, _3, _2, _1, _0) +#define DISPATCH_COUNT_ARGS1(z, a, b, c, d, e, f, g, h, cnt, ...) cnt + +#if BYTE_ORDER == LITTLE_ENDIAN +#define DISPATCH_STRUCT_LE_2(a, b) struct { a; b; } +#define DISPATCH_STRUCT_LE_3(a, b, c) struct { a; b; c; } +#define DISPATCH_STRUCT_LE_4(a, b, c, d) struct { a; b; c; d; } +#else +#define DISPATCH_STRUCT_LE_2(a, b) struct { b; a; } +#define DISPATCH_STRUCT_LE_3(a, b, c) struct { c; b; a; } +#define DISPATCH_STRUCT_LE_4(a, b, c, d) struct { d; c; b; a; } +#endif +#if __has_feature(c_startic_assert) +#define DISPATCH_UNION_ASSERT(alias, st) \ + _Static_assert(sizeof(struct { alias; }) == sizeof(st), "bogus union"); +#else +#define DISPATCH_UNION_ASSERT(alias, st) +#endif +#define DISPATCH_UNION_LE(alias, ...) \ + DISPATCH_UNION_ASSERT(alias, DISPATCH_CONCAT(DISPATCH_STRUCT_LE, \ + DISPATCH_COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__)) \ + union { alias; DISPATCH_CONCAT(DISPATCH_STRUCT_LE, \ + DISPATCH_COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__); } + // workaround 6368156 #ifdef NSEC_PER_SEC #undef NSEC_PER_SEC @@ -280,88 +353,149 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void); /* I wish we had __builtin_expect_range() */ #if __GNUC__ -#define fastpath(x) ((typeof(x))__builtin_expect((long)(x), ~0l)) -#define slowpath(x) ((typeof(x))__builtin_expect((long)(x), 0l)) +#define _safe_cast_to_long(x) \ + ({ _Static_assert(sizeof(typeof(x)) <= sizeof(long), \ + "__builtin_expect doesn't support types wider than long"); \ + (long)(x); }) +#define fastpath(x) ((typeof(x))__builtin_expect(_safe_cast_to_long(x), ~0l)) +#define slowpath(x) ((typeof(x))__builtin_expect(_safe_cast_to_long(x), 0l)) +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) #else #define fastpath(x) (x) #define slowpath(x) (x) +#define likely(x) (!!(x)) +#define unlikely(x) (!!(x)) #endif // __GNUC__ -DISPATCH_NOINLINE +#define _TAILQ_IS_ENQUEUED(elm, field) \ + ((elm)->field.tqe_prev != NULL) +#define _TAILQ_MARK_NOT_ENQUEUED(elm, field) \ + do { (elm)->field.tqe_prev = NULL; } while (0) + +#if DISPATCH_DEBUG +// sys/queue.h debugging +#if defined(__linux__) +#define QUEUE_MACRO_DEBUG 1 +#else +#undef TRASHIT +#define TRASHIT(x) do {(x) = (void *)-1;} while (0) +#endif +#endif // DISPATCH_DEBUG +#define _TAILQ_TRASH_ENTRY(elm, field) do { \ + TRASHIT((elm)->field.tqe_next); \ + TRASHIT((elm)->field.tqe_prev); \ + } while (0) +#define _TAILQ_TRASH_HEAD(head) do { \ + TRASHIT((head)->tqh_first); \ + TRASHIT((head)->tqh_last); \ + } while (0) + +DISPATCH_EXPORT DISPATCH_NOINLINE void _dispatch_bug(size_t line, long val); -#if HAVE_MACH DISPATCH_NOINLINE void _dispatch_bug_client(const char* msg); +#if HAVE_MACH DISPATCH_NOINLINE void _dispatch_bug_mach_client(const char *msg, mach_msg_return_t kr); +#endif // HAVE_MACH DISPATCH_NOINLINE void _dispatch_bug_kevent_client(const char* msg, const char* filter, const char *operation, int err); -#endif + +DISPATCH_NOINLINE +void _dispatch_bug_deprecated(const char *msg); DISPATCH_NOINLINE DISPATCH_NORETURN void _dispatch_abort(size_t line, long val); -#if !defined(DISPATCH_USE_OS_TRACE) && DISPATCH_DEBUG -#if __has_include() -#define DISPATCH_USE_OS_TRACE 1 -#include +#if !defined(DISPATCH_USE_OS_DEBUG_LOG) && DISPATCH_DEBUG +#if __has_include() +#define DISPATCH_USE_OS_DEBUG_LOG 1 +#include +#endif +#endif // DISPATCH_USE_OS_DEBUG_LOG + +#if !defined(DISPATCH_USE_SIMPLE_ASL) && !DISPATCH_USE_OS_DEBUG_LOG +#if __has_include(<_simple.h>) +#define DISPATCH_USE_SIMPLE_ASL 1 +#include <_simple.h> +#endif +#endif // DISPATCH_USE_SIMPLE_ASL + +#if !DISPATCH_USE_SIMPLE_ASL && !DISPATCH_USE_OS_DEBUG_LOG && !TARGET_OS_WIN32 +#include #endif -#endif // DISPATCH_USE_OS_TRACE -#if DISPATCH_USE_OS_TRACE -#define _dispatch_log(msg, ...) os_trace("libdispatch", msg, ## __VA_ARGS__) +#if DISPATCH_USE_OS_DEBUG_LOG +#define _dispatch_log(msg, ...) os_debug_log("libdispatch", msg, ## __VA_ARGS__) #else -DISPATCH_NOINLINE __attribute__((__format__(__printf__,1,2))) +DISPATCH_EXPORT DISPATCH_NOINLINE __attribute__((__format__(__printf__,1,2))) void _dispatch_log(const char *msg, ...); -#endif // DISPATCH_USE_OS_TRACE +#endif // DISPATCH_USE_OS_DEBUG_LOG + +#define dsnprintf(buf, siz, ...) \ + ({ size_t _siz = siz; int _r = snprintf(buf, _siz, __VA_ARGS__); \ + _r < 0 ? 0u : ((size_t)_r > _siz ? _siz : (size_t)_r); }) + +#if __GNUC__ +#define dispatch_static_assert(e) ({ \ + char __compile_time_assert__[(bool)(e) ? 1 : -1] DISPATCH_UNUSED; \ + }) +#else +#define dispatch_static_assert(e) +#endif -#define dsnprintf(...) \ - ({ int _r = snprintf(__VA_ARGS__); _r < 0 ? 0u : (size_t)_r; }) +#define DISPATCH_BAD_INPUT ((void *_Nonnull)0) +#define DISPATCH_OUT_OF_MEMORY ((void *_Nonnull)0) /* * For reporting bugs within libdispatch when using the "_debug" version of the * library. */ -#if __GNUC__ +#if __APPLE__ #define dispatch_assert(e) do { \ if (__builtin_constant_p(e)) { \ - char __compile_time_assert__[(bool)(e) ? 1 : -1] DISPATCH_UNUSED; \ + dispatch_static_assert(e); \ } else { \ - typeof(e) _e = fastpath(e); /* always eval 'e' */ \ - if (DISPATCH_DEBUG && !_e) { \ + typeof(e) _e = (e); /* always eval 'e' */ \ + if (unlikely(DISPATCH_DEBUG && !_e)) { \ _dispatch_abort(__LINE__, (long)_e); \ } \ } \ } while (0) #else -static inline void _dispatch_assert(long e, long line) { +static inline void +_dispatch_assert(long e, size_t line) +{ if (DISPATCH_DEBUG && !e) _dispatch_abort(line, e); } #define dispatch_assert(e) _dispatch_assert((long)(e), __LINE__) #endif /* __GNUC__ */ -#if __GNUC__ +#if __APPLE__ /* * A lot of API return zero upon success and not-zero on fail. Let's capture * and log the non-zero value */ #define dispatch_assert_zero(e) do { \ if (__builtin_constant_p(e)) { \ - char __compile_time_assert__[(bool)(e) ? -1 : 1] DISPATCH_UNUSED; \ + dispatch_static_assert(e); \ } else { \ - typeof(e) _e = slowpath(e); /* always eval 'e' */ \ - if (DISPATCH_DEBUG && _e) { \ + typeof(e) _e = (e); /* always eval 'e' */ \ + if (unlikely(DISPATCH_DEBUG && _e)) { \ _dispatch_abort(__LINE__, (long)_e); \ } \ } \ } while (0) #else -static inline void _dispatch_assert_zero(long e, long line) { +static inline void +_dispatch_assert_zero(long e, size_t line) +{ if (DISPATCH_DEBUG && e) _dispatch_abort(line, e); } -#define dispatch_assert_zero(e) _dispatch_assert((long)(e), __LINE__) +#define dispatch_assert_zero(e) _dispatch_assert_zero((long)(e), __LINE__) #endif /* __GNUC__ */ /* @@ -372,18 +506,19 @@ static inline void _dispatch_assert_zero(long e, long line) { */ #if __GNUC__ #define dispatch_assume(e) ({ \ - typeof(e) _e = fastpath(e); /* always eval 'e' */ \ - if (!_e) { \ + typeof(e) _e = (e); /* always eval 'e' */ \ + if (unlikely(!_e)) { \ if (__builtin_constant_p(e)) { \ - char __compile_time_assert__[(bool)(e) ? 1 : -1]; \ - (void)__compile_time_assert__; \ + dispatch_static_assert(e); \ } \ _dispatch_bug(__LINE__, (long)_e); \ } \ _e; \ }) #else -static inline long _dispatch_assume(long e, long line) { +static inline long +_dispatch_assume(long e, long line) +{ if (!e) _dispatch_bug(line, e); return e; } @@ -396,18 +531,19 @@ static inline long _dispatch_assume(long e, long line) { */ #if __GNUC__ #define dispatch_assume_zero(e) ({ \ - typeof(e) _e = slowpath(e); /* always eval 'e' */ \ - if (_e) { \ + typeof(e) _e = (e); /* always eval 'e' */ \ + if (unlikely(_e)) { \ if (__builtin_constant_p(e)) { \ - char __compile_time_assert__[(bool)(e) ? -1 : 1]; \ - (void)__compile_time_assert__; \ + dispatch_static_assert(e); \ } \ _dispatch_bug(__LINE__, (long)_e); \ } \ _e; \ }) #else -static inline long _dispatch_assume_zero(long e, long line) { +static inline long +_dispatch_assume_zero(long e, long line) +{ if (e) _dispatch_bug(line, e); return e; } @@ -420,10 +556,10 @@ static inline long _dispatch_assume_zero(long e, long line) { #if __GNUC__ #define dispatch_debug_assert(e, msg, args...) do { \ if (__builtin_constant_p(e)) { \ - char __compile_time_assert__[(bool)(e) ? 1 : -1] DISPATCH_UNUSED; \ + dispatch_static_assert(e); \ } else { \ - typeof(e) _e = fastpath(e); /* always eval 'e' */ \ - if (DISPATCH_DEBUG && !_e) { \ + typeof(e) _e = (e); /* always eval 'e' */ \ + if (unlikely(DISPATCH_DEBUG && !_e)) { \ _dispatch_log("%s() 0x%lx: " msg, __func__, (long)_e, ##args); \ abort(); \ } \ @@ -431,8 +567,8 @@ static inline long _dispatch_assume_zero(long e, long line) { } while (0) #else #define dispatch_debug_assert(e, msg, args...) do { \ - long _e = (long)fastpath(e); /* always eval 'e' */ \ - if (DISPATCH_DEBUG && !_e) { \ + typeof(e) _e = (e); /* always eval 'e' */ \ + if (unlikely(DISPATCH_DEBUG && !_e)) { \ _dispatch_log("%s() 0x%lx: " msg, __FUNCTION__, _e, ##args); \ abort(); \ } \ @@ -447,13 +583,6 @@ static inline long _dispatch_assume_zero(long e, long line) { } \ } while (0) -#if DISPATCH_DEBUG -#if HAVE_MACH -DISPATCH_NOINLINE DISPATCH_USED -void dispatch_debug_machport(mach_port_t name, const char* str); -#endif -#endif - #if DISPATCH_DEBUG /* This is the private version of the deprecated dispatch_debug() */ DISPATCH_NONNULL2 DISPATCH_NOTHROW @@ -464,201 +593,132 @@ _dispatch_object_debug(dispatch_object_t object, const char *message, ...); #define _dispatch_object_debug(object, message, ...) #endif // DISPATCH_DEBUG -#if DISPATCH_USE_CLIENT_CALLOUT - -DISPATCH_NOTHROW void -_dispatch_client_callout(void *ctxt, dispatch_function_t f); -DISPATCH_NOTHROW void -_dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t)); -DISPATCH_NOTHROW bool -_dispatch_client_callout3(void *ctxt, dispatch_data_t region, size_t offset, - const void *buffer, size_t size, dispatch_data_applier_function_t f); -DISPATCH_NOTHROW void -_dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason, - dispatch_mach_msg_t dmsg, mach_error_t error, - dispatch_mach_handler_function_t f); - -#else // !DISPATCH_USE_CLIENT_CALLOUT - -DISPATCH_ALWAYS_INLINE -static inline void -_dispatch_client_callout(void *ctxt, dispatch_function_t f) -{ - return f(ctxt); -} - -DISPATCH_ALWAYS_INLINE -static inline void -_dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t)) -{ - return f(ctxt, i); -} - -DISPATCH_ALWAYS_INLINE -static inline bool -_dispatch_client_callout3(void *ctxt, dispatch_data_t region, size_t offset, - const void *buffer, size_t size, dispatch_data_applier_function_t f) -{ - return f(ctxt, region, offset, buffer, size); -} - -DISPATCH_ALWAYS_INLINE -static inline void -_dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason, - dispatch_mach_msg_t dmsg, mach_error_t error, - dispatch_mach_handler_function_t f); -{ - return f(ctxt, reason, dmsg, error); -} - -#endif // !DISPATCH_USE_CLIENT_CALLOUT - #ifdef __BLOCKS__ #define _dispatch_Block_invoke(bb) \ ((dispatch_function_t)((struct Block_layout *)bb)->invoke) -DISPATCH_ALWAYS_INLINE -static inline void -_dispatch_client_callout_block(dispatch_block_t b) -{ - return _dispatch_client_callout(b, _dispatch_Block_invoke(b)); -} - +void *_dispatch_Block_copy(void *block); #if __GNUC__ -dispatch_block_t _dispatch_Block_copy(dispatch_block_t block); #define _dispatch_Block_copy(x) ((typeof(x))_dispatch_Block_copy(x)) -#else -dispatch_block_t _dispatch_Block_copy(const void *block); #endif - void _dispatch_call_block_and_release(void *block); #endif /* __BLOCKS__ */ void _dispatch_temporary_resource_shortage(void); void *_dispatch_calloc(size_t num_items, size_t size); +const char *_dispatch_strdup_if_mutable(const char *str); void _dispatch_vtable_init(void); char *_dispatch_get_build(void); +int _dispatch_sigmask(void); uint64_t _dispatch_timeout(dispatch_time_t when); +uint64_t _dispatch_time_nanoseconds_since_epoch(dispatch_time_t when); -extern bool _dispatch_safe_fork, _dispatch_child_of_unsafe_fork; +#define _DISPATCH_UNSAFE_FORK_MULTITHREADED ((uint8_t)1) +#define _DISPATCH_UNSAFE_FORK_PROHIBIT ((uint8_t)2) +extern uint8_t _dispatch_unsafe_fork; +extern bool _dispatch_child_of_unsafe_fork; +void _dispatch_fork_becomes_unsafe_slow(void); -extern struct _dispatch_hw_config_s { - uint32_t cc_max_active; - uint32_t cc_max_logical; - uint32_t cc_max_physical; -} _dispatch_hw_config; +#define _dispatch_is_multithreaded_inline() \ + ((_dispatch_unsafe_fork & _DISPATCH_UNSAFE_FORK_MULTITHREADED) != 0) -#if !defined(DISPATCH_USE_OS_SEMAPHORE_CACHE) && !(TARGET_IPHONE_SIMULATOR && \ - IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1090) -#if __has_include() -#define DISPATCH_USE_OS_SEMAPHORE_CACHE 1 -#include -#endif +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_fork_becomes_unsafe(void) +{ + if (unlikely(!_dispatch_is_multithreaded_inline())) { + _dispatch_fork_becomes_unsafe_slow(); + DISPATCH_COMPILER_CAN_ASSUME(_dispatch_is_multithreaded_inline()); + } +} + +#if DISPATCH_INTROSPECTION +#undef DISPATCH_PERF_MON +#define DISPATCH_PERF_MON 0 #endif /* #includes dependent on internal.h */ #include "shims.h" +#include "event/event_internal.h" // Older Mac OS X and iOS Simulator fallbacks -#if HAVE_PTHREAD_WORKQUEUES -#ifndef WORKQ_BG_PRIOQUEUE -#define WORKQ_BG_PRIOQUEUE 3 +#if HAVE__PTHREAD_WORKQUEUE_INIT && PTHREAD_WORKQUEUE_SPI_VERSION >= 20140213 \ + && !defined(HAVE_PTHREAD_WORKQUEUE_QOS) +#define HAVE_PTHREAD_WORKQUEUE_QOS 1 #endif -#ifndef WORKQ_ADDTHREADS_OPTION_OVERCOMMIT -#define WORKQ_ADDTHREADS_OPTION_OVERCOMMIT 0x00000001 +#if HAVE__PTHREAD_WORKQUEUE_INIT && PTHREAD_WORKQUEUE_SPI_VERSION >= 20150304 \ + && !defined(HAVE_PTHREAD_WORKQUEUE_KEVENT) +#define HAVE_PTHREAD_WORKQUEUE_KEVENT 1 #endif -#if TARGET_IPHONE_SIMULATOR && IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1070 -#ifndef DISPATCH_NO_BG_PRIORITY -#define DISPATCH_NO_BG_PRIORITY 1 + + +#ifndef HAVE_PTHREAD_WORKQUEUE_NARROWING +#if !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(109900) +#define HAVE_PTHREAD_WORKQUEUE_NARROWING 0 +#else +#define HAVE_PTHREAD_WORKQUEUE_NARROWING 1 #endif #endif -#if TARGET_IPHONE_SIMULATOR && IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1080 -#ifndef DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK -#define DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK 1 + +#ifdef EVFILT_MEMORYSTATUS +#ifndef DISPATCH_USE_MEMORYSTATUS +#define DISPATCH_USE_MEMORYSTATUS 1 #endif +#endif // EVFILT_MEMORYSTATUS + +#if TARGET_OS_SIMULATOR +#undef DISPATCH_USE_MEMORYPRESSURE_SOURCE +#define DISPATCH_USE_MEMORYPRESSURE_SOURCE 0 +#endif // TARGET_OS_SIMULATOR +#if !defined(DISPATCH_USE_MEMORYPRESSURE_SOURCE) && DISPATCH_USE_MEMORYSTATUS +#define DISPATCH_USE_MEMORYPRESSURE_SOURCE 1 #endif -#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED < 1080 -#undef HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP -#define HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP 0 +#if DISPATCH_USE_MEMORYPRESSURE_SOURCE +#if __has_include() +#include +#else +extern void malloc_memory_event_handler(unsigned long); +#endif // __has_include( -#endif -#endif // HAVE_DECL_NOTE_REAP +#if defined(MACH_SEND_SYNC_OVERRIDE) && defined(MACH_RCV_SYNC_WAIT) && \ + DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(109900) && \ + !defined(DISPATCH_USE_MACH_SEND_SYNC_OVERRIDE) +#define DISPATCH_USE_MACH_SEND_SYNC_OVERRIDE 1 +#endif #if defined(F_SETNOSIGPIPE) && defined(F_GETNOSIGPIPE) -#if TARGET_IPHONE_SIMULATOR && IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1070 -#undef DISPATCH_USE_SETNOSIGPIPE -#define DISPATCH_USE_SETNOSIGPIPE 0 -#endif #ifndef DISPATCH_USE_SETNOSIGPIPE #define DISPATCH_USE_SETNOSIGPIPE 1 #endif #endif // F_SETNOSIGPIPE +#if defined(MACH_SEND_NOIMPORTANCE) +#ifndef DISPATCH_USE_CHECKIN_NOIMPORTANCE +#define DISPATCH_USE_CHECKIN_NOIMPORTANCE 1 // rdar://problem/16996737 +#endif +#ifndef DISPATCH_USE_NOIMPORTANCE_QOS +#define DISPATCH_USE_NOIMPORTANCE_QOS 1 // rdar://problem/21414476 +#endif +#endif // MACH_SEND_NOIMPORTANCE + #if HAVE_LIBPROC_INTERNAL_H #include #include -#if TARGET_IPHONE_SIMULATOR && IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1090 -#undef DISPATCH_USE_IMPORTANCE_ASSERTION -#define DISPATCH_USE_IMPORTANCE_ASSERTION 0 -#endif -#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED < 1090 -#undef DISPATCH_USE_IMPORTANCE_ASSERTION -#define DISPATCH_USE_IMPORTANCE_ASSERTION 0 -#endif #ifndef DISPATCH_USE_IMPORTANCE_ASSERTION #define DISPATCH_USE_IMPORTANCE_ASSERTION 1 #endif @@ -666,23 +726,149 @@ extern struct _dispatch_hw_config_s { #if HAVE_SYS_GUARDED_H #include -#if TARGET_IPHONE_SIMULATOR && IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1090 -#undef DISPATCH_USE_GUARDED_FD -#define DISPATCH_USE_GUARDED_FD 0 -#endif #ifndef DISPATCH_USE_GUARDED_FD #define DISPATCH_USE_GUARDED_FD 1 #endif -// change_fdguard_np() requires GUARD_DUP -#if DISPATCH_USE_GUARDED_FD && RDAR_11814513 -#define DISPATCH_USE_GUARDED_FD_CHANGE_FDGUARD 1 -#endif #endif // HAVE_SYS_GUARDED_H -#define _dispatch_hardware_crash() __builtin_trap() +#if DISPATCH_USE_DTRACE || DISPATCH_USE_DTRACE_INTROSPECTION +typedef struct dispatch_trace_timer_params_s { + int64_t deadline, interval, leeway; +} *dispatch_trace_timer_params_t; + +#include "provider.h" +#endif // DISPATCH_USE_DTRACE || DISPATCH_USE_DTRACE_INTROSPECTION + +#if __has_include() +#include +#ifndef DBG_DISPATCH +#define DBG_DISPATCH 46 +#endif +#ifndef KDBG_CODE +#define KDBG_CODE(...) 0 +#define DBG_FUNC_START 0 +#define DBG_FUNC_END 0 +#endif +#define DISPATCH_CODE(subclass, code) \ + KDBG_CODE(DBG_DISPATCH, DISPATCH_TRACE_SUBCLASS_##subclass, code) +#define DISPATCH_CODE_START(subclass, code) \ + (DISPATCH_CODE(subclass, code) | DBG_FUNC_START) +#define DISPATCH_CODE_END(subclass, code) \ + (DISPATCH_CODE(subclass, code) | DBG_FUNC_END) +#ifdef ARIADNEDBG_CODE +#define ARIADNE_ENTER_DISPATCH_MAIN_CODE ARIADNEDBG_CODE(220, 2) +#else +#define ARIADNE_ENTER_DISPATCH_MAIN_CODE 0 +#endif +#if !defined(DISPATCH_USE_VOUCHER_KDEBUG_TRACE) && \ + (DISPATCH_INTROSPECTION || DISPATCH_PROFILE || DISPATCH_DEBUG) +#define DISPATCH_USE_VOUCHER_KDEBUG_TRACE 1 +#endif + +#define DISPATCH_TRACE_SUBCLASS_DEFAULT 0 +#define DISPATCH_TRACE_SUBCLASS_VOUCHER 1 +#define DISPATCH_TRACE_SUBCLASS_PERF 2 +#define DISPATCH_TRACE_SUBCLASS_MACH_MSG 3 +#define DISPATCH_TRACE_SUBCLASS_PERF_MON 4 + +#define DISPATCH_PERF_non_leaf_retarget DISPATCH_CODE(PERF, 1) +#define DISPATCH_PERF_post_activate_retarget DISPATCH_CODE(PERF, 2) +#define DISPATCH_PERF_post_activate_mutation DISPATCH_CODE(PERF, 3) +#define DISPATCH_PERF_delayed_registration DISPATCH_CODE(PERF, 4) +#define DISPATCH_PERF_mutable_target DISPATCH_CODE(PERF, 5) +#define DISPATCH_PERF_strict_bg_timer DISPATCH_CODE(PERF, 6) + +#define DISPATCH_MACH_MSG_hdr_move DISPATCH_CODE(MACH_MSG, 1) -#define _dispatch_set_crash_log_message(x) +#define DISPATCH_PERF_MON_worker_thread_start DISPATCH_CODE_START(PERF_MON, 1) +#define DISPATCH_PERF_MON_worker_thread_end DISPATCH_CODE_END(PERF_MON, 1) +#define DISPATCH_PERF_MON_worker_useless DISPATCH_CODE(PERF_MON, 2) + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_ktrace_impl(uint32_t code, uint64_t a, uint64_t b, + uint64_t c, uint64_t d) +{ + if (!code) return; +#ifdef _COMM_PAGE_KDEBUG_ENABLE + if (likely(*(volatile uint32_t *)_COMM_PAGE_KDEBUG_ENABLE == 0)) return; +#endif + kdebug_trace(code, a, b, c, d); +} +#define _dispatch_cast_to_uint64(e) \ + __builtin_choose_expr(sizeof(e) > 4, \ + ((uint64_t)(e)), ((uint64_t)(uintptr_t)(e))) +#define _dispatch_ktrace(code, a, b, c, d) _dispatch_ktrace_impl(code, \ + _dispatch_cast_to_uint64(a), _dispatch_cast_to_uint64(b), \ + _dispatch_cast_to_uint64(c), _dispatch_cast_to_uint64(d)) + +#else // __has_include() +#define DISPATCH_CODE(subclass, code) 0 +#define ARIADNE_ENTER_DISPATCH_MAIN_CODE 0 +#define DISPATCH_USE_VOUCHER_KDEBUG_TRACE 0 +#define _dispatch_ktrace(code, a, b, c, d) +#endif // !__has_include() +#define _dispatch_ktrace4(code, a, b, c, d) _dispatch_ktrace(code, a, b, c, d) +#define _dispatch_ktrace3(code, a, b, c) _dispatch_ktrace(code, a, b, c, 0) +#define _dispatch_ktrace2(code, a, b) _dispatch_ktrace(code, a, b, 0, 0) +#define _dispatch_ktrace1(code, a) _dispatch_ktrace(code, a, 0, 0, 0) +#define _dispatch_ktrace0(code) _dispatch_ktrace(code, 0, 0, 0, 0) + +#ifndef MACH_MSGH_BITS_VOUCHER_MASK +#define MACH_MSGH_BITS_VOUCHER_MASK 0x001f0000 +#define MACH_MSGH_BITS_SET_PORTS(remote, local, voucher) \ + (((remote) & MACH_MSGH_BITS_REMOTE_MASK) | \ + (((local) << 8) & MACH_MSGH_BITS_LOCAL_MASK) | \ + (((voucher) << 16) & MACH_MSGH_BITS_VOUCHER_MASK)) +#define MACH_MSGH_BITS_VOUCHER(bits) \ + (((bits) & MACH_MSGH_BITS_VOUCHER_MASK) >> 16) +#define MACH_MSGH_BITS_HAS_VOUCHER(bits) \ + (MACH_MSGH_BITS_VOUCHER(bits) != MACH_MSGH_BITS_ZERO) +#define msgh_voucher_port msgh_reserved +#define mach_voucher_t mach_port_t +#define MACH_VOUCHER_NULL MACH_PORT_NULL +#define MACH_SEND_INVALID_VOUCHER 0x10000005 +#endif + +#ifndef VOUCHER_USE_MACH_VOUCHER +#if __has_include() +#define VOUCHER_USE_MACH_VOUCHER 1 +#endif +#endif // VOUCHER_USE_MACH_VOUCHER + +#ifndef VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER +#if RDAR_24272659 // FIXME: +#define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 1 +#else // RDAR_24272659 +#define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 0 +#endif // RDAR_24272659 +#endif + +#ifndef VOUCHER_USE_PERSONA +#if VOUCHER_USE_MACH_VOUCHER && defined(BANK_PERSONA_TOKEN) && \ + TARGET_OS_IOS && !TARGET_OS_SIMULATOR +#define VOUCHER_USE_PERSONA 1 +#else +#define VOUCHER_USE_PERSONA 0 +#endif +#endif // VOUCHER_USE_PERSONA + +#if VOUCHER_USE_MACH_VOUCHER +#undef DISPATCH_USE_IMPORTANCE_ASSERTION +#define DISPATCH_USE_IMPORTANCE_ASSERTION 0 +#else +#undef MACH_RCV_VOUCHER +#define MACH_RCV_VOUCHER 0 +#define VOUCHER_USE_PERSONA 0 +#endif // VOUCHER_USE_MACH_VOUCHER + +#define _dispatch_hardware_crash() \ + __asm__(""); __builtin_trap() // + +#define _dispatch_set_crash_log_cause_and_message(ac, msg) ((void)(ac)) +#define _dispatch_set_crash_log_message(msg) +#define _dispatch_set_crash_log_message_dynamic(msg) #if HAVE_MACH // MIG_REPLY_MISMATCH means either: @@ -691,19 +877,22 @@ extern struct _dispatch_hw_config_s { // 2) A hand crafted call to mach_msg*() screwed up. Use MIG. #define DISPATCH_VERIFY_MIG(x) do { \ if ((x) == MIG_REPLY_MISMATCH) { \ - _dispatch_set_crash_log_message("MIG_REPLY_MISMATCH"); \ + _dispatch_set_crash_log_cause_and_message((x), \ + "MIG_REPLY_MISMATCH"); \ _dispatch_hardware_crash(); \ } \ } while (0) #endif -#define DISPATCH_CRASH(x) do { \ - _dispatch_set_crash_log_message("BUG IN LIBDISPATCH: " x); \ +#define DISPATCH_INTERNAL_CRASH(c, x) do { \ + _dispatch_set_crash_log_cause_and_message((c), \ + "BUG IN LIBDISPATCH: " x); \ _dispatch_hardware_crash(); \ } while (0) -#define DISPATCH_CLIENT_CRASH(x) do { \ - _dispatch_set_crash_log_message("BUG IN CLIENT OF LIBDISPATCH: " x); \ +#define DISPATCH_CLIENT_CRASH(c, x) do { \ + _dispatch_set_crash_log_cause_and_message((c), \ + "BUG IN CLIENT OF LIBDISPATCH: " x); \ _dispatch_hardware_crash(); \ } while (0) @@ -712,16 +901,74 @@ extern struct _dispatch_hw_config_s { _dispatch_hardware_crash(); \ } while (0) +#define DISPATCH_ASSERTION_FAILED_MESSAGE \ + "BUG IN CLIENT OF LIBDISPATCH: Assertion failed: " + +#define _dispatch_assert_crash(msg) do { \ + const char *__msg = (msg); \ + _dispatch_log("%s", __msg); \ + _dispatch_set_crash_log_message_dynamic(__msg); \ + _dispatch_hardware_crash(); \ + } while (0) + +#define _dispatch_client_assert_fail(fmt, ...) do { \ + char *_msg = NULL; \ + asprintf(&_msg, "%s" fmt, DISPATCH_ASSERTION_FAILED_MESSAGE, \ + ##__VA_ARGS__); \ + _dispatch_assert_crash(_msg); \ + free(_msg); \ + } while (0) + +#define DISPATCH_NO_VOUCHER ((voucher_t)(void*)~0ul) +#define DISPATCH_NO_PRIORITY ((pthread_priority_t)~0ul) +DISPATCH_ENUM(dispatch_thread_set_self, unsigned long, + DISPATCH_PRIORITY_ENFORCE = 0x1, + DISPATCH_VOUCHER_REPLACE = 0x2, + DISPATCH_VOUCHER_CONSUME = 0x4, + DISPATCH_THREAD_PARK = 0x8, +); +DISPATCH_WARN_RESULT +static inline voucher_t _dispatch_adopt_priority_and_set_voucher( + pthread_priority_t priority, voucher_t voucher, + dispatch_thread_set_self_t flags); +#if HAVE_MACH +mach_port_t _dispatch_get_mach_host_port(void); +#endif + +#if HAVE_PTHREAD_WORKQUEUE_QOS +#if DISPATCH_DEBUG +extern int _dispatch_set_qos_class_enabled; +#else +#define _dispatch_set_qos_class_enabled (1) +#endif +#endif // HAVE_PTHREAD_WORKQUEUE_QOS +#if DISPATCH_USE_KEVENT_WORKQUEUE +#if !HAVE_PTHREAD_WORKQUEUE_QOS || !EV_UDATA_SPECIFIC +#error Invalid build configuration +#endif +#if DISPATCH_USE_MGR_THREAD +extern int _dispatch_kevent_workqueue_enabled; +#else +#define _dispatch_kevent_workqueue_enabled (1) +#endif +#else +#define _dispatch_kevent_workqueue_enabled (0) +#endif // DISPATCH_USE_KEVENT_WORKQUEUE + + /* #includes dependent on internal.h */ #include "object_internal.h" #include "semaphore_internal.h" #include "introspection_internal.h" #include "queue_internal.h" #include "source_internal.h" +#include "mach_internal.h" +#include "voucher_internal.h" #include "data_internal.h" #if !TARGET_OS_WIN32 #include "io_internal.h" #endif -#include "trace.h" +#include "inline_internal.h" +#include "firehose/firehose_internal.h" #endif /* __DISPATCH_INTERNAL__ */