libdispatch-913.30.4.tar.gz

[apple/libdispatch.git] / src / internal.h

diff --git a/src/internal.h b/src/internal.h
index 98626c6436f0cce66808af71003d9929a1035a74..286e53458d3adca6db76e6766ee37fa7c940b58a 100644 (file)
--- a/src/internal.h
+++ b/src/internal.h
@@ -27,16 +27,48 @@
 #ifndef __DISPATCH_INTERNAL__
 #define __DISPATCH_INTERNAL__
 
 #ifndef __DISPATCH_INTERNAL__
 #define __DISPATCH_INTERNAL__
 

+#if __has_include(<config/config_ac.h>)
+#include <config/config_ac.h>
+#else

 #include <config/config.h>
 #include <config/config.h>

+#endif

 
 #define __DISPATCH_BUILDING_DISPATCH__
 #define __DISPATCH_INDIRECT__
 
 #ifdef __APPLE__
 #include <Availability.h>
 
 #define __DISPATCH_BUILDING_DISPATCH__
 #define __DISPATCH_INDIRECT__
 
 #ifdef __APPLE__
 #include <Availability.h>

+#include <os/availability.h>

 #include <TargetConditionals.h>
 #include <TargetConditionals.h>

+
+#ifndef TARGET_OS_MAC_DESKTOP
+#define TARGET_OS_MAC_DESKTOP  (TARGET_OS_MAC && \
+               !TARGET_OS_SIMULATOR && !TARGET_OS_IPHONE && !TARGET_OS_EMBEDDED)
+#endif
+
+#if TARGET_OS_MAC_DESKTOP
+#  define DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(x) \
+               (__MAC_OS_X_VERSION_MIN_REQUIRED >= (x))
+#  if !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
+#    error "OS X hosts older than OS X 10.12 aren't supported anymore"
+#  endif // !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
+#elif TARGET_OS_SIMULATOR
+#  define DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(x) \
+               (IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED >= (x))
+#  if !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
+#    error "Simulator hosts older than OS X 10.12 aren't supported anymore"
+#  endif // !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
+#else
+#  define DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(x) 1
+#  if __IPHONE_OS_VERSION_MIN_REQUIRED < 90000
+#    error "iOS hosts older than iOS 9.0 aren't supported anymore"
+#  endif

 #endif
 
 #endif
 

+#else // !__APPLE__
+#define DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(x) 0
+#endif // !__APPLE__
+

 
 #if !defined(DISPATCH_MACH_SPI) && TARGET_OS_MAC
 #define DISPATCH_MACH_SPI 1
 
 #if !defined(DISPATCH_MACH_SPI) && TARGET_OS_MAC
 #define DISPATCH_MACH_SPI 1


@@ -47,29 +79,38 @@
 #if !defined(OS_VOUCHER_ACTIVITY_SPI) && TARGET_OS_MAC
 #define OS_VOUCHER_ACTIVITY_SPI 1
 #endif
 #if !defined(OS_VOUCHER_ACTIVITY_SPI) && TARGET_OS_MAC
 #define OS_VOUCHER_ACTIVITY_SPI 1
 #endif

-#if !defined(OS_VOUCHER_ACTIVITY_BUFFER_SPI) && TARGET_OS_MAC && \
-               __has_include(<atm/atm_types.h>)
-#define OS_VOUCHER_ACTIVITY_BUFFER_SPI 1
+#if !defined(OS_VOUCHER_ACTIVITY_GENERATE_SWAPS)
+#define OS_VOUCHER_ACTIVITY_GENERATE_SWAPS 0
+#endif
+#if !defined(OS_FIREHOSE_SPI) && TARGET_OS_MAC
+#define OS_FIREHOSE_SPI 1

 #endif
 #if !defined(DISPATCH_LAYOUT_SPI) && TARGET_OS_MAC
 #define DISPATCH_LAYOUT_SPI 1
 #endif
 
 #endif
 #if !defined(DISPATCH_LAYOUT_SPI) && TARGET_OS_MAC
 #define DISPATCH_LAYOUT_SPI 1
 #endif
 

-#if !defined(USE_OBJC) && HAVE_OBJC
-#define USE_OBJC 1
+#if __has_include(<mach-o/dyld_priv.h>)
+#include <mach-o/dyld_priv.h>
+#if !defined(HAVE_DYLD_IS_MEMORY_IMMUTABLE)
+#if defined(DYLD_MACOSX_VERSION_10_12) || defined(DYLD_IOS_VERSION_10_0)
+#define HAVE_DYLD_IS_MEMORY_IMMUTABLE 1
+#else
+#define HAVE_DYLD_IS_MEMORY_IMMUTABLE 0

 #endif
 #endif

+#endif // !defined(HAVE_DYLD_IS_MEMORY_IMMUTABLE)
+#endif // __has_include(<mach-o/dyld_priv.h>)

 
 

-#if USE_OBJC && ((!TARGET_IPHONE_SIMULATOR && defined(__i386__)) || \
-               (!TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED < 1080))
-// Disable Objective-C support on platforms with legacy objc runtime
-#undef USE_OBJC
-#define USE_OBJC 0
+#if !defined(USE_OBJC) && HAVE_OBJC
+#define USE_OBJC 1

 #endif
 
 #if USE_OBJC
 #define OS_OBJECT_HAVE_OBJC_SUPPORT 1
 #endif
 
 #if USE_OBJC
 #define OS_OBJECT_HAVE_OBJC_SUPPORT 1

-#if __OBJC__
+#if defined(__OBJC__)

 #define OS_OBJECT_USE_OBJC 1
 #define OS_OBJECT_USE_OBJC 1

+// Force internal Objective-C sources to use class-visible headers
+// even when not compiling in Swift.
+#define OS_OBJECT_SWIFT3 1

 #else
 #define OS_OBJECT_USE_OBJC 0
 #endif // __OBJC__
 #else
 #define OS_OBJECT_USE_OBJC 0
 #endif // __OBJC__


@@ -80,6 +121,22 @@
 #include <dispatch/dispatch.h>
 #include <dispatch/base.h>
 
 #include <dispatch/dispatch.h>
 #include <dispatch/base.h>
 

+#define __DISPATCH_HIDE_SYMBOL(sym, version) \
+       __asm__(".section __TEXT,__const\n\t" \
+                       ".globl $ld$hide$os" #version "$_" #sym "\n\t" \
+                       "$ld$hide$os" #version "$_" #sym ":\n\t" \
+                       "    .byte 0\n\t" \
+                       ".previous")
+
+
+#ifndef DISPATCH_HIDE_SYMBOL
+#if TARGET_OS_MAC && !TARGET_OS_IPHONE
+#define DISPATCH_HIDE_SYMBOL(sym, osx, ios, tvos, watchos) \
+               __DISPATCH_HIDE_SYMBOL(sym, osx)
+#else
+#define DISPATCH_HIDE_SYMBOL(sym, osx, ios, tvos, watchos)
+#endif
+#endif

 
 #include <os/object.h>
 #include <dispatch/time.h>
 
 #include <os/object.h>
 #include <dispatch/time.h>


@@ -95,47 +152,24 @@
 #include <dispatch/io.h>
 #endif
 
 #include <dispatch/io.h>
 #endif
 

-#define DISPATCH_STRUCT_DECL(type, name, ...) \
-       struct type __VA_ARGS__ name
-
-// Visual Studio C++ does not support C99 designated initializers.
-// This means that static declarations should be zero initialized and cannot
-// be const since we must fill in the values during DLL initialization.
-#if !TARGET_OS_WIN32
-#define DISPATCH_STRUCT_INSTANCE(type, name, ...) \
-struct type name = { \
-__VA_ARGS__ \
-}
-#else
-#define DISPATCH_STRUCT_INSTANCE(type, name, ...) \
-struct type name = { 0 }
-#endif
-
-#if !TARGET_OS_WIN32
-#define DISPATCH_CONST_STRUCT_DECL(type, name, ...) \
-       const DISPATCH_STRUCT_DECL(type, name, __VA_ARGS__)
-
-#define DISPATCH_CONST_STRUCT_INSTANCE(type, name, ...) \
-       const DISPATCH_STRUCT_INSTANCE(type, name, __VA_ARGS__)
+#if defined(__OBJC__) || defined(__cplusplus)
+#define DISPATCH_PURE_C 0

 #else
 #else

-#define DISPATCH_CONST_STRUCT_DECL(type, name, ...) \
-       DISPATCH_STRUCT_DECL(type, name, __VA_ARGS__)
-
-#define DISPATCH_CONST_STRUCT_INSTANCE(type, name, ...) \
-       DISPATCH_STRUCT_INSTANCE(type, name, __VA_ARGS__)
+#define DISPATCH_PURE_C 1

 #endif
 
 /* private.h must be included last to avoid picking up installed headers. */
 #endif
 
 /* private.h must be included last to avoid picking up installed headers. */

-#include "object_private.h"
+#include <pthread.h>
+#include "os/object_private.h"

 #include "queue_private.h"
 #include "source_private.h"
 #include "mach_private.h"
 #include "data_private.h"
 #include "queue_private.h"
 #include "source_private.h"
 #include "mach_private.h"
 #include "data_private.h"

+#include "os/voucher_private.h"
+#include "os/voucher_activity_private.h"

 #if !TARGET_OS_WIN32
 #include "io_private.h"
 #endif
 #if !TARGET_OS_WIN32
 #include "io_private.h"
 #endif

-#include "voucher_private.h"
-#include "voucher_activity_private.h"

 #include "layout_private.h"
 #include "benchmark.h"
 #include "private.h"
 #include "layout_private.h"
 #include "benchmark.h"
 #include "private.h"


@@ -155,6 +189,8 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void);
 #define DISPATCH_USE_CLIENT_CALLOUT 1
 #endif
 
 #define DISPATCH_USE_CLIENT_CALLOUT 1
 #endif
 

+#define DISPATCH_ALLOW_NON_LEAF_RETARGET 1
+

 /* The "_debug" library build */
 #ifndef DISPATCH_DEBUG
 #define DISPATCH_DEBUG 0
 /* The "_debug" library build */
 #ifndef DISPATCH_DEBUG
 #define DISPATCH_DEBUG 0


@@ -174,6 +210,10 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void);
 #define DISPATCH_USE_DTRACE_INTROSPECTION 1
 #endif
 
 #define DISPATCH_USE_DTRACE_INTROSPECTION 1
 #endif
 

+#ifndef DISPATCH_DEBUG_QOS
+#define DISPATCH_DEBUG_QOS DISPATCH_DEBUG
+#endif
+

 #if HAVE_LIBKERN_OSCROSSENDIAN_H
 #include <libkern/OSCrossEndian.h>
 #endif
 #if HAVE_LIBKERN_OSCROSSENDIAN_H
 #include <libkern/OSCrossEndian.h>
 #endif


@@ -205,21 +245,26 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void);
 
 #include <sys/stat.h>
 
 
 #include <sys/stat.h>
 

-#if !TARGET_OS_WIN32 
-#include <sys/event.h>
+#if !TARGET_OS_WIN32

 #include <sys/mount.h>
 #include <sys/queue.h>
 #include <sys/mount.h>
 #include <sys/queue.h>

+#ifdef __ANDROID__
+#include <linux/sysctl.h>
+#else

 #include <sys/sysctl.h>
 #include <sys/sysctl.h>

+#endif /* __ANDROID__ */

 #include <sys/socket.h>
 #include <sys/time.h>
 #include <sys/mman.h>
 #include <netinet/in.h>
 #include <sys/socket.h>
 #include <sys/time.h>
 #include <sys/mman.h>
 #include <netinet/in.h>

-#else
-#include "sys_queue.h"

 #endif
 
 #ifdef __BLOCKS__
 #endif
 
 #ifdef __BLOCKS__

+#if __has_include(<Block_private.h>)

 #include <Block_private.h>
 #include <Block_private.h>

+#else
+#include "BlocksRuntime/Block_private.h"
+#endif // __has_include(<Block_private.h>)

 #include <Block.h>
 #endif /* __BLOCKS__ */
 
 #include <Block.h>
 #endif /* __BLOCKS__ */
 


@@ -244,25 +289,13 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void);
 #include <unistd.h>
 #endif
 
 #include <unistd.h>
 #endif
 

-#ifndef __has_builtin
-#define __has_builtin(x) 0
-#endif
-#ifndef __has_include
-#define __has_include(x) 0
-#endif
-#ifndef __has_feature
-#define __has_feature(x) 0
-#endif
-#ifndef __has_attribute
-#define __has_attribute(x) 0
-#endif
-

 #if __GNUC__
 #define DISPATCH_NOINLINE __attribute__((__noinline__))
 #define DISPATCH_USED __attribute__((__used__))
 #define DISPATCH_UNUSED __attribute__((__unused__))
 #define DISPATCH_WEAK __attribute__((__weak__))
 #define DISPATCH_OVERLOADABLE __attribute__((__overloadable__))
 #if __GNUC__
 #define DISPATCH_NOINLINE __attribute__((__noinline__))
 #define DISPATCH_USED __attribute__((__used__))
 #define DISPATCH_UNUSED __attribute__((__unused__))
 #define DISPATCH_WEAK __attribute__((__weak__))
 #define DISPATCH_OVERLOADABLE __attribute__((__overloadable__))

+#define DISPATCH_PACKED __attribute__((__packed__))

 #if DISPATCH_DEBUG
 #define DISPATCH_ALWAYS_INLINE_NDEBUG
 #else
 #if DISPATCH_DEBUG
 #define DISPATCH_ALWAYS_INLINE_NDEBUG
 #else


@@ -279,6 +312,31 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void);
 #define DISPATCH_CONCAT(x,y) DISPATCH_CONCAT1(x,y)
 #define DISPATCH_CONCAT1(x,y) x ## y
 
 #define DISPATCH_CONCAT(x,y) DISPATCH_CONCAT1(x,y)
 #define DISPATCH_CONCAT1(x,y) x ## y
 

+#define DISPATCH_COUNT_ARGS(...) DISPATCH_COUNT_ARGS1(, ## __VA_ARGS__, \
+               _8, _7, _6, _5, _4, _3, _2, _1, _0)
+#define DISPATCH_COUNT_ARGS1(z, a, b, c, d, e, f, g, h, cnt, ...) cnt
+
+#if BYTE_ORDER == LITTLE_ENDIAN
+#define DISPATCH_STRUCT_LE_2(a, b)        struct { a; b; }
+#define DISPATCH_STRUCT_LE_3(a, b, c)     struct { a; b; c; }
+#define DISPATCH_STRUCT_LE_4(a, b, c, d)  struct { a; b; c; d; }
+#else
+#define DISPATCH_STRUCT_LE_2(a, b)        struct { b; a; }
+#define DISPATCH_STRUCT_LE_3(a, b, c)     struct { c; b; a; }
+#define DISPATCH_STRUCT_LE_4(a, b, c, d)  struct { d; c; b; a; }
+#endif
+#if __has_feature(c_startic_assert)
+#define DISPATCH_UNION_ASSERT(alias, st) \
+               _Static_assert(sizeof(struct { alias; }) == sizeof(st), "bogus union");
+#else
+#define DISPATCH_UNION_ASSERT(alias, st)
+#endif
+#define DISPATCH_UNION_LE(alias, ...) \
+               DISPATCH_UNION_ASSERT(alias, DISPATCH_CONCAT(DISPATCH_STRUCT_LE, \
+                               DISPATCH_COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__)) \
+               union { alias; DISPATCH_CONCAT(DISPATCH_STRUCT_LE, \
+                               DISPATCH_COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__); }
+

 // workaround 6368156
 #ifdef NSEC_PER_SEC
 #undef NSEC_PER_SEC
 // workaround 6368156
 #ifdef NSEC_PER_SEC
 #undef NSEC_PER_SEC


@@ -295,17 +353,34 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void);
 
 /* I wish we had __builtin_expect_range() */
 #if __GNUC__
 
 /* I wish we had __builtin_expect_range() */
 #if __GNUC__

-#define fastpath(x) ((typeof(x))__builtin_expect((long)(x), ~0l))
-#define slowpath(x) ((typeof(x))__builtin_expect((long)(x), 0l))
+#define _safe_cast_to_long(x) \
+               ({ _Static_assert(sizeof(typeof(x)) <= sizeof(long), \
+                               "__builtin_expect doesn't support types wider than long"); \
+                               (long)(x); })
+#define fastpath(x) ((typeof(x))__builtin_expect(_safe_cast_to_long(x), ~0l))
+#define slowpath(x) ((typeof(x))__builtin_expect(_safe_cast_to_long(x), 0l))
+#define likely(x) __builtin_expect(!!(x), 1)
+#define unlikely(x) __builtin_expect(!!(x), 0)

 #else
 #define fastpath(x) (x)
 #define slowpath(x) (x)
 #else
 #define fastpath(x) (x)
 #define slowpath(x) (x)

+#define likely(x) (!!(x))
+#define unlikely(x) (!!(x))

 #endif // __GNUC__
 
 #endif // __GNUC__
 

+#define _TAILQ_IS_ENQUEUED(elm, field) \
+               ((elm)->field.tqe_prev != NULL)
+#define _TAILQ_MARK_NOT_ENQUEUED(elm, field) \
+               do { (elm)->field.tqe_prev = NULL; } while (0)
+

 #if DISPATCH_DEBUG
 // sys/queue.h debugging
 #if DISPATCH_DEBUG
 // sys/queue.h debugging

+#if defined(__linux__)
+#define QUEUE_MACRO_DEBUG 1
+#else

 #undef TRASHIT
 #define TRASHIT(x) do {(x) = (void *)-1;} while (0)
 #undef TRASHIT
 #define TRASHIT(x) do {(x) = (void *)-1;} while (0)

+#endif

 #endif // DISPATCH_DEBUG
 #define _TAILQ_TRASH_ENTRY(elm, field) do { \
                        TRASHIT((elm)->field.tqe_next); \
 #endif // DISPATCH_DEBUG
 #define _TAILQ_TRASH_ENTRY(elm, field) do { \
                        TRASHIT((elm)->field.tqe_next); \


@@ -316,18 +391,21 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void);
                        TRASHIT((head)->tqh_last); \
                } while (0)
 
                        TRASHIT((head)->tqh_last); \
                } while (0)
 

-DISPATCH_NOINLINE
+DISPATCH_EXPORT DISPATCH_NOINLINE

 void _dispatch_bug(size_t line, long val);
 
 void _dispatch_bug(size_t line, long val);
 

-#if HAVE_MACH

 DISPATCH_NOINLINE
 void _dispatch_bug_client(const char* msg);
 DISPATCH_NOINLINE
 void _dispatch_bug_client(const char* msg);

+#if HAVE_MACH

 DISPATCH_NOINLINE
 void _dispatch_bug_mach_client(const char *msg, mach_msg_return_t kr);
 DISPATCH_NOINLINE
 void _dispatch_bug_mach_client(const char *msg, mach_msg_return_t kr);

+#endif // HAVE_MACH

 DISPATCH_NOINLINE
 void _dispatch_bug_kevent_client(const char* msg, const char* filter,
                const char *operation, int err);
 DISPATCH_NOINLINE
 void _dispatch_bug_kevent_client(const char* msg, const char* filter,
                const char *operation, int err);

-#endif
+
+DISPATCH_NOINLINE
+void _dispatch_bug_deprecated(const char *msg);

 
 DISPATCH_NOINLINE DISPATCH_NORETURN
 void _dispatch_abort(size_t line, long val);
 
 DISPATCH_NOINLINE DISPATCH_NORETURN
 void _dispatch_abort(size_t line, long val);


@@ -353,12 +431,13 @@ void _dispatch_abort(size_t line, long val);
 #if DISPATCH_USE_OS_DEBUG_LOG
 #define _dispatch_log(msg, ...) os_debug_log("libdispatch", msg, ## __VA_ARGS__)
 #else
 #if DISPATCH_USE_OS_DEBUG_LOG
 #define _dispatch_log(msg, ...) os_debug_log("libdispatch", msg, ## __VA_ARGS__)
 #else

-DISPATCH_NOINLINE __attribute__((__format__(__printf__,1,2)))
+DISPATCH_EXPORT DISPATCH_NOINLINE __attribute__((__format__(__printf__,1,2)))

 void _dispatch_log(const char *msg, ...);
 #endif // DISPATCH_USE_OS_DEBUG_LOG
 
 void _dispatch_log(const char *msg, ...);
 #endif // DISPATCH_USE_OS_DEBUG_LOG
 

-#define dsnprintf(...) \
-               ({ int _r = snprintf(__VA_ARGS__); _r < 0 ? 0u : (size_t)_r; })
+#define dsnprintf(buf, siz, ...) \
+               ({ size_t _siz = siz; int _r = snprintf(buf, _siz, __VA_ARGS__); \
+                _r < 0 ? 0u : ((size_t)_r > _siz ? _siz : (size_t)_r); })

 
 #if __GNUC__
 #define dispatch_static_assert(e) ({ \
 
 #if __GNUC__
 #define dispatch_static_assert(e) ({ \


@@ -368,29 +447,34 @@ void _dispatch_log(const char *msg, ...);
 #define dispatch_static_assert(e)
 #endif
 
 #define dispatch_static_assert(e)
 #endif
 

+#define DISPATCH_BAD_INPUT             ((void *_Nonnull)0)
+#define DISPATCH_OUT_OF_MEMORY ((void *_Nonnull)0)
+

 /*
  * For reporting bugs within libdispatch when using the "_debug" version of the
  * library.
  */
 /*
  * For reporting bugs within libdispatch when using the "_debug" version of the
  * library.
  */

-#if __GNUC__
+#if __APPLE__

 #define dispatch_assert(e) do { \
                if (__builtin_constant_p(e)) { \
                        dispatch_static_assert(e); \
                } else { \
 #define dispatch_assert(e) do { \
                if (__builtin_constant_p(e)) { \
                        dispatch_static_assert(e); \
                } else { \

-                       typeof(e) _e = fastpath(e); /* always eval 'e' */ \
-                       if (DISPATCH_DEBUG && !_e) { \
+                       typeof(e) _e = (e); /* always eval 'e' */ \
+                       if (unlikely(DISPATCH_DEBUG && !_e)) { \

                                _dispatch_abort(__LINE__, (long)_e); \
                        } \
                } \
        } while (0)
 #else
                                _dispatch_abort(__LINE__, (long)_e); \
                        } \
                } \
        } while (0)
 #else

-static inline void _dispatch_assert(long e, long line) {
+static inline void
+_dispatch_assert(long e, size_t line)
+{

        if (DISPATCH_DEBUG && !e) _dispatch_abort(line, e);
 }
 #define dispatch_assert(e) _dispatch_assert((long)(e), __LINE__)
 #endif /* __GNUC__ */
 
        if (DISPATCH_DEBUG && !e) _dispatch_abort(line, e);
 }
 #define dispatch_assert(e) _dispatch_assert((long)(e), __LINE__)
 #endif /* __GNUC__ */
 

-#if __GNUC__
+#if __APPLE__

 /*
  * A lot of API return zero upon success and not-zero on fail. Let's capture
  * and log the non-zero value
 /*
  * A lot of API return zero upon success and not-zero on fail. Let's capture
  * and log the non-zero value


@@ -399,17 +483,19 @@ static inline void _dispatch_assert(long e, long line) {
                if (__builtin_constant_p(e)) { \
                        dispatch_static_assert(e); \
                } else { \
                if (__builtin_constant_p(e)) { \
                        dispatch_static_assert(e); \
                } else { \

-                       typeof(e) _e = slowpath(e); /* always eval 'e' */ \
-                       if (DISPATCH_DEBUG && _e) { \
+                       typeof(e) _e = (e); /* always eval 'e' */ \
+                       if (unlikely(DISPATCH_DEBUG && _e)) { \

                                _dispatch_abort(__LINE__, (long)_e); \
                        } \
                } \
        } while (0)
 #else
                                _dispatch_abort(__LINE__, (long)_e); \
                        } \
                } \
        } while (0)
 #else

-static inline void _dispatch_assert_zero(long e, long line) {
+static inline void
+_dispatch_assert_zero(long e, size_t line)
+{

        if (DISPATCH_DEBUG && e) _dispatch_abort(line, e);
 }
        if (DISPATCH_DEBUG && e) _dispatch_abort(line, e);
 }

-#define dispatch_assert_zero(e) _dispatch_assert((long)(e), __LINE__) 
+#define dispatch_assert_zero(e) _dispatch_assert_zero((long)(e), __LINE__)

 #endif /* __GNUC__ */
 
 /*
 #endif /* __GNUC__ */
 
 /*


@@ -420,8 +506,8 @@ static inline void _dispatch_assert_zero(long e, long line) {
  */
 #if __GNUC__
 #define dispatch_assume(e) ({ \
  */
 #if __GNUC__
 #define dispatch_assume(e) ({ \

-               typeof(e) _e = fastpath(e); /* always eval 'e' */ \
-               if (!_e) { \
+               typeof(e) _e = (e); /* always eval 'e' */ \
+               if (unlikely(!_e)) { \

                        if (__builtin_constant_p(e)) { \
                                dispatch_static_assert(e); \
                        } \
                        if (__builtin_constant_p(e)) { \
                                dispatch_static_assert(e); \
                        } \


@@ -430,7 +516,9 @@ static inline void _dispatch_assert_zero(long e, long line) {
                _e; \
        })
 #else
                _e; \
        })
 #else

-static inline long _dispatch_assume(long e, long line) {
+static inline long
+_dispatch_assume(long e, long line)
+{

        if (!e) _dispatch_bug(line, e);
        return e;
 }
        if (!e) _dispatch_bug(line, e);
        return e;
 }


@@ -443,8 +531,8 @@ static inline long _dispatch_assume(long e, long line) {
  */
 #if __GNUC__
 #define dispatch_assume_zero(e) ({ \
  */
 #if __GNUC__
 #define dispatch_assume_zero(e) ({ \

-               typeof(e) _e = slowpath(e); /* always eval 'e' */ \
-               if (_e) { \
+               typeof(e) _e = (e); /* always eval 'e' */ \
+               if (unlikely(_e)) { \

                        if (__builtin_constant_p(e)) { \
                                dispatch_static_assert(e); \
                        } \
                        if (__builtin_constant_p(e)) { \
                                dispatch_static_assert(e); \
                        } \


@@ -453,7 +541,9 @@ static inline long _dispatch_assume(long e, long line) {
                _e; \
        })
 #else
                _e; \
        })
 #else

-static inline long _dispatch_assume_zero(long e, long line) {
+static inline long
+_dispatch_assume_zero(long e, long line)
+{

        if (e) _dispatch_bug(line, e);
        return e;
 }
        if (e) _dispatch_bug(line, e);
        return e;
 }


@@ -468,8 +558,8 @@ static inline long _dispatch_assume_zero(long e, long line) {
                if (__builtin_constant_p(e)) { \
                        dispatch_static_assert(e); \
                } else { \
                if (__builtin_constant_p(e)) { \
                        dispatch_static_assert(e); \
                } else { \

-                       typeof(e) _e = fastpath(e); /* always eval 'e' */ \
-                       if (DISPATCH_DEBUG && !_e) { \
+                       typeof(e) _e = (e); /* always eval 'e' */ \
+                       if (unlikely(DISPATCH_DEBUG && !_e)) { \

                                _dispatch_log("%s() 0x%lx: " msg, __func__, (long)_e, ##args); \
                                abort(); \
                        } \
                                _dispatch_log("%s() 0x%lx: " msg, __func__, (long)_e, ##args); \
                                abort(); \
                        } \


@@ -477,8 +567,8 @@ static inline long _dispatch_assume_zero(long e, long line) {
        } while (0)
 #else
 #define dispatch_debug_assert(e, msg, args...) do { \
        } while (0)
 #else
 #define dispatch_debug_assert(e, msg, args...) do { \

-       long _e = (long)fastpath(e); /* always eval 'e' */ \
-       if (DISPATCH_DEBUG && !_e) { \
+       typeof(e) _e = (e); /* always eval 'e' */ \
+       if (unlikely(DISPATCH_DEBUG && !_e)) { \

                _dispatch_log("%s() 0x%lx: " msg, __FUNCTION__, _e, ##args); \
                abort(); \
        } \
                _dispatch_log("%s() 0x%lx: " msg, __FUNCTION__, _e, ##args); \
                abort(); \
        } \


@@ -493,13 +583,6 @@ static inline long _dispatch_assume_zero(long e, long line) {
        } \
 } while (0)
 
        } \
 } while (0)
 

-#if DISPATCH_DEBUG
-#if HAVE_MACH
-DISPATCH_NOINLINE DISPATCH_USED
-void dispatch_debug_machport(mach_port_t name, const char* str);
-#endif
-#endif
-

 #if DISPATCH_DEBUG
 /* This is the private version of the deprecated dispatch_debug() */
 DISPATCH_NONNULL2 DISPATCH_NOTHROW
 #if DISPATCH_DEBUG
 /* This is the private version of the deprecated dispatch_debug() */
 DISPATCH_NONNULL2 DISPATCH_NOTHROW


@@ -513,73 +596,70 @@ _dispatch_object_debug(dispatch_object_t object, const char *message, ...);
 #ifdef __BLOCKS__
 #define _dispatch_Block_invoke(bb) \
                ((dispatch_function_t)((struct Block_layout *)bb)->invoke)
 #ifdef __BLOCKS__
 #define _dispatch_Block_invoke(bb) \
                ((dispatch_function_t)((struct Block_layout *)bb)->invoke)

+void *_dispatch_Block_copy(void *block);

 #if __GNUC__
 #if __GNUC__

-dispatch_block_t _dispatch_Block_copy(dispatch_block_t block);

 #define _dispatch_Block_copy(x) ((typeof(x))_dispatch_Block_copy(x))
 #define _dispatch_Block_copy(x) ((typeof(x))_dispatch_Block_copy(x))

-#else
-dispatch_block_t _dispatch_Block_copy(const void *block);

 #endif
 void _dispatch_call_block_and_release(void *block);
 #endif /* __BLOCKS__ */
 
 void _dispatch_temporary_resource_shortage(void);
 void *_dispatch_calloc(size_t num_items, size_t size);
 #endif
 void _dispatch_call_block_and_release(void *block);
 #endif /* __BLOCKS__ */
 
 void _dispatch_temporary_resource_shortage(void);
 void *_dispatch_calloc(size_t num_items, size_t size);

+const char *_dispatch_strdup_if_mutable(const char *str);

 void _dispatch_vtable_init(void);
 char *_dispatch_get_build(void);
 void _dispatch_vtable_init(void);
 char *_dispatch_get_build(void);

+int _dispatch_sigmask(void);

 
 uint64_t _dispatch_timeout(dispatch_time_t when);
 
 uint64_t _dispatch_timeout(dispatch_time_t when);

+uint64_t _dispatch_time_nanoseconds_since_epoch(dispatch_time_t when);
+
+#define _DISPATCH_UNSAFE_FORK_MULTITHREADED  ((uint8_t)1)
+#define _DISPATCH_UNSAFE_FORK_PROHIBIT       ((uint8_t)2)
+extern uint8_t _dispatch_unsafe_fork;
+extern bool _dispatch_child_of_unsafe_fork;
+void _dispatch_fork_becomes_unsafe_slow(void);
+
+#define _dispatch_is_multithreaded_inline() \
+       ((_dispatch_unsafe_fork & _DISPATCH_UNSAFE_FORK_MULTITHREADED) != 0)
+
+DISPATCH_ALWAYS_INLINE
+static inline void
+_dispatch_fork_becomes_unsafe(void)
+{
+       if (unlikely(!_dispatch_is_multithreaded_inline())) {
+               _dispatch_fork_becomes_unsafe_slow();
+               DISPATCH_COMPILER_CAN_ASSUME(_dispatch_is_multithreaded_inline());
+       }
+}

 
 

-extern bool _dispatch_safe_fork, _dispatch_child_of_unsafe_fork;
-
-#if !defined(DISPATCH_USE_OS_SEMAPHORE_CACHE) && !(TARGET_IPHONE_SIMULATOR)
-// rdar://problem/15492045
-#if __has_include(<os/semaphore_private.h>)
-#define DISPATCH_USE_OS_SEMAPHORE_CACHE 1
-#include <os/semaphore_private.h>
-#endif
+#if DISPATCH_INTROSPECTION
+#undef DISPATCH_PERF_MON
+#define DISPATCH_PERF_MON 0

 #endif
 
 /* #includes dependent on internal.h */
 #include "shims.h"
 #endif
 
 /* #includes dependent on internal.h */
 #include "shims.h"

+#include "event/event_internal.h"

 
 // Older Mac OS X and iOS Simulator fallbacks
 
 
 // Older Mac OS X and iOS Simulator fallbacks
 

-#if HAVE_PTHREAD_WORKQUEUES
-#ifndef WORKQ_ADDTHREADS_OPTION_OVERCOMMIT
-#define WORKQ_ADDTHREADS_OPTION_OVERCOMMIT 0x00000001
-#endif
-#if TARGET_IPHONE_SIMULATOR && IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1080
-#ifndef DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK
-#define DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK 1
-#endif
-#endif
-#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED < 1080
-#undef HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP
-#define HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP 0
-#endif
-#if TARGET_IPHONE_SIMULATOR && \
-               IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101000
-#ifndef DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK
-#define DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK 1
-#endif
-#endif
-#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED < 101000
-#undef HAVE__PTHREAD_WORKQUEUE_INIT
-#define HAVE__PTHREAD_WORKQUEUE_INIT 0
-#endif
-#endif // HAVE_PTHREAD_WORKQUEUES

 #if HAVE__PTHREAD_WORKQUEUE_INIT && PTHREAD_WORKQUEUE_SPI_VERSION >= 20140213 \
                && !defined(HAVE_PTHREAD_WORKQUEUE_QOS)
 #define HAVE_PTHREAD_WORKQUEUE_QOS 1
 #endif
 #if HAVE__PTHREAD_WORKQUEUE_INIT && PTHREAD_WORKQUEUE_SPI_VERSION >= 20140213 \
                && !defined(HAVE_PTHREAD_WORKQUEUE_QOS)
 #define HAVE_PTHREAD_WORKQUEUE_QOS 1
 #endif

+#if HAVE__PTHREAD_WORKQUEUE_INIT && PTHREAD_WORKQUEUE_SPI_VERSION >= 20150304 \
+               && !defined(HAVE_PTHREAD_WORKQUEUE_KEVENT)
+#define HAVE_PTHREAD_WORKQUEUE_KEVENT 1
+#endif

 
 

-#if HAVE_MACH
-#if !defined(MACH_NOTIFY_SEND_POSSIBLE) || (TARGET_IPHONE_SIMULATOR && \
-               IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1070)
-#undef MACH_NOTIFY_SEND_POSSIBLE
-#define MACH_NOTIFY_SEND_POSSIBLE MACH_NOTIFY_DEAD_NAME
+
+#ifndef HAVE_PTHREAD_WORKQUEUE_NARROWING
+#if !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(109900)
+#define HAVE_PTHREAD_WORKQUEUE_NARROWING 0
+#else
+#define HAVE_PTHREAD_WORKQUEUE_NARROWING 1
+#endif

 #endif
 #endif

-#endif // HAVE_MACH

 
 #ifdef EVFILT_MEMORYSTATUS
 #ifndef DISPATCH_USE_MEMORYSTATUS
 
 #ifdef EVFILT_MEMORYSTATUS
 #ifndef DISPATCH_USE_MEMORYSTATUS


@@ -587,96 +667,40 @@ extern bool _dispatch_safe_fork, _dispatch_child_of_unsafe_fork;
 #endif
 #endif // EVFILT_MEMORYSTATUS
 
 #endif
 #endif // EVFILT_MEMORYSTATUS
 

-#if defined(EVFILT_VM) && !DISPATCH_USE_MEMORYSTATUS
-#ifndef DISPATCH_USE_VM_PRESSURE
-#define DISPATCH_USE_VM_PRESSURE 1
-#endif
-#endif // EVFILT_VM
-
-#if TARGET_IPHONE_SIMULATOR
-#undef DISPATCH_USE_MEMORYSTATUS_SOURCE
-#define DISPATCH_USE_MEMORYSTATUS_SOURCE 0
-#undef DISPATCH_USE_VM_PRESSURE_SOURCE
-#define DISPATCH_USE_VM_PRESSURE_SOURCE 0
-#endif // TARGET_IPHONE_SIMULATOR
-#if !defined(DISPATCH_USE_MEMORYSTATUS_SOURCE) && DISPATCH_USE_MEMORYSTATUS
-#define DISPATCH_USE_MEMORYSTATUS_SOURCE 1
-#elif !defined(DISPATCH_USE_VM_PRESSURE_SOURCE) && DISPATCH_USE_VM_PRESSURE
-#define DISPATCH_USE_VM_PRESSURE_SOURCE 1
-#endif
-
-#if !defined(NOTE_LEEWAY) || (TARGET_IPHONE_SIMULATOR && \
-               IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1090)
-#undef NOTE_LEEWAY
-#define NOTE_LEEWAY 0
-#undef NOTE_CRITICAL
-#define NOTE_CRITICAL 0
-#undef NOTE_BACKGROUND
-#define NOTE_BACKGROUND 0
-#endif // NOTE_LEEWAY
-
-#if HAVE_DECL_NOTE_REAP
-#if defined(NOTE_REAP) && defined(__APPLE__)
-#undef NOTE_REAP
-#define NOTE_REAP 0x10000000 // <rdar://problem/13338526>
-#endif
-#endif // HAVE_DECL_NOTE_REAP
-
-#if !defined(EV_UDATA_SPECIFIC) || (TARGET_IPHONE_SIMULATOR && \
-               IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101100) || \
-               (!TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED < 101100)
-#undef DISPATCH_USE_EV_UDATA_SPECIFIC
-#define DISPATCH_USE_EV_UDATA_SPECIFIC 0
-#elif !defined(DISPATCH_USE_EV_UDATA_SPECIFIC)
-#define DISPATCH_USE_EV_UDATA_SPECIFIC 1
-#endif // EV_UDATA_SPECIFIC
-
-#if !DISPATCH_USE_EV_UDATA_SPECIFIC
-#undef EV_UDATA_SPECIFIC
-#define EV_UDATA_SPECIFIC 0
-#undef DISPATCH_DYNAMIC_SELECT_FALLBACK
-#define DISPATCH_DYNAMIC_SELECT_FALLBACK 0
-#undef DISPATCH_USE_SELECT_FALLBACK
-#define DISPATCH_USE_SELECT_FALLBACK 1
-#endif // !DISPATCH_USE_EV_UDATA_SPECIFIC
-
-#if !defined(EV_SET_QOS) || (TARGET_IPHONE_SIMULATOR && \
-               IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101100) || \
-               (!TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED < 101100)
-#undef DISPATCH_USE_KEVENT_QOS
-#define DISPATCH_USE_KEVENT_QOS 0
-#elif !defined(DISPATCH_USE_KEVENT_QOS)
-#define DISPATCH_USE_KEVENT_QOS 1
-#endif // EV_SET_QOS
-
-#if DISPATCH_USE_KEVENT_QOS
-typedef struct kevent_qos_s _dispatch_kevent_qos_s;
-#else // DISPATCH_USE_KEVENT_QOS
-#ifndef KEVENT_FLAG_IMMEDIATE
-#define KEVENT_FLAG_NONE 0x00
-#define KEVENT_FLAG_IMMEDIATE 0x01
-#define KEVENT_FLAG_ERROR_EVENTS 0x02
-#endif // KEVENT_FLAG_IMMEDIATE
-typedef struct kevent64_s _dispatch_kevent_qos_s;
-#define kevent_qos(_kq, _changelist, _nchanges, _eventlist, _nevents, \
-               _data_out, _data_available, _flags) \
-               ({ unsigned int _f = (_flags); _dispatch_kevent_qos_s _kev_copy; \
-               const _dispatch_kevent_qos_s *_cl = (_changelist); \
-               int _n = (_nchanges); const struct timespec _timeout_immediately = {}; \
-               dispatch_static_assert(!(_data_out) && !(_data_available)); \
-               if (_f & KEVENT_FLAG_ERROR_EVENTS) { \
-                       dispatch_static_assert(_n == 1); \
-                       _kev_copy = *_cl; _kev_copy.flags |= EV_RECEIPT; } \
-               kevent64((_kq), _f & KEVENT_FLAG_ERROR_EVENTS ? &_kev_copy : _cl, _n, \
-                       (_eventlist), (_nevents), 0, \
-                       _f & KEVENT_FLAG_IMMEDIATE ? &_timeout_immediately : NULL); })
-#endif // DISPATCH_USE_KEVENT_QOS
+#if TARGET_OS_SIMULATOR
+#undef DISPATCH_USE_MEMORYPRESSURE_SOURCE
+#define DISPATCH_USE_MEMORYPRESSURE_SOURCE 0
+#endif // TARGET_OS_SIMULATOR
+#if !defined(DISPATCH_USE_MEMORYPRESSURE_SOURCE) && DISPATCH_USE_MEMORYSTATUS
+#define DISPATCH_USE_MEMORYPRESSURE_SOURCE 1
+#endif
+#if DISPATCH_USE_MEMORYPRESSURE_SOURCE
+#if __has_include(<malloc_private.h>)
+#include <malloc_private.h>
+#else
+extern void malloc_memory_event_handler(unsigned long);
+#endif // __has_include(<malloc_private.h)
+extern bool _dispatch_memory_warn;
+#endif
+
+#if HAVE_PTHREAD_WORKQUEUE_KEVENT && defined(KEVENT_FLAG_WORKQ) && \
+               !defined(DISPATCH_USE_KEVENT_WORKQUEUE)
+#define DISPATCH_USE_KEVENT_WORKQUEUE 1
+#endif

 
 

-#if defined(F_SETNOSIGPIPE) && defined(F_GETNOSIGPIPE)
-#if TARGET_IPHONE_SIMULATOR && IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1070
-#undef DISPATCH_USE_SETNOSIGPIPE
-#define DISPATCH_USE_SETNOSIGPIPE 0
+#if (!DISPATCH_USE_KEVENT_WORKQUEUE || DISPATCH_DEBUG || DISPATCH_PROFILE) && \
+               !defined(DISPATCH_USE_MGR_THREAD)
+#define DISPATCH_USE_MGR_THREAD 1

 #endif
 #endif

+
+
+#if defined(MACH_SEND_SYNC_OVERRIDE) && defined(MACH_RCV_SYNC_WAIT) && \
+               DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(109900) && \
+               !defined(DISPATCH_USE_MACH_SEND_SYNC_OVERRIDE)
+#define DISPATCH_USE_MACH_SEND_SYNC_OVERRIDE 1
+#endif
+
+#if defined(F_SETNOSIGPIPE) && defined(F_GETNOSIGPIPE)

 #ifndef DISPATCH_USE_SETNOSIGPIPE
 #define DISPATCH_USE_SETNOSIGPIPE 1
 #endif
 #ifndef DISPATCH_USE_SETNOSIGPIPE
 #define DISPATCH_USE_SETNOSIGPIPE 1
 #endif


@@ -695,14 +719,6 @@ typedef struct kevent64_s _dispatch_kevent_qos_s;
 #if HAVE_LIBPROC_INTERNAL_H
 #include <libproc.h>
 #include <libproc_internal.h>
 #if HAVE_LIBPROC_INTERNAL_H
 #include <libproc.h>
 #include <libproc_internal.h>

-#if TARGET_IPHONE_SIMULATOR && IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1090
-#undef DISPATCH_USE_IMPORTANCE_ASSERTION
-#define DISPATCH_USE_IMPORTANCE_ASSERTION 0
-#endif
-#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED < 1090
-#undef DISPATCH_USE_IMPORTANCE_ASSERTION
-#define DISPATCH_USE_IMPORTANCE_ASSERTION 0
-#endif

 #ifndef DISPATCH_USE_IMPORTANCE_ASSERTION
 #define DISPATCH_USE_IMPORTANCE_ASSERTION 1
 #endif
 #ifndef DISPATCH_USE_IMPORTANCE_ASSERTION
 #define DISPATCH_USE_IMPORTANCE_ASSERTION 1
 #endif


@@ -710,20 +726,95 @@ typedef struct kevent64_s _dispatch_kevent_qos_s;
 
 #if HAVE_SYS_GUARDED_H
 #include <sys/guarded.h>
 
 #if HAVE_SYS_GUARDED_H
 #include <sys/guarded.h>

-#if TARGET_IPHONE_SIMULATOR && IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1090
-#undef DISPATCH_USE_GUARDED_FD
-#define DISPATCH_USE_GUARDED_FD 0
-#endif

 #ifndef DISPATCH_USE_GUARDED_FD
 #define DISPATCH_USE_GUARDED_FD 1
 #endif
 #ifndef DISPATCH_USE_GUARDED_FD
 #define DISPATCH_USE_GUARDED_FD 1
 #endif

-// change_fdguard_np() requires GUARD_DUP <rdar://problem/11814513>
-#if DISPATCH_USE_GUARDED_FD && RDAR_11814513
-#define DISPATCH_USE_GUARDED_FD_CHANGE_FDGUARD 1
-#endif

 #endif // HAVE_SYS_GUARDED_H
 
 
 #endif // HAVE_SYS_GUARDED_H
 
 

+#if DISPATCH_USE_DTRACE || DISPATCH_USE_DTRACE_INTROSPECTION
+typedef struct dispatch_trace_timer_params_s {
+       int64_t deadline, interval, leeway;
+} *dispatch_trace_timer_params_t;
+
+#include "provider.h"
+#endif // DISPATCH_USE_DTRACE || DISPATCH_USE_DTRACE_INTROSPECTION
+
+#if __has_include(<sys/kdebug.h>)
+#include <sys/kdebug.h>
+#ifndef DBG_DISPATCH
+#define DBG_DISPATCH 46
+#endif
+#ifndef KDBG_CODE
+#define KDBG_CODE(...) 0
+#define DBG_FUNC_START 0
+#define DBG_FUNC_END 0
+#endif
+#define DISPATCH_CODE(subclass, code) \
+               KDBG_CODE(DBG_DISPATCH, DISPATCH_TRACE_SUBCLASS_##subclass, code)
+#define DISPATCH_CODE_START(subclass, code) \
+               (DISPATCH_CODE(subclass, code) | DBG_FUNC_START)
+#define DISPATCH_CODE_END(subclass, code) \
+               (DISPATCH_CODE(subclass, code) | DBG_FUNC_END)
+#ifdef ARIADNEDBG_CODE
+#define ARIADNE_ENTER_DISPATCH_MAIN_CODE ARIADNEDBG_CODE(220, 2)
+#else
+#define ARIADNE_ENTER_DISPATCH_MAIN_CODE 0
+#endif
+#if !defined(DISPATCH_USE_VOUCHER_KDEBUG_TRACE) && \
+               (DISPATCH_INTROSPECTION || DISPATCH_PROFILE || DISPATCH_DEBUG)
+#define DISPATCH_USE_VOUCHER_KDEBUG_TRACE 1
+#endif
+
+#define DISPATCH_TRACE_SUBCLASS_DEFAULT 0
+#define DISPATCH_TRACE_SUBCLASS_VOUCHER 1
+#define DISPATCH_TRACE_SUBCLASS_PERF 2
+#define DISPATCH_TRACE_SUBCLASS_MACH_MSG 3
+#define DISPATCH_TRACE_SUBCLASS_PERF_MON 4
+
+#define DISPATCH_PERF_non_leaf_retarget DISPATCH_CODE(PERF, 1)
+#define DISPATCH_PERF_post_activate_retarget DISPATCH_CODE(PERF, 2)
+#define DISPATCH_PERF_post_activate_mutation DISPATCH_CODE(PERF, 3)
+#define DISPATCH_PERF_delayed_registration DISPATCH_CODE(PERF, 4)
+#define DISPATCH_PERF_mutable_target DISPATCH_CODE(PERF, 5)
+#define DISPATCH_PERF_strict_bg_timer DISPATCH_CODE(PERF, 6)
+
+#define DISPATCH_MACH_MSG_hdr_move DISPATCH_CODE(MACH_MSG, 1)
+
+#define DISPATCH_PERF_MON_worker_thread_start DISPATCH_CODE_START(PERF_MON, 1)
+#define DISPATCH_PERF_MON_worker_thread_end DISPATCH_CODE_END(PERF_MON, 1)
+#define DISPATCH_PERF_MON_worker_useless DISPATCH_CODE(PERF_MON, 2)
+
+DISPATCH_ALWAYS_INLINE
+static inline void
+_dispatch_ktrace_impl(uint32_t code, uint64_t a, uint64_t b,
+               uint64_t c, uint64_t d)
+{
+       if (!code) return;
+#ifdef _COMM_PAGE_KDEBUG_ENABLE
+       if (likely(*(volatile uint32_t *)_COMM_PAGE_KDEBUG_ENABLE == 0)) return;
+#endif
+       kdebug_trace(code, a, b, c, d);
+}
+#define _dispatch_cast_to_uint64(e) \
+               __builtin_choose_expr(sizeof(e) > 4, \
+                               ((uint64_t)(e)), ((uint64_t)(uintptr_t)(e)))
+#define _dispatch_ktrace(code, a, b, c, d)  _dispatch_ktrace_impl(code, \
+               _dispatch_cast_to_uint64(a), _dispatch_cast_to_uint64(b), \
+               _dispatch_cast_to_uint64(c), _dispatch_cast_to_uint64(d))
+
+#else // __has_include(<sys/kdebug.h>)
+#define DISPATCH_CODE(subclass, code) 0
+#define ARIADNE_ENTER_DISPATCH_MAIN_CODE 0
+#define DISPATCH_USE_VOUCHER_KDEBUG_TRACE 0
+#define _dispatch_ktrace(code, a, b, c, d)
+#endif // !__has_include(<sys/kdebug.h>)
+#define _dispatch_ktrace4(code, a, b, c, d) _dispatch_ktrace(code, a, b, c, d)
+#define _dispatch_ktrace3(code, a, b, c)    _dispatch_ktrace(code, a, b, c, 0)
+#define _dispatch_ktrace2(code, a, b)       _dispatch_ktrace(code, a, b, 0, 0)
+#define _dispatch_ktrace1(code, a)          _dispatch_ktrace(code, a, 0, 0, 0)
+#define _dispatch_ktrace0(code)             _dispatch_ktrace(code, 0, 0, 0, 0)
+

 #ifndef MACH_MSGH_BITS_VOUCHER_MASK
 #define MACH_MSGH_BITS_VOUCHER_MASK    0x001f0000
 #define        MACH_MSGH_BITS_SET_PORTS(remote, local, voucher)        \
 #ifndef MACH_MSGH_BITS_VOUCHER_MASK
 #define MACH_MSGH_BITS_VOUCHER_MASK    0x001f0000
 #define        MACH_MSGH_BITS_SET_PORTS(remote, local, voucher)        \


@@ -740,9 +831,42 @@ typedef struct kevent64_s _dispatch_kevent_qos_s;
 #define MACH_SEND_INVALID_VOUCHER 0x10000005
 #endif
 
 #define MACH_SEND_INVALID_VOUCHER 0x10000005
 #endif
 

+#ifndef VOUCHER_USE_MACH_VOUCHER
+#if __has_include(<mach/mach_voucher.h>)
+#define VOUCHER_USE_MACH_VOUCHER 1
+#endif
+#endif // VOUCHER_USE_MACH_VOUCHER
+
+#ifndef VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
+#if RDAR_24272659 // FIXME: <rdar://problem/24272659>
+#define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 1
+#else // RDAR_24272659
+#define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 0
+#endif // RDAR_24272659
+#endif
+
+#ifndef VOUCHER_USE_PERSONA
+#if VOUCHER_USE_MACH_VOUCHER && defined(BANK_PERSONA_TOKEN) && \
+               TARGET_OS_IOS && !TARGET_OS_SIMULATOR
+#define VOUCHER_USE_PERSONA 1
+#else
+#define VOUCHER_USE_PERSONA 0
+#endif
+#endif // VOUCHER_USE_PERSONA
+
+#if VOUCHER_USE_MACH_VOUCHER
+#undef DISPATCH_USE_IMPORTANCE_ASSERTION
+#define DISPATCH_USE_IMPORTANCE_ASSERTION 0
+#else
+#undef MACH_RCV_VOUCHER
+#define MACH_RCV_VOUCHER 0
+#define VOUCHER_USE_PERSONA 0
+#endif // VOUCHER_USE_MACH_VOUCHER
+

 #define _dispatch_hardware_crash() \
                __asm__(""); __builtin_trap() // <rdar://problem/17464981>
 
 #define _dispatch_hardware_crash() \
                __asm__(""); __builtin_trap() // <rdar://problem/17464981>
 

+#define _dispatch_set_crash_log_cause_and_message(ac, msg) ((void)(ac))

 #define _dispatch_set_crash_log_message(msg)
 #define _dispatch_set_crash_log_message_dynamic(msg)
 
 #define _dispatch_set_crash_log_message(msg)
 #define _dispatch_set_crash_log_message_dynamic(msg)
 


@@ -753,19 +877,22 @@ typedef struct kevent64_s _dispatch_kevent_qos_s;
 // 2) A hand crafted call to mach_msg*() screwed up. Use MIG.
 #define DISPATCH_VERIFY_MIG(x) do { \
                if ((x) == MIG_REPLY_MISMATCH) { \
 // 2) A hand crafted call to mach_msg*() screwed up. Use MIG.
 #define DISPATCH_VERIFY_MIG(x) do { \
                if ((x) == MIG_REPLY_MISMATCH) { \

-                       _dispatch_set_crash_log_message("MIG_REPLY_MISMATCH"); \
+                       _dispatch_set_crash_log_cause_and_message((x), \
+                                       "MIG_REPLY_MISMATCH"); \

                        _dispatch_hardware_crash(); \
                } \
        } while (0)
 #endif
 
                        _dispatch_hardware_crash(); \
                } \
        } while (0)
 #endif
 

-#define DISPATCH_CRASH(x) do { \
-               _dispatch_set_crash_log_message("BUG IN LIBDISPATCH: " x); \
+#define DISPATCH_INTERNAL_CRASH(c, x) do { \
+               _dispatch_set_crash_log_cause_and_message((c), \
+                               "BUG IN LIBDISPATCH: " x); \

                _dispatch_hardware_crash(); \
        } while (0)
 
                _dispatch_hardware_crash(); \
        } while (0)
 

-#define DISPATCH_CLIENT_CRASH(x) do { \
-               _dispatch_set_crash_log_message("BUG IN CLIENT OF LIBDISPATCH: " x); \
+#define DISPATCH_CLIENT_CRASH(c, x) do { \
+               _dispatch_set_crash_log_cause_and_message((c), \
+                               "BUG IN CLIENT OF LIBDISPATCH: " x); \

                _dispatch_hardware_crash(); \
        } while (0)
 
                _dispatch_hardware_crash(); \
        } while (0)
 


@@ -774,19 +901,60 @@ typedef struct kevent64_s _dispatch_kevent_qos_s;
                _dispatch_hardware_crash(); \
        } while (0)
 
                _dispatch_hardware_crash(); \
        } while (0)
 

-extern int _dispatch_set_qos_class_enabled;
+#define DISPATCH_ASSERTION_FAILED_MESSAGE \
+               "BUG IN CLIENT OF LIBDISPATCH: Assertion failed: "
+
+#define _dispatch_assert_crash(msg)  do { \
+               const char *__msg = (msg); \
+               _dispatch_log("%s", __msg); \
+               _dispatch_set_crash_log_message_dynamic(__msg); \
+               _dispatch_hardware_crash(); \
+       } while (0)
+
+#define _dispatch_client_assert_fail(fmt, ...)  do { \
+               char *_msg = NULL; \
+               asprintf(&_msg, "%s" fmt, DISPATCH_ASSERTION_FAILED_MESSAGE, \
+                               ##__VA_ARGS__); \
+               _dispatch_assert_crash(_msg); \
+               free(_msg); \
+       } while (0)
+

 #define DISPATCH_NO_VOUCHER ((voucher_t)(void*)~0ul)
 #define DISPATCH_NO_PRIORITY ((pthread_priority_t)~0ul)
 #define DISPATCH_NO_VOUCHER ((voucher_t)(void*)~0ul)
 #define DISPATCH_NO_PRIORITY ((pthread_priority_t)~0ul)

-#define DISPATCH_PRIORITY_ENFORCE 0x1
-#define DISPATCH_VOUCHER_IGNORE_QUEUE_OVERRIDE 0x2
-static inline void _dispatch_adopt_priority_and_replace_voucher(
-               pthread_priority_t priority, voucher_t voucher, unsigned long flags);
+DISPATCH_ENUM(dispatch_thread_set_self, unsigned long,
+       DISPATCH_PRIORITY_ENFORCE = 0x1,
+       DISPATCH_VOUCHER_REPLACE = 0x2,
+       DISPATCH_VOUCHER_CONSUME = 0x4,
+       DISPATCH_THREAD_PARK = 0x8,
+);
+DISPATCH_WARN_RESULT
+static inline voucher_t _dispatch_adopt_priority_and_set_voucher(
+               pthread_priority_t priority, voucher_t voucher,
+               dispatch_thread_set_self_t flags);

 #if HAVE_MACH
 #if HAVE_MACH

-static inline void _dispatch_set_priority_and_mach_voucher(
-               pthread_priority_t priority, mach_voucher_t kv);

 mach_port_t _dispatch_get_mach_host_port(void);
 #endif
 
 mach_port_t _dispatch_get_mach_host_port(void);
 #endif
 

+#if HAVE_PTHREAD_WORKQUEUE_QOS
+#if DISPATCH_DEBUG
+extern int _dispatch_set_qos_class_enabled;
+#else
+#define _dispatch_set_qos_class_enabled (1)
+#endif
+#endif // HAVE_PTHREAD_WORKQUEUE_QOS
+#if DISPATCH_USE_KEVENT_WORKQUEUE
+#if !HAVE_PTHREAD_WORKQUEUE_QOS || !EV_UDATA_SPECIFIC
+#error Invalid build configuration
+#endif
+#if DISPATCH_USE_MGR_THREAD
+extern int _dispatch_kevent_workqueue_enabled;
+#else
+#define _dispatch_kevent_workqueue_enabled (1)
+#endif
+#else
+#define _dispatch_kevent_workqueue_enabled (0)
+#endif // DISPATCH_USE_KEVENT_WORKQUEUE
+

 
 /* #includes dependent on internal.h */
 #include "object_internal.h"
 
 /* #includes dependent on internal.h */
 #include "object_internal.h"


@@ -794,11 +962,13 @@ mach_port_t _dispatch_get_mach_host_port(void);
 #include "introspection_internal.h"
 #include "queue_internal.h"
 #include "source_internal.h"
 #include "introspection_internal.h"
 #include "queue_internal.h"
 #include "source_internal.h"

+#include "mach_internal.h"

 #include "voucher_internal.h"
 #include "data_internal.h"
 #if !TARGET_OS_WIN32
 #include "io_internal.h"
 #endif
 #include "inline_internal.h"
 #include "voucher_internal.h"
 #include "data_internal.h"
 #if !TARGET_OS_WIN32
 #include "io_internal.h"
 #endif
 #include "inline_internal.h"

+#include "firehose/firehose_internal.h"

 
 #endif /* __DISPATCH_INTERNAL__ */
 
 #endif /* __DISPATCH_INTERNAL__ */