From 45201a42773e3f9687334f0f931d12bc311c2ceb Mon Sep 17 00:00:00 2001 From: Apple Date: Fri, 4 Sep 2015 17:28:44 +0000 Subject: [PATCH] libdispatch-500.1.5.tar.gz --- dispatch/dispatch.h | 2 +- dispatch/object.h | 6 +- dispatch/queue.h | 7 +- dispatch/source.h | 3 +- libdispatch.xcodeproj/project.pbxproj | 813 ++++++++++- man/dispatch_source_create.3 | 4 +- os/object.h | 6 +- os/object_private.h | 5 + private/private.h | 2 +- private/queue_private.h | 42 + private/source_private.h | 4 + private/voucher_activity_private.h | 219 ++- private/voucher_private.h | 64 +- src/allocator.c | 6 + src/allocator_internal.h | 15 +- src/block.cpp | 110 ++ src/data.c | 412 ++++-- src/data.m | 19 + src/data_internal.h | 89 +- src/init.c | 188 +-- src/inline_internal.h | 270 +++- src/internal.h | 89 +- src/introspection.c | 3 +- src/introspection_internal.h | 6 +- src/io.c | 23 +- src/io_internal.h | 11 +- src/object.c | 26 +- src/object.m | 64 +- src/object_internal.h | 86 +- src/queue.c | 402 ++++-- src/queue_internal.h | 66 +- src/semaphore.c | 27 +- src/semaphore_internal.h | 10 +- src/shims/tsd.h | 10 +- src/source.c | 716 +++++++--- src/source_internal.h | 21 +- src/trace.h | 8 +- src/voucher.c | 1255 +++++++---------- src/voucher_internal.h | 300 ++-- .../libdispatch-introspection.xcconfig | 2 +- xcodeconfig/libdispatch-resolved.xcconfig | 2 +- xcodeconfig/libdispatch.xcconfig | 27 +- xcodescripts/install-dtrace.sh | 3 +- 43 files changed, 3752 insertions(+), 1691 deletions(-) create mode 100644 src/block.cpp diff --git a/dispatch/dispatch.h b/dispatch/dispatch.h index 722b0c9..bb32bdf 100644 --- a/dispatch/dispatch.h +++ b/dispatch/dispatch.h @@ -37,7 +37,7 @@ #define __OSX_AVAILABLE_STARTING(x, y) #endif -#define DISPATCH_API_VERSION 20140804 +#define DISPATCH_API_VERSION 20141121 #ifndef __DISPATCH_BUILDING_DISPATCH__ diff --git a/dispatch/object.h b/dispatch/object.h index c637189..a9b805e 100644 --- a/dispatch/object.h +++ b/dispatch/object.h @@ -56,7 +56,7 @@ _dispatch_object_validate(dispatch_object_t object) { void *isa = *(void* volatile*)(OS_OBJECT_BRIDGE void*)object; (void)isa; } -#elif defined(__cplusplus) +#elif defined(__cplusplus) && !defined(__DISPATCH_BUILDING_DISPATCH__) /* * Dispatch objects are NOT C++ objects. Nevertheless, we can at least keep C++ * aware of type compatibility. @@ -99,6 +99,7 @@ typedef union { #define DISPATCH_RETURNS_RETAINED #endif +#ifdef __BLOCKS__ /*! * @typedef dispatch_block_t * @@ -141,6 +142,7 @@ typedef union { * function or by sending it a -[copy] message. */ typedef void (^dispatch_block_t)(void); +#endif // __BLOCKS__ __BEGIN_DECLS @@ -295,6 +297,7 @@ DISPATCH_EXPORT DISPATCH_NONNULL_ALL DISPATCH_NOTHROW void dispatch_resume(dispatch_object_t object); +#ifdef __BLOCKS__ /*! * @function dispatch_wait * @@ -427,6 +430,7 @@ dispatch_testcancel(void *object); dispatch_source_t:dispatch_source_testcancel \ )((object)) #endif +#endif // __BLOCKS__ /*! * @function dispatch_debug diff --git a/dispatch/queue.h b/dispatch/queue.h index cc3ca94..b3cb54f 100644 --- a/dispatch/queue.h +++ b/dispatch/queue.h @@ -930,7 +930,7 @@ dispatch_barrier_sync_f(dispatch_queue_t queue, * is NULL. */ __OSX_AVAILABLE_STARTING(__MAC_10_7,__IPHONE_5_0) -DISPATCH_EXPORT DISPATCH_NONNULL1 DISPATCH_NONNULL2 DISPATCH_NOTHROW +DISPATCH_EXPORT DISPATCH_NONNULL1 DISPATCH_NOTHROW void dispatch_queue_set_specific(dispatch_queue_t queue, const void *key, void *context, dispatch_function_t destructor); @@ -959,7 +959,7 @@ dispatch_queue_set_specific(dispatch_queue_t queue, const void *key, * The context for the specified key or NULL if no context was found. */ __OSX_AVAILABLE_STARTING(__MAC_10_7,__IPHONE_5_0) -DISPATCH_EXPORT DISPATCH_NONNULL_ALL DISPATCH_PURE DISPATCH_WARN_RESULT +DISPATCH_EXPORT DISPATCH_NONNULL1 DISPATCH_PURE DISPATCH_WARN_RESULT DISPATCH_NOTHROW void * dispatch_queue_get_specific(dispatch_queue_t queue, const void *key); @@ -986,8 +986,7 @@ dispatch_queue_get_specific(dispatch_queue_t queue, const void *key); * The context for the specified key or NULL if no context was found. */ __OSX_AVAILABLE_STARTING(__MAC_10_7,__IPHONE_5_0) -DISPATCH_EXPORT DISPATCH_NONNULL_ALL DISPATCH_PURE DISPATCH_WARN_RESULT -DISPATCH_NOTHROW +DISPATCH_EXPORT DISPATCH_PURE DISPATCH_WARN_RESULT DISPATCH_NOTHROW void * dispatch_get_specific(const void *key); diff --git a/dispatch/source.h b/dispatch/source.h index 411ed06..f01fd93 100644 --- a/dispatch/source.h +++ b/dispatch/source.h @@ -54,6 +54,8 @@ */ DISPATCH_DECL(dispatch_source); +__BEGIN_DECLS + /*! * @typedef dispatch_source_type_t * @@ -319,7 +321,6 @@ typedef unsigned long dispatch_source_vnode_flags_t; typedef unsigned long dispatch_source_timer_flags_t; -__BEGIN_DECLS /*! * @function dispatch_source_create diff --git a/libdispatch.xcodeproj/project.pbxproj b/libdispatch.xcodeproj/project.pbxproj index ff12a47..898ffca 100644 --- a/libdispatch.xcodeproj/project.pbxproj +++ b/libdispatch.xcodeproj/project.pbxproj @@ -18,6 +18,17 @@ name = libdispatch_Sim; productName = libdispatch_Sim; }; + 4552540A19B1389700B88766 /* libdispatch_tests */ = { + isa = PBXAggregateTarget; + buildConfigurationList = 4552540B19B1389700B88766 /* Build configuration list for PBXAggregateTarget "libdispatch_tests" */; + buildPhases = ( + ); + dependencies = ( + 4552540F19B138B700B88766 /* PBXTargetDependency */, + ); + name = libdispatch_tests; + productName = libdispatch_tests; + }; C927F35A10FD7F0600C5AB8B /* libdispatch_tools */ = { isa = PBXAggregateTarget; buildConfigurationList = C927F35E10FD7F0B00C5AB8B /* Build configuration list for PBXAggregateTarget "libdispatch_tools" */; @@ -84,6 +95,12 @@ E43570B9126E93380097AB9F /* provider.d in Sources */ = {isa = PBXBuildFile; fileRef = E43570B8126E93380097AB9F /* provider.d */; }; E43570BA126E93380097AB9F /* provider.d in Sources */ = {isa = PBXBuildFile; fileRef = E43570B8126E93380097AB9F /* provider.d */; }; E43A710615783F7E0012D38D /* data_private.h in Headers */ = {isa = PBXBuildFile; fileRef = C913AC0E143BD34800B78976 /* data_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; + E43A72501AF85BBC00BAA921 /* block.cpp in Sources */ = {isa = PBXBuildFile; fileRef = E43A724F1AF85BBC00BAA921 /* block.cpp */; }; + E43A72841AF85BCB00BAA921 /* block.cpp in Sources */ = {isa = PBXBuildFile; fileRef = E43A724F1AF85BBC00BAA921 /* block.cpp */; }; + E43A72851AF85BCC00BAA921 /* block.cpp in Sources */ = {isa = PBXBuildFile; fileRef = E43A724F1AF85BBC00BAA921 /* block.cpp */; }; + E43A72861AF85BCC00BAA921 /* block.cpp in Sources */ = {isa = PBXBuildFile; fileRef = E43A724F1AF85BBC00BAA921 /* block.cpp */; }; + E43A72871AF85BCD00BAA921 /* block.cpp in Sources */ = {isa = PBXBuildFile; fileRef = E43A724F1AF85BBC00BAA921 /* block.cpp */; }; + E43A72881AF85BE900BAA921 /* block.cpp in Sources */ = {isa = PBXBuildFile; fileRef = E43A724F1AF85BBC00BAA921 /* block.cpp */; }; E44757DA17F4572600B82CA1 /* inline_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = E44757D917F4572600B82CA1 /* inline_internal.h */; }; E44757DB17F4573500B82CA1 /* inline_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = E44757D917F4572600B82CA1 /* inline_internal.h */; }; E44757DC17F4573600B82CA1 /* inline_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = E44757D917F4572600B82CA1 /* inline_internal.h */; }; @@ -267,6 +284,356 @@ /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ + 455253A819B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = DF80F67E10B5C71600FAB5AE; + remoteInfo = dispatch_test; + }; + 455253AA19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01C78108E68D400FAA873; + remoteInfo = dispatch_apply; + }; + 455253AC19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4EB36CD1088F0B000C33AD4; + remoteInfo = dispatch_api; + }; + 455253AE19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01CA7108E6C5000FAA873; + remoteInfo = dispatch_c99; + }; + 455253B019B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4C72A26115C3F65009F3CE1; + remoteInfo = dispatch_cf_main; + }; + 455253B219B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01CB9108E6C7200FAA873; + remoteInfo = dispatch_deadname; + }; + 455253B419B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01CC3108E6CC300FAA873; + remoteInfo = dispatch_debug; + }; + 455253B619B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01CCC108E6CD400FAA873; + remoteInfo = dispatch_group; + }; + 455253B819B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01CD5108E6CE300FAA873; + remoteInfo = dispatch_overcommit; + }; + 455253BA19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01CDE108E6CF300FAA873; + remoteInfo = dispatch_pingpong; + }; + 455253BC19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01CE7108E6D0500FAA873; + remoteInfo = dispatch_plusplus; + }; + 455253BE19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01CF0108E6D2900FAA873; + remoteInfo = dispatch_priority; + }; + 455253C019B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01CF9108E6D3800FAA873; + remoteInfo = dispatch_priority2; + }; + 455253C219B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E46D768811D0365F00615518; + remoteInfo = dispatch_concur; + }; + 455253C419B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4324AAC12250F0800A3CAD5; + remoteInfo = dispatch_context_for_key; + }; + 455253C619B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01D02108E6D5600FAA873; + remoteInfo = dispatch_proc; + }; + 455253C819B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01D0B108E6D6000FAA873; + remoteInfo = dispatch_queue_finalizer; + }; + 455253CA19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01D14108E6D7300FAA873; + remoteInfo = dispatch_read; + }; + 455253CC19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01D1D108E6D8B00FAA873; + remoteInfo = dispatch_read2; + }; + 455253CE19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01D26108E6D9A00FAA873; + remoteInfo = dispatch_after; + }; + 455253D019B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01D2F108E6DA700FAA873; + remoteInfo = dispatch_timer; + }; + 455253D219B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4CE9BC31151AB2A00D710C0; + remoteInfo = dispatch_timer_short; + }; + 455253D419B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = 5A2BA66D11D0369E0081FF89; + remoteInfo = dispatch_timer_timeout; + }; + 455253D619B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01D38108E6DB200FAA873; + remoteInfo = dispatch_suspend_timer; + }; + 455253D819B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01D41108E6DBF00FAA873; + remoteInfo = dispatch_sema; + }; + 455253DA19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01D53108E6DDC00FAA873; + remoteInfo = dispatch_timer_bit31; + }; + 455253DC19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01D5C108E6E0400FAA873; + remoteInfo = dispatch_timer_bit63; + }; + 455253DE19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01D74108E6E4B00FAA873; + remoteInfo = dispatch_timer_set_time; + }; + 455253E019B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01D7D108E6E6600FAA873; + remoteInfo = dispatch_drift; + }; + 455253E219B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01D86108E6E7200FAA873; + remoteInfo = dispatch_starfish; + }; + 455253E419B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01D8F108E6E7E00FAA873; + remoteInfo = dispatch_cascade; + }; + 455253E619B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01D98108E6E9500FAA873; + remoteInfo = dispatch_readsync; + }; + 455253E819B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4E24A0710E0020B00C3C692; + remoteInfo = dispatch_sync_on_main; + }; + 455253EA19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4E24A1810E0021C00C3C692; + remoteInfo = dispatch_sync_gc; + }; + 455253EC19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4E24C3210E01DF800C3C692; + remoteInfo = dispatch_apply_gc; + }; + 455253EE19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = 5AAB464A10D330C5004407EA; + remoteInfo = dispatch_data; + }; + 455253F019B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = 5A11B20E10DB124C000FAD7A; + remoteInfo = dispatch_io; + }; + 455253F219B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = 5AA78BAB114821D0009A233B; + remoteInfo = dispatch_io_net; + }; + 455253F419B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = 5AF00EF51135FA1300CA14CE; + remoteInfo = dispatch_vm; + }; + 455253F619B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4E33EB6121C9C9400F4B71C; + remoteInfo = dispatch_vnode; + }; + 455253F819B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = C9E804AF1963EC5F00C2B970; + remoteInfo = dispatch_qos; + }; + 455253FA19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = C9B1FF84113F458A00843414; + remoteInfo = dispatch_select; + }; + 455253FC19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = C985912B143D584100718FE3; + remoteInfo = dispatch_transform; + }; + 455253FE19B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01DA1108E6EE000FAA873; + remoteInfo = nsoperation; + }; + 4552540019B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D01CB0108E6C6300FAA873; + remoteInfo = cffd; + }; + 4552540219B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E4D020B4108F73E000FAA873; + remoteInfo = bench; + }; + 4552540419B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E454823616C1D8E50042EC2D; + remoteInfo = jsgc_bench; + }; + 4552540619B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E454824516C1F0EF0042EC2D; + remoteInfo = async_bench; + }; + 4552540819B1384900B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = E454824F16C1F0FE0042EC2D; + remoteInfo = apply_bench; + }; + 4552540E19B138B700B88766 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + proxyType = 1; + remoteGlobalIDString = E4D01DC5108E708E00FAA873; + remoteInfo = all; + }; C927F36610FD7F1000C5AB8B /* PBXContainerItemProxy */ = { isa = PBXContainerItemProxy; containerPortal = C927F35F10FD7F1000C5AB8B /* ddt.xcodeproj */; @@ -322,6 +689,7 @@ 2BBF5A5F154B64D8002B20F9 /* allocator_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = allocator_internal.h; sourceTree = ""; }; 2BBF5A62154B64F5002B20F9 /* allocator.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = allocator.c; sourceTree = ""; }; 2BE17C6318EA305E002CA4E8 /* layout_private.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = layout_private.h; sourceTree = ""; }; + 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = libdispatchtest.xcodeproj; path = tests/libdispatchtest.xcodeproj; sourceTree = ""; }; 5A0095A110F274B0000E2A31 /* io_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = io_internal.h; sourceTree = ""; }; 5A27262510F26F1900751FBC /* io.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; lineEnding = 0; path = io.c; sourceTree = ""; xcLanguageSpecificationIdentifier = xcode.lang.c; }; 5A5D13AB0F6B280500197CC3 /* semaphore_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = semaphore_internal.h; sourceTree = ""; }; @@ -369,6 +737,7 @@ E422DA3614D2A7E7003C6EE4 /* libdispatch.aliases */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = libdispatch.aliases; sourceTree = ""; }; E422DA3714D2AE11003C6EE4 /* libdispatch.unexport */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = libdispatch.unexport; sourceTree = ""; }; E43570B8126E93380097AB9F /* provider.d */ = {isa = PBXFileReference; explicitFileType = sourcecode.dtrace; fileEncoding = 4; path = provider.d; sourceTree = ""; }; + E43A724F1AF85BBC00BAA921 /* block.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = block.cpp; sourceTree = ""; }; E43D93F11097917E004F6A62 /* libdispatch.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = libdispatch.xcconfig; sourceTree = ""; }; E44757D917F4572600B82CA1 /* inline_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = inline_internal.h; sourceTree = ""; }; E448727914C6215D00BB45C2 /* libdispatch.order */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = libdispatch.order; sourceTree = ""; }; @@ -463,6 +832,8 @@ 08FB7795FE84155DC02AAC07 /* Source */, C6A0FF2B0290797F04C91782 /* Documentation */, 1AB674ADFE9D54B511CA2CBB /* Products */, + C927F35F10FD7F1000C5AB8B /* ddt.xcodeproj */, + 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */, ); indentWidth = 4; name = libdispatch; @@ -476,6 +847,7 @@ 2BBF5A62154B64F5002B20F9 /* allocator.c */, 9676A0E00F3E755D00713ADB /* apply.c */, 965CD6340F3E806200D4E28D /* benchmark.c */, + E43A724F1AF85BBC00BAA921 /* block.cpp */, 5AAB45BF10D30B79004407EA /* data.c */, E420866F16027AE500EEE210 /* data.m */, E44EBE3B1251659900645D88 /* init.c */, @@ -510,6 +882,62 @@ name = Products; sourceTree = ""; }; + 4552536F19B1384900B88766 /* Products */ = { + isa = PBXGroup; + children = ( + 455253A919B1384900B88766 /* libdispatch_test.a */, + 455253AB19B1384900B88766 /* dispatch_apply */, + 455253AD19B1384900B88766 /* dispatch_api */, + 455253AF19B1384900B88766 /* dispatch_c99 */, + 455253B119B1384900B88766 /* dispatch_cf_main */, + 455253B319B1384900B88766 /* dispatch_deadname */, + 455253B519B1384900B88766 /* dispatch_debug */, + 455253B719B1384900B88766 /* dispatch_group */, + 455253B919B1384900B88766 /* dispatch_overcommit */, + 455253BB19B1384900B88766 /* dispatch_pingpong */, + 455253BD19B1384900B88766 /* dispatch_plusplus */, + 455253BF19B1384900B88766 /* dispatch_priority */, + 455253C119B1384900B88766 /* dispatch_priority2 */, + 455253C319B1384900B88766 /* dispatch_concur */, + 455253C519B1384900B88766 /* dispatch_context_for_key */, + 455253C719B1384900B88766 /* dispatch_proc */, + 455253C919B1384900B88766 /* dispatch_queue_finalizer */, + 455253CB19B1384900B88766 /* dispatch_read */, + 455253CD19B1384900B88766 /* dispatch_read2 */, + 455253CF19B1384900B88766 /* dispatch_after */, + 455253D119B1384900B88766 /* dispatch_timer */, + 455253D319B1384900B88766 /* dispatch_timer_short */, + 455253D519B1384900B88766 /* dispatch_timer_timeout */, + 455253D719B1384900B88766 /* dispatch_suspend_timer */, + 455253D919B1384900B88766 /* dispatch_sema */, + 455253DB19B1384900B88766 /* dispatch_timer_bit31 */, + 455253DD19B1384900B88766 /* dispatch_timer_bit63 */, + 455253DF19B1384900B88766 /* dispatch_timer_set_time */, + 455253E119B1384900B88766 /* dispatch_drift */, + 455253E319B1384900B88766 /* dispatch_starfish */, + 455253E519B1384900B88766 /* dispatch_cascade */, + 455253E719B1384900B88766 /* dispatch_readsync */, + 455253E919B1384900B88766 /* dispatch_sync_on_main */, + 455253EB19B1384900B88766 /* dispatch_sync_gc */, + 455253ED19B1384900B88766 /* dispatch_apply_gc */, + 455253EF19B1384900B88766 /* dispatch_data */, + 455253F119B1384900B88766 /* dispatch_io */, + 455253F319B1384900B88766 /* dispatch_io_net */, + 455253F519B1384900B88766 /* dispatch_vm */, + 455253F719B1384900B88766 /* dispatch_vnode */, + 455253F919B1384900B88766 /* dispatch_qos */, + 455253FB19B1384900B88766 /* dispatch_select */, + 455253FD19B1384900B88766 /* dispatch_transform */, + 455253FF19B1384900B88766 /* nsoperation */, + 4552540119B1384900B88766 /* cffd */, + 4552540319B1384900B88766 /* bench */, + 4552540519B1384900B88766 /* jsgc_bench */, + 4552540719B1384900B88766 /* async_bench */, + 4552540919B1384900B88766 /* apply_bench */, + ); + name = Products; + sourceTree = ""; + }; C6A0FF2B0290797F04C91782 /* Documentation */ = { isa = PBXGroup; children = ( @@ -570,7 +998,6 @@ E40041E4125E71150022B135 /* xcodeconfig */, E49F259C125D664F0057C971 /* xcodescripts */, E47D6BCA125F10F70070D91C /* resolver */, - C927F35F10FD7F1000C5AB8B /* ddt.xcodeproj */, ); name = "Build Support"; sourceTree = ""; @@ -957,7 +1384,7 @@ isa = PBXProject; attributes = { BuildIndependentTargetsInParallel = YES; - LastUpgradeCheck = 0600; + LastUpgradeCheck = 0700; }; buildConfigurationList = 1DEB91EF08733DB70010E9CD /* Build configuration list for PBXProject "libdispatch" */; compatibilityVersion = "Xcode 3.2"; @@ -976,6 +1403,10 @@ ProductGroup = C927F36010FD7F1000C5AB8B /* Products */; ProjectRef = C927F35F10FD7F1000C5AB8B /* ddt.xcodeproj */; }, + { + ProductGroup = 4552536F19B1384900B88766 /* Products */; + ProjectRef = 4552536E19B1384900B88766 /* libdispatchtest.xcodeproj */; + }, ); projectRoot = ""; targets = ( @@ -987,11 +1418,355 @@ E46DBC1A14EE10C80001F9F6 /* libdispatch static */, 3F3C9326128E637B0042B1F7 /* libdispatch_Sim */, C927F35A10FD7F0600C5AB8B /* libdispatch_tools */, + 4552540A19B1389700B88766 /* libdispatch_tests */, ); }; /* End PBXProject section */ /* Begin PBXReferenceProxy section */ + 455253A919B1384900B88766 /* libdispatch_test.a */ = { + isa = PBXReferenceProxy; + fileType = archive.ar; + path = libdispatch_test.a; + remoteRef = 455253A819B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253AB19B1384900B88766 /* dispatch_apply */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_apply; + remoteRef = 455253AA19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253AD19B1384900B88766 /* dispatch_api */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_api; + remoteRef = 455253AC19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253AF19B1384900B88766 /* dispatch_c99 */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_c99; + remoteRef = 455253AE19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253B119B1384900B88766 /* dispatch_cf_main */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_cf_main; + remoteRef = 455253B019B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253B319B1384900B88766 /* dispatch_deadname */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_deadname; + remoteRef = 455253B219B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253B519B1384900B88766 /* dispatch_debug */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_debug; + remoteRef = 455253B419B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253B719B1384900B88766 /* dispatch_group */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_group; + remoteRef = 455253B619B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253B919B1384900B88766 /* dispatch_overcommit */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_overcommit; + remoteRef = 455253B819B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253BB19B1384900B88766 /* dispatch_pingpong */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_pingpong; + remoteRef = 455253BA19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253BD19B1384900B88766 /* dispatch_plusplus */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_plusplus; + remoteRef = 455253BC19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253BF19B1384900B88766 /* dispatch_priority */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_priority; + remoteRef = 455253BE19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253C119B1384900B88766 /* dispatch_priority2 */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_priority2; + remoteRef = 455253C019B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253C319B1384900B88766 /* dispatch_concur */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_concur; + remoteRef = 455253C219B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253C519B1384900B88766 /* dispatch_context_for_key */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_context_for_key; + remoteRef = 455253C419B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253C719B1384900B88766 /* dispatch_proc */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_proc; + remoteRef = 455253C619B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253C919B1384900B88766 /* dispatch_queue_finalizer */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_queue_finalizer; + remoteRef = 455253C819B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253CB19B1384900B88766 /* dispatch_read */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_read; + remoteRef = 455253CA19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253CD19B1384900B88766 /* dispatch_read2 */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_read2; + remoteRef = 455253CC19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253CF19B1384900B88766 /* dispatch_after */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_after; + remoteRef = 455253CE19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253D119B1384900B88766 /* dispatch_timer */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_timer; + remoteRef = 455253D019B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253D319B1384900B88766 /* dispatch_timer_short */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_timer_short; + remoteRef = 455253D219B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253D519B1384900B88766 /* dispatch_timer_timeout */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_timer_timeout; + remoteRef = 455253D419B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253D719B1384900B88766 /* dispatch_suspend_timer */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_suspend_timer; + remoteRef = 455253D619B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253D919B1384900B88766 /* dispatch_sema */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_sema; + remoteRef = 455253D819B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253DB19B1384900B88766 /* dispatch_timer_bit31 */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_timer_bit31; + remoteRef = 455253DA19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253DD19B1384900B88766 /* dispatch_timer_bit63 */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_timer_bit63; + remoteRef = 455253DC19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253DF19B1384900B88766 /* dispatch_timer_set_time */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_timer_set_time; + remoteRef = 455253DE19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253E119B1384900B88766 /* dispatch_drift */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_drift; + remoteRef = 455253E019B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253E319B1384900B88766 /* dispatch_starfish */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_starfish; + remoteRef = 455253E219B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253E519B1384900B88766 /* dispatch_cascade */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_cascade; + remoteRef = 455253E419B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253E719B1384900B88766 /* dispatch_readsync */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_readsync; + remoteRef = 455253E619B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253E919B1384900B88766 /* dispatch_sync_on_main */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_sync_on_main; + remoteRef = 455253E819B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253EB19B1384900B88766 /* dispatch_sync_gc */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_sync_gc; + remoteRef = 455253EA19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253ED19B1384900B88766 /* dispatch_apply_gc */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_apply_gc; + remoteRef = 455253EC19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253EF19B1384900B88766 /* dispatch_data */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_data; + remoteRef = 455253EE19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253F119B1384900B88766 /* dispatch_io */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_io; + remoteRef = 455253F019B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253F319B1384900B88766 /* dispatch_io_net */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_io_net; + remoteRef = 455253F219B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253F519B1384900B88766 /* dispatch_vm */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_vm; + remoteRef = 455253F419B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253F719B1384900B88766 /* dispatch_vnode */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_vnode; + remoteRef = 455253F619B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253F919B1384900B88766 /* dispatch_qos */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_qos; + remoteRef = 455253F819B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253FB19B1384900B88766 /* dispatch_select */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_select; + remoteRef = 455253FA19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253FD19B1384900B88766 /* dispatch_transform */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = dispatch_transform; + remoteRef = 455253FC19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 455253FF19B1384900B88766 /* nsoperation */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = nsoperation; + remoteRef = 455253FE19B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 4552540119B1384900B88766 /* cffd */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = cffd; + remoteRef = 4552540019B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 4552540319B1384900B88766 /* bench */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = bench; + remoteRef = 4552540219B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 4552540519B1384900B88766 /* jsgc_bench */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = jsgc_bench; + remoteRef = 4552540419B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 4552540719B1384900B88766 /* async_bench */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = async_bench; + remoteRef = 4552540619B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 4552540919B1384900B88766 /* apply_bench */ = { + isa = PBXReferenceProxy; + fileType = "compiled.mach-o.executable"; + path = apply_bench; + remoteRef = 4552540819B1384900B88766 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; C927F36710FD7F1000C5AB8B /* ddt */ = { isa = PBXReferenceProxy; fileType = "compiled.mach-o.executable"; @@ -1223,6 +1998,7 @@ 96032E4B0F5CC8C700241C5F /* time.c in Sources */, 5AAB45C010D30B79004407EA /* data.c in Sources */, 5A27262610F26F1900751FBC /* io.c in Sources */, + E43A72501AF85BBC00BAA921 /* block.cpp in Sources */, C9C5F80E143C1771006DC718 /* transform.c in Sources */, E4FC3264145F46C9002FBDDB /* object.m in Sources */, 2BBF5A63154B64F5002B20F9 /* allocator.c in Sources */, @@ -1239,6 +2015,7 @@ E46DBC4114EE10C80001F9F6 /* resolver.c in Sources */, E46DBC4214EE10C80001F9F6 /* init.c in Sources */, E46DBC4314EE10C80001F9F6 /* queue.c in Sources */, + E43A72881AF85BE900BAA921 /* block.cpp in Sources */, E46DBC4414EE10C80001F9F6 /* semaphore.c in Sources */, E46DBC4514EE10C80001F9F6 /* once.c in Sources */, E44A8E701805C3E0009FFDB6 /* voucher.c in Sources */, @@ -1272,6 +2049,7 @@ E49F24D2125D57FA0057C971 /* time.c in Sources */, E49F24D3125D57FA0057C971 /* data.c in Sources */, E49F24D4125D57FA0057C971 /* io.c in Sources */, + E43A72841AF85BCB00BAA921 /* block.cpp in Sources */, C93D6165143E190E00EB9023 /* transform.c in Sources */, E4FC3265145F46C9002FBDDB /* object.m in Sources */, 2BBF5A64154B64F5002B20F9 /* allocator.c in Sources */, @@ -1291,6 +2069,7 @@ E4B515C1164B2DA300E003AF /* queue.c in Sources */, E4B515C2164B2DA300E003AF /* semaphore.c in Sources */, E4B515C3164B2DA300E003AF /* once.c in Sources */, + E43A72871AF85BCD00BAA921 /* block.cpp in Sources */, E4B515C4164B2DA300E003AF /* apply.c in Sources */, E4B515C5164B2DA300E003AF /* object.c in Sources */, E4B515C6164B2DA300E003AF /* benchmark.c in Sources */, @@ -1325,6 +2104,7 @@ E4EC11B512514302000DDBD1 /* time.c in Sources */, E4EC11B712514302000DDBD1 /* data.c in Sources */, E4EC11B812514302000DDBD1 /* io.c in Sources */, + E43A72861AF85BCC00BAA921 /* block.cpp in Sources */, C93D6166143E190F00EB9023 /* transform.c in Sources */, E4FC3266145F46C9002FBDDB /* object.m in Sources */, 2BBF5A65154B64F5002B20F9 /* allocator.c in Sources */, @@ -1351,6 +2131,7 @@ E4EC122112514715000DDBD1 /* time.c in Sources */, E4EC122312514715000DDBD1 /* data.c in Sources */, E4EC122412514715000DDBD1 /* io.c in Sources */, + E43A72851AF85BCC00BAA921 /* block.cpp in Sources */, C93D6167143E190F00EB9023 /* transform.c in Sources */, E4FC3267145F46C9002FBDDB /* object.m in Sources */, 2BBF5A66154B64F5002B20F9 /* allocator.c in Sources */, @@ -1362,6 +2143,11 @@ /* End PBXSourcesBuildPhase section */ /* Begin PBXTargetDependency section */ + 4552540F19B138B700B88766 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + name = all; + targetProxy = 4552540E19B138B700B88766 /* PBXContainerItemProxy */; + }; C927F36910FD7F1A00C5AB8B /* PBXTargetDependency */ = { isa = PBXTargetDependency; name = ddt; @@ -1421,6 +2207,20 @@ }; name = Debug; }; + 4552540C19B1389700B88766 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; + 4552540D19B1389700B88766 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; C927F35B10FD7F0600C5AB8B /* Release */ = { isa = XCBuildConfiguration; buildSettings = { @@ -1551,6 +2351,15 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + 4552540B19B1389700B88766 /* Build configuration list for PBXAggregateTarget "libdispatch_tests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 4552540C19B1389700B88766 /* Release */, + 4552540D19B1389700B88766 /* Debug */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; C927F35E10FD7F0B00C5AB8B /* Build configuration list for PBXAggregateTarget "libdispatch_tools" */ = { isa = XCConfigurationList; buildConfigurations = ( diff --git a/man/dispatch_source_create.3 b/man/dispatch_source_create.3 index a17e868..b954bcf 100644 --- a/man/dispatch_source_create.3 +++ b/man/dispatch_source_create.3 @@ -297,7 +297,9 @@ The data returned by .Fn dispatch_source_get_data indicates which of the events in the .Fa mask -were observed. +were observed. Note that because this source type will request notifications on the provided port, it should not be mixed with the use of +.Fn mach_port_request_notification +on the same port. .Pp .Vt DISPATCH_SOURCE_TYPE_MACH_RECV .Pp diff --git a/os/object.h b/os/object.h index 944d331..e07aaec 100644 --- a/os/object.h +++ b/os/object.h @@ -74,12 +74,14 @@ #if OS_OBJECT_USE_OBJC #import #define OS_OBJECT_CLASS(name) OS_##name -#define OS_OBJECT_DECL(name, ...) \ +#define OS_OBJECT_DECL_IMPL(name, ...) \ @protocol OS_OBJECT_CLASS(name) __VA_ARGS__ \ @end \ typedef NSObject *name##_t +#define OS_OBJECT_DECL(name, ...) \ + OS_OBJECT_DECL_IMPL(name, __VA_ARGS__) #define OS_OBJECT_DECL_SUBCLASS(name, super) \ - OS_OBJECT_DECL(name, ) + OS_OBJECT_DECL_IMPL(name, ) #if defined(__has_attribute) #if __has_attribute(ns_returns_retained) #define OS_OBJECT_RETURNS_RETAINED __attribute__((__ns_returns_retained__)) diff --git a/os/object_private.h b/os/object_private.h index f5d3268..0f2f01d 100644 --- a/os/object_private.h +++ b/os/object_private.h @@ -121,6 +121,11 @@ OS_OBJECT_EXPORT OS_OBJECT_NONNULL OS_OBJECT_NOTHROW _os_object_t _os_object_retain(_os_object_t object); +__OSX_AVAILABLE_STARTING(__MAC_10_11,__IPHONE_9_0) +OS_OBJECT_EXPORT OS_OBJECT_NONNULL OS_OBJECT_NOTHROW +_os_object_t +_os_object_retain_with_resurrect(_os_object_t obj); + __OSX_AVAILABLE_STARTING(__MAC_10_8,__IPHONE_6_0) OS_OBJECT_EXPORT OS_OBJECT_NONNULL OS_OBJECT_NOTHROW void diff --git a/private/private.h b/private/private.h index 8fd5abc..46d0e5d 100644 --- a/private/private.h +++ b/private/private.h @@ -64,7 +64,7 @@ #endif /* !__DISPATCH_BUILDING_DISPATCH__ */ // Check that public and private dispatch headers match -#if DISPATCH_API_VERSION != 20140804 // Keep in sync with +#if DISPATCH_API_VERSION != 20141121 // Keep in sync with #error "Dispatch header mismatch between /usr/include and /usr/local/include" #endif diff --git a/private/queue_private.h b/private/queue_private.h index e813036..f2bb691 100644 --- a/private/queue_private.h +++ b/private/queue_private.h @@ -79,6 +79,8 @@ dispatch_queue_attr_make_with_overcommit(dispatch_queue_attr_t attr, * This priority level is intended for user-initiated application activity that * is long-running and CPU or IO intensive and that the user is actively waiting * on, but that should not interfere with interactive use of the application. + * + * This global queue priority level is mapped to QOS_CLASS_UTILITY. */ #define DISPATCH_QUEUE_PRIORITY_NON_INTERACTIVE INT8_MIN @@ -237,6 +239,8 @@ dispatch_pthread_root_queue_create(const char *label, unsigned long flags, * is no guarantee that the specified number will be reached. * Pass 0 to specify that a default pool size determined by the system should * be used. + * NOTE: passing pool_size == 1 does NOT make the pthread root queue equivalent + * to a serial queue. * * @result * The flags argument to pass to dispatch_pthread_root_queue_create(). @@ -328,6 +332,44 @@ dispatch_assert_queue_not(dispatch_queue_t queue); #define dispatch_assert_queue_not_debug(q) dispatch_assert_queue_not(q) #endif +/*! + * @function dispatch_async_enforce_qos_class_f + * + * @abstract + * Submits a function for asynchronous execution on a dispatch queue. + * + * @discussion + * See dispatch_async() for details. The QOS will be enforced as if + * this was called: + * + * dispatch_async(queue, dispatch_block_create(DISPATCH_BLOCK_ENFORCE_QOS_CLASS, ^{ + * work(context); + * }); + * + * + * @param queue + * The target dispatch queue to which the function is submitted. + * The system will hold a reference on the target queue until the function + * has returned. + * The result of passing NULL in this parameter is undefined. + * + * @param context + * The application-defined context parameter to pass to the function. + * + * @param work + * The application-defined function to invoke on the target queue. The first + * parameter passed to this function is the context provided to + * dispatch_async_f(). + * The result of passing NULL in this parameter is undefined. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_11,__IPHONE_9_0) +DISPATCH_EXPORT DISPATCH_NONNULL1 DISPATCH_NONNULL3 DISPATCH_NOTHROW +void +dispatch_async_enforce_qos_class_f(dispatch_queue_t queue, + void *context, + dispatch_function_t work); + + __END_DECLS #endif diff --git a/private/source_private.h b/private/source_private.h index c4ce1d4..e8373ba 100644 --- a/private/source_private.h +++ b/private/source_private.h @@ -32,6 +32,8 @@ #include // for HeaderDoc #endif +__BEGIN_DECLS + /*! * @const DISPATCH_SOURCE_TYPE_TIMER_WITH_AGGREGATE * @discussion A dispatch timer source that is part of a timer aggregate. @@ -108,6 +110,8 @@ DISPATCH_EXPORT const struct dispatch_source_type_s __OSX_AVAILABLE_STARTING(__MAC_10_8, __IPHONE_6_0) DISPATCH_EXPORT const struct dispatch_source_type_s _dispatch_source_type_sock; +__END_DECLS + /*! * @enum dispatch_source_sock_flags_t * diff --git a/private/voucher_activity_private.h b/private/voucher_activity_private.h index c02b236..8a13e76 100644 --- a/private/voucher_activity_private.h +++ b/private/voucher_activity_private.h @@ -27,7 +27,7 @@ #include #endif -#define OS_VOUCHER_ACTIVITY_SPI_VERSION 20140708 +#define OS_VOUCHER_ACTIVITY_SPI_VERSION 20150318 #if OS_VOUCHER_WEAK_IMPORT #define OS_VOUCHER_EXPORT OS_EXPORT OS_WEAK_IMPORT @@ -77,6 +77,9 @@ OS_ENUM(voucher_activity_tracepoint_type, uint8_t, OS_ENUM(voucher_activity_flag, unsigned long, voucher_activity_flag_default = 0, voucher_activity_flag_force = 0x1, + voucher_activity_flag_debug = 0x2, + voucher_activity_flag_persist = 0x4, + voucher_activity_flag_stream = 0x8, ); /*! @@ -277,6 +280,47 @@ uint64_t voucher_activity_trace(voucher_activity_trace_id_t trace_id, uint64_t location, void *buffer, size_t length); +/*! + * @function voucher_activity_trace_strings + * + * @abstract + * Add a tracepoint with strings data to trace buffer of the current activity. + * + * @param trace_id + * Tracepoint identifier returned by voucher_activity_trace_id() + * + * @param location + * Tracepoint location. + * + * @param buffer + * Pointer to packed buffer of tracepoint data. + * + * @param length + * Length of data at 'buffer'. + * + * @param strings + * NULL-terminated array of strings data. + * + * @param string_lengths + * Array of string lengths (required to have the same number of elements as the + * 'strings' array): string_lengths[i] is the maximum number of characters to + * copy from strings[i], excluding the NUL-terminator (may be smaller than the + * length of the string present in strings[i]). + * + * @param total_strings_size + * Total size of all strings data to be copied from strings array (including + * all NUL-terminators). + * + * @result + * Timestamp recorded in tracepoint or 0 if no tracepoint was recorded. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_11,__IPHONE_9_0) +OS_VOUCHER_EXPORT OS_NOTHROW +uint64_t +voucher_activity_trace_strings(voucher_activity_trace_id_t trace_id, + uint64_t location, void *buffer, size_t length, const char *strings[], + size_t string_lengths[], size_t total_strings_size); + /*! * @function voucher_activity_trace_args * @@ -349,7 +393,7 @@ voucher_activity_mode_t voucher_activity_get_mode(void); /*! - * @function voucher_activity_set_mode_4libtrace(void) + * @function voucher_activity_set_mode_4libtrace * * @abstract * Set the current mode of voucher activity subsystem. @@ -400,6 +444,7 @@ OS_ENUM(_voucher_activity_tracepoint_flag, uint16_t, _voucher_activity_trace_flag_buffer_empty = 0, _voucher_activity_trace_flag_tracepoint = (1u << 0), _voucher_activity_trace_flag_tracepoint_args = (1u << 1), + _voucher_activity_trace_flag_tracepoint_strings = (1u << 2), _voucher_activity_trace_flag_wide_first = (1u << 6), _voucher_activity_trace_flag_wide_second = (1u << 6) | (1u << 7), _voucher_activity_trace_flag_start = (1u << 8), @@ -426,7 +471,13 @@ typedef struct _voucher_activity_tracepoint_s { uint64_t vat_thread; // pthread_t uint64_t vat_timestamp; // absolute time uint64_t vat_location; // tracepoint PC - uint64_t vat_data[4]; // trace data + union { + uint64_t vat_data[4]; // trace data + struct { + uint16_t vats_offset; // offset to string data (from buffer end) + uint8_t vats_data[30]; // trace data + } vat_stroff; // iff _vat_flag_tracepoint_strings present + }; } *_voucher_activity_tracepoint_t; /*! @@ -439,20 +490,33 @@ typedef struct _voucher_activity_tracepoint_s { #include #include -static const atm_subaid32_t _voucher_default_activity_subid = - ATM_SUBAID32_MAX-1; - static const size_t _voucher_activity_buffer_size = 4096; static const size_t _voucher_activity_tracepoints_per_buffer = _voucher_activity_buffer_size / sizeof(struct _voucher_activity_tracepoint_s); +static const size_t _voucher_activity_buffer_header_size = + sizeof(struct _voucher_activity_tracepoint_s); +static const size_t _voucher_activity_strings_header_size = 0; // TODO + typedef uint8_t _voucher_activity_buffer_t[_voucher_activity_buffer_size]; +static const size_t _voucher_activity_buffers_per_heap = 512; +typedef unsigned long _voucher_activity_bitmap_base_t; +static const size_t _voucher_activity_bits_per_bitmap_base_t = + 8 * sizeof(_voucher_activity_bitmap_base_t); +static const size_t _voucher_activity_bitmaps_per_heap = + _voucher_activity_buffers_per_heap / + _voucher_activity_bits_per_bitmap_base_t; +typedef _voucher_activity_bitmap_base_t + _voucher_activity_bitmap_t[_voucher_activity_bitmaps_per_heap] + __attribute__((__aligned__(64))); + struct _voucher_activity_self_metadata_s { struct _voucher_activity_metadata_opaque_s *vasm_baseaddr; + _voucher_activity_bitmap_t volatile vam_buffer_bitmap; }; + typedef struct _voucher_activity_metadata_opaque_s { - _voucher_activity_buffer_t vam_kernel_metadata; _voucher_activity_buffer_t vam_client_metadata; union { struct _voucher_activity_self_metadata_s vam_self_metadata; @@ -462,80 +526,91 @@ typedef struct _voucher_activity_metadata_opaque_s { typedef os_lock_handoff_s _voucher_activity_lock_s; -typedef struct _voucher_atm_s { - int32_t volatile vatm_refcnt; - mach_voucher_t vatm_kvoucher; - atm_aid_t vatm_id; - atm_mailbox_offset_t vatm_mailbox_offset; - TAILQ_ENTRY(_voucher_atm_s) vatm_list; -#if __LP64__ - uintptr_t vatm_pad[3]; - // cacheline -#endif - _voucher_activity_lock_s vatm_activities_lock; - TAILQ_HEAD(_voucher_atm_activities_s, _voucher_activity_s) vatm_activities; - TAILQ_HEAD(, _voucher_activity_s) vatm_used_activities; -} *_voucher_atm_t; +OS_ENUM(_voucher_activity_buffer_atomic_flags, uint8_t, + _voucher_activity_buffer_full = (1u << 0), + _voucher_activity_buffer_pushing = (1u << 1), +); + +typedef union { + uint64_t vabp_atomic_pos; + struct { + uint16_t vabp_refcnt; + uint8_t vabp_flags; + uint8_t vabp_unused; + uint16_t vabp_next_tracepoint_idx; + uint16_t vabp_string_offset; // offset from the _end_ of the buffer + } vabp_pos; +} _voucher_activity_buffer_position_u; // must match layout of _voucher_activity_tracepoint_s typedef struct _voucher_activity_buffer_header_s { uint16_t vabh_flags; // _voucher_activity_trace_flag_buffer_header - uint8_t vabh_unused[6]; - uint64_t vabh_thread; - uint64_t vabh_timestamp; - uint32_t volatile vabh_next_tracepoint_idx; - uint32_t vabh_sequence_no; + uint8_t vat_type; + uint8_t vat_namespace; + uint32_t vat_code; + uint64_t vat_thread; + uint64_t vat_timestamp; + uint64_t vat_location; voucher_activity_id_t vabh_activity_id; - uint64_t vabh_reserved; + _voucher_activity_buffer_position_u volatile vabh_pos; TAILQ_ENTRY(_voucher_activity_buffer_header_s) vabh_list; } *_voucher_activity_buffer_header_t; -// must match layout of _voucher_activity_buffer_header_s -typedef struct _voucher_activity_s { - // first tracepoint entry - // must match layout of _voucher_activity_tracepoint_s - uint16_t va_flags; // _voucher_activity_trace_flag_buffer_header | - // _voucher_activity_trace_flag_activity | - // _voucher_activity_trace_flag_start | - // _voucher_activity_trace_flag_wide_first - uint8_t va_type; - uint8_t va_namespace; - uint32_t va_code; - uint64_t va_thread; - uint64_t va_timestamp; - uint32_t volatile vabh_next_tracepoint_idx; - uint32_t volatile va_max_sequence_no; - voucher_activity_id_t va_id; - int32_t volatile va_use_count; - uint32_t va_buffer_limit; - TAILQ_HEAD(_voucher_activity_buffer_list_s, - _voucher_activity_buffer_header_s) va_buffers; -#if !__LP64__ - uint64_t va_pad; -#endif +/*! + * @enum _voucher_activity_buffer_hook_reason + * + * @constant _voucher_activity_buffer_hook_reason_full + * Specified activity buffer is full. + * Will be reported reused or freed later. + * + * @constant _voucher_activity_buffer_hook_reason_reuse + * Specified activity buffer is about to be reused. + * Was previously reported as full. + * + * @constant _voucher_activity_buffer_hook_reason_free + * Specified activity buffer is about to be freed. + * May have been previously reported as full or may be only partially filled. + */ +typedef enum _voucher_activity_buffer_hook_reason { + _voucher_activity_buffer_hook_reason_full = 0x1, + _voucher_activity_buffer_hook_reason_reuse = 0x2, + _voucher_activity_buffer_hook_reason_free = 0x4, +} _voucher_activity_buffer_hook_reason; - // second tracepoint entry - // must match layout of _voucher_activity_tracepoint_s - uint16_t va_flags2; - uint8_t va_unused2[2]; - int32_t volatile va_refcnt; - uint64_t va_location; - _voucher_activity_buffer_header_t volatile va_current_buffer; - _voucher_atm_t va_atm; - _voucher_activity_lock_s va_buffers_lock; - uintptr_t va_pad2[2]; - -#if __LP64__ - // third tracepoint entry - // must match layout of _voucher_activity_tracepoint_s - uint16_t va_flags3; - uint8_t va_unused3[6]; - uintptr_t va_pad3; -#endif - TAILQ_ENTRY(_voucher_activity_s) va_list; - TAILQ_ENTRY(_voucher_activity_s) va_atm_list; - TAILQ_ENTRY(_voucher_activity_s) va_atm_used_list; -} *_voucher_activity_t; +/*! + * @typedef _voucher_activity_buffer_hook_t + * + * @abstract + * A function pointer called when an activity buffer is full or being freed. + * NOTE: callbacks occur under an activity-wide handoff lock and work done + * inside the callback function must not block or otherwise cause that lock to + * be held for a extended period of time. + * + * @param reason + * Reason for callback. + * + * @param buffer + * Pointer to activity buffer. + */ +typedef void (*_voucher_activity_buffer_hook_t)( + _voucher_activity_buffer_hook_reason reason, + _voucher_activity_buffer_header_t buffer); + +/*! + * @function voucher_activity_buffer_hook_install_4libtrace + * + * @abstract + * Install activity buffer hook callback function. + * Must be called from the libtrace initializer, and at most once. + * + * @param hook + * Hook function to install. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_11,__IPHONE_9_0) +OS_VOUCHER_EXPORT OS_NOTHROW +void +voucher_activity_buffer_hook_install_4libtrace( + _voucher_activity_buffer_hook_t hook); #endif // OS_VOUCHER_ACTIVITY_BUFFER_SPI diff --git a/private/voucher_private.h b/private/voucher_private.h index fcc28f1..e4c31a6 100644 --- a/private/voucher_private.h +++ b/private/voucher_private.h @@ -24,7 +24,7 @@ #include #include -#define OS_VOUCHER_SPI_VERSION 20140425 +#define OS_VOUCHER_SPI_VERSION 20141203 #if OS_VOUCHER_WEAK_IMPORT #define OS_VOUCHER_EXPORT OS_EXPORT OS_WEAK_IMPORT @@ -161,7 +161,7 @@ void voucher_decrement_importance_count4CF(voucher_t voucher); /*! - * @group Dispatch block objects + * @group Voucher dispatch block SPI */ #ifndef __DISPATCH_BUILDING_DISPATCH__ @@ -334,6 +334,66 @@ dispatch_block_create_with_voucher_and_qos_class(dispatch_block_flags_t flags, voucher_t voucher, dispatch_qos_class_t qos_class, int relative_priority, dispatch_block_t block); +/*! + * @group Voucher dispatch queue SPI + */ + +/*! + * @function dispatch_queue_create_with_accounting_override_voucher + * + * @abstract + * Creates a new dispatch queue with an accounting override voucher created + * from the specified voucher. + * + * @discussion + * See dispatch_queue_create() headerdoc for generic details on queue creation. + * + * The resource accounting attributes of the specified voucher are extracted + * and used to create an accounting override voucher for the new queue. + * + * Every block executed on the returned queue will initially have this override + * voucher adopted, any voucher automatically associated with or explicitly + * assigned to the block will NOT be used and released immediately before block + * execution starts. + * + * The accounting override voucher will be automatically propagated to any + * asynchronous work generated from the queue following standard voucher + * propagation rules. + * + * NOTE: this SPI should only be used in special circumstances when a subsystem + * has complete control over all workitems submitted to a queue (e.g. no client + * block is ever submitted to the queue) and if and only if such queues have a + * one-to-one mapping with resource accounting identities. + * + * CAUTION: use of this SPI represents a potential voucher propagation hole. It + * is the responsibility of the caller to ensure that any callbacks into client + * code from the queue have the correct client voucher applied (rather than the + * automatically propagated accounting override voucher), e.g. by use of the + * dispatch_block_create() API to capture client state at the time the callback + * is registered. + * + * @param label + * A string label to attach to the queue. + * This parameter is optional and may be NULL. + * + * @param attr + * DISPATCH_QUEUE_SERIAL, DISPATCH_QUEUE_CONCURRENT, or the result of a call to + * the function dispatch_queue_attr_make_with_qos_class(). + * + * @param voucher + * A voucher whose resource accounting attributes are used to create the + * accounting override voucher attached to the queue. + * + * @result + * The newly created dispatch queue. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_11,__IPHONE_9_0) +DISPATCH_EXPORT DISPATCH_MALLOC DISPATCH_RETURNS_RETAINED DISPATCH_WARN_RESULT +DISPATCH_NOTHROW +dispatch_queue_t +dispatch_queue_create_with_accounting_override_voucher(const char *label, + dispatch_queue_attr_t attr, voucher_t voucher); + /*! * @group Voucher Mach SPI * SPI intended for clients that need to interact with mach messages or mach diff --git a/src/allocator.c b/src/allocator.c index af1f3c1..d6db272 100644 --- a/src/allocator.c +++ b/src/allocator.c @@ -643,6 +643,12 @@ _dispatch_alloc_init(void) sizeof(((struct dispatch_magazine_s *)0x0)->fp_conts) == DISPATCH_ALLOCATOR_PAGE_SIZE); #endif // PACK_FIRST_PAGE_WITH_CONTINUATIONS + // Make sure our alignment will be correct: that is, that we are correctly + // aligning to both. + dispatch_assert(ROUND_UP_TO_BITMAP_ALIGNMENT(ROUND_UP_TO_BITMAP_ALIGNMENT_AND_CONTINUATION_SIZE(1)) == + ROUND_UP_TO_BITMAP_ALIGNMENT_AND_CONTINUATION_SIZE(1)); + dispatch_assert(ROUND_UP_TO_CONTINUATION_SIZE(ROUND_UP_TO_BITMAP_ALIGNMENT_AND_CONTINUATION_SIZE(1)) == + ROUND_UP_TO_BITMAP_ALIGNMENT_AND_CONTINUATION_SIZE(1)); } #elif (DISPATCH_ALLOCATOR && DISPATCH_CONTINUATION_MALLOC) \ || (DISPATCH_CONTINUATION_MALLOC && DISPATCH_USE_MALLOCZONE) diff --git a/src/allocator_internal.h b/src/allocator_internal.h index f4c8ba0..893ba82 100644 --- a/src/allocator_internal.h +++ b/src/allocator_internal.h @@ -141,6 +141,16 @@ typedef unsigned long bitmap_t; #define HEAP_MASK (~(uintptr_t)(BYTES_PER_HEAP - 1)) #define MAGAZINE_MASK (~(uintptr_t)(BYTES_PER_MAGAZINE - 1)) +// this will round up such that first_bitmap_in_same_page() can mask the address +// of a bitmap_t in the maps to obtain the first bitmap for that same page +#define ROUND_UP_TO_BITMAP_ALIGNMENT(x) \ + (((x) + ((BITMAPS_PER_PAGE * BYTES_PER_BITMAP) - 1u)) & \ + ~((BITMAPS_PER_PAGE * BYTES_PER_BITMAP) - 1u)) +// Since these are both powers of two, we end up with not only the max alignment, +// but happily the least common multiple, which will be the greater of the two. +#define ROUND_UP_TO_BITMAP_ALIGNMENT_AND_CONTINUATION_SIZE(x) (ROUND_UP_TO_CONTINUATION_SIZE(ROUND_UP_TO_BITMAP_ALIGNMENT(x))) +#define PADDING_TO_BITMAP_ALIGNMENT_AND_CONTINUATION_SIZE(x) (ROUND_UP_TO_BITMAP_ALIGNMENT_AND_CONTINUATION_SIZE(x) - (x)) + #define PADDING_TO_CONTINUATION_SIZE(x) (ROUND_UP_TO_CONTINUATION_SIZE(x) - (x)) #if defined(__LP64__) @@ -155,8 +165,11 @@ typedef unsigned long bitmap_t; // header is expected to end on supermap's required alignment #define HEADER_TO_SUPERMAPS_PADDING 0 -#define SUPERMAPS_TO_MAPS_PADDING (PADDING_TO_CONTINUATION_SIZE( \ +// we want to align the maps to a continuation size, but we must also have proper padding +// so that we can perform first_bitmap_in_same_page() +#define SUPERMAPS_TO_MAPS_PADDING (PADDING_TO_BITMAP_ALIGNMENT_AND_CONTINUATION_SIZE( \ SIZEOF_SUPERMAPS + HEADER_TO_SUPERMAPS_PADDING + SIZEOF_HEADER)) + #define MAPS_TO_FPMAPS_PADDING (PADDING_TO_CONTINUATION_SIZE(SIZEOF_MAPS)) #define BYTES_LEFT_IN_FIRST_PAGE (BYTES_PER_PAGE - \ diff --git a/src/block.cpp b/src/block.cpp new file mode 100644 index 0000000..83fff54 --- /dev/null +++ b/src/block.cpp @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2015 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +#ifdef __BLOCKS__ + +#if __cplusplus < 201103L +#error Must build with C++11 or later +#endif + +#if __has_feature(cxx_exceptions) +#error Must build without C++ exceptions +#endif + +extern "C" { +#include "internal.h" +} + +#if DISPATCH_DEBUG && DISPATCH_BLOCK_PRIVATE_DATA_DEBUG +#define _dispatch_block_private_data_debug(msg, ...) \ + _dispatch_debug("block_private[%p]: " msg, (this), ##__VA_ARGS__) +#else +#define _dispatch_block_private_data_debug(msg, ...) +#endif + +#pragma mark - +#pragma mark _dispatch_block_create + +// rdar://20766742 C++ helpers to enable block capture of vouchers and groups + +struct dispatch_block_private_data_s { + DISPATCH_BLOCK_PRIVATE_DATA_HEADER(); + static void* operator new(size_t) = delete; + static void* operator new [] (size_t) = delete; + explicit inline DISPATCH_ALWAYS_INLINE dispatch_block_private_data_s( + dispatch_block_flags_t flags, voucher_t voucher, + pthread_priority_t priority, dispatch_block_t block) noexcept : + dbpd_magic(), dbpd_flags(flags), dbpd_atomic_flags(), + dbpd_performed(), dbpd_priority(priority), dbpd_voucher(voucher), + dbpd_block(block), dbpd_group(), dbpd_queue(), dbpd_thread() + { + // stack structure constructor, no releases on destruction + _dispatch_block_private_data_debug("create, block: %p", dbpd_block); + } + inline DISPATCH_ALWAYS_INLINE dispatch_block_private_data_s( + dispatch_block_private_data_s const &o) noexcept : + dbpd_magic(DISPATCH_BLOCK_PRIVATE_DATA_MAGIC), + dbpd_flags(o.dbpd_flags), dbpd_atomic_flags(), dbpd_performed(), + dbpd_priority(o.dbpd_priority), dbpd_voucher(o.dbpd_voucher), + dbpd_block(), dbpd_group(), dbpd_queue(), dbpd_thread() + { + // copy constructor, create copy with retained references + if (dbpd_voucher) voucher_retain(dbpd_voucher); + if (o.dbpd_block) dbpd_block = _dispatch_Block_copy(o.dbpd_block); + _dispatch_block_private_data_debug("copy from %p, block: %p from %p", + &o, dbpd_block, o.dbpd_block); + if (!o.dbpd_magic) return; // No group in initial copy of stack object + dbpd_group = _dispatch_group_create_and_enter(); + } + inline DISPATCH_ALWAYS_INLINE ~dispatch_block_private_data_s() noexcept + { + _dispatch_block_private_data_debug("destroy%s, block: %p", + dbpd_magic ? "" : " (stack)", dbpd_block); + if (dbpd_magic != DISPATCH_BLOCK_PRIVATE_DATA_MAGIC) return; + if (dbpd_group) { + if (!dbpd_performed) dispatch_group_leave(dbpd_group); + ((void (*)(dispatch_group_t))dispatch_release)(dbpd_group); + } + if (dbpd_block) Block_release(dbpd_block); + if (dbpd_voucher) voucher_release(dbpd_voucher); + } +}; + +dispatch_block_t +_dispatch_block_create(dispatch_block_flags_t flags, voucher_t voucher, + pthread_priority_t pri, dispatch_block_t block) +{ + struct dispatch_block_private_data_s dbpds(flags, voucher, pri, block); + return _dispatch_Block_copy(^{ + // Capture stack object: invokes copy constructor (17094902) + (void)dbpds; + _dispatch_block_invoke(&dbpds); + }); +} + +extern "C" { +// The compiler hides the name of the function it generates, and changes it if +// we try to reference it directly, but the linker still sees it. +extern void DISPATCH_BLOCK_SPECIAL_INVOKE(void *) + asm("____dispatch_block_create_block_invoke"); +void (*_dispatch_block_special_invoke)(void*) = DISPATCH_BLOCK_SPECIAL_INVOKE; +} + +#endif // __BLOCKS__ diff --git a/src/data.c b/src/data.c index feb6012..e65399f 100644 --- a/src/data.c +++ b/src/data.c @@ -20,14 +20,77 @@ #include "internal.h" -// Dispatch data objects are dispatch objects with standard retain/release -// memory management. A dispatch data object either points to a number of other -// dispatch data objects or is a leaf data object. A leaf data object contains -// a pointer to represented memory. A composite data object specifies the total -// size of data it represents and list of constituent records. -// -// A leaf data object always points to a full represented buffer, a composite -// dispatch data object is needed to represent a subrange of a memory region. +/* + * Dispatch data objects are dispatch objects with standard retain/release + * memory management. A dispatch data object either points to a number of other + * dispatch data objects or is a leaf data object. + * A composite data object specifies the total size of data it represents + * and list of constituent records. + * + ******************************************************************************* + * + * CURRENT IMPLEMENTATION DETAILS + * + * There are actually 3 kinds of composite objects + * - trivial subranges + * - unflattened composite data objects + * - flattened composite data objects + * + * LEAVES (num_records == 0, destructor != nil) + * + * Those objects have a pointer to represented memory in `buf`. + * + * UNFLATTENED (num_records > 1, buf == nil, destructor == nil) + * + * This is the generic case of a composite object. + * + * FLATTENED (num_records > 1, buf != nil, destructor == nil) + * + * Those objects are non trivial composite objects whose `buf` pointer + * is a contiguous representation (copied) of the memory it represents. + * + * Such objects are created when used as an NSData and -bytes is called and + * where the dispatch data object is an unflattened composite object. + * The underlying implementation is _dispatch_data_get_flattened_bytes + * + * TRIVIAL SUBRANGES (num_records == 1, buf == nil, destructor == nil) + * + * Those objects point to a single leaf, never to flattened objects. + * + ******************************************************************************* + * + * Non trivial invariants: + * + * It is forbidden to point into a composite data object and ignore entire + * records from it. (for example by having `from` longer than the first + * record length). + * + * dispatch_data_t's are either leaves, or composite objects pointing to + * leaves. Depth is never greater than 1. + * + ******************************************************************************* + * + * There are 4 dispatch_data_t constructors who may create non leaf objects, + * and ensure proper invariants. + * + * dispatch_data_copy_region() + * This function first sees through trivial subranges, and may in turn + * generate new trivial subranges. + * + * dispatch_data_create_map() + * This function either returns existing data objects, or a leaf. + * + * dispatch_data_create_subrange() + * This function treats flattened objects like unflattened ones, + * and recurses into trivial subranges, it can create trivial subranges. + * + * dispatch_data_create_concat() + * This function unwraps the top-level composite objects, trivial or not, + * and else concatenates the two arguments range lists, hence always creating + * unflattened objects, unless one of the arguments was empty. + * + ******************************************************************************* + */ #if USE_OBJC #define _dispatch_data_retain(x) _dispatch_objc_retain(x) @@ -68,7 +131,7 @@ _dispatch_data_alloc(size_t n, size_t extra) { dispatch_data_t data = _dispatch_alloc(DISPATCH_DATA_CLASS, sizeof(struct dispatch_data_s) + extra + - (n ? n * sizeof(range_record) - sizeof(data->buf) : 0)); + n * sizeof(range_record)); data->num_records = n; #if !USE_OBJC data->do_targetq = dispatch_get_global_queue( @@ -107,10 +170,6 @@ _dispatch_data_init(dispatch_data_t data, const void *buffer, size_t size, data->buf = buffer; data->size = size; data->destructor = destructor; -#if DISPATCH_DATA_USE_LEAF_MEMBER - data->leaf = true; - data->num_records = 1; -#endif if (queue) { _dispatch_retain(queue); data->do_targetq = queue; @@ -210,15 +269,15 @@ out: void _dispatch_data_dispose(dispatch_data_t dd) { - dispatch_block_t destructor = dd->destructor; - if (destructor == NULL) { + if (_dispatch_data_leaf(dd)) { + _dispatch_data_destroy_buffer(dd->buf, dd->size, dd->do_targetq, + dd->destructor); + } else { size_t i; for (i = 0; i < _dispatch_data_num_records(dd); ++i) { _dispatch_data_release(dd->records[i].data_object); } - } else { - _dispatch_data_destroy_buffer(dd->buf, dd->size, dd->do_targetq, - destructor); + free((void *)dd->buf); } } @@ -234,6 +293,10 @@ _dispatch_data_debug(dispatch_data_t dd, char* buf, size_t bufsiz) offset += dsnprintf(&buf[offset], bufsiz - offset, "composite, size = %zd, num_records = %zd ", dd->size, _dispatch_data_num_records(dd)); + if (dd->buf) { + offset += dsnprintf(&buf[offset], bufsiz - offset, + ", flatbuf = %p ", dd->buf); + } size_t i; for (i = 0; i < _dispatch_data_num_records(dd); ++i) { range_record r = dd->records[i]; @@ -264,6 +327,7 @@ dispatch_data_create_concat(dispatch_data_t dd1, dispatch_data_t dd2) _dispatch_data_retain(dd1); return dd1; } + data = _dispatch_data_alloc(_dispatch_data_num_records(dd1) + _dispatch_data_num_records(dd2), 0); data->size = dd1->size + dd2->size; @@ -305,6 +369,13 @@ dispatch_data_create_subrange(dispatch_data_t dd, size_t offset, _dispatch_data_retain(dd); return dd; } + /* + * we must only optimize leaves and not flattened objects + * because lots of users want to keep the end of a buffer and release + * as much memory as they can from the beginning of it + * + * Using the flatbuf here would be very wrong with respect to that goal + */ if (_dispatch_data_leaf(dd)) { data = _dispatch_data_alloc(1, 0); data->size = length; @@ -314,36 +385,88 @@ dispatch_data_create_subrange(dispatch_data_t dd, size_t offset, _dispatch_data_retain(dd); return data; } - // Subrange of a composite dispatch data object: find the record containing - // the specified offset - data = dispatch_data_empty; - size_t i = 0, bytes_left = length; - while (i < _dispatch_data_num_records(dd) && - offset >= dd->records[i].length) { + + // Subrange of a composite dispatch data object + const size_t dd_num_records = _dispatch_data_num_records(dd); + bool to_the_end = (offset + length == dd->size); + size_t i = 0; + + // find the record containing the specified offset + while (i < dd_num_records && offset >= dd->records[i].length) { offset -= dd->records[i++].length; } - while (i < _dispatch_data_num_records(dd)) { - size_t record_len = dd->records[i].length - offset; - if (record_len > bytes_left) { - record_len = bytes_left; - } - dispatch_data_t subrange = dispatch_data_create_subrange( - dd->records[i].data_object, dd->records[i].from + offset, - record_len); - dispatch_data_t concat = dispatch_data_create_concat(data, subrange); - _dispatch_data_release(data); - _dispatch_data_release(subrange); - data = concat; - bytes_left -= record_len; - if (!bytes_left) { - return data; + + // Crashing here indicates memory corruption of passed in data object + if (slowpath(i >= dd_num_records)) { + DISPATCH_CRASH("dispatch_data_create_subrange out of bounds"); + return NULL; + } + + // if everything is from a single dispatch data object, avoid boxing it + if (offset + length <= dd->records[i].length) { + return dispatch_data_create_subrange(dd->records[i].data_object, + dd->records[i].from + offset, length); + } + + // find the record containing the end of the current range + // and optimize the case when you just remove bytes at the origin + size_t count, last_length; + + if (to_the_end) { + count = dd_num_records - i; + } else { + last_length = length - (dd->records[i].length - offset); + count = 1; + + while (i + count < dd_num_records) { + size_t record_length = dd->records[i + count++].length; + + if (last_length <= record_length) { + break; + } + last_length -= record_length; + + // Crashing here indicates memory corruption of passed in data object + if (slowpath(i + count >= dd_num_records)) { + DISPATCH_CRASH("dispatch_data_create_subrange out of bounds"); + return NULL; + } } - offset = 0; - i++; } - // Crashing here indicates memory corruption of passed in data object - DISPATCH_CRASH("dispatch_data_create_subrange out of bounds"); - return NULL; + + data = _dispatch_data_alloc(count, 0); + data->size = length; + memcpy(data->records, dd->records + i, count * sizeof(range_record)); + + if (offset) { + data->records[0].from += offset; + data->records[0].length -= offset; + } + if (!to_the_end) { + data->records[count - 1].length = last_length; + } + + for (i = 0; i < count; i++) { + _dispatch_data_retain(data->records[i].data_object); + } + return data; +} + +static void* +_dispatch_data_flatten(dispatch_data_t dd) +{ + void *buffer = malloc(dd->size); + + // Composite data object, copy the represented buffers + if (buffer) { + dispatch_data_apply(dd, ^(dispatch_data_t region DISPATCH_UNUSED, + size_t off, const void* buf, size_t len) { + memcpy(buffer + off, buf, len); + return (bool)true; + }); + } + + return buffer; } // When mapping a leaf object or a subrange of a leaf object, return a direct @@ -354,37 +477,30 @@ dispatch_data_t dispatch_data_create_map(dispatch_data_t dd, const void **buffer_ptr, size_t *size_ptr) { - dispatch_data_t data = dd; + dispatch_data_t data = NULL; const void *buffer = NULL; - size_t size = dd->size, offset = 0; + size_t size = dd->size; + if (!size) { data = dispatch_data_empty; goto out; } - if (!_dispatch_data_leaf(dd) && _dispatch_data_num_records(dd) == 1 && - _dispatch_data_leaf(dd->records[0].data_object)) { - offset = dd->records[0].from; - dd = dd->records[0].data_object; - } - if (_dispatch_data_leaf(dd)) { - _dispatch_data_retain(data); - buffer = dd->buf + offset; + + buffer = _dispatch_data_map_direct(dd, 0, NULL, NULL); + if (buffer) { + _dispatch_data_retain(dd); + data = dd; goto out; } - // Composite data object, copy the represented buffers - buffer = malloc(size); - if (!buffer) { - data = NULL; + + buffer = _dispatch_data_flatten(dd); + if (fastpath(buffer)) { + data = dispatch_data_create(buffer, size, NULL, + DISPATCH_DATA_DESTRUCTOR_FREE); + } else { size = 0; - goto out; } - dispatch_data_apply(dd, ^(dispatch_data_t region DISPATCH_UNUSED, - size_t off, const void* buf, size_t len) { - memcpy((void*)buffer + off, buf, len); - return (bool)true; - }); - data = dispatch_data_create(buffer, size, NULL, - DISPATCH_DATA_DESTRUCTOR_FREE); + out: if (buffer_ptr) { *buffer_ptr = buffer; @@ -395,24 +511,63 @@ out: return data; } +const void * +_dispatch_data_get_flattened_bytes(dispatch_data_t dd) +{ + const void *buffer; + size_t offset = 0; + + if (slowpath(!dd->size)) { + return NULL; + } + + buffer = _dispatch_data_map_direct(dd, 0, &dd, &offset); + if (buffer) { + return buffer; + } + + void *flatbuf = _dispatch_data_flatten(dd); + if (fastpath(flatbuf)) { + // we need a release so that readers see the content of the buffer + if (slowpath(!dispatch_atomic_cmpxchgv2o(dd, buf, NULL, flatbuf, + &buffer, release))) { + free(flatbuf); + } else { + buffer = flatbuf; + } + } else { + return NULL; + } + + return buffer + offset; +} + +#if DISPATCH_USE_CLIENT_CALLOUT +DISPATCH_NOINLINE +#else +DISPATCH_ALWAYS_INLINE +#endif +static bool +_dispatch_data_apply_client_callout(void *ctxt, dispatch_data_t region, size_t offset, + const void *buffer, size_t size, dispatch_data_applier_function_t f) +{ + return f(ctxt, region, offset, buffer, size); +} + + static bool _dispatch_data_apply(dispatch_data_t dd, size_t offset, size_t from, size_t size, void *ctxt, dispatch_data_applier_function_t applier) { bool result = true; - dispatch_data_t data = dd; const void *buffer; - dispatch_assert(dd->size); - if (!_dispatch_data_leaf(dd) && _dispatch_data_num_records(dd) == 1 && - _dispatch_data_leaf(dd->records[0].data_object)) { - from = dd->records[0].from; - dd = dd->records[0].data_object; - } - if (_dispatch_data_leaf(dd)) { - buffer = dd->buf + from; - return _dispatch_client_callout3(ctxt, data, offset, buffer, size, - applier); + + buffer = _dispatch_data_map_direct(dd, 0, NULL, NULL); + if (buffer) { + return _dispatch_data_apply_client_callout(ctxt, dd, + offset, buffer + from, size, applier); } + size_t i; for (i = 0; i < _dispatch_data_num_records(dd) && result; ++i) { result = _dispatch_data_apply(dd->records[i].data_object, @@ -443,58 +598,73 @@ dispatch_data_apply(dispatch_data_t dd, dispatch_data_applier_t applier) (dispatch_data_applier_function_t)_dispatch_Block_invoke(applier)); } +static dispatch_data_t +_dispatch_data_copy_region(dispatch_data_t dd, size_t from, size_t size, + size_t location, size_t *offset_ptr) +{ + dispatch_data_t reusable_dd = NULL; + size_t offset = 0; + + if (from == 0 && size == dd->size) { + reusable_dd = dd; + } + + if (_dispatch_data_map_direct(dd, from, &dd, &from)) { + if (reusable_dd) { + _dispatch_data_retain(reusable_dd); + return reusable_dd; + } + + _dispatch_data_retain(dd); + if (from == 0 && size == dd->size) { + return dd; + } + + dispatch_data_t data = _dispatch_data_alloc(1, 0); + data->size = size; + data->records[0].from = from; + data->records[0].length = size; + data->records[0].data_object = dd; + return data; + } + + size_t i; + for (i = 0; i < _dispatch_data_num_records(dd); ++i) { + size_t length = dd->records[i].length; + + if (from >= length) { + from -= length; + continue; + } + + length -= from; + if (location >= offset + length) { + offset += length; + from = 0; + continue; + } + + from += dd->records[i].from; + dd = dd->records[i].data_object; + *offset_ptr += offset; + location -= offset; + return _dispatch_data_copy_region(dd, from, length, location, offset_ptr); + } + + DISPATCH_CRASH("dispatch_data_copy_region out of bounds"); +} + // Returs either a leaf object or an object composed of a single leaf object dispatch_data_t dispatch_data_copy_region(dispatch_data_t dd, size_t location, size_t *offset_ptr) { if (location >= dd->size) { - *offset_ptr = 0; + *offset_ptr = dd->size; return dispatch_data_empty; } - dispatch_data_t data; - size_t size = dd->size, offset = 0, from = 0; - while (true) { - if (_dispatch_data_leaf(dd)) { - _dispatch_data_retain(dd); - *offset_ptr = offset; - if (size == dd->size) { - return dd; - } else { - // Create a new object for the requested subrange of the leaf - data = _dispatch_data_alloc(1, 0); - data->size = size; - data->records[0].from = from; - data->records[0].length = size; - data->records[0].data_object = dd; - return data; - } - } else { - // Find record at the specified location - size_t i, pos; - for (i = 0; i < _dispatch_data_num_records(dd); ++i) { - pos = offset + dd->records[i].length; - if (location < pos) { - size = dd->records[i].length; - from = dd->records[i].from; - data = dd->records[i].data_object; - if (_dispatch_data_num_records(dd) == 1 && - _dispatch_data_leaf(data)) { - // Return objects composed of a single leaf node - *offset_ptr = offset; - _dispatch_data_retain(dd); - return dd; - } else { - // Drill down into other objects - dd = data; - break; - } - } else { - offset = pos; - } - } - } - } + *offset_ptr = 0; + return _dispatch_data_copy_region(dd, 0, dd->size, location, offset_ptr); } #if HAVE_MACH diff --git a/src/data.m b/src/data.m index c76f26a..92bc1e2 100644 --- a/src/data.m +++ b/src/data.m @@ -32,9 +32,13 @@ #include @interface DISPATCH_CLASS(data) () +@property (readonly) NSUInteger length; +@property (readonly) const void *bytes NS_RETURNS_INNER_POINTER; + - (id)initWithBytes:(void *)bytes length:(NSUInteger)length copy:(BOOL)copy freeWhenDone:(BOOL)freeBytes bytesAreVM:(BOOL)vm; - (BOOL)_bytesAreVM; +- (BOOL)_isCompact; @end @interface DISPATCH_CLASS(data_empty) : DISPATCH_CLASS(data) @@ -132,6 +136,21 @@ class_getName([self class]), buf]; } +- (NSUInteger)length { + struct dispatch_data_s *dd = (void*)self; + return dd->size; +} + +- (const void *)bytes { + struct dispatch_data_s *dd = (void*)self; + return _dispatch_data_get_flattened_bytes(dd); +} + +- (BOOL)_isCompact { + struct dispatch_data_s *dd = (void*)self; + return !dd->size || _dispatch_data_map_direct(dd, 0, NULL, NULL) != NULL; +} + @end @implementation DISPATCH_CLASS(data_empty) diff --git a/src/data_internal.h b/src/data_internal.h index d0de8bb..40a780c 100644 --- a/src/data_internal.h +++ b/src/data_internal.h @@ -32,11 +32,6 @@ #include // for HeaderDoc #endif -#if defined(__LP64__) && !defined(DISPATCH_DATA_USE_LEAF_MEMBER) && !USE_OBJC -// explicit leaf member is free on 64bit due to padding -#define DISPATCH_DATA_USE_LEAF_MEMBER 1 -#endif - typedef struct range_record_s { dispatch_data_t data_object; size_t from; @@ -67,25 +62,31 @@ struct dispatch_data_s { #else // USE_OBJC DISPATCH_STRUCT_HEADER(data); #endif // USE_OBJC -#if DISPATCH_DATA_USE_LEAF_MEMBER - bool leaf; -#endif + const void *buf; dispatch_block_t destructor; size_t size, num_records; - union { - const void* buf; - range_record records[0]; - }; + range_record records[0]; }; -#if DISPATCH_DATA_USE_LEAF_MEMBER -#define _dispatch_data_leaf(d) ((d)->leaf) -#define _dispatch_data_num_records(d) ((d)->num_records) -#else -#define _dispatch_data_leaf(d) ((d)->num_records ? 0 : ((d)->size ? 1 : 0)) -#define _dispatch_data_num_records(d) \ - (_dispatch_data_leaf(d) ? 1 : (d)->num_records) -#endif // DISPATCH_DATA_USE_LEAF_MEMBER +DISPATCH_ALWAYS_INLINE +static inline bool +_dispatch_data_leaf(struct dispatch_data_s *dd) +{ + return dd->num_records == 0; +} + +/* + * This is about the number of records required to hold that dispatch data + * if it's not a leaf. Callers either want that value, or have to special + * case the case when the dispatch data *is* a leaf before (and that the actual + * embeded record count of that dispatch data is 0) + */ +DISPATCH_ALWAYS_INLINE +static inline size_t +_dispatch_data_num_records(struct dispatch_data_s *dd) +{ + return dd->num_records ?: 1; +} typedef dispatch_data_t (*dispatch_transform_t)(dispatch_data_t data); @@ -101,27 +102,49 @@ void dispatch_data_init(dispatch_data_t data, const void *buffer, size_t size, dispatch_block_t destructor); void _dispatch_data_dispose(dispatch_data_t data); size_t _dispatch_data_debug(dispatch_data_t data, char* buf, size_t bufsiz); -const dispatch_block_t _dispatch_data_destructor_inline; -#define DISPATCH_DATA_DESTRUCTOR_INLINE (_dispatch_data_destructor_inline) +const void* +_dispatch_data_get_flattened_bytes(struct dispatch_data_s *dd); +#if !defined(__cplusplus) #if !__OBJC2__ +const dispatch_block_t _dispatch_data_destructor_inline; +#define DISPATCH_DATA_DESTRUCTOR_INLINE (_dispatch_data_destructor_inline) +#endif // !__OBJC2__ +/* + * the out parameters are about seeing "through" trivial subranges + * so for something like this: dd = { subrange [ dd1, offset1 ] }, + * this will return { dd1, offset + offset1 } + * + * If the dispatch object isn't a trivial subrange, it returns { dd, offset } + */ +DISPATCH_ALWAYS_INLINE static inline const void* -_dispatch_data_map_direct(dispatch_data_t dd) +_dispatch_data_map_direct(struct dispatch_data_s *dd, size_t offset, + struct dispatch_data_s **dd_out, size_t *from_out) { - size_t offset = 0; - if (slowpath(!dd->size)) { - return NULL; - } + const void *buffer = NULL; + + dispatch_assert(dd->size); if (slowpath(!_dispatch_data_leaf(dd)) && - _dispatch_data_num_records(dd) == 1 && - _dispatch_data_leaf(dd->records[0].data_object)) { - offset = dd->records[0].from; - dd = dd->records[0].data_object; + _dispatch_data_num_records(dd) == 1) { + offset += dd->records[0].from; + dd = (struct dispatch_data_s *)dd->records[0].data_object; } - return fastpath(_dispatch_data_leaf(dd)) ? (dd->buf + offset) : NULL; + + if (fastpath(_dispatch_data_leaf(dd))) { + buffer = dd->buf + offset; + } else { + buffer = dispatch_atomic_load((void **)&dd->buf, relaxed); + if (buffer) { + buffer += offset; + } + } + if (dd_out) *dd_out = dd; + if (from_out) *from_out = offset; + return buffer; } -#endif // !__OBJC2__ +#endif // !defined(__cplusplus) #endif // __DISPATCH_DATA_INTERNAL__ diff --git a/src/init.c b/src/init.c index 5cbf805..0aff191 100644 --- a/src/init.c +++ b/src/init.c @@ -82,6 +82,7 @@ pthread_key_t dispatch_bcounter_key; #if VOUCHER_USE_MACH_VOUCHER dispatch_once_t _voucher_task_mach_voucher_pred; mach_voucher_t _voucher_task_mach_voucher; +_voucher_atm_t _voucher_task_atm; _voucher_activity_t _voucher_activity_default; #endif voucher_activity_mode_t _voucher_activity_mode; @@ -180,6 +181,7 @@ struct dispatch_queue_s _dispatch_main_q = { .dq_running = 1, .dq_width = 1, .dq_is_thread_bound = 1, + .dq_override_voucher = DISPATCH_NO_VOUCHER, .dq_serialnum = 1, }; @@ -194,20 +196,24 @@ struct dispatch_queue_s _dispatch_main_q = { .do_next = DISPATCH_OBJECT_LISTLESS, \ .dqa_qos_class = (qos), \ .dqa_relative_priority = (qos) ? (prio) : 0, \ - .dqa_overcommit = (overcommit), \ + .dqa_overcommit = _dispatch_queue_attr_overcommit_##overcommit, \ .dqa_concurrent = (concurrent), \ } #define DISPATCH_QUEUE_ATTR_KIND_INIT(qos, prio) \ { \ [DQA_INDEX_NON_OVERCOMMIT][DQA_INDEX_CONCURRENT] = \ - DISPATCH_QUEUE_ATTR_INITIALIZER(qos, prio, 0, 1), \ + DISPATCH_QUEUE_ATTR_INITIALIZER(qos, prio, disabled, 1), \ [DQA_INDEX_NON_OVERCOMMIT][DQA_INDEX_SERIAL] = \ - DISPATCH_QUEUE_ATTR_INITIALIZER(qos, prio, 0, 0), \ + DISPATCH_QUEUE_ATTR_INITIALIZER(qos, prio, disabled, 0), \ [DQA_INDEX_OVERCOMMIT][DQA_INDEX_CONCURRENT] = \ - DISPATCH_QUEUE_ATTR_INITIALIZER(qos, prio, 1, 1), \ + DISPATCH_QUEUE_ATTR_INITIALIZER(qos, prio, enabled, 1), \ [DQA_INDEX_OVERCOMMIT][DQA_INDEX_SERIAL] = \ - DISPATCH_QUEUE_ATTR_INITIALIZER(qos, prio, 1, 0), \ + DISPATCH_QUEUE_ATTR_INITIALIZER(qos, prio, enabled, 0), \ + [DQA_INDEX_UNSPECIFIED_OVERCOMMIT][DQA_INDEX_CONCURRENT] = \ + DISPATCH_QUEUE_ATTR_INITIALIZER(qos, prio, unspecified, 1),\ + [DQA_INDEX_UNSPECIFIED_OVERCOMMIT][DQA_INDEX_SERIAL] = \ + DISPATCH_QUEUE_ATTR_INITIALIZER(qos, prio, unspecified, 0),\ } #define DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, prio) \ @@ -237,8 +243,12 @@ struct dispatch_queue_s _dispatch_main_q = { [DQA_INDEX_QOS_CLASS_##qos] = \ DISPATCH_QUEUE_ATTR_PRIO_INIT(_DISPATCH_QOS_CLASS_##qos) +// DISPATCH_QUEUE_CONCURRENT resp. _dispatch_queue_attr_concurrent is aliased +// to array member [0][0][0][0] and their properties must match! const struct dispatch_queue_attr_s _dispatch_queue_attrs[] - [DISPATCH_QUEUE_ATTR_PRIO_COUNT][2][2] = { + [DISPATCH_QUEUE_ATTR_PRIO_COUNT] + [DISPATCH_QUEUE_ATTR_OVERCOMMIT_COUNT] + [DISPATCH_QUEUE_ATTR_CONCURRENCY_COUNT] = { DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(UNSPECIFIED), DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(MAINTENANCE), DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(BACKGROUND), @@ -466,6 +476,9 @@ _dispatch_abort(size_t line, long val) static int dispatch_logfile = -1; static bool dispatch_log_disabled; +#if DISPATCH_DEBUG +static uint64_t dispatch_log_basetime; +#endif static dispatch_once_t _dispatch_logv_pred; static void @@ -502,6 +515,9 @@ _dispatch_logv_init(void *context DISPATCH_UNUSED) if (dispatch_logfile != -1) { struct timeval tv; gettimeofday(&tv, NULL); +#if DISPATCH_DEBUG + dispatch_log_basetime = mach_absolute_time(); +#endif dprintf(dispatch_logfile, "=== log file opened for %s[%u] at " "%ld.%06u ===\n", getprogname() ?: "", getpid(), tv.tv_sec, tv.tv_usec); @@ -527,13 +543,20 @@ static void _dispatch_logv_file(const char *msg, va_list ap) { char buf[2048]; - int r = vsnprintf(buf, sizeof(buf), msg, ap); + size_t bufsiz = sizeof(buf), offset = 0; + int r; + +#if DISPATCH_DEBUG + offset += dsnprintf(&buf[offset], bufsiz - offset, "%llu\t", + mach_absolute_time() - dispatch_log_basetime); +#endif + r = vsnprintf(&buf[offset], bufsiz - offset, msg, ap); if (r < 0) return; - size_t len = (size_t)r; - if (len > sizeof(buf) - 1) { - len = sizeof(buf) - 1; + offset += (size_t)r; + if (offset > bufsiz - 1) { + offset = bufsiz - 1; } - _dispatch_log_file(buf, len); + _dispatch_log_file(buf, offset); } #if DISPATCH_USE_SIMPLE_ASL @@ -618,22 +641,27 @@ static void _dispatch_debugv(dispatch_object_t dou, const char *msg, va_list ap) { char buf[2048]; + size_t bufsiz = sizeof(buf), offset = 0; int r; - size_t offs; +#if DISPATCH_DEBUG && !DISPATCH_USE_OS_DEBUG_LOG + offset += dsnprintf(&buf[offset], bufsiz - offset, "%llu\t\t%p\t", + mach_absolute_time() - dispatch_log_basetime, + (void *)_dispatch_thread_self()); +#endif if (dou._do) { - offs = _dispatch_object_debug2(dou, buf, sizeof(buf)); - dispatch_assert(offs + 2 < sizeof(buf)); - buf[offs++] = ':'; - buf[offs++] = ' '; - buf[offs] = '\0'; + offset += _dispatch_object_debug2(dou, &buf[offset], bufsiz - offset); + dispatch_assert(offset + 2 < bufsiz); + buf[offset++] = ':'; + buf[offset++] = ' '; + buf[offset] = '\0'; } else { - offs = strlcpy(buf, "NULL: ", sizeof(buf)); + offset += strlcpy(&buf[offset], "NULL: ", bufsiz - offset); } - r = vsnprintf(buf + offs, sizeof(buf) - offs, msg, ap); + r = vsnprintf(&buf[offset], bufsiz - offset, msg, ap); #if !DISPATCH_USE_OS_DEBUG_LOG - size_t len = offs + (r < 0 ? 0 : (size_t)r); - if (len > sizeof(buf) - 1) { - len = sizeof(buf) - 1; + size_t len = offset + (r < 0 ? 0 : (size_t)r); + if (len > bufsiz - 1) { + len = bufsiz - 1; } _dispatch_logv(buf, len, NULL); #else @@ -720,40 +748,6 @@ _dispatch_call_block_and_release(void *block) Block_release(b); } -#pragma mark - -#pragma mark _dispatch_block_create no_objc - -#if !USE_OBJC - -// The compiler hides the name of the function it generates, and changes it if -// we try to reference it directly, but the linker still sees it. -extern void DISPATCH_BLOCK_SPECIAL_INVOKE(void *) - asm("____dispatch_block_create_block_invoke"); -void (*_dispatch_block_special_invoke)(void*) = DISPATCH_BLOCK_SPECIAL_INVOKE; - -dispatch_block_t -_dispatch_block_create(dispatch_block_flags_t flags, voucher_t voucher, - pthread_priority_t pri, dispatch_block_t block) -{ - dispatch_block_t copy_block = _dispatch_Block_copy(block); // 17094902 - (void)voucher; // No voucher capture! (requires ObjC runtime) - struct dispatch_block_private_data_s dbpds = - DISPATCH_BLOCK_PRIVATE_DATA_INITIALIZER(flags, NULL, pri, copy_block); - dispatch_block_t new_block = _dispatch_Block_copy(^{ - // Capture object references, which retains copy_block. - // All retained objects must be captured by the *block*. We - // cannot borrow any references, because the block might be - // called zero or several times, so Block_release() is the - // only place that can release retained objects. - (void)copy_block; - _dispatch_block_invoke(&dbpds); - }); - Block_release(copy_block); - return new_block; -} - -#endif // !USE_OBJC - #endif // __BLOCKS__ #pragma mark - @@ -797,21 +791,6 @@ _dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t)) _dispatch_set_unwind_tsd(u); } -#undef _dispatch_client_callout3 -bool -_dispatch_client_callout3(void *ctxt, dispatch_data_t region, size_t offset, - const void *buffer, size_t size, dispatch_data_applier_function_t f) -{ - _dispatch_get_tsd_base(); - void *u = _dispatch_get_unwind_tsd(); - if (fastpath(!u)) return f(ctxt, region, offset, buffer, size); - _dispatch_set_unwind_tsd(NULL); - bool res = f(ctxt, region, offset, buffer, size); - _dispatch_free_unwind_tsd(); - _dispatch_set_unwind_tsd(u); - return res; -} - #undef _dispatch_client_callout4 void _dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason, @@ -871,6 +850,7 @@ _os_object_dealloc(_os_object_t obj) void _os_object_xref_dispose(_os_object_t obj) { + _os_object_xrefcnt_dispose_barrier(obj); if (fastpath(obj->os_obj_isa->_os_obj_xref_dispose)) { return obj->os_obj_isa->_os_obj_xref_dispose(obj); } @@ -880,6 +860,7 @@ _os_object_xref_dispose(_os_object_t obj) void _os_object_dispose(_os_object_t obj) { + _os_object_refcnt_dispose_barrier(obj); if (fastpath(obj->os_obj_isa->_os_obj_dispose)) { return obj->os_obj_isa->_os_obj_dispose(obj); } @@ -1003,11 +984,30 @@ const struct dispatch_source_type_s _dispatch_source_type_interval = { .init = dispatch_source_type_interval_init, }; +#if !DISPATCH_USE_SELECT_FALLBACK || DISPATCH_DYNAMIC_SELECT_FALLBACK +static void +dispatch_source_type_readwrite_init(dispatch_source_t ds, + dispatch_source_type_t type DISPATCH_UNUSED, + uintptr_t handle DISPATCH_UNUSED, + unsigned long mask DISPATCH_UNUSED, + dispatch_queue_t q DISPATCH_UNUSED) +{ + ds->ds_dkev->dk_kevent.flags |= EV_UDATA_SPECIFIC; + ds->ds_is_direct_kevent = true; + // bypass kernel check for device kqueue support rdar://19004921 + ds->ds_dkev->dk_kevent.fflags = NOTE_LOWAT; + ds->ds_dkev->dk_kevent.data = 1; +} +#else +#define dispatch_source_type_readwrite_init NULL +#endif + const struct dispatch_source_type_s _dispatch_source_type_read = { .ke = { .filter = EVFILT_READ, .flags = EV_DISPATCH, }, + .init = dispatch_source_type_readwrite_init, }; const struct dispatch_source_type_s _dispatch_source_type_write = { @@ -1015,6 +1015,7 @@ const struct dispatch_source_type_s _dispatch_source_type_write = { .filter = EVFILT_WRITE, .flags = EV_DISPATCH, }, + .init = dispatch_source_type_readwrite_init, }; #if DISPATCH_USE_MEMORYSTATUS @@ -1063,7 +1064,7 @@ dispatch_source_type_memorystatus_init(dispatch_source_t ds, const struct dispatch_source_type_s _dispatch_source_type_memorystatus = { .ke = { .filter = EVFILT_MEMORYSTATUS, - .flags = EV_DISPATCH, + .flags = EV_DISPATCH|EV_UDATA_SPECIFIC, }, .mask = NOTE_MEMORYSTATUS_PRESSURE_NORMAL|NOTE_MEMORYSTATUS_PRESSURE_WARN |NOTE_MEMORYSTATUS_PRESSURE_CRITICAL|NOTE_MEMORYSTATUS_LOW_SWAP, @@ -1088,7 +1089,7 @@ dispatch_source_type_vm_init(dispatch_source_t ds, const struct dispatch_source_type_s _dispatch_source_type_vm = { .ke = { .filter = EVFILT_MEMORYSTATUS, - .flags = EV_DISPATCH, + .flags = EV_DISPATCH|EV_UDATA_SPECIFIC, }, .mask = NOTE_VM_PRESSURE, .init = dispatch_source_type_vm_init, @@ -1109,7 +1110,7 @@ dispatch_source_type_vm_init(dispatch_source_t ds, const struct dispatch_source_type_s _dispatch_source_type_vm = { .ke = { .filter = EVFILT_VM, - .flags = EV_DISPATCH, + .flags = EV_DISPATCH|EV_UDATA_SPECIFIC, }, .mask = NOTE_VM_PRESSURE, .init = dispatch_source_type_vm_init, @@ -1117,10 +1118,20 @@ const struct dispatch_source_type_s _dispatch_source_type_vm = { #endif // DISPATCH_USE_VM_PRESSURE +static void +dispatch_source_type_proc_init(dispatch_source_t ds, + dispatch_source_type_t type DISPATCH_UNUSED, + uintptr_t handle DISPATCH_UNUSED, + unsigned long mask DISPATCH_UNUSED, + dispatch_queue_t q DISPATCH_UNUSED) +{ + ds->ds_dkev->dk_kevent.fflags |= NOTE_EXIT; // rdar://16655831 +} + const struct dispatch_source_type_s _dispatch_source_type_proc = { .ke = { .filter = EVFILT_PROC, - .flags = EV_CLEAR, + .flags = EV_CLEAR|EV_UDATA_SPECIFIC, }, .mask = NOTE_EXIT|NOTE_FORK|NOTE_EXEC #if HAVE_DECL_NOTE_SIGNAL @@ -1130,18 +1141,20 @@ const struct dispatch_source_type_s _dispatch_source_type_proc = { |NOTE_REAP #endif , + .init = dispatch_source_type_proc_init, }; const struct dispatch_source_type_s _dispatch_source_type_signal = { .ke = { .filter = EVFILT_SIGNAL, + .flags = EV_UDATA_SPECIFIC, }, }; const struct dispatch_source_type_s _dispatch_source_type_vnode = { .ke = { .filter = EVFILT_VNODE, - .flags = EV_CLEAR, + .flags = EV_CLEAR|EV_UDATA_SPECIFIC, }, .mask = NOTE_DELETE|NOTE_WRITE|NOTE_EXTEND|NOTE_ATTRIB|NOTE_LINK| NOTE_RENAME|NOTE_REVOKE @@ -1154,7 +1167,7 @@ const struct dispatch_source_type_s _dispatch_source_type_vnode = { const struct dispatch_source_type_s _dispatch_source_type_vfs = { .ke = { .filter = EVFILT_FS, - .flags = EV_CLEAR, + .flags = EV_CLEAR|EV_UDATA_SPECIFIC, }, .mask = VQ_NOTRESP|VQ_NEEDAUTH|VQ_LOWDISK|VQ_MOUNT|VQ_UNMOUNT|VQ_DEAD| VQ_ASSIST|VQ_NOTRESPLOCK @@ -1171,7 +1184,7 @@ const struct dispatch_source_type_s _dispatch_source_type_sock = { #ifdef EVFILT_SOCK .ke = { .filter = EVFILT_SOCK, - .flags = EV_CLEAR, + .flags = EV_CLEAR|EV_UDATA_SPECIFIC, }, .mask = NOTE_CONNRESET | NOTE_READCLOSED | NOTE_WRITECLOSED | NOTE_TIMEOUT | NOTE_NOSRCADDR | NOTE_IFDENIED | NOTE_SUSPEND | @@ -1186,18 +1199,35 @@ const struct dispatch_source_type_s _dispatch_source_type_sock = { #endif // EVFILT_SOCK }; +#if DISPATCH_USE_EV_UDATA_SPECIFIC +static void +dispatch_source_type_data_init(dispatch_source_t ds, + dispatch_source_type_t type DISPATCH_UNUSED, + uintptr_t handle DISPATCH_UNUSED, + unsigned long mask DISPATCH_UNUSED, + dispatch_queue_t q DISPATCH_UNUSED) +{ + ds->ds_needs_rearm = false; // not registered with kevent +} +#else +#define dispatch_source_type_data_init NULL +#endif + const struct dispatch_source_type_s _dispatch_source_type_data_add = { .ke = { .filter = DISPATCH_EVFILT_CUSTOM_ADD, + .flags = EV_UDATA_SPECIFIC, }, + .init = dispatch_source_type_data_init, }; const struct dispatch_source_type_s _dispatch_source_type_data_or = { .ke = { .filter = DISPATCH_EVFILT_CUSTOM_OR, - .flags = EV_CLEAR, + .flags = EV_CLEAR|EV_UDATA_SPECIFIC, .fflags = ~0u, }, + .init = dispatch_source_type_data_init, }; #if HAVE_MACH diff --git a/src/inline_internal.h b/src/inline_internal.h index ea6953c..5cc4cd8 100644 --- a/src/inline_internal.h +++ b/src/inline_internal.h @@ -38,9 +38,6 @@ DISPATCH_NOTHROW void _dispatch_client_callout(void *ctxt, dispatch_function_t f); DISPATCH_NOTHROW void _dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t)); -DISPATCH_NOTHROW bool -_dispatch_client_callout3(void *ctxt, dispatch_data_t region, size_t offset, - const void *buffer, size_t size, dispatch_data_applier_function_t f); DISPATCH_NOTHROW void _dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason, dispatch_mach_msg_t dmsg, mach_error_t error, @@ -62,14 +59,6 @@ _dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t)) return f(ctxt, i); } -DISPATCH_ALWAYS_INLINE -static inline bool -_dispatch_client_callout3(void *ctxt, dispatch_data_t region, size_t offset, - const void *buffer, size_t size, dispatch_data_applier_function_t f) -{ - return f(ctxt, region, offset, buffer, size); -} - DISPATCH_ALWAYS_INLINE static inline void _dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason, @@ -81,7 +70,7 @@ _dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason, #endif // !DISPATCH_USE_CLIENT_CALLOUT -#if !(USE_OBJC && __OBJC2__) +#if !(USE_OBJC && __OBJC2__) && !defined(__cplusplus) #pragma mark - #pragma mark _os_object_t & dispatch_object_t @@ -90,11 +79,7 @@ DISPATCH_ALWAYS_INLINE static inline _os_object_t _os_object_retain_internal_inline(_os_object_t obj) { - int ref_cnt = obj->os_obj_ref_cnt; - if (slowpath(ref_cnt == _OS_OBJECT_GLOBAL_REFCNT)) { - return obj; // global object - } - ref_cnt = dispatch_atomic_inc2o(obj, os_obj_ref_cnt, relaxed); + int ref_cnt = _os_object_refcnt_inc(obj); if (slowpath(ref_cnt <= 0)) { DISPATCH_CRASH("Resurrection of an object"); } @@ -105,11 +90,7 @@ DISPATCH_ALWAYS_INLINE static inline void _os_object_release_internal_inline(_os_object_t obj) { - int ref_cnt = obj->os_obj_ref_cnt; - if (slowpath(ref_cnt == _OS_OBJECT_GLOBAL_REFCNT)) { - return; // global object - } - ref_cnt = dispatch_atomic_dec2o(obj, os_obj_ref_cnt, relaxed); + int ref_cnt = _os_object_refcnt_dec(obj); if (fastpath(ref_cnt >= 0)) { return; } @@ -121,6 +102,7 @@ _os_object_release_internal_inline(_os_object_t obj) DISPATCH_CRASH("Release while external references exist"); } #endif + // _os_object_refcnt_dispose_barrier() is in _os_object_dispose() return _os_object_dispose(obj); } @@ -204,8 +186,16 @@ static inline pthread_priority_t _dispatch_queue_reset_override_priority( static inline pthread_priority_t _dispatch_get_defaultpriority(void); static inline void _dispatch_set_defaultpriority_override(void); static inline void _dispatch_reset_defaultpriority(pthread_priority_t priority); +static inline pthread_priority_t _dispatch_get_priority(void); static inline void _dispatch_set_priority(pthread_priority_t priority); +DISPATCH_ALWAYS_INLINE +static inline dispatch_queue_t +_dispatch_queue_get_current(void) +{ + return (dispatch_queue_t)_dispatch_thread_getspecific(dispatch_queue_key); +} + DISPATCH_ALWAYS_INLINE static inline void _dispatch_queue_set_thread(dispatch_queue_t dq) @@ -286,44 +276,135 @@ _dispatch_queue_push_wakeup(dispatch_queue_t dq, dispatch_object_t _tail, } } +struct _dispatch_identity_s { + pthread_priority_t old_pri; + pthread_priority_t old_pp; + dispatch_queue_t old_dq; +}; + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_root_queue_identity_assume(struct _dispatch_identity_s *di, + dispatch_queue_t assumed_rq) +{ + di->old_dq = _dispatch_queue_get_current(); + di->old_pri = _dispatch_get_priority(); + di->old_pp = _dispatch_get_defaultpriority(); + + dispatch_assert(dx_type(di->old_dq) == DISPATCH_QUEUE_ROOT_TYPE); + dispatch_assert(dx_type(assumed_rq) == DISPATCH_QUEUE_ROOT_TYPE); + + _dispatch_wqthread_override_start(_dispatch_thread_port(), di->old_pri); + _dispatch_set_priority(assumed_rq->dq_priority); + _dispatch_reset_defaultpriority(assumed_rq->dq_priority); + _dispatch_thread_setspecific(dispatch_queue_key, assumed_rq); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_root_queue_identity_restore(struct _dispatch_identity_s *di) +{ + _dispatch_thread_setspecific(dispatch_queue_key, di->old_dq); + _dispatch_set_priority(di->old_pri); + _dispatch_reset_defaultpriority(di->old_pp); + // Ensure that the root queue sees that this thread was overridden. + _dispatch_set_defaultpriority_override(); +} + +typedef dispatch_queue_t +_dispatch_queue_class_invoke_handler_t(dispatch_object_t, + _dispatch_thread_semaphore_t*); + DISPATCH_ALWAYS_INLINE static inline void _dispatch_queue_class_invoke(dispatch_object_t dou, - dispatch_queue_t (*invoke)(dispatch_object_t, - _dispatch_thread_semaphore_t*)) + dispatch_continuation_t dc, dispatch_invoke_flags_t flags, + _dispatch_queue_class_invoke_handler_t invoke) { pthread_priority_t p = 0; dispatch_queue_t dq = dou._dq; + bool owning = !slowpath(flags & DISPATCH_INVOKE_STEALING); + bool overriding = slowpath(flags & DISPATCH_INVOKE_OVERRIDING); + if (!slowpath(DISPATCH_OBJECT_SUSPENDED(dq)) && fastpath(dispatch_atomic_cmpxchg2o(dq, dq_running, 0, 1, acquire))){ _dispatch_queue_set_thread(dq); + dispatch_queue_t tq = NULL; _dispatch_thread_semaphore_t sema = 0; + struct _dispatch_identity_s di; + + if (overriding) { + _dispatch_object_debug(dq, "stolen onto thread 0x%x, 0x%lx", + dq->dq_thread, _dispatch_get_defaultpriority()); + _dispatch_root_queue_identity_assume(&di, dc->dc_other); + } + tq = invoke(dq, &sema); _dispatch_queue_clear_thread(dq); - p = _dispatch_queue_reset_override_priority(dq); - if (p > (dq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK)) { - // Ensure that the root queue sees that this thread was overridden. - _dispatch_set_defaultpriority_override(); + + if (!owning && !sema && tq && tq != dq->do_targetq) { + /* + * When (tq && tq != dq->do_targetq) this is a source or mach + * channel asking to get to their manager queue. + * + * Since stealers cannot call _dispatch_queue_push_queue and + * retarget those, they need ot destroy the override so that + * when waking those sources or mach channels on their target queue + * we don't risk a stealer taking them over and not be able to + * retarget again, effectively live-locking them. + * + * Also, we're in the `overriding` case so the thread will be marked + * dirty by _dispatch_root_queue_identity_restore anyway + * so forgetting about p is fine. + */ + (void)_dispatch_queue_reset_override_priority(dq); + p = 0; + } else if (sema || tq || DISPATCH_OBJECT_SUSPENDED(dq)) { + p = _dispatch_queue_get_override_priority(dq); + } else { + p = _dispatch_queue_reset_override_priority(dq); + } + if (overriding) { + _dispatch_root_queue_identity_restore(&di); + } else { + if (p > (dq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK)) { + // Ensure that the root queue sees that this thread was overridden. + _dispatch_set_defaultpriority_override(); + } } - // We do not need to check the result. - // When the suspend-count lock is dropped, then the check will happen. - (void)dispatch_atomic_dec2o(dq, dq_running, release); + + uint32_t running = dispatch_atomic_dec2o(dq, dq_running, release); if (sema) { _dispatch_thread_semaphore_signal(sema); - } else if (tq) { + } else if (owning && tq) { + _dispatch_introspection_queue_item_complete(dq); + return _dispatch_queue_push_queue(tq, dq, p); + } + if (!owning && running == 0) { _dispatch_introspection_queue_item_complete(dq); - return _dispatch_queue_push(tq, dq, p); + return _dispatch_queue_wakeup_with_qos_and_release(dq, p); + } + } else if (overriding) { + mach_port_t th = dq->dq_thread; + if (th) { + p = _dispatch_queue_get_override_priority(dq); + _dispatch_object_debug(dq, "overriding thr 0x%x to priority 0x%lx", + th, p); + _dispatch_wqthread_override_start(th, p); } } - dq->do_next = DISPATCH_OBJECT_LISTLESS; + _dispatch_introspection_queue_item_complete(dq); - if (!dispatch_atomic_sub2o(dq, do_suspend_cnt, - DISPATCH_OBJECT_SUSPEND_LOCK, seq_cst)) { - // seq_cst with atomic store to suspend_cnt - if (dispatch_atomic_load2o(dq, dq_running, seq_cst) == 0) { - // verify that the queue is idle - return _dispatch_queue_wakeup_with_qos_and_release(dq, p); + if (owning) { + dq->do_next = DISPATCH_OBJECT_LISTLESS; + if (!dispatch_atomic_sub2o(dq, do_suspend_cnt, + DISPATCH_OBJECT_SUSPEND_LOCK, seq_cst)) { + // seq_cst with atomic store to suspend_cnt + if (dispatch_atomic_load2o(dq, dq_running, seq_cst) == 0) { + // verify that the queue is idle + return _dispatch_queue_wakeup_with_qos_and_release(dq, p); + } } } _dispatch_release(dq); // added when the queue is put on the list @@ -351,13 +432,6 @@ _dispatch_object_suspended(dispatch_object_t dou) return slowpath(suspend_cnt >= DISPATCH_OBJECT_SUSPEND_INTERVAL); } -DISPATCH_ALWAYS_INLINE -static inline dispatch_queue_t -_dispatch_queue_get_current(void) -{ - return (dispatch_queue_t)_dispatch_thread_getspecific(dispatch_queue_key); -} - DISPATCH_ALWAYS_INLINE DISPATCH_CONST static inline dispatch_queue_t _dispatch_get_root_queue(qos_class_t priority, bool overcommit) @@ -409,6 +483,7 @@ _dispatch_queue_init(dispatch_queue_t dq) dq->dq_running = 0; dq->dq_width = 1; + dq->dq_override_voucher = DISPATCH_NO_VOUCHER; dq->dq_serialnum = dispatch_atomic_inc_orig(&_dispatch_queue_serial_numbers, relaxed); } @@ -438,6 +513,23 @@ _dispatch_queue_get_bound_thread(dispatch_queue_t dq) return dq->dq_thread; } +DISPATCH_ALWAYS_INLINE +static inline dispatch_pthread_root_queue_observer_hooks_t +_dispatch_get_pthread_root_queue_observer_hooks(void) +{ + return _dispatch_thread_getspecific( + dispatch_pthread_root_queue_observer_hooks_key); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_set_pthread_root_queue_observer_hooks( + dispatch_pthread_root_queue_observer_hooks_t observer_hooks) +{ + _dispatch_thread_setspecific(dispatch_pthread_root_queue_observer_hooks_key, + observer_hooks); +} + #pragma mark - #pragma mark dispatch_priority @@ -516,7 +608,8 @@ _dispatch_queue_priority_inherit_from_target(dispatch_queue_t dq, const pthread_priority_t rootqueue_flag = _PTHREAD_PRIORITY_ROOTQUEUE_FLAG; const pthread_priority_t inherited_flag = _PTHREAD_PRIORITY_INHERIT_FLAG; pthread_priority_t dqp = dq->dq_priority, tqp = tq->dq_priority; - if ((!dqp || (dqp & inherited_flag)) && (tqp & rootqueue_flag)) { + if ((!(dqp & ~_PTHREAD_PRIORITY_FLAGS_MASK) || (dqp & inherited_flag)) && + (tqp & rootqueue_flag)) { dq->dq_priority = (tqp & ~rootqueue_flag) | inherited_flag; } #else @@ -646,13 +739,31 @@ _dispatch_set_priority_and_adopt_voucher(pthread_priority_t priority, DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT static inline voucher_t _dispatch_adopt_priority_and_voucher(pthread_priority_t priority, - voucher_t voucher, unsigned long flags) + voucher_t v, unsigned long flags) { pthread_priority_t p = 0; if (priority != DISPATCH_NO_PRIORITY) { p = _dispatch_priority_adopt(priority, flags); } - return _dispatch_set_priority_and_adopt_voucher(p, voucher); + if (!(flags & DISPATCH_VOUCHER_IGNORE_QUEUE_OVERRIDE)) { + dispatch_queue_t dq = _dispatch_queue_get_current(); + if (dq && dq->dq_override_voucher != DISPATCH_NO_VOUCHER) { + if (v != DISPATCH_NO_VOUCHER && v) _voucher_release(v); + v = dq->dq_override_voucher; + if (v) _voucher_retain(v); + } + } + return _dispatch_set_priority_and_adopt_voucher(p, v); +} + +DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT +static inline voucher_t +_dispatch_adopt_queue_override_voucher(dispatch_queue_t dq) +{ + voucher_t v = dq->dq_override_voucher; + if (v == DISPATCH_NO_VOUCHER) return DISPATCH_NO_VOUCHER; + if (v) _voucher_retain(v); + return _dispatch_set_priority_and_adopt_voucher(DISPATCH_NO_PRIORITY, v); } DISPATCH_ALWAYS_INLINE @@ -667,7 +778,7 @@ _dispatch_adopt_priority_and_replace_voucher(pthread_priority_t priority, DISPATCH_ALWAYS_INLINE static inline void -_dispatch_set_priority_and_replace_voucher(pthread_priority_t priority, +_dispatch_reset_priority_and_voucher(pthread_priority_t priority, voucher_t voucher) { voucher_t ov; @@ -675,6 +786,13 @@ _dispatch_set_priority_and_replace_voucher(pthread_priority_t priority, if (voucher != DISPATCH_NO_VOUCHER && ov) _voucher_release(ov); } +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_reset_voucher(voucher_t voucher) +{ + return _dispatch_reset_priority_and_voucher(DISPATCH_NO_PRIORITY, voucher); +} + DISPATCH_ALWAYS_INLINE static inline void _dispatch_set_priority(pthread_priority_t priority) @@ -716,11 +834,25 @@ _dispatch_queue_need_override_retain(dispatch_queue_t dq, pthread_priority_t pp) DISPATCH_ALWAYS_INLINE static inline bool -_dispatch_queue_override_priority(dispatch_queue_t dq, pthread_priority_t pp) +_dispatch_queue_override_priority(dispatch_queue_t dq, pthread_priority_t *pp, + bool *was_overridden) { - uint32_t p = (pp & _PTHREAD_PRIORITY_QOS_CLASS_MASK); uint32_t o = dq->dq_override; - if (o < p) o = dispatch_atomic_or_orig2o(dq, dq_override, p, relaxed); + uint32_t p = (*pp & _PTHREAD_PRIORITY_QOS_CLASS_MASK); + if (o < p) { + o = dispatch_atomic_or_orig2o(dq, dq_override, p, relaxed); + if (was_overridden) { + o = (uint32_t)_dispatch_priority_normalize(o); + } + *pp = _dispatch_priority_normalize(o | p); + } else { + o = (uint32_t)_dispatch_priority_normalize(o); + *pp = o; + } + if (was_overridden) { + *was_overridden = + (dq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK) < o; + } return (o < p); } @@ -812,13 +944,7 @@ _dispatch_block_get_data(const dispatch_block_t db) uint8_t *x = (uint8_t *)db; // x points to base of struct Block_layout x += sizeof(struct Block_layout); - // x points to addresss of captured block - x += sizeof(dispatch_block_t); -#if USE_OBJC - // x points to addresss of captured voucher - x += sizeof(voucher_t); -#endif - // x points to base of captured dispatch_block_private_data_s structure + // x points to base of captured dispatch_block_private_data_s object dispatch_block_private_data_t dbpd = (dispatch_block_private_data_t)x; if (dbpd->dbpd_magic != DISPATCH_BLOCK_PRIVATE_DATA_MAGIC) { DISPATCH_CRASH("Corruption of dispatch block object"); @@ -905,16 +1031,16 @@ _dispatch_continuation_free(dispatch_continuation_t dc) #include "trace.h" -DISPATCH_ALWAYS_INLINE_NDEBUG +DISPATCH_ALWAYS_INLINE static inline void -_dispatch_continuation_pop(dispatch_object_t dou) +_dispatch_continuation_invoke(dispatch_object_t dou, dispatch_queue_t dq) { dispatch_continuation_t dc = dou._dc, dc1; dispatch_group_t dg; - _dispatch_trace_continuation_pop(_dispatch_queue_get_current(), dou); + _dispatch_trace_continuation_pop(dq, dou); if (DISPATCH_OBJ_IS_VTABLE(dou._do)) { - return dx_invoke(dou._do); + return dx_invoke(dou._do, NULL, DISPATCH_INVOKE_NONE); } // Add the item back to the cache before calling the function. This @@ -945,6 +1071,18 @@ _dispatch_continuation_pop(dispatch_object_t dou) } } +DISPATCH_ALWAYS_INLINE_NDEBUG +static inline void +_dispatch_continuation_pop(dispatch_object_t dou) +{ + dispatch_queue_t dq = _dispatch_queue_get_current(); + dispatch_pthread_root_queue_observer_hooks_t observer_hooks = + _dispatch_get_pthread_root_queue_observer_hooks(); + if (observer_hooks) observer_hooks->queue_will_execute(dq); + _dispatch_continuation_invoke(dou, dq); + if (observer_hooks) observer_hooks->queue_did_execute(dq); +} + DISPATCH_ALWAYS_INLINE static inline void _dispatch_continuation_priority_set(dispatch_continuation_t dc, @@ -991,6 +1129,6 @@ _dispatch_continuation_get_override_priority(dispatch_queue_t dq, #endif } -#endif // !(USE_OBJC && __OBJC2__) +#endif // !(USE_OBJC && __OBJC2__) && !defined(__cplusplus) #endif /* __DISPATCH_INLINE_INTERNAL__ */ diff --git a/src/internal.h b/src/internal.h index 33fcedb..98626c6 100644 --- a/src/internal.h +++ b/src/internal.h @@ -302,6 +302,20 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void); #define slowpath(x) (x) #endif // __GNUC__ +#if DISPATCH_DEBUG +// sys/queue.h debugging +#undef TRASHIT +#define TRASHIT(x) do {(x) = (void *)-1;} while (0) +#endif // DISPATCH_DEBUG +#define _TAILQ_TRASH_ENTRY(elm, field) do { \ + TRASHIT((elm)->field.tqe_next); \ + TRASHIT((elm)->field.tqe_prev); \ + } while (0) +#define _TAILQ_TRASH_HEAD(head) do { \ + TRASHIT((head)->tqh_first); \ + TRASHIT((head)->tqh_last); \ + } while (0) + DISPATCH_NOINLINE void _dispatch_bug(size_t line, long val); @@ -346,6 +360,14 @@ void _dispatch_log(const char *msg, ...); #define dsnprintf(...) \ ({ int _r = snprintf(__VA_ARGS__); _r < 0 ? 0u : (size_t)_r; }) +#if __GNUC__ +#define dispatch_static_assert(e) ({ \ + char __compile_time_assert__[(bool)(e) ? 1 : -1] DISPATCH_UNUSED; \ + }) +#else +#define dispatch_static_assert(e) +#endif + /* * For reporting bugs within libdispatch when using the "_debug" version of the * library. @@ -353,7 +375,7 @@ void _dispatch_log(const char *msg, ...); #if __GNUC__ #define dispatch_assert(e) do { \ if (__builtin_constant_p(e)) { \ - char __compile_time_assert__[(bool)(e) ? 1 : -1] DISPATCH_UNUSED; \ + dispatch_static_assert(e); \ } else { \ typeof(e) _e = fastpath(e); /* always eval 'e' */ \ if (DISPATCH_DEBUG && !_e) { \ @@ -375,7 +397,7 @@ static inline void _dispatch_assert(long e, long line) { */ #define dispatch_assert_zero(e) do { \ if (__builtin_constant_p(e)) { \ - char __compile_time_assert__[(bool)(e) ? -1 : 1] DISPATCH_UNUSED; \ + dispatch_static_assert(e); \ } else { \ typeof(e) _e = slowpath(e); /* always eval 'e' */ \ if (DISPATCH_DEBUG && _e) { \ @@ -401,8 +423,7 @@ static inline void _dispatch_assert_zero(long e, long line) { typeof(e) _e = fastpath(e); /* always eval 'e' */ \ if (!_e) { \ if (__builtin_constant_p(e)) { \ - char __compile_time_assert__[(bool)(e) ? 1 : -1]; \ - (void)__compile_time_assert__; \ + dispatch_static_assert(e); \ } \ _dispatch_bug(__LINE__, (long)_e); \ } \ @@ -425,8 +446,7 @@ static inline long _dispatch_assume(long e, long line) { typeof(e) _e = slowpath(e); /* always eval 'e' */ \ if (_e) { \ if (__builtin_constant_p(e)) { \ - char __compile_time_assert__[(bool)(e) ? -1 : 1]; \ - (void)__compile_time_assert__; \ + dispatch_static_assert(e); \ } \ _dispatch_bug(__LINE__, (long)_e); \ } \ @@ -446,7 +466,7 @@ static inline long _dispatch_assume_zero(long e, long line) { #if __GNUC__ #define dispatch_debug_assert(e, msg, args...) do { \ if (__builtin_constant_p(e)) { \ - char __compile_time_assert__[(bool)(e) ? 1 : -1] DISPATCH_UNUSED; \ + dispatch_static_assert(e); \ } else { \ typeof(e) _e = fastpath(e); /* always eval 'e' */ \ if (DISPATCH_DEBUG && !_e) { \ @@ -602,6 +622,56 @@ extern bool _dispatch_safe_fork, _dispatch_child_of_unsafe_fork; #endif #endif // HAVE_DECL_NOTE_REAP +#if !defined(EV_UDATA_SPECIFIC) || (TARGET_IPHONE_SIMULATOR && \ + IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101100) || \ + (!TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED < 101100) +#undef DISPATCH_USE_EV_UDATA_SPECIFIC +#define DISPATCH_USE_EV_UDATA_SPECIFIC 0 +#elif !defined(DISPATCH_USE_EV_UDATA_SPECIFIC) +#define DISPATCH_USE_EV_UDATA_SPECIFIC 1 +#endif // EV_UDATA_SPECIFIC + +#if !DISPATCH_USE_EV_UDATA_SPECIFIC +#undef EV_UDATA_SPECIFIC +#define EV_UDATA_SPECIFIC 0 +#undef DISPATCH_DYNAMIC_SELECT_FALLBACK +#define DISPATCH_DYNAMIC_SELECT_FALLBACK 0 +#undef DISPATCH_USE_SELECT_FALLBACK +#define DISPATCH_USE_SELECT_FALLBACK 1 +#endif // !DISPATCH_USE_EV_UDATA_SPECIFIC + +#if !defined(EV_SET_QOS) || (TARGET_IPHONE_SIMULATOR && \ + IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101100) || \ + (!TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED < 101100) +#undef DISPATCH_USE_KEVENT_QOS +#define DISPATCH_USE_KEVENT_QOS 0 +#elif !defined(DISPATCH_USE_KEVENT_QOS) +#define DISPATCH_USE_KEVENT_QOS 1 +#endif // EV_SET_QOS + +#if DISPATCH_USE_KEVENT_QOS +typedef struct kevent_qos_s _dispatch_kevent_qos_s; +#else // DISPATCH_USE_KEVENT_QOS +#ifndef KEVENT_FLAG_IMMEDIATE +#define KEVENT_FLAG_NONE 0x00 +#define KEVENT_FLAG_IMMEDIATE 0x01 +#define KEVENT_FLAG_ERROR_EVENTS 0x02 +#endif // KEVENT_FLAG_IMMEDIATE +typedef struct kevent64_s _dispatch_kevent_qos_s; +#define kevent_qos(_kq, _changelist, _nchanges, _eventlist, _nevents, \ + _data_out, _data_available, _flags) \ + ({ unsigned int _f = (_flags); _dispatch_kevent_qos_s _kev_copy; \ + const _dispatch_kevent_qos_s *_cl = (_changelist); \ + int _n = (_nchanges); const struct timespec _timeout_immediately = {}; \ + dispatch_static_assert(!(_data_out) && !(_data_available)); \ + if (_f & KEVENT_FLAG_ERROR_EVENTS) { \ + dispatch_static_assert(_n == 1); \ + _kev_copy = *_cl; _kev_copy.flags |= EV_RECEIPT; } \ + kevent64((_kq), _f & KEVENT_FLAG_ERROR_EVENTS ? &_kev_copy : _cl, _n, \ + (_eventlist), (_nevents), 0, \ + _f & KEVENT_FLAG_IMMEDIATE ? &_timeout_immediately : NULL); }) +#endif // DISPATCH_USE_KEVENT_QOS + #if defined(F_SETNOSIGPIPE) && defined(F_GETNOSIGPIPE) #if TARGET_IPHONE_SIMULATOR && IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1070 #undef DISPATCH_USE_SETNOSIGPIPE @@ -616,6 +686,9 @@ extern bool _dispatch_safe_fork, _dispatch_child_of_unsafe_fork; #ifndef DISPATCH_USE_CHECKIN_NOIMPORTANCE #define DISPATCH_USE_CHECKIN_NOIMPORTANCE 1 // rdar://problem/16996737 #endif +#ifndef DISPATCH_USE_NOIMPORTANCE_QOS +#define DISPATCH_USE_NOIMPORTANCE_QOS 1 // rdar://problem/21414476 +#endif #endif // MACH_SEND_NOIMPORTANCE @@ -671,6 +744,7 @@ extern bool _dispatch_safe_fork, _dispatch_child_of_unsafe_fork; __asm__(""); __builtin_trap() // #define _dispatch_set_crash_log_message(msg) +#define _dispatch_set_crash_log_message_dynamic(msg) #if HAVE_MACH // MIG_REPLY_MISMATCH means either: @@ -704,6 +778,7 @@ extern int _dispatch_set_qos_class_enabled; #define DISPATCH_NO_VOUCHER ((voucher_t)(void*)~0ul) #define DISPATCH_NO_PRIORITY ((pthread_priority_t)~0ul) #define DISPATCH_PRIORITY_ENFORCE 0x1 +#define DISPATCH_VOUCHER_IGNORE_QUEUE_OVERRIDE 0x2 static inline void _dispatch_adopt_priority_and_replace_voucher( pthread_priority_t priority, voucher_t voucher, unsigned long flags); #if HAVE_MACH diff --git a/src/introspection.c b/src/introspection.c index e907f85..35b0b57 100644 --- a/src/introspection.c +++ b/src/introspection.c @@ -311,7 +311,8 @@ dispatch_introspection_queue_item_get_info(dispatch_queue_t dq, type != DISPATCH_QUEUE_SPECIFIC_TYPE) { diqi.type = dispatch_introspection_queue_item_type_queue; diqi.queue = dispatch_introspection_queue_get_info(dou._dq); - } else if (metatype == _DISPATCH_SOURCE_TYPE) { + } else if (metatype == _DISPATCH_SOURCE_TYPE && + type != DISPATCH_MACH_CHANNEL_TYPE) { diqi.type = dispatch_introspection_queue_item_type_source; diqi.source = _dispatch_introspection_source_get_info(dou._ds); } else { diff --git a/src/introspection_internal.h b/src/introspection_internal.h index 7b015aa..4ed951e 100644 --- a/src/introspection_internal.h +++ b/src/introspection_internal.h @@ -46,7 +46,7 @@ void _dispatch_introspection_queue_item_complete(dispatch_object_t dou); void _dispatch_introspection_callout_entry(void *ctxt, dispatch_function_t f); void _dispatch_introspection_callout_return(void *ctxt, dispatch_function_t f); -#if !__OBJC2__ +#if !__OBJC2__ && !defined(__cplusplus) DISPATCH_ALWAYS_INLINE static inline void @@ -70,9 +70,9 @@ _dispatch_introspection_queue_pop(dispatch_queue_t dq, dispatch_object_t dou) { _dispatch_introspection_queue_item_dequeue(dq, dou); }; -#endif +#endif // !__OBJC2__ && !defined(__cplusplus) -#else +#else // DISPATCH_INTROSPECTION #define DISPATCH_INTROSPECTION_QUEUE_LIST #define DISPATCH_INTROSPECTION_QUEUE_LIST_SIZE 0 diff --git a/src/io.c b/src/io.c index d66873b..0ad5b53 100644 --- a/src/io.c +++ b/src/io.c @@ -122,13 +122,6 @@ enum { #pragma mark - #pragma mark dispatch_io_hashtables -#if TARGET_OS_EMBEDDED -#define DIO_HASH_SIZE 64u // must be a power of two -#else -#define DIO_HASH_SIZE 256u // must be a power of two -#endif -#define DIO_HASH(x) ((uintptr_t)(x) & (DIO_HASH_SIZE - 1)) - // Global hashtable of dev_t -> disk_s mappings DISPATCH_CACHELINE_ALIGN static TAILQ_HEAD(, dispatch_disk_s) _dispatch_io_devs[DIO_HASH_SIZE]; @@ -173,10 +166,10 @@ enum { }; static struct dispatch_io_defaults_s { - size_t chunk_pages, low_water_chunks, max_pending_io_reqs; + size_t chunk_size, low_water_chunks, max_pending_io_reqs; bool initial_delivery; } dispatch_io_defaults = { - .chunk_pages = DIO_MAX_CHUNK_PAGES, + .chunk_size = DIO_MAX_CHUNK_SIZE, .low_water_chunks = DIO_DEFAULT_LOW_WATER_CHUNKS, .max_pending_io_reqs = DIO_MAX_PENDING_IO_REQS, }; @@ -190,7 +183,7 @@ _dispatch_iocntl(uint32_t param, uint64_t value) { switch (param) { case DISPATCH_IOCNTL_CHUNK_PAGES: - _dispatch_iocntl_set_default(chunk_pages, value); + _dispatch_iocntl_set_default(chunk_size, value * PAGE_SIZE); break; case DISPATCH_IOCNTL_LOW_WATER_CHUNKS: _dispatch_iocntl_set_default(low_water_chunks, value); @@ -217,7 +210,7 @@ _dispatch_io_create(dispatch_io_type_t type) channel->params.type = type; channel->params.high = SIZE_MAX; channel->params.low = dispatch_io_defaults.low_water_chunks * - dispatch_io_defaults.chunk_pages * PAGE_SIZE; + dispatch_io_defaults.chunk_size; channel->queue = dispatch_queue_create("com.apple.libdispatch-io.channelq", NULL); return channel; @@ -371,7 +364,7 @@ dispatch_io_create_with_path(dispatch_io_type_t type, const char *path, void (^cleanup_handler)(int error)) { if ((type != DISPATCH_IO_STREAM && type != DISPATCH_IO_RANDOM) || - !(path && *path == '/')) { + !(*path == '/')) { return NULL; } size_t pathlen = strlen(path); @@ -1992,7 +1985,7 @@ static void _dispatch_disk_perform(void *ctxt) { dispatch_disk_t disk = ctxt; - size_t chunk_size = dispatch_io_defaults.chunk_pages * PAGE_SIZE; + size_t chunk_size = dispatch_io_defaults.chunk_size; _dispatch_fd_debug("disk perform", -1); dispatch_operation_t op; size_t i = disk->advise_idx, j = disk->free_idx; @@ -2109,10 +2102,10 @@ _dispatch_operation_perform(dispatch_operation_t op) _dispatch_object_debug(op, "%s", __func__); if (!op->buf) { size_t max_buf_siz = op->params.high; - size_t chunk_siz = dispatch_io_defaults.chunk_pages * PAGE_SIZE; + size_t chunk_siz = dispatch_io_defaults.chunk_size; if (op->direction == DOP_DIR_READ) { // If necessary, create a buffer for the ongoing operation, large - // enough to fit chunk_pages but at most high-water + // enough to fit chunk_size but at most high-water size_t data_siz = dispatch_data_get_size(op->data); if (data_siz) { dispatch_assert(data_siz < max_buf_siz); diff --git a/src/io_internal.h b/src/io_internal.h index fbb27c5..ecdc775 100644 --- a/src/io_internal.h +++ b/src/io_internal.h @@ -35,11 +35,15 @@ #define _DISPATCH_IO_LABEL_SIZE 16 #if TARGET_OS_EMBEDDED // rdar://problem/9032036 -#define DIO_MAX_CHUNK_PAGES 128u // 512kB chunk size +#define DIO_MAX_CHUNK_SIZE (512u * 1024) +#define DIO_HASH_SIZE 64u // must be a power of two #else -#define DIO_MAX_CHUNK_PAGES 256u // 1024kB chunk size +#define DIO_MAX_CHUNK_SIZE (1024u * 1024) +#define DIO_HASH_SIZE 256u // must be a power of two #endif +#define DIO_HASH(x) ((uintptr_t)(x) & (DIO_HASH_SIZE - 1)) + #define DIO_DEFAULT_LOW_WATER_CHUNKS 1u // default low-water mark #define DIO_MAX_PENDING_IO_REQS 6u // Pending I/O read advises @@ -93,7 +97,6 @@ struct dispatch_stat_s { DISPATCH_CLASS_DECL(disk); struct dispatch_disk_s { DISPATCH_STRUCT_HEADER(disk); - dev_t dev; TAILQ_HEAD(dispatch_disk_operations_s, dispatch_operation_s) operations; dispatch_operation_t cur_rq; dispatch_queue_t pick_queue; @@ -101,8 +104,8 @@ struct dispatch_disk_s { size_t free_idx; size_t req_idx; size_t advise_idx; + dev_t dev; bool io_active; - int err; TAILQ_ENTRY(dispatch_disk_s) disk_list; size_t advise_list_depth; dispatch_operation_t advise_list[]; diff --git a/src/object.c b/src/object.c index 5b09de7..4089ba0 100644 --- a/src/object.c +++ b/src/object.c @@ -51,26 +51,32 @@ DISPATCH_NOINLINE _os_object_t _os_object_retain(_os_object_t obj) { - int xref_cnt = obj->os_obj_xref_cnt; - if (slowpath(xref_cnt == _OS_OBJECT_GLOBAL_REFCNT)) { - return obj; // global object - } - xref_cnt = dispatch_atomic_inc2o(obj, os_obj_xref_cnt, relaxed); + int xref_cnt = _os_object_xrefcnt_inc(obj); if (slowpath(xref_cnt <= 0)) { _OS_OBJECT_CLIENT_CRASH("Resurrection of an object"); } return obj; } +DISPATCH_NOINLINE +_os_object_t +_os_object_retain_with_resurrect(_os_object_t obj) +{ + int xref_cnt = _os_object_xrefcnt_inc(obj); + if (slowpath(xref_cnt < 0)) { + _OS_OBJECT_CLIENT_CRASH("Resurrection of an overreleased object"); + } + if (slowpath(xref_cnt == 0)) { + _os_object_retain_internal(obj); + } + return obj; +} + DISPATCH_NOINLINE void _os_object_release(_os_object_t obj) { - int xref_cnt = obj->os_obj_xref_cnt; - if (slowpath(xref_cnt == _OS_OBJECT_GLOBAL_REFCNT)) { - return; // global object - } - xref_cnt = dispatch_atomic_dec2o(obj, os_obj_xref_cnt, relaxed); + int xref_cnt = _os_object_xrefcnt_dec(obj); if (fastpath(xref_cnt >= 0)) { return; } diff --git a/src/object.m b/src/object.m index 953cb0b..1a98d7e 100644 --- a/src/object.m +++ b/src/object.m @@ -142,6 +142,10 @@ _os_objc_destructInstance(id obj) return obj; } +#if DISPATCH_COCOA_COMPAT +static bool _os_object_debug_missing_pools; +#endif + void _os_object_init(void) { @@ -155,6 +159,10 @@ _os_object_init(void) (void (*)(const void *))&_os_objc_destructInstance }; _Block_use_RR2(&callbacks); +#if DISPATCH_COCOA_COMPAT + const char *v = getenv("OBJC_DEBUG_MISSING_POOLS"); + _os_object_debug_missing_pools = v && !strcmp(v, "YES"); +#endif } _os_object_t @@ -181,12 +189,16 @@ _os_object_dealloc(_os_object_t obj) void _os_object_xref_dispose(_os_object_t obj) { + struct _os_object_s *o = (struct _os_object_s *)obj; + _os_object_xrefcnt_dispose_barrier(o); [obj _xref_dispose]; } void _os_object_dispose(_os_object_t obj) { + struct _os_object_s *o = (struct _os_object_s *)obj; + _os_object_refcnt_dispose_barrier(o); [obj _dispose]; } @@ -463,12 +475,17 @@ DISPATCH_OBJC_LOAD() void * _dispatch_autorelease_pool_push(void) { - return objc_autoreleasePoolPush(); + if (!slowpath(_os_object_debug_missing_pools)) { + return objc_autoreleasePoolPush(); + } + return NULL; } void _dispatch_autorelease_pool_pop(void *context) { - return objc_autoreleasePoolPop(context); + if (!slowpath(_os_object_debug_missing_pools)) { + return objc_autoreleasePoolPop(context); + } } #endif // DISPATCH_COCOA_COMPAT @@ -507,19 +524,6 @@ _dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t)) } } -#undef _dispatch_client_callout3 -bool -_dispatch_client_callout3(void *ctxt, dispatch_data_t region, size_t offset, - const void *buffer, size_t size, dispatch_data_applier_function_t f) -{ - @try { - return f(ctxt, region, offset, buffer, size); - } - @catch (...) { - objc_terminate(); - } -} - #undef _dispatch_client_callout4 void _dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason, @@ -536,34 +540,4 @@ _dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason, #endif // DISPATCH_USE_CLIENT_CALLOUT -#pragma mark - -#pragma mark _dispatch_block_create - -// The compiler hides the name of the function it generates, and changes it if -// we try to reference it directly, but the linker still sees it. -extern void DISPATCH_BLOCK_SPECIAL_INVOKE(void *) - asm("____dispatch_block_create_block_invoke"); -void (*_dispatch_block_special_invoke)(void*) = DISPATCH_BLOCK_SPECIAL_INVOKE; - -dispatch_block_t -_dispatch_block_create(dispatch_block_flags_t flags, voucher_t voucher, - pthread_priority_t pri, dispatch_block_t block) -{ - dispatch_block_t copy_block = _dispatch_Block_copy(block); // 17094902 - struct dispatch_block_private_data_s dbpds = - DISPATCH_BLOCK_PRIVATE_DATA_INITIALIZER(flags, voucher, pri, copy_block); - dispatch_block_t new_block = _dispatch_Block_copy(^{ - // Capture object references, which retains copy_block and voucher. - // All retained objects must be captured by the *block*. We - // cannot borrow any references, because the block might be - // called zero or several times, so Block_release() is the - // only place that can release retained objects. - (void)copy_block; - (void)voucher; - _dispatch_block_invoke(&dbpds); - }); - Block_release(copy_block); - return new_block; -} - #endif // USE_OBJC diff --git a/src/object_internal.h b/src/object_internal.h index c0d17ae..4778f4c 100644 --- a/src/object_internal.h +++ b/src/object_internal.h @@ -40,6 +40,18 @@ #define DISPATCH_DECL_SUBCLASS_INTERFACE(name, super) #endif // OS_OBJECT_USE_OBJC +DISPATCH_ENUM(dispatch_invoke_flags, unsigned long, + DISPATCH_INVOKE_NONE = 0x00, + /* This invoke is a stealer, meaning that it doesn't own the + * enqueue lock, and is not allowed to requeue elsewhere + */ + DISPATCH_INVOKE_STEALING = 0x01, + /* The `dc` argument is a dispatch continuation wrapper + * created by _dispatch_queue_push_override + */ + DISPATCH_INVOKE_OVERRIDING = 0x02, +); + #if USE_OBJC #define DISPATCH_CLASS(name) OS_OBJECT_CLASS(dispatch_##name) // ObjC classes and dispatch vtables are co-located via linker order and alias @@ -84,7 +96,8 @@ unsigned long const do_type; \ const char *const do_kind; \ size_t (*const do_debug)(struct dispatch_##x##_s *, char *, size_t); \ - void (*const do_invoke)(struct dispatch_##x##_s *); \ + void (*const do_invoke)(struct dispatch_##x##_s *, dispatch_object_t dc, \ + dispatch_invoke_flags_t); \ unsigned long (*const do_probe)(struct dispatch_##x##_s *); \ void (*const do_dispose)(struct dispatch_##x##_s *); #else @@ -93,7 +106,8 @@ unsigned long do_type; \ const char *do_kind; \ size_t (*do_debug)(struct dispatch_##x##_s *, char *, size_t); \ - void (*do_invoke)(struct dispatch_##x##_s *); \ + void (*do_invoke)(struct dispatch_##x##_s *, dispatch_object_t dc, \ + dispatch_invoke_flags_t); \ unsigned long (*do_probe)(struct dispatch_##x##_s *); \ void (*do_dispose)(struct dispatch_##x##_s *); #endif @@ -103,7 +117,7 @@ #define dx_kind(x) (x)->do_vtable->do_kind #define dx_debug(x, y, z) (x)->do_vtable->do_debug((x), (y), (z)) #define dx_dispose(x) (x)->do_vtable->do_dispose(x) -#define dx_invoke(x) (x)->do_vtable->do_invoke(x) +#define dx_invoke(x, y, z) (x)->do_vtable->do_invoke(x, y, z) #define dx_probe(x) (x)->do_vtable->do_probe(x) #define DISPATCH_STRUCT_HEADER(x) \ @@ -243,6 +257,72 @@ size_t _dispatch_objc_debug(dispatch_object_t dou, char* buf, size_t bufsiz); #pragma mark - #pragma mark _os_object_s +/* + * Low level _os_atomic_refcnt_* actions + * + * _os_atomic_refcnt_inc2o(o, f): + * performs a refcount increment and returns the new refcount value + * + * _os_atomic_refcnt_dec2o(o, f): + * performs a refcount decrement and returns the new refcount value + * + * _os_atomic_refcnt_dispose_barrier2o(o, f): + * a barrier to perform prior to tearing down an object when the refcount + * reached -1. + */ +#define _os_atomic_refcnt_perform2o(o, f, op, m) ({ \ + typeof(o) _o = (o); \ + int _ref_cnt = _o->f; \ + if (fastpath(_ref_cnt != _OS_OBJECT_GLOBAL_REFCNT)) { \ + _ref_cnt = dispatch_atomic_##op##2o(_o, f, m); \ + } \ + _ref_cnt; \ + }) + +#define _os_atomic_refcnt_inc2o(o, m) \ + _os_atomic_refcnt_perform2o(o, m, inc, relaxed) + +#define _os_atomic_refcnt_dec2o(o, m) \ + _os_atomic_refcnt_perform2o(o, m, dec, release) + +#define _os_atomic_refcnt_dispose_barrier2o(o, m) \ + (void)dispatch_atomic_load2o(o, m, acquire) + + +/* + * Higher level _os_object_{x,}refcnt_* actions + * + * _os_atomic_{x,}refcnt_inc(o): + * increment the external (resp. internal) refcount and + * returns the new refcount value + * + * _os_atomic_{x,}refcnt_dec(o): + * decrement the external (resp. internal) refcount and + * returns the new refcount value + * + * _os_atomic_{x,}refcnt_dispose_barrier(o): + * performs the pre-teardown barrier for the external + * (resp. internal) refcount + * + */ +#define _os_object_xrefcnt_inc(o) \ + _os_atomic_refcnt_inc2o(o, os_obj_xref_cnt) + +#define _os_object_xrefcnt_dec(o) \ + _os_atomic_refcnt_dec2o(o, os_obj_xref_cnt) + +#define _os_object_xrefcnt_dispose_barrier(o) \ + _os_atomic_refcnt_dispose_barrier2o(o, os_obj_xref_cnt) + +#define _os_object_refcnt_inc(o) \ + _os_atomic_refcnt_inc2o(o, os_obj_ref_cnt) + +#define _os_object_refcnt_dec(o) \ + _os_atomic_refcnt_dec2o(o, os_obj_ref_cnt) + +#define _os_object_refcnt_dispose_barrier(o) \ + _os_atomic_refcnt_dispose_barrier2o(o, os_obj_ref_cnt) + typedef struct _os_object_class_s { _OS_OBJECT_CLASS_HEADER(); } _os_object_class_s; diff --git a/src/queue.c b/src/queue.c index b8b4ad9..5868e87 100644 --- a/src/queue.c +++ b/src/queue.c @@ -60,7 +60,7 @@ static inline _dispatch_thread_semaphore_t static inline bool _dispatch_queue_prepare_override(dispatch_queue_t dq, dispatch_queue_t tq, pthread_priority_t p); static inline void _dispatch_queue_push_override(dispatch_queue_t dq, - dispatch_queue_t tq, pthread_priority_t p); + dispatch_queue_t tq, pthread_priority_t p, bool owning); #if HAVE_PTHREAD_WORKQUEUES static void _dispatch_worker_thread4(void *context); #if HAVE_PTHREAD_WORKQUEUE_QOS @@ -93,6 +93,7 @@ struct dispatch_pthread_root_queue_context_s { pthread_attr_t dpq_thread_attr; dispatch_block_t dpq_thread_configure; struct dispatch_semaphore_s dpq_thread_mediator; + dispatch_pthread_root_queue_observer_hooks_s dpq_observer_hooks; }; typedef struct dispatch_pthread_root_queue_context_s * dispatch_pthread_root_queue_context_t; @@ -349,6 +350,7 @@ struct dispatch_queue_s _dispatch_root_queues[] = { .dq_label = "com.apple.root.maintenance-qos", .dq_running = 2, .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_override_voucher = DISPATCH_NO_VOUCHER, .dq_serialnum = 4, }, [DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS_OVERCOMMIT] = { @@ -361,6 +363,7 @@ struct dispatch_queue_s _dispatch_root_queues[] = { .dq_label = "com.apple.root.maintenance-qos.overcommit", .dq_running = 2, .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_override_voucher = DISPATCH_NO_VOUCHER, .dq_serialnum = 5, }, [DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS] = { @@ -373,6 +376,7 @@ struct dispatch_queue_s _dispatch_root_queues[] = { .dq_label = "com.apple.root.background-qos", .dq_running = 2, .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_override_voucher = DISPATCH_NO_VOUCHER, .dq_serialnum = 6, }, [DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS_OVERCOMMIT] = { @@ -385,6 +389,7 @@ struct dispatch_queue_s _dispatch_root_queues[] = { .dq_label = "com.apple.root.background-qos.overcommit", .dq_running = 2, .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_override_voucher = DISPATCH_NO_VOUCHER, .dq_serialnum = 7, }, [DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS] = { @@ -397,6 +402,7 @@ struct dispatch_queue_s _dispatch_root_queues[] = { .dq_label = "com.apple.root.utility-qos", .dq_running = 2, .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_override_voucher = DISPATCH_NO_VOUCHER, .dq_serialnum = 8, }, [DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS_OVERCOMMIT] = { @@ -409,6 +415,7 @@ struct dispatch_queue_s _dispatch_root_queues[] = { .dq_label = "com.apple.root.utility-qos.overcommit", .dq_running = 2, .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_override_voucher = DISPATCH_NO_VOUCHER, .dq_serialnum = 9, }, [DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS] = { @@ -421,6 +428,7 @@ struct dispatch_queue_s _dispatch_root_queues[] = { .dq_label = "com.apple.root.default-qos", .dq_running = 2, .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_override_voucher = DISPATCH_NO_VOUCHER, .dq_serialnum = 10, }, [DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT] = { @@ -433,6 +441,7 @@ struct dispatch_queue_s _dispatch_root_queues[] = { .dq_label = "com.apple.root.default-qos.overcommit", .dq_running = 2, .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_override_voucher = DISPATCH_NO_VOUCHER, .dq_serialnum = 11, }, [DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS] = { @@ -445,6 +454,7 @@ struct dispatch_queue_s _dispatch_root_queues[] = { .dq_label = "com.apple.root.user-initiated-qos", .dq_running = 2, .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_override_voucher = DISPATCH_NO_VOUCHER, .dq_serialnum = 12, }, [DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS_OVERCOMMIT] = { @@ -457,6 +467,7 @@ struct dispatch_queue_s _dispatch_root_queues[] = { .dq_label = "com.apple.root.user-initiated-qos.overcommit", .dq_running = 2, .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_override_voucher = DISPATCH_NO_VOUCHER, .dq_serialnum = 13, }, [DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS] = { @@ -469,6 +480,7 @@ struct dispatch_queue_s _dispatch_root_queues[] = { .dq_label = "com.apple.root.user-interactive-qos", .dq_running = 2, .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_override_voucher = DISPATCH_NO_VOUCHER, .dq_serialnum = 14, }, [DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS_OVERCOMMIT] = { @@ -481,6 +493,7 @@ struct dispatch_queue_s _dispatch_root_queues[] = { .dq_label = "com.apple.root.user-interactive-qos.overcommit", .dq_running = 2, .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_override_voucher = DISPATCH_NO_VOUCHER, .dq_serialnum = 15, }, }; @@ -559,6 +572,7 @@ struct dispatch_queue_s _dispatch_mgr_q = { .dq_label = "com.apple.libdispatch-manager", .dq_width = 1, .dq_is_thread_bound = 1, + .dq_override_voucher = DISPATCH_NO_VOUCHER, .dq_serialnum = 2, }; @@ -572,7 +586,7 @@ dispatch_get_global_queue(long priority, unsigned long flags) _dispatch_root_queues_init); qos_class_t qos; switch (priority) { -#if !RDAR_17878963 || DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK +#if DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK case _DISPATCH_QOS_CLASS_MAINTENANCE: if (!_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS] .dq_priority) { @@ -582,7 +596,7 @@ dispatch_get_global_queue(long priority, unsigned long flags) qos = (qos_class_t)priority; } break; -#endif // RDAR_17878963 || DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK +#endif // DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK case DISPATCH_QUEUE_PRIORITY_BACKGROUND: qos = _dispatch_priority2qos[DISPATCH_PRIORITY_IDX_BACKGROUND]; break; @@ -653,7 +667,7 @@ _dispatch_assert_queue_fail(dispatch_queue_t dq, bool expected) expected ? "Expected" : "Unexpected", dq, dq->dq_label ? dq->dq_label : ""); _dispatch_log("%s", msg); - _dispatch_set_crash_log_message(msg); + _dispatch_set_crash_log_message_dynamic(msg); _dispatch_hardware_crash(); free(msg); } @@ -661,7 +675,7 @@ _dispatch_assert_queue_fail(dispatch_queue_t dq, bool expected) void dispatch_assert_queue(dispatch_queue_t dq) { - if (slowpath(!dq) || slowpath(!(dx_metatype(dq) == _DISPATCH_QUEUE_TYPE))) { + if (slowpath(!(dx_metatype(dq) == _DISPATCH_QUEUE_TYPE))) { DISPATCH_CLIENT_CRASH("invalid queue passed to " "dispatch_assert_queue()"); } @@ -675,7 +689,7 @@ dispatch_assert_queue(dispatch_queue_t dq) void dispatch_assert_queue_not(dispatch_queue_t dq) { - if (slowpath(!dq) || slowpath(!(dx_metatype(dq) == _DISPATCH_QUEUE_TYPE))) { + if (slowpath(!(dx_metatype(dq) == _DISPATCH_QUEUE_TYPE))) { DISPATCH_CLIENT_CRASH("invalid queue passed to " "dispatch_assert_queue_not()"); } @@ -934,6 +948,8 @@ libdispatch_init(void) _dispatch_thread_key_create(&dispatch_io_key, NULL); _dispatch_thread_key_create(&dispatch_apply_key, NULL); _dispatch_thread_key_create(&dispatch_defaultpriority_key, NULL); + _dispatch_thread_key_create(&dispatch_pthread_root_queue_observer_hooks_key, + NULL); #if DISPATCH_PERF_MON && !DISPATCH_INTROSPECTION _dispatch_thread_key_create(&dispatch_bcounter_key, NULL); #endif @@ -1067,18 +1083,21 @@ _dispatch_queue_attr_index_qos_class_t _dispatch_queue_attr_qos2idx[] = { }; #define DISPATCH_QUEUE_ATTR_OVERCOMMIT2IDX(overcommit) \ - (overcommit ? DQA_INDEX_OVERCOMMIT : DQA_INDEX_NON_OVERCOMMIT) + ((overcommit) == _dispatch_queue_attr_overcommit_disabled ? \ + DQA_INDEX_NON_OVERCOMMIT : \ + ((overcommit) == _dispatch_queue_attr_overcommit_enabled ? \ + DQA_INDEX_OVERCOMMIT : DQA_INDEX_UNSPECIFIED_OVERCOMMIT)) #define DISPATCH_QUEUE_ATTR_CONCURRENT2IDX(concurrent) \ - (concurrent ? DQA_INDEX_CONCURRENT : DQA_INDEX_SERIAL) + ((concurrent) ? DQA_INDEX_CONCURRENT : DQA_INDEX_SERIAL) #define DISPATCH_QUEUE_ATTR_PRIO2IDX(prio) (-(prio)) #define DISPATCH_QUEUE_ATTR_QOS2IDX(qos) (_dispatch_queue_attr_qos2idx[(qos)]) static inline dispatch_queue_attr_t -_dispatch_get_queue_attr(qos_class_t qos, int prio, bool overcommit, - bool concurrent) +_dispatch_get_queue_attr(qos_class_t qos, int prio, + _dispatch_queue_attr_overcommit_t overcommit, bool concurrent) { return (dispatch_queue_attr_t)&_dispatch_queue_attrs [DISPATCH_QUEUE_ATTR_QOS2IDX(qos)] @@ -1093,7 +1112,8 @@ dispatch_queue_attr_make_with_qos_class(dispatch_queue_attr_t dqa, { if (!_dispatch_qos_class_valid(qos_class, relative_priority)) return NULL; if (!slowpath(dqa)) { - dqa = _dispatch_get_queue_attr(0, 0, false, false); + dqa = _dispatch_get_queue_attr(0, 0, + _dispatch_queue_attr_overcommit_unspecified, false); } else if (dqa->do_vtable != DISPATCH_VTABLE(queue_attr)) { DISPATCH_CLIENT_CRASH("Invalid queue attribute"); } @@ -1106,12 +1126,15 @@ dispatch_queue_attr_make_with_overcommit(dispatch_queue_attr_t dqa, bool overcommit) { if (!slowpath(dqa)) { - dqa = _dispatch_get_queue_attr(0, 0, false, false); + dqa = _dispatch_get_queue_attr(0, 0, + _dispatch_queue_attr_overcommit_unspecified, false); } else if (dqa->do_vtable != DISPATCH_VTABLE(queue_attr)) { DISPATCH_CLIENT_CRASH("Invalid queue attribute"); } return _dispatch_get_queue_attr(dqa->dqa_qos_class, - dqa->dqa_relative_priority, overcommit, dqa->dqa_concurrent); + dqa->dqa_relative_priority, overcommit ? + _dispatch_queue_attr_overcommit_enabled : + _dispatch_queue_attr_overcommit_disabled, dqa->dqa_concurrent); } #pragma mark - @@ -1136,7 +1159,8 @@ dispatch_queue_create_with_target(const char *label, dispatch_queue_attr_t dqa, #endif bool disallow_tq = (slowpath(dqa) && dqa != DISPATCH_QUEUE_CONCURRENT); if (!slowpath(dqa)) { - dqa = _dispatch_get_queue_attr(0, 0, false, false); + dqa = _dispatch_get_queue_attr(0, 0, + _dispatch_queue_attr_overcommit_unspecified, false); } else if (dqa->do_vtable != DISPATCH_VTABLE(queue_attr)) { DISPATCH_CLIENT_CRASH("Invalid queue attribute"); } @@ -1147,16 +1171,20 @@ dispatch_queue_create_with_target(const char *label, dispatch_queue_attr_t dqa, dq->dq_label = strdup(label); } qos_class_t qos = dqa->dqa_qos_class; - bool overcommit = dqa->dqa_overcommit; + _dispatch_queue_attr_overcommit_t overcommit = dqa->dqa_overcommit; + if (overcommit == _dispatch_queue_attr_overcommit_unspecified) { + // Serial queues default to overcommit! + overcommit = dqa->dqa_concurrent ? + _dispatch_queue_attr_overcommit_disabled : + _dispatch_queue_attr_overcommit_enabled; + } #if HAVE_PTHREAD_WORKQUEUE_QOS dq->dq_priority = _pthread_qos_class_encode(qos, dqa->dqa_relative_priority, - overcommit); + overcommit == _dispatch_queue_attr_overcommit_enabled ? + _PTHREAD_PRIORITY_OVERCOMMIT_FLAG : 0); #endif if (dqa->dqa_concurrent) { dq->dq_width = DISPATCH_QUEUE_WIDTH_MAX; - } else { - // Default serial queue target queue is overcommit! - overcommit = true; } if (!tq) { if (qos == _DISPATCH_QOS_CLASS_UNSPECIFIED) { @@ -1181,7 +1209,8 @@ dispatch_queue_create_with_target(const char *label, dispatch_queue_attr_t dqa, } } - tq = _dispatch_get_root_queue(qos, overcommit); + tq = _dispatch_get_root_queue(qos, overcommit == + _dispatch_queue_attr_overcommit_enabled); if (slowpath(!tq)) { DISPATCH_CLIENT_CRASH("Invalid queue attribute"); } @@ -1207,6 +1236,16 @@ dispatch_queue_create(const char *label, dispatch_queue_attr_t attr) DISPATCH_TARGET_QUEUE_DEFAULT); } +dispatch_queue_t +dispatch_queue_create_with_accounting_override_voucher(const char *label, + dispatch_queue_attr_t attr, voucher_t voucher) +{ + dispatch_queue_t dq = dispatch_queue_create_with_target(label, attr, + DISPATCH_TARGET_QUEUE_DEFAULT); + dq->dq_override_voucher = _voucher_create_accounting_voucher(voucher); + return dq; +} + void _dispatch_queue_destroy(dispatch_object_t dou) { @@ -1226,6 +1265,10 @@ _dispatch_queue_destroy(dispatch_object_t dou) if (dqsq) { _dispatch_release(dqsq); } + if (dq->dq_override_voucher != DISPATCH_NO_VOUCHER) { + if (dq->dq_override_voucher) _voucher_release(dq->dq_override_voucher); + dq->dq_override_voucher = DISPATCH_NO_VOUCHER; + } } // 6618342 Contact the team that owns the Instrument DTrace probe before @@ -1319,13 +1362,17 @@ static void _dispatch_set_target_queue2(void *ctxt) { dispatch_queue_t prev_dq, dq = _dispatch_queue_get_current(), tq = ctxt; +#if HAVE_PTHREAD_WORKQUEUE_QOS + // see _dispatch_queue_wakeup_with_qos_slow + mach_msg_timeout_t timeout = 1; mach_port_t th; while (!dispatch_atomic_cmpxchgv2o(dq, dq_tqthread, MACH_PORT_NULL, _dispatch_thread_port(), &th, acquire)) { _dispatch_thread_switch(th, DISPATCH_YIELD_THREAD_SWITCH_OPTION, - DISPATCH_CONTENTION_USLEEP_START); + timeout++); } +#endif _dispatch_queue_priority_inherit_from_target(dq, tq); prev_dq = dq->do_targetq; dq->do_targetq = tq; @@ -1383,6 +1430,7 @@ static struct dispatch_root_queue_context_s .dgq_ctxt = &_dispatch_mgr_root_queue_pthread_context, .dgq_thread_pool_size = 1, }}}; + static struct dispatch_queue_s _dispatch_mgr_root_queue = { .do_vtable = DISPATCH_VTABLE(queue_root), .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, @@ -1392,16 +1440,32 @@ static struct dispatch_queue_s _dispatch_mgr_root_queue = { .dq_label = "com.apple.root.libdispatch-manager", .dq_running = 2, .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_override_voucher = DISPATCH_NO_VOUCHER, .dq_serialnum = 3, }; + static struct { volatile int prio; + volatile qos_class_t qos; int default_prio; int policy; pthread_t tid; } _dispatch_mgr_sched; + static dispatch_once_t _dispatch_mgr_sched_pred; +// TODO: switch to "event-reflector thread" property + +// Must be kept in sync with list of qos classes in sys/qos.h +static const int _dispatch_mgr_sched_qos2prio[] = { + [_DISPATCH_QOS_CLASS_MAINTENANCE] = 4, + [_DISPATCH_QOS_CLASS_BACKGROUND] = 4, + [_DISPATCH_QOS_CLASS_UTILITY] = 20, + [_DISPATCH_QOS_CLASS_DEFAULT] = 31, + [_DISPATCH_QOS_CLASS_USER_INITIATED] = 37, + [_DISPATCH_QOS_CLASS_USER_INTERACTIVE] = 47, +}; + static void _dispatch_mgr_sched_init(void *ctxt DISPATCH_UNUSED) { @@ -1412,7 +1476,16 @@ _dispatch_mgr_sched_init(void *ctxt DISPATCH_UNUSED) (void)dispatch_assume_zero(pthread_attr_getschedpolicy(attr, &_dispatch_mgr_sched.policy)); (void)dispatch_assume_zero(pthread_attr_getschedparam(attr, ¶m)); - // legacy priority calls allowed when requesting above default priority +#if HAVE_PTHREAD_WORKQUEUE_QOS + qos_class_t qos = qos_class_main(); + if (qos == _DISPATCH_QOS_CLASS_DEFAULT) { + qos = _DISPATCH_QOS_CLASS_USER_INITIATED; // rdar://problem/17279292 + } + if (qos) { + _dispatch_mgr_sched.qos = qos; + param.sched_priority = _dispatch_mgr_sched_qos2prio[qos]; + } +#endif _dispatch_mgr_sched.default_prio = param.sched_priority; _dispatch_mgr_sched.prio = _dispatch_mgr_sched.default_prio; } @@ -1431,9 +1504,12 @@ _dispatch_mgr_root_queue_init(void) (void)dispatch_assume_zero(pthread_attr_setstacksize(attr, 64 * 1024)); #endif #if HAVE_PTHREAD_WORKQUEUE_QOS - if (_dispatch_set_qos_class_enabled) { - qos_class_t qos = qos_class_main(); - (void)dispatch_assume_zero(pthread_attr_set_qos_class_np(attr, qos, 0)); + qos_class_t qos = _dispatch_mgr_sched.qos; + if (qos) { + if (_dispatch_set_qos_class_enabled) { + (void)dispatch_assume_zero(pthread_attr_set_qos_class_np(attr, + qos, 0)); + } _dispatch_mgr_q.dq_priority = _pthread_qos_class_encode(qos, 0, 0); _dispatch_queue_set_override_priority(&_dispatch_mgr_q); } @@ -1467,6 +1543,17 @@ _dispatch_mgr_priority_init(void) pthread_attr_t *attr; attr = &_dispatch_mgr_root_queue_pthread_context.dpq_thread_attr; (void)dispatch_assume_zero(pthread_attr_getschedparam(attr, ¶m)); +#if HAVE_PTHREAD_WORKQUEUE_QOS + qos_class_t qos = 0; + (void)pthread_attr_get_qos_class_np(attr, &qos, NULL); + if (_dispatch_mgr_sched.qos > qos && _dispatch_set_qos_class_enabled) { + (void)pthread_set_qos_class_self_np(_dispatch_mgr_sched.qos, 0); + int p = _dispatch_mgr_sched_qos2prio[_dispatch_mgr_sched.qos]; + if (p > param.sched_priority) { + param.sched_priority = p; + } + } +#endif if (slowpath(_dispatch_mgr_sched.prio > param.sched_priority)) { return _dispatch_mgr_priority_apply(); } @@ -1479,6 +1566,18 @@ _dispatch_mgr_priority_raise(const pthread_attr_t *attr) dispatch_once_f(&_dispatch_mgr_sched_pred, NULL, _dispatch_mgr_sched_init); struct sched_param param; (void)dispatch_assume_zero(pthread_attr_getschedparam(attr, ¶m)); +#if HAVE_PTHREAD_WORKQUEUE_QOS + qos_class_t qos = 0; + (void)pthread_attr_get_qos_class_np((pthread_attr_t *)attr, &qos, NULL); + if (qos) { + param.sched_priority = _dispatch_mgr_sched_qos2prio[qos]; + qos_class_t q = _dispatch_mgr_sched.qos; + do if (q >= qos) { + break; + } while (slowpath(!dispatch_atomic_cmpxchgvw2o(&_dispatch_mgr_sched, + qos, q, qos, &q, relaxed))); + } +#endif int p = _dispatch_mgr_sched.prio; do if (p >= param.sched_priority) { return; @@ -1489,9 +1588,10 @@ _dispatch_mgr_priority_raise(const pthread_attr_t *attr) } } -dispatch_queue_t -dispatch_pthread_root_queue_create(const char *label, unsigned long flags, - const pthread_attr_t *attr, dispatch_block_t configure) +static dispatch_queue_t +_dispatch_pthread_root_queue_create(const char *label, unsigned long flags, + const pthread_attr_t *attr, dispatch_block_t configure, + dispatch_pthread_root_queue_observer_hooks_t observer_hooks) { dispatch_queue_t dq; dispatch_root_queue_context_t qc; @@ -1527,14 +1627,6 @@ dispatch_pthread_root_queue_create(const char *label, unsigned long flags, if (attr) { memcpy(&pqc->dpq_thread_attr, attr, sizeof(pthread_attr_t)); -#if HAVE_PTHREAD_WORKQUEUE_QOS - qos_class_t qos = 0; - if (!pthread_attr_get_qos_class_np(&pqc->dpq_thread_attr, &qos, NULL) - && qos > _DISPATCH_QOS_CLASS_DEFAULT) { - DISPATCH_CLIENT_CRASH("pthread root queues do not support " - "explicit QoS attributes"); - } -#endif _dispatch_mgr_priority_raise(&pqc->dpq_thread_attr); } else { (void)dispatch_assume_zero(pthread_attr_init(&pqc->dpq_thread_attr)); @@ -1544,10 +1636,23 @@ dispatch_pthread_root_queue_create(const char *label, unsigned long flags, if (configure) { pqc->dpq_thread_configure = _dispatch_Block_copy(configure); } + if (observer_hooks) { + pqc->dpq_observer_hooks = *observer_hooks; + } _dispatch_object_debug(dq, "%s", __func__); return _dispatch_introspection_queue_create(dq); } -#endif + +dispatch_queue_t +dispatch_pthread_root_queue_create(const char *label, unsigned long flags, + const pthread_attr_t *attr, dispatch_block_t configure) +{ + return _dispatch_pthread_root_queue_create(label, flags, attr, configure, + NULL); +} + + +#endif // DISPATCH_ENABLE_PTHREAD_ROOT_QUEUES void _dispatch_pthread_root_queue_dispose(dispatch_queue_t dq) @@ -1737,6 +1842,7 @@ dispatch_get_specific(const void *key) return ctxt; } + #pragma mark - #pragma mark dispatch_queue_debug @@ -1745,12 +1851,12 @@ _dispatch_queue_debug_attr(dispatch_queue_t dq, char* buf, size_t bufsiz) { size_t offset = 0; dispatch_queue_t target = dq->do_targetq; - offset += dsnprintf(buf, bufsiz, "target = %s[%p], width = 0x%x, " - "running = 0x%x, barrier = %d ", target && target->dq_label ? - target->dq_label : "", target, dq->dq_width / 2, - dq->dq_running / 2, dq->dq_running & 1); + offset += dsnprintf(&buf[offset], bufsiz - offset, "target = %s[%p], " + "width = 0x%x, running = 0x%x, barrier = %d ", + target && target->dq_label ? target->dq_label : "", target, + dq->dq_width / 2, dq->dq_running / 2, dq->dq_running & 1); if (dq->dq_is_thread_bound) { - offset += dsnprintf(buf, bufsiz, ", thread = 0x%x ", + offset += dsnprintf(&buf[offset], bufsiz - offset, ", thread = 0x%x ", _dispatch_queue_get_bound_thread(dq)); } return offset; @@ -1868,6 +1974,8 @@ _dispatch_continuation_redirect(dispatch_queue_t dq, dispatch_object_t dou) if (!DISPATCH_OBJ_IS_VTABLE(dc) && (long)dc->do_vtable & DISPATCH_OBJ_SYNC_SLOW_BIT) { _dispatch_trace_continuation_pop(dq, dou); + _dispatch_wqthread_override_start((mach_port_t)dc->dc_data, + _dispatch_queue_get_override_priority(dq)); _dispatch_thread_semaphore_signal( (_dispatch_thread_semaphore_t)dc->dc_other); _dispatch_introspection_queue_item_complete(dou); @@ -1985,12 +2093,11 @@ dispatch_block_perform(dispatch_block_flags_t flags, dispatch_block_t block) } flags = _dispatch_block_normalize_flags(flags); struct dispatch_block_private_data_s dbpds = - DISPATCH_BLOCK_PRIVATE_DATA_INITIALIZER(flags, NULL, 0, block); - dbpds.dbpd_atomic_flags |= DBF_PERFORM; // no group_leave at end of invoke + DISPATCH_BLOCK_PRIVATE_DATA_PERFORM_INITIALIZER(flags, block); return _dispatch_block_invoke(&dbpds); } -#define _dbpd_group(dbpd) ((dispatch_group_t)&(dbpd)->dbpd_group) +#define _dbpd_group(dbpd) ((dbpd)->dbpd_group) void _dispatch_block_invoke(const struct dispatch_block_private_data_s *dbcpd) @@ -2005,11 +2112,11 @@ _dispatch_block_invoke(const struct dispatch_block_private_data_s *dbcpd) if (atomic_flags & DBF_CANCELED) goto out; pthread_priority_t op = DISPATCH_NO_PRIORITY, p = DISPATCH_NO_PRIORITY; - unsigned long override = 0; + unsigned long override = DISPATCH_VOUCHER_IGNORE_QUEUE_OVERRIDE; if (flags & DISPATCH_BLOCK_HAS_PRIORITY) { op = _dispatch_get_priority(); p = dbpd->dbpd_priority; - override = (flags & DISPATCH_BLOCK_ENFORCE_QOS_CLASS) || + override |= (flags & DISPATCH_BLOCK_ENFORCE_QOS_CLASS) || !(flags & DISPATCH_BLOCK_INHERIT_QOS_CLASS) ? DISPATCH_PRIORITY_ENFORCE : 0; } @@ -2020,8 +2127,9 @@ _dispatch_block_invoke(const struct dispatch_block_private_data_s *dbcpd) } ov = _dispatch_adopt_priority_and_voucher(p, v, override); dbpd->dbpd_thread = _dispatch_thread_port(); - dbpd->dbpd_block(); - _dispatch_set_priority_and_replace_voucher(op, ov); + _dispatch_client_callout(dbpd->dbpd_block, + _dispatch_Block_invoke(dbpd->dbpd_block)); + _dispatch_reset_priority_and_voucher(op, ov); out: if ((atomic_flags & DBF_PERFORM) == 0) { if (dispatch_atomic_inc2o(dbpd, dbpd_performed, acquire) == 1) { @@ -2059,7 +2167,7 @@ _dispatch_block_sync_invoke(void *block) } ov = _dispatch_adopt_priority_and_voucher(p, v, override); dbpd->dbpd_block(); - _dispatch_set_priority_and_replace_voucher(op, ov); + _dispatch_reset_priority_and_voucher(op, ov); out: if ((atomic_flags & DBF_PERFORM) == 0) { if (dispatch_atomic_inc2o(dbpd, dbpd_performed, acquire) == 1) { @@ -2485,6 +2593,15 @@ dispatch_async_f(dispatch_queue_t dq, void *ctxt, dispatch_function_t func) return _dispatch_async_f(dq, ctxt, func, 0, 0); } +DISPATCH_NOINLINE +void +dispatch_async_enforce_qos_class_f(dispatch_queue_t dq, void *ctxt, + dispatch_function_t func) +{ + return _dispatch_async_f(dq, ctxt, func, 0, + DISPATCH_BLOCK_ENFORCE_QOS_CLASS); +} + #ifdef __BLOCKS__ void dispatch_async(dispatch_queue_t dq, void (^work)(void)) @@ -2578,11 +2695,28 @@ dispatch_group_async(dispatch_group_t dg, dispatch_queue_t dq, static void _dispatch_sync_f(dispatch_queue_t dq, void *ctxt, dispatch_function_t func, pthread_priority_t pp); +DISPATCH_NOINLINE +static void +_dispatch_function_invoke_slow(dispatch_queue_t dq, void *ctxt, + dispatch_function_t func) +{ + dispatch_queue_t old_dq = _dispatch_thread_getspecific(dispatch_queue_key); + _dispatch_thread_setspecific(dispatch_queue_key, dq); + voucher_t ov = _dispatch_adopt_queue_override_voucher(dq); + _dispatch_client_callout(ctxt, func); + _dispatch_perfmon_workitem_inc(); + _dispatch_reset_voucher(ov); + _dispatch_thread_setspecific(dispatch_queue_key, old_dq); +} + DISPATCH_ALWAYS_INLINE static inline void _dispatch_function_invoke(dispatch_queue_t dq, void *ctxt, dispatch_function_t func) { + if (slowpath(dq->dq_override_voucher != DISPATCH_NO_VOUCHER)) { + return _dispatch_function_invoke_slow(dq, ctxt, func); + } dispatch_queue_t old_dq = _dispatch_thread_getspecific(dispatch_queue_key); _dispatch_thread_setspecific(dispatch_queue_key, dq); _dispatch_client_callout(ctxt, func); @@ -2718,6 +2852,12 @@ _dispatch_barrier_sync_f_slow(dispatch_queue_t dq, void *ctxt, _dispatch_thread_semaphore_wait(sema); // acquire _dispatch_put_thread_semaphore(sema); + pthread_priority_t p = _dispatch_queue_get_override_priority(dq); + if (p > (pp & _PTHREAD_PRIORITY_QOS_CLASS_MASK)) { + // Ensure that the root queue sees that this thread was overridden. + _dispatch_set_defaultpriority_override(); + } + #if DISPATCH_COCOA_COMPAT // Queue bound to a non-dispatch thread if (dc.dc_func == NULL) { @@ -2949,6 +3089,12 @@ _dispatch_sync_f_slow(dispatch_queue_t dq, void *ctxt, dispatch_function_t func, _dispatch_thread_semaphore_wait(sema); _dispatch_put_thread_semaphore(sema); + pthread_priority_t p = _dispatch_queue_get_override_priority(dq); + if (p > (pp & _PTHREAD_PRIORITY_QOS_CLASS_MASK)) { + // Ensure that the root queue sees that this thread was overridden. + _dispatch_set_defaultpriority_override(); + } + if (slowpath(dq->do_targetq->do_targetq)) { _dispatch_function_recurse(dq, ctxt, func, pp); } else { @@ -3467,9 +3613,10 @@ dispatch_queue_invoke2(dispatch_object_t dou, // renaming this symbol DISPATCH_NOINLINE void -_dispatch_queue_invoke(dispatch_queue_t dq) +_dispatch_queue_invoke(dispatch_queue_t dq, dispatch_object_t dou, + dispatch_invoke_flags_t flags) { - _dispatch_queue_class_invoke(dq, dispatch_queue_invoke2); + _dispatch_queue_class_invoke(dq, dou._dc, flags, dispatch_queue_invoke2); } #pragma mark - @@ -3602,10 +3749,15 @@ _dispatch_main_queue_drain(void) out: if (next_dc) { _dispatch_main_queue_wakeup(); + } else { + pthread_priority_t p = _dispatch_queue_reset_override_priority(dq); + + if (p > (dq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK)) { + _dispatch_thread_override_end(dq->dq_thread); + } } _dispatch_voucher_debug("main queue restore", voucher); - _dispatch_set_priority_and_replace_voucher(old_pri, voucher); - _dispatch_queue_reset_override_priority(dq); + _dispatch_reset_priority_and_voucher(old_pri, voucher); _dispatch_reset_defaultpriority(old_dp); _dispatch_thread_setspecific(dispatch_queue_key, old_dq); _dispatch_perfmon_end(); @@ -3632,7 +3784,7 @@ _dispatch_runloop_queue_drain_one(dispatch_queue_t dq) _dispatch_perfmon_workitem_inc(); _dispatch_voucher_debug("runloop queue restore", voucher); - _dispatch_set_priority_and_replace_voucher(old_pri, voucher); + _dispatch_reset_priority_and_voucher(old_pri, voucher); _dispatch_reset_defaultpriority(old_dp); _dispatch_thread_setspecific(dispatch_queue_key, old_dq); _dispatch_perfmon_end(); @@ -3690,20 +3842,28 @@ _dispatch_queue_wakeup_with_qos_slow(dispatch_queue_t dq, pthread_priority_t pp, if (retained) _dispatch_release(dq); return NULL; } - pp &= _PTHREAD_PRIORITY_QOS_CLASS_MASK; - bool override = _dispatch_queue_override_priority(dq, pp); - if (override && dq->dq_running > 1) { - override = false; - } - if (!dispatch_atomic_cmpxchg2o(dq, do_suspend_cnt, 0, DISPATCH_OBJECT_SUSPEND_LOCK, acquire)) { + bool was_overridden, override; + + override = _dispatch_queue_override_priority(dq, &pp, &was_overridden); + if (override && dq->dq_running > 1) { + override = false; + } + #if DISPATCH_COCOA_COMPAT if (dq == &_dispatch_main_q && dq->dq_is_thread_bound) { + if (override) { + _dispatch_thread_override_start(dq->dq_thread, pp); + if (was_overridden) { + _dispatch_thread_override_end(dq->dq_thread); + } + } return _dispatch_main_queue_wakeup(); } #endif if (override) { +#if HAVE_PTHREAD_WORKQUEUE_QOS mach_port_t th; // to traverse the tq chain safely we must // lock it to ensure it cannot change, unless the queue is running @@ -3713,31 +3873,27 @@ _dispatch_queue_wakeup_with_qos_slow(dispatch_queue_t dq, pthread_priority_t pp, } else if (!dispatch_atomic_cmpxchgv2o(dq, dq_tqthread, MACH_PORT_NULL, _dispatch_thread_port(), &th, acquire)) { // already locked, override the owner, trysync will do a queue - // wakeup when it returns. + // wakeup when it returns, see _dispatch_set_target_queue2 _dispatch_wqthread_override_start(th, pp); } else { dispatch_queue_t tq = dq->do_targetq; if (_dispatch_queue_prepare_override(dq, tq, pp)) { - _dispatch_queue_push_override(dq, tq, pp); + _dispatch_queue_push_override(dq, tq, pp, false); } else { _dispatch_queue_wakeup_with_qos(tq, pp); } dispatch_atomic_store2o(dq, dq_tqthread, MACH_PORT_NULL, release); } +#endif } if (retained) _dispatch_release(dq); return NULL; } + dispatch_queue_t tq = dq->do_targetq; if (!retained) _dispatch_retain(dq); - if (override) { - override = _dispatch_queue_prepare_override(dq, tq, pp); - } - _dispatch_queue_push(tq, dq, pp); - if (override) { - _dispatch_queue_push_override(dq, tq, pp); - } + _dispatch_queue_push_queue(tq, dq, pp); return tq; // libdispatch does not need this, but the Instrument DTrace // probe does } @@ -3748,7 +3904,7 @@ _dispatch_queue_wakeup_with_qos2(dispatch_queue_t dq, pthread_priority_t pp, bool retained) { if (_dispatch_object_suspended(dq)) { - _dispatch_queue_override_priority(dq, pp); + _dispatch_queue_override_priority(dq, &pp, NULL); if (retained) _dispatch_release(dq); return NULL; } @@ -3770,6 +3926,14 @@ _dispatch_queue_wakeup_with_qos(dispatch_queue_t dq, pthread_priority_t pp) (void)_dispatch_queue_wakeup_with_qos2(dq, pp, false); } +DISPATCH_NOINLINE +void +_dispatch_queue_wakeup_and_release(dispatch_queue_t dq) +{ + (void)_dispatch_queue_wakeup_with_qos2(dq, + _dispatch_queue_get_override_priority(dq), true); +} + DISPATCH_NOINLINE dispatch_queue_t _dispatch_queue_wakeup(dispatch_queue_t dq) @@ -3779,53 +3943,26 @@ _dispatch_queue_wakeup(dispatch_queue_t dq) } #if HAVE_PTHREAD_WORKQUEUE_QOS +DISPATCH_NOINLINE static void -_dispatch_queue_override_invoke(void *ctxt) +_dispatch_queue_override_invoke_stealing(void *ctxt) { dispatch_continuation_t dc = (dispatch_continuation_t)ctxt; dispatch_queue_t dq = dc->dc_data; - pthread_priority_t p = 0; - if (!slowpath(DISPATCH_OBJECT_SUSPENDED(dq)) && - fastpath(dispatch_atomic_cmpxchg2o(dq, dq_running, 0, 1, acquire))) { - _dispatch_queue_set_thread(dq); - - _dispatch_object_debug(dq, "stolen onto thread 0x%x, 0x%lx", - dq->dq_thread, _dispatch_get_defaultpriority()); - - pthread_priority_t old_dp = _dispatch_get_defaultpriority(); - _dispatch_reset_defaultpriority(dc->dc_priority); - - dispatch_queue_t tq = NULL; - _dispatch_thread_semaphore_t sema = 0; - tq = dispatch_queue_invoke2(dq, &sema); + dx_invoke(dq, dc, DISPATCH_INVOKE_OVERRIDING | DISPATCH_INVOKE_STEALING); +} - _dispatch_queue_clear_thread(dq); - _dispatch_reset_defaultpriority(old_dp); +DISPATCH_NOINLINE +static void +_dispatch_queue_override_invoke_owning(void *ctxt) +{ + dispatch_continuation_t dc = (dispatch_continuation_t)ctxt; + dispatch_queue_t dq = dc->dc_data; - uint32_t running = dispatch_atomic_dec2o(dq, dq_running, release); - if (sema) { - _dispatch_thread_semaphore_signal(sema); - } else if (!tq && running == 0) { - p = _dispatch_queue_reset_override_priority(dq); - if (p > (dq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK)) { - _dispatch_wqthread_override_reset(); - } - } - _dispatch_introspection_queue_item_complete(dq); - if (running == 0) { - return _dispatch_queue_wakeup_with_qos_and_release(dq, p); - } - } else { - mach_port_t th = dq->dq_thread; - if (th) { - p = _dispatch_queue_get_override_priority(dq); - _dispatch_object_debug(dq, "overriding thr 0x%x to priority 0x%lx", - th, p); - _dispatch_wqthread_override_start(th, p); - } - } - _dispatch_release(dq); // added when we pushed the override block + // balance the fake continuation push in _dispatch_queue_push_override + _dispatch_trace_continuation_pop(dc->dc_other, dc->dc_data); + dx_invoke(dq, dc, DISPATCH_INVOKE_OVERRIDING); } #endif @@ -3843,7 +3980,6 @@ _dispatch_queue_prepare_override(dispatch_queue_t dq, dispatch_queue_t tq, if (p <= (tq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK)) { return false; } - _dispatch_retain(dq); return true; #else (void)dq; (void)tq; (void)p; @@ -3853,7 +3989,7 @@ _dispatch_queue_prepare_override(dispatch_queue_t dq, dispatch_queue_t tq, static inline void _dispatch_queue_push_override(dispatch_queue_t dq, dispatch_queue_t tq, - pthread_priority_t p) + pthread_priority_t p, bool owning) { #if HAVE_PTHREAD_WORKQUEUE_QOS unsigned int qosbit, idx, overcommit; @@ -3868,12 +4004,19 @@ _dispatch_queue_push_override(dispatch_queue_t dq, dispatch_queue_t tq, dispatch_continuation_t dc = _dispatch_continuation_alloc(); dc->do_vtable = (void *)(DISPATCH_OBJ_ASYNC_BIT | DISPATCH_OBJ_BARRIER_BIT); - dc->dc_func = _dispatch_queue_override_invoke; + if (owning) { + // fake that we queued `dq` on `tq` for introspection purposes + _dispatch_trace_continuation_push(tq, dq); + dc->dc_func = _dispatch_queue_override_invoke_owning; + } else { + dc->dc_func = _dispatch_queue_override_invoke_stealing; + _dispatch_retain(dq); + } dc->dc_ctxt = dc; - dc->dc_priority = tq->dq_priority; + dc->dc_priority = 0; + dc->dc_other = tq; dc->dc_voucher = NULL; dc->dc_data = dq; - // dq retained by _dispatch_queue_prepare_override _dispatch_queue_push(rq, dc, 0); #else @@ -3881,6 +4024,18 @@ _dispatch_queue_push_override(dispatch_queue_t dq, dispatch_queue_t tq, #endif } +void +_dispatch_queue_push_queue(dispatch_queue_t tq, dispatch_queue_t dq, + pthread_priority_t pp) +{ + _dispatch_queue_override_priority(dq, &pp, NULL); + if (_dispatch_queue_prepare_override(dq, tq, pp)) { + _dispatch_queue_push_override(dq, tq, pp, true); + } else { + _dispatch_queue_push(tq, dq, pp); + } +} + #pragma mark - #pragma mark dispatch_root_queue_drain @@ -4023,7 +4178,7 @@ _dispatch_root_queue_drain(dispatch_queue_t dq) reset = _dispatch_reset_defaultpriority_override(); } _dispatch_voucher_debug("root queue clear", NULL); - _dispatch_set_priority_and_replace_voucher(old_pri, NULL); + _dispatch_reset_priority_and_voucher(old_pri, NULL); _dispatch_reset_defaultpriority(old_dp); _dispatch_perfmon_end(); @@ -4104,6 +4259,10 @@ _dispatch_worker_thread(void *context) dispatch_root_queue_context_t qc = dq->do_ctxt; dispatch_pthread_root_queue_context_t pqc = qc->dgq_ctxt; + if (pqc->dpq_observer_hooks.queue_will_execute) { + _dispatch_set_pthread_root_queue_observer_hooks( + &pqc->dpq_observer_hooks); + } if (pqc->dpq_thread_configure) { pqc->dpq_thread_configure(); } @@ -4366,12 +4525,13 @@ _dispatch_queue_cleanup2(void) { dispatch_queue_t dq = &_dispatch_main_q; (void)dispatch_atomic_dec2o(dq, dq_running, relaxed); - unsigned int suspend_cnt = dispatch_atomic_sub2o(dq, do_suspend_cnt, + (void)dispatch_atomic_sub2o(dq, do_suspend_cnt, DISPATCH_OBJECT_SUSPEND_LOCK, release); + _dispatch_queue_clear_bound_thread(dq); dq->dq_is_thread_bound = 0; - if (suspend_cnt == 0) { - _dispatch_queue_wakeup(dq); - } + // no need to drop the override, the thread will die anyway + _dispatch_queue_wakeup_with_qos(dq, + _dispatch_queue_reset_override_priority(dq)); // overload the "probably" variable to mean that dispatch_main() or // similar non-POSIX API was called diff --git a/src/queue_internal.h b/src/queue_internal.h index d76b15f..143ab1e 100644 --- a/src/queue_internal.h +++ b/src/queue_internal.h @@ -56,10 +56,11 @@ dispatch_queue_t dq_specific_q; \ uint16_t dq_width; \ uint16_t dq_is_thread_bound:1; \ + uint32_t volatile dq_override; \ pthread_priority_t dq_priority; \ mach_port_t dq_thread; \ mach_port_t volatile dq_tqthread; \ - uint32_t volatile dq_override; \ + voucher_t dq_override_voucher; \ unsigned long dq_serialnum; \ const char *dq_label; \ DISPATCH_INTROSPECTION_QUEUE_LIST; @@ -74,16 +75,18 @@ + DISPATCH_CACHELINE_SIZE) % DISPATCH_CACHELINE_SIZE) #else #define DISPATCH_QUEUE_CACHELINE_PAD (( \ - (13*sizeof(void*) - DISPATCH_INTROSPECTION_QUEUE_LIST_SIZE) \ + (12*sizeof(void*) - DISPATCH_INTROSPECTION_QUEUE_LIST_SIZE) \ + DISPATCH_CACHELINE_SIZE) % DISPATCH_CACHELINE_SIZE) #endif DISPATCH_CLASS_DECL(queue); +#if !(defined(__cplusplus) && DISPATCH_INTROSPECTION) struct dispatch_queue_s { DISPATCH_STRUCT_HEADER(queue); DISPATCH_QUEUE_HEADER; DISPATCH_QUEUE_CACHELINE_PADDING; // for static queues only }; +#endif // !(defined(__cplusplus) && DISPATCH_INTROSPECTION) DISPATCH_INTERNAL_SUBCLASS_DECL(queue_root, queue); DISPATCH_INTERNAL_SUBCLASS_DECL(queue_runloop, queue); @@ -94,7 +97,8 @@ DISPATCH_CLASS_DECL(queue_specific_queue); void _dispatch_queue_destroy(dispatch_object_t dou); void _dispatch_queue_dispose(dispatch_queue_t dq); -void _dispatch_queue_invoke(dispatch_queue_t dq); +void _dispatch_queue_invoke(dispatch_queue_t dq, dispatch_object_t dou, + dispatch_invoke_flags_t flags); void _dispatch_queue_push_list_slow(dispatch_queue_t dq, pthread_priority_t pp, struct dispatch_object_s *obj, unsigned int n, bool retained); @@ -103,10 +107,13 @@ void _dispatch_queue_push_slow(dispatch_queue_t dq, unsigned long _dispatch_queue_probe(dispatch_queue_t dq); dispatch_queue_t _dispatch_wakeup(dispatch_object_t dou); dispatch_queue_t _dispatch_queue_wakeup(dispatch_queue_t dq); +void _dispatch_queue_wakeup_and_release(dispatch_queue_t dq); void _dispatch_queue_wakeup_with_qos(dispatch_queue_t dq, pthread_priority_t pp); void _dispatch_queue_wakeup_with_qos_and_release(dispatch_queue_t dq, pthread_priority_t pp); +void _dispatch_queue_push_queue(dispatch_queue_t tq, dispatch_queue_t dq, + pthread_priority_t pp); _dispatch_thread_semaphore_t _dispatch_queue_drain(dispatch_object_t dou); void _dispatch_queue_specific_queue_dispose(dispatch_queue_specific_queue_t dqsq); @@ -175,25 +182,34 @@ extern pthread_priority_t _dispatch_user_initiated_priority; #pragma mark - #pragma mark dispatch_queue_attr_t +typedef enum { + _dispatch_queue_attr_overcommit_unspecified = 0, + _dispatch_queue_attr_overcommit_enabled, + _dispatch_queue_attr_overcommit_disabled, +} _dispatch_queue_attr_overcommit_t; + DISPATCH_CLASS_DECL(queue_attr); struct dispatch_queue_attr_s { DISPATCH_STRUCT_HEADER(queue_attr); qos_class_t dqa_qos_class; int dqa_relative_priority; - unsigned int dqa_overcommit:1, dqa_concurrent:1; + unsigned int dqa_overcommit:2, dqa_concurrent:1; }; enum { DQA_INDEX_NON_OVERCOMMIT = 0, DQA_INDEX_OVERCOMMIT, + DQA_INDEX_UNSPECIFIED_OVERCOMMIT, }; +#define DISPATCH_QUEUE_ATTR_OVERCOMMIT_COUNT 3 + enum { DQA_INDEX_CONCURRENT = 0, DQA_INDEX_SERIAL, }; -#define DISPATCH_QUEUE_ATTR_PRIO_COUNT (1 - QOS_MIN_RELATIVE_PRIORITY) +#define DISPATCH_QUEUE_ATTR_CONCURRENCY_COUNT 2 typedef enum { DQA_INDEX_QOS_CLASS_UNSPECIFIED = 0, @@ -205,8 +221,12 @@ typedef enum { DQA_INDEX_QOS_CLASS_USER_INTERACTIVE, } _dispatch_queue_attr_index_qos_class_t; +#define DISPATCH_QUEUE_ATTR_PRIO_COUNT (1 - QOS_MIN_RELATIVE_PRIORITY) + extern const struct dispatch_queue_attr_s _dispatch_queue_attrs[] - [DISPATCH_QUEUE_ATTR_PRIO_COUNT][2][2]; + [DISPATCH_QUEUE_ATTR_PRIO_COUNT] + [DISPATCH_QUEUE_ATTR_OVERCOMMIT_COUNT] + [DISPATCH_QUEUE_ATTR_CONCURRENCY_COUNT]; #pragma mark - #pragma mark dispatch_continuation_t @@ -279,7 +299,7 @@ typedef struct dispatch_continuation_s *dispatch_continuation_t; #define DISPATCH_CONTINUATION_CACHE_LIMIT 112 // one 256k heap for 64 threads #define DISPATCH_CONTINUATION_CACHE_LIMIT_MEMORYSTATUS_PRESSURE_WARN 16 #else -#define DISPATCH_CONTINUATION_CACHE_LIMIT 65536 +#define DISPATCH_CONTINUATION_CACHE_LIMIT 1024 #define DISPATCH_CONTINUATION_CACHE_LIMIT_MEMORYSTATUS_PRESSURE_WARN 128 #endif #endif @@ -317,18 +337,23 @@ typedef struct dispatch_apply_s *dispatch_apply_t; #define DISPATCH_BLOCK_HAS_VOUCHER (1u << 31) #define DISPATCH_BLOCK_HAS_PRIORITY (1u << 30) -struct dispatch_block_private_data_s { - unsigned long dbpd_magic; - dispatch_block_flags_t dbpd_flags; - unsigned int volatile dbpd_atomic_flags; - int volatile dbpd_performed; - pthread_priority_t dbpd_priority; - voucher_t dbpd_voucher; - dispatch_block_t dbpd_block; - struct dispatch_semaphore_s dbpd_group; - dispatch_queue_t volatile dbpd_queue; +#define DISPATCH_BLOCK_PRIVATE_DATA_HEADER() \ + unsigned long dbpd_magic; \ + dispatch_block_flags_t dbpd_flags; \ + unsigned int volatile dbpd_atomic_flags; \ + int volatile dbpd_performed; \ + pthread_priority_t dbpd_priority; \ + voucher_t dbpd_voucher; \ + dispatch_block_t dbpd_block; \ + dispatch_group_t dbpd_group; \ + dispatch_queue_t volatile dbpd_queue; \ mach_port_t dbpd_thread; + +#if !defined(__cplusplus) +struct dispatch_block_private_data_s { + DISPATCH_BLOCK_PRIVATE_DATA_HEADER(); }; +#endif typedef struct dispatch_block_private_data_s *dispatch_block_private_data_t; // dbpd_atomic_flags bits @@ -339,14 +364,13 @@ typedef struct dispatch_block_private_data_s *dispatch_block_private_data_t; #define DISPATCH_BLOCK_PRIVATE_DATA_MAGIC 0xD159B10C // 0xDISPatch_BLOCk -#define DISPATCH_BLOCK_PRIVATE_DATA_INITIALIZER(flags, voucher, prio, block) \ +// struct for synchronous perform: no group_leave at end of invoke +#define DISPATCH_BLOCK_PRIVATE_DATA_PERFORM_INITIALIZER(flags, block) \ { \ .dbpd_magic = DISPATCH_BLOCK_PRIVATE_DATA_MAGIC, \ .dbpd_flags = (flags), \ - .dbpd_priority = (prio), \ - .dbpd_voucher = (voucher), \ + .dbpd_atomic_flags = DBF_PERFORM, \ .dbpd_block = (block), \ - .dbpd_group = DISPATCH_GROUP_INITIALIZER(1), \ } dispatch_block_t _dispatch_block_create(dispatch_block_flags_t flags, diff --git a/src/semaphore.c b/src/semaphore.c index f9bfdbe..f356fb8 100644 --- a/src/semaphore.c +++ b/src/semaphore.c @@ -415,15 +415,28 @@ dispatch_semaphore_wait(dispatch_semaphore_t dsema, dispatch_time_t timeout) #pragma mark - #pragma mark dispatch_group_t -dispatch_group_t -dispatch_group_create(void) +DISPATCH_ALWAYS_INLINE +static inline dispatch_group_t +_dispatch_group_create_with_count(long count) { dispatch_group_t dg = (dispatch_group_t)_dispatch_alloc( DISPATCH_VTABLE(group), sizeof(struct dispatch_semaphore_s)); - _dispatch_semaphore_init(LONG_MAX, dg); + _dispatch_semaphore_init(LONG_MAX - count, dg); return dg; } +dispatch_group_t +dispatch_group_create(void) +{ + return _dispatch_group_create_with_count(0); +} + +dispatch_group_t +_dispatch_group_create_and_enter(void) +{ + return _dispatch_group_create_with_count(1); +} + void dispatch_group_enter(dispatch_group_t dg) { @@ -507,7 +520,7 @@ DISPATCH_NOINLINE static long _dispatch_group_wait_slow(dispatch_semaphore_t dsema, dispatch_time_t timeout) { - long orig; + long orig, value; #if USE_MACH_SEM mach_timespec_t _timeout; @@ -525,7 +538,8 @@ _dispatch_group_wait_slow(dispatch_semaphore_t dsema, dispatch_time_t timeout) again: // check before we cause another signal to be sent by incrementing // dsema->dsema_group_waiters - if (dsema->dsema_value == LONG_MAX) { + value = dispatch_atomic_load2o(dsema, dsema_value, seq_cst); // 19296565 + if (value == LONG_MAX) { return _dispatch_group_wake(dsema); } // Mach semaphores appear to sometimes spuriously wake up. Therefore, @@ -533,7 +547,8 @@ again: // signaled (6880961). (void)dispatch_atomic_inc2o(dsema, dsema_group_waiters, relaxed); // check the values again in case we need to wake any threads - if (dsema->dsema_value == LONG_MAX) { + value = dispatch_atomic_load2o(dsema, dsema_value, seq_cst); // 19296565 + if (value == LONG_MAX) { return _dispatch_group_wake(dsema); } diff --git a/src/semaphore_internal.h b/src/semaphore_internal.h index 01179cb..11261c3 100644 --- a/src/semaphore_internal.h +++ b/src/semaphore_internal.h @@ -53,15 +53,7 @@ struct dispatch_semaphore_s { DISPATCH_CLASS_DECL(group); -#define DISPATCH_GROUP_INITIALIZER(s) \ - { \ - .do_vtable = (const void*)DISPATCH_VTABLE(group), \ - .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, \ - .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, \ - .dsema_value = LONG_MAX - (s), \ - .dsema_orig = LONG_MAX, \ - } - +dispatch_group_t _dispatch_group_create_and_enter(void); void _dispatch_semaphore_dispose(dispatch_object_t dou); size_t _dispatch_semaphore_debug(dispatch_object_t dou, char *buf, size_t bufsiz); diff --git a/src/shims/tsd.h b/src/shims/tsd.h index 25de7e4..cf52385 100644 --- a/src/shims/tsd.h +++ b/src/shims/tsd.h @@ -49,15 +49,17 @@ static const unsigned long dispatch_io_key = __PTK_LIBDISPATCH_KEY3; static const unsigned long dispatch_apply_key = __PTK_LIBDISPATCH_KEY4; static const unsigned long dispatch_defaultpriority_key =__PTK_LIBDISPATCH_KEY5; #if DISPATCH_INTROSPECTION -static const unsigned long dispatch_introspection_key =__PTK_LIBDISPATCH_KEY5+1; +static const unsigned long dispatch_introspection_key = __PTK_LIBDISPATCH_KEY6; #elif DISPATCH_PERF_MON -static const unsigned long dispatch_bcounter_key = __PTK_LIBDISPATCH_KEY5+1; +static const unsigned long dispatch_bcounter_key = __PTK_LIBDISPATCH_KEY6; #endif #if DISPATCH_USE_OS_SEMAPHORE_CACHE static const unsigned long dispatch_sema4_key = __TSD_SEMAPHORE_CACHE; #else -static const unsigned long dispatch_sema4_key = __PTK_LIBDISPATCH_KEY5+2; +static const unsigned long dispatch_sema4_key = __PTK_LIBDISPATCH_KEY7; #endif +static const unsigned long dispatch_pthread_root_queue_observer_hooks_key = + __PTK_LIBDISPATCH_KEY8; #ifndef __TSD_THREAD_QOS_CLASS #define __TSD_THREAD_QOS_CLASS 4 @@ -88,7 +90,7 @@ extern pthread_key_t dispatch_introspection_key; #elif DISPATCH_PERF_MON extern pthread_key_t dispatch_bcounter_key; #endif - +exern pthread_key_t dispatch_pthread_root_queue_observer_hooks_key; DISPATCH_TSD_INLINE static inline void diff --git a/src/source.c b/src/source.c index b593ae0..dde7db9 100644 --- a/src/source.c +++ b/src/source.c @@ -25,15 +25,19 @@ #endif #include +#define DKEV_DISPOSE_IMMEDIATE_DELETE 0x1 +#define DKEV_DISPOSE_IGNORE_ENOENT 0x2 + static void _dispatch_source_merge_kevent(dispatch_source_t ds, - const struct kevent64_s *ke); + const _dispatch_kevent_qos_s *ke); static bool _dispatch_kevent_register(dispatch_kevent_t *dkp, uint32_t *flgp); -static void _dispatch_kevent_unregister(dispatch_kevent_t dk, uint32_t flg); -static bool _dispatch_kevent_resume(dispatch_kevent_t dk, uint32_t new_flags, +static long _dispatch_kevent_unregister(dispatch_kevent_t dk, uint32_t flg, + int options); +static long _dispatch_kevent_resume(dispatch_kevent_t dk, uint32_t new_flags, uint32_t del_flags); -static void _dispatch_kevent_drain(struct kevent64_s *ke); -static void _dispatch_kevent_merge(struct kevent64_s *ke); -static void _dispatch_timers_kevent(struct kevent64_s *ke); +static void _dispatch_kevent_drain(_dispatch_kevent_qos_s *ke); +static void _dispatch_kevent_merge(_dispatch_kevent_qos_s *ke); +static void _dispatch_timers_kevent(_dispatch_kevent_qos_s *ke); static void _dispatch_timers_unregister(dispatch_source_t ds, dispatch_kevent_t dk); static void _dispatch_timers_update(dispatch_source_t ds); @@ -45,7 +49,7 @@ static void _dispatch_timer_aggregates_unregister(dispatch_source_t ds, unsigned int tidx); static inline unsigned long _dispatch_source_timer_data( dispatch_source_refs_t dr, unsigned long prev); -static long _dispatch_kq_update(const struct kevent64_s *); +static long _dispatch_kq_update(const _dispatch_kevent_qos_s *); static void _dispatch_memorystatus_init(void); #if HAVE_MACH static void _dispatch_mach_host_calendar_change_register(void); @@ -54,23 +58,34 @@ static kern_return_t _dispatch_kevent_machport_resume(dispatch_kevent_t dk, uint32_t new_flags, uint32_t del_flags); static kern_return_t _dispatch_kevent_mach_notify_resume(dispatch_kevent_t dk, uint32_t new_flags, uint32_t del_flags); -static inline void _dispatch_kevent_mach_portset(struct kevent64_s *ke); +static inline void _dispatch_kevent_mach_portset(_dispatch_kevent_qos_s *ke); #else static inline void _dispatch_mach_host_calendar_change_register(void) {} static inline void _dispatch_mach_recv_msg_buf_init(void) {} #endif static const char * _evfiltstr(short filt); #if DISPATCH_DEBUG -static void _dispatch_kevent_debug(struct kevent64_s* kev, const char* str); +static void _dispatch_kevent_debug(const _dispatch_kevent_qos_s* kev, + const char* str); static void _dispatch_kevent_debugger(void *context); #define DISPATCH_ASSERT_ON_MANAGER_QUEUE() \ dispatch_assert(_dispatch_queue_get_current() == &_dispatch_mgr_q) #else static inline void -_dispatch_kevent_debug(struct kevent64_s* kev DISPATCH_UNUSED, +_dispatch_kevent_debug(const _dispatch_kevent_qos_s* kev DISPATCH_UNUSED, const char* str DISPATCH_UNUSED) {} #define DISPATCH_ASSERT_ON_MANAGER_QUEUE() #endif +#ifndef DISPATCH_MGR_QUEUE_DEBUG +#define DISPATCH_MGR_QUEUE_DEBUG 0 +#endif +#if DISPATCH_MGR_QUEUE_DEBUG +#define _dispatch_kevent_mgr_debug _dispatch_kevent_debug +#else +static inline void +_dispatch_kevent_mgr_debug(_dispatch_kevent_qos_s* kev DISPATCH_UNUSED, + const char* str DISPATCH_UNUSED) {} +#endif #pragma mark - #pragma mark dispatch_source_t @@ -79,9 +94,9 @@ dispatch_source_t dispatch_source_create(dispatch_source_type_t type, uintptr_t handle, unsigned long mask, - dispatch_queue_t q) + dispatch_queue_t dq) { - const struct kevent64_s *proto_kev = &type->ke; + const _dispatch_kevent_qos_s *proto_kev = &type->ke; dispatch_source_t ds; dispatch_kevent_t dk; @@ -127,9 +142,6 @@ dispatch_source_create(dispatch_source_type_t type, ds->do_ref_cnt++; // the reference the manager queue holds ds->do_ref_cnt++; // since source is created suspended ds->do_suspend_cnt = DISPATCH_OBJECT_SUSPEND_INTERVAL; - // The initial target queue is the manager queue, in order to get - // the source installed. - ds->do_targetq = &_dispatch_mgr_q; dk = _dispatch_calloc(1ul, sizeof(struct dispatch_kevent_s)); dk->dk_kevent = *proto_kev; @@ -149,9 +161,15 @@ dispatch_source_create(dispatch_source_type_t type, // we cheat and use EV_CLEAR to mean a "flag thingy" ds->ds_is_adder = true; } + if (EV_UDATA_SPECIFIC & proto_kev->flags) { + dispatch_assert(!(EV_ONESHOT & proto_kev->flags)); + dk->dk_kevent.flags |= EV_DISPATCH; + ds->ds_is_direct_kevent = true; + ds->ds_needs_rearm = true; + } // Some sources require special processing if (type->init != NULL) { - type->init(ds, type, handle, mask, q); + type->init(ds, type, handle, mask, dq); } dispatch_assert(!(ds->ds_is_level && ds->ds_is_adder)); @@ -161,12 +179,36 @@ dispatch_source_create(dispatch_source_type_t type, } ds->ds_refs->dr_source_wref = _dispatch_ptr2wref(ds); - // First item on the queue sets the user-specified target queue - dispatch_set_target_queue(ds, q); + if (!ds->ds_is_direct_kevent) { + // The initial target queue is the manager queue, in order to get + // the source installed. + ds->do_targetq = &_dispatch_mgr_q; + // First item on the queue sets the user-specified target queue + dispatch_set_target_queue(ds, dq); + } else { + if (slowpath(!dq)) { + dq = _dispatch_get_root_queue(_DISPATCH_QOS_CLASS_DEFAULT, true); + } else { + _dispatch_retain(dq); + } + ds->do_targetq = dq; + _dispatch_queue_priority_inherit_from_target((dispatch_queue_t)ds, dq); + _dispatch_queue_set_override_priority(dq); + } _dispatch_object_debug(ds, "%s", __func__); return ds; } +DISPATCH_ALWAYS_INLINE +static inline dispatch_queue_t +_dispatch_source_get_kevent_queue(dispatch_source_t ds) +{ + if (ds->ds_is_direct_kevent) { + return ds->do_targetq; + } + return &_dispatch_mgr_q; +} + void _dispatch_source_dispose(dispatch_source_t ds) { @@ -247,7 +289,7 @@ dispatch_source_get_data(dispatch_source_t ds) void dispatch_source_merge_data(dispatch_source_t ds, unsigned long val) { - struct kevent64_s kev = { + _dispatch_kevent_qos_s kev = { .fflags = (typeof(kev.fflags))val, .data = (typeof(kev.data))val, }; @@ -498,11 +540,9 @@ _dispatch_source_latch_and_call(dispatch_source_t ds) return; } pthread_priority_t old_dp = _dispatch_set_defaultpriority(ds->dq_priority); - _dispatch_trace_continuation_pop(_dispatch_queue_get_current(), dc); voucher_t voucher = dc->dc_voucher ? _voucher_retain(dc->dc_voucher) : NULL; _dispatch_continuation_voucher_adopt(dc); // consumes voucher reference - _dispatch_client_callout(dc->dc_ctxt, dc->dc_func); - _dispatch_introspection_queue_item_complete(dc); + _dispatch_continuation_pop(dc); if (voucher) dc->dc_voucher = voucher; _dispatch_reset_defaultpriority(old_dp); } @@ -511,19 +551,45 @@ static void _dispatch_source_kevent_unregister(dispatch_source_t ds) { _dispatch_object_debug(ds, "%s", __func__); + uint32_t flags = (uint32_t)ds->ds_pending_data_mask; dispatch_kevent_t dk = ds->ds_dkev; - ds->ds_dkev = NULL; - switch (dk->dk_kevent.filter) { - case DISPATCH_EVFILT_TIMER: + if (ds->ds_atomic_flags & DSF_DELETED) { + dk->dk_kevent.flags |= EV_DELETE; // already deleted + dk->dk_kevent.flags &= ~(EV_ADD|EV_ENABLE); + } + if (dk->dk_kevent.filter == DISPATCH_EVFILT_TIMER) { + ds->ds_dkev = NULL; _dispatch_timers_unregister(ds, dk); - break; - default: + } else if (!ds->ds_is_direct_kevent) { + ds->ds_dkev = NULL; TAILQ_REMOVE(&dk->dk_sources, ds->ds_refs, dr_list); - _dispatch_kevent_unregister(dk, (uint32_t)ds->ds_pending_data_mask); - break; + _dispatch_kevent_unregister(dk, flags, 0); + } else { + int dkev_dispose_options = 0; + if (ds->ds_needs_rearm && !(ds->ds_atomic_flags & DSF_ARMED)) { + dkev_dispose_options |= DKEV_DISPOSE_IMMEDIATE_DELETE; + } + if (ds->ds_needs_mgr) { + dkev_dispose_options |= DKEV_DISPOSE_IGNORE_ENOENT; + ds->ds_needs_mgr = false; + } + long r = _dispatch_kevent_unregister(dk, flags, dkev_dispose_options); + if (r == EINPROGRESS) { + _dispatch_debug("kevent-source[%p]: deferred delete kevent[%p]", + ds, dk); + ds->ds_pending_delete = true; + return; // deferred unregistration + } else if (r == ENOENT) { + _dispatch_debug("kevent-source[%p]: ENOENT delete kevent[%p]", + ds, dk); + ds->ds_needs_mgr = true; + return; // potential concurrent EV_DELETE delivery rdar://22047283 + } + ds->ds_dkev = NULL; + _TAILQ_TRASH_ENTRY(ds->ds_refs, dr_list); } - (void)dispatch_atomic_and2o(ds, ds_atomic_flags, ~DSF_ARMED, relaxed); + _dispatch_debug("kevent-source[%p]: disarmed kevent[%p]", ds, ds->ds_dkev); ds->ds_needs_rearm = false; // re-arm is pointless and bad now _dispatch_release(ds); // the retain is done at creation time } @@ -533,14 +599,19 @@ _dispatch_source_kevent_resume(dispatch_source_t ds, uint32_t new_flags) { switch (ds->ds_dkev->dk_kevent.filter) { case DISPATCH_EVFILT_TIMER: - return _dispatch_timers_update(ds); + _dispatch_timers_update(ds); + (void)dispatch_atomic_or2o(ds, ds_atomic_flags, DSF_ARMED, relaxed); + _dispatch_debug("kevent-source[%p]: rearmed kevent[%p]", ds, + ds->ds_dkev); + return; case EVFILT_MACHPORT: if (ds->ds_pending_data_mask & DISPATCH_MACH_RECV_MESSAGE) { new_flags |= DISPATCH_MACH_RECV_MESSAGE; // emulate EV_DISPATCH } break; } - if (_dispatch_kevent_resume(ds->ds_dkev, new_flags, 0)) { + if ((ds->ds_atomic_flags & DSF_DELETED) || + _dispatch_kevent_resume(ds->ds_dkev, new_flags, 0)) { _dispatch_source_kevent_unregister(ds); } } @@ -551,15 +622,19 @@ _dispatch_source_kevent_register(dispatch_source_t ds) dispatch_assert_zero(ds->ds_is_installed); switch (ds->ds_dkev->dk_kevent.filter) { case DISPATCH_EVFILT_TIMER: - return _dispatch_timers_update(ds); + _dispatch_timers_update(ds); + (void)dispatch_atomic_or2o(ds, ds_atomic_flags, DSF_ARMED, relaxed); + _dispatch_debug("kevent-source[%p]: armed kevent[%p]", ds, ds->ds_dkev); + return; } uint32_t flags; bool do_resume = _dispatch_kevent_register(&ds->ds_dkev, &flags); TAILQ_INSERT_TAIL(&ds->ds_dkev->dk_sources, ds->ds_refs, dr_list); + (void)dispatch_atomic_or2o(ds, ds_atomic_flags, DSF_ARMED, relaxed); + _dispatch_debug("kevent-source[%p]: armed kevent[%p]", ds, ds->ds_dkev); if (do_resume || ds->ds_needs_rearm) { _dispatch_source_kevent_resume(ds, flags); } - (void)dispatch_atomic_or2o(ds, ds_atomic_flags, DSF_ARMED, relaxed); _dispatch_object_debug(ds, "%s", __func__); } @@ -569,8 +644,10 @@ _dispatch_source_invoke2(dispatch_object_t dou, _dispatch_thread_semaphore_t *sema_ptr DISPATCH_UNUSED) { dispatch_source_t ds = dou._ds; - if (slowpath(_dispatch_queue_drain(ds))) { - DISPATCH_CLIENT_CRASH("Sync onto source"); + if (_dispatch_queue_class_probe(ds)) { + if (slowpath(_dispatch_queue_drain(ds))) { + DISPATCH_CLIENT_CRASH("Sync onto source"); + } } // This function performs all source actions. Each action is responsible @@ -581,12 +658,13 @@ _dispatch_source_invoke2(dispatch_object_t dou, // The order of tests here in invoke and in probe should be consistent. dispatch_queue_t dq = _dispatch_queue_get_current(); + dispatch_queue_t dkq = _dispatch_source_get_kevent_queue(ds); dispatch_source_refs_t dr = ds->ds_refs; if (!ds->ds_is_installed) { - // The source needs to be installed on the manager queue. - if (dq != &_dispatch_mgr_q) { - return &_dispatch_mgr_q; + // The source needs to be installed on the kevent queue. + if (dq != dkq) { + return dkq; } _dispatch_source_kevent_register(ds); ds->ds_is_installed = true; @@ -594,7 +672,7 @@ _dispatch_source_invoke2(dispatch_object_t dou, return ds->do_targetq; } if (slowpath(ds->do_xref_cnt == -1)) { - return &_dispatch_mgr_q; // rdar://problem/9558246 + return dkq; // rdar://problem/9558246 } } else if (slowpath(DISPATCH_OBJECT_SUSPENDED(ds))) { // Source suspended by an item drained from the source queue. @@ -608,17 +686,56 @@ _dispatch_source_invoke2(dispatch_object_t dou, // clears ds_registration_handler _dispatch_source_registration_callout(ds); if (slowpath(ds->do_xref_cnt == -1)) { - return &_dispatch_mgr_q; // rdar://problem/9558246 + return dkq; // rdar://problem/9558246 + } + } else if ((ds->ds_atomic_flags & DSF_DELETED) && (ds->ds_pending_delete || + (ds->ds_atomic_flags & DSF_ONESHOT))) { + // Pending source kevent unregistration has been completed + if (ds->ds_needs_mgr) { + dkq = &_dispatch_mgr_q; } - } else if ((ds->ds_atomic_flags & DSF_CANCELED) || (ds->do_xref_cnt == -1)){ + if (dq != dkq) { + return dkq; + } + ds->ds_pending_delete = false; + if (ds->ds_atomic_flags & DSF_ONESHOT) { + (void)dispatch_atomic_and2o(ds, ds_atomic_flags, ~DSF_ONESHOT, + relaxed); + } + if (ds->ds_dkev) { + _dispatch_source_kevent_unregister(ds); + if (ds->ds_needs_mgr) { + return &_dispatch_mgr_q; + } + } + if (dr->ds_handler[DS_EVENT_HANDLER] || + dr->ds_handler[DS_CANCEL_HANDLER] || + dr->ds_handler[DS_REGISTN_HANDLER]) { + return ds->do_targetq; + } + } else if (((ds->ds_atomic_flags & DSF_CANCELED) || (ds->do_xref_cnt == -1)) + && !ds->ds_pending_delete) { // The source has been cancelled and needs to be uninstalled from the - // manager queue. After uninstallation, the cancellation handler needs + // kevent queue. After uninstallation, the cancellation handler needs // to be delivered to the target queue. if (ds->ds_dkev) { - if (dq != &_dispatch_mgr_q) { - return &_dispatch_mgr_q; + if (ds->ds_needs_mgr) { + dkq = &_dispatch_mgr_q; + } + if (dq != dkq) { + return dkq; } _dispatch_source_kevent_unregister(ds); + if (ds->ds_needs_mgr) { + return &_dispatch_mgr_q; + } + if (ds->ds_pending_delete) { + // deferred unregistration + if (ds->ds_needs_rearm) { + return dkq; + } + return NULL; + } } if (dr->ds_handler[DS_EVENT_HANDLER] || dr->ds_handler[DS_CANCEL_HANDLER] || @@ -628,24 +745,26 @@ _dispatch_source_invoke2(dispatch_object_t dou, } } _dispatch_source_cancel_callout(ds); - } else if (ds->ds_pending_data) { + } else if (ds->ds_pending_data && !ds->ds_pending_delete) { // The source has pending data to deliver via the event handler callback - // on the target queue. Some sources need to be rearmed on the manager + // on the target queue. Some sources need to be rearmed on the kevent // queue after event delivery. if (dq != ds->do_targetq) { return ds->do_targetq; } _dispatch_source_latch_and_call(ds); if (ds->ds_needs_rearm) { - return &_dispatch_mgr_q; + return dkq; } } else if (ds->ds_needs_rearm && !(ds->ds_atomic_flags & DSF_ARMED)) { - // The source needs to be rearmed on the manager queue. - if (dq != &_dispatch_mgr_q) { - return &_dispatch_mgr_q; + // The source needs to be rearmed on the kevent queue. + if (dq != dkq) { + return dkq; } - _dispatch_source_kevent_resume(ds, 0); (void)dispatch_atomic_or2o(ds, ds_atomic_flags, DSF_ARMED, relaxed); + _dispatch_debug("kevent-source[%p]: rearmed kevent[%p]", ds, + ds->ds_dkev); + _dispatch_source_kevent_resume(ds, 0); } return NULL; @@ -653,9 +772,10 @@ _dispatch_source_invoke2(dispatch_object_t dou, DISPATCH_NOINLINE void -_dispatch_source_invoke(dispatch_source_t ds) +_dispatch_source_invoke(dispatch_source_t ds, dispatch_object_t dou, + dispatch_invoke_flags_t flags) { - _dispatch_queue_class_invoke(ds, _dispatch_source_invoke2); + _dispatch_queue_class_invoke(ds, dou._dc, flags, _dispatch_source_invoke2); } unsigned long @@ -666,13 +786,18 @@ _dispatch_source_probe(dispatch_source_t ds) dispatch_source_refs_t dr = ds->ds_refs; if (!ds->ds_is_installed) { - // The source needs to be installed on the manager queue. + // The source needs to be installed on the kevent queue. return true; } else if (dr->ds_handler[DS_REGISTN_HANDLER]) { // The registration handler needs to be delivered to the target queue. return true; - } else if ((ds->ds_atomic_flags & DSF_CANCELED) || (ds->do_xref_cnt == -1)){ - // The source needs to be uninstalled from the manager queue, or the + } else if ((ds->ds_atomic_flags & DSF_DELETED) && (ds->ds_pending_delete || + (ds->ds_atomic_flags & DSF_ONESHOT))) { + // Pending source kevent unregistration has been completed + return true; + } else if (((ds->ds_atomic_flags & DSF_CANCELED) || (ds->do_xref_cnt == -1)) + && !ds->ds_pending_delete) { + // The source needs to be uninstalled from the kevent queue, or the // cancellation handler needs to be delivered to the target queue. // Note: cancellation assumes installation. if (ds->ds_dkev || dr->ds_handler[DS_EVENT_HANDLER] || @@ -680,21 +805,37 @@ _dispatch_source_probe(dispatch_source_t ds) dr->ds_handler[DS_REGISTN_HANDLER]) { return true; } - } else if (ds->ds_pending_data) { + } else if (ds->ds_pending_data && !ds->ds_pending_delete) { // The source has pending data to deliver to the target queue. return true; } else if (ds->ds_needs_rearm && !(ds->ds_atomic_flags & DSF_ARMED)) { - // The source needs to be rearmed on the manager queue. + // The source needs to be rearmed on the kevent queue. return true; } return _dispatch_queue_class_probe(ds); } static void -_dispatch_source_merge_kevent(dispatch_source_t ds, const struct kevent64_s *ke) +_dispatch_source_merge_kevent(dispatch_source_t ds, + const _dispatch_kevent_qos_s *ke) { + _dispatch_object_debug(ds, "%s", __func__); + bool retained = false; + if ((ke->flags & EV_UDATA_SPECIFIC) && (ke->flags & EV_ONESHOT) && + !(ke->flags & EV_DELETE)) { + _dispatch_debug("kevent-source[%p]: deferred delete oneshot kevent[%p]", + ds, (void*)ke->udata); + (void)dispatch_atomic_or2o(ds, ds_atomic_flags, DSF_ONESHOT, relaxed); + } else if ((ke->flags & EV_DELETE) || (ke->flags & EV_ONESHOT)) { + _dispatch_debug("kevent-source[%p]: delete kevent[%p]", + ds, (void*)ke->udata); + retained = true; + _dispatch_retain(ds); + (void)dispatch_atomic_or2o(ds, ds_atomic_flags, DSF_DELETED, relaxed); + if (ke->flags & EV_DELETE) goto done; + } if ((ds->ds_atomic_flags & DSF_CANCELED) || (ds->do_xref_cnt == -1)) { - return; + goto done; // rdar://20204025 } if (ds->ds_is_level) { // ke->data is signed and "negative available data" makes no sense @@ -710,12 +851,22 @@ _dispatch_source_merge_kevent(dispatch_source_t ds, const struct kevent64_s *ke) (void)dispatch_atomic_or2o(ds, ds_pending_data, ke->fflags & ds->ds_pending_data_mask, relaxed); } +done: // EV_DISPATCH and EV_ONESHOT sources are no longer armed after delivery if (ds->ds_needs_rearm) { + if (!retained) { + retained = true; + _dispatch_retain(ds); // rdar://20382435 + } (void)dispatch_atomic_and2o(ds, ds_atomic_flags, ~DSF_ARMED, relaxed); + _dispatch_debug("kevent-source[%p]: disarmed kevent[%p] ", + ds, (void*)ke->udata); + } + if (retained) { + _dispatch_queue_wakeup_and_release((dispatch_queue_t)ds); + } else { + _dispatch_queue_wakeup((dispatch_queue_t)ds); } - - _dispatch_wakeup(ds); } #pragma mark - @@ -729,6 +880,7 @@ static inline void _dispatch_kevent_guard(dispatch_kevent_t dk) { (void)dk; } static inline void _dispatch_kevent_unguard(dispatch_kevent_t dk) { (void)dk; } #endif +#if !DISPATCH_USE_EV_UDATA_SPECIFIC static struct dispatch_kevent_s _dispatch_kevent_data_or = { .dk_kevent = { .filter = DISPATCH_EVFILT_CUSTOM_OR, @@ -742,6 +894,7 @@ static struct dispatch_kevent_s _dispatch_kevent_data_add = { }, .dk_sources = TAILQ_HEAD_INITIALIZER(_dispatch_kevent_data_add.dk_sources), }; +#endif // !DISPATCH_USE_EV_UDATA_SPECIFIC #define DSL_HASH(x) ((x) & (DSL_HASH_SIZE - 1)) @@ -756,6 +909,7 @@ _dispatch_kevent_init() TAILQ_INIT(&_dispatch_sources[i]); } +#if !DISPATCH_USE_EV_UDATA_SPECIFIC TAILQ_INSERT_TAIL(&_dispatch_sources[0], &_dispatch_kevent_data_or, dk_list); TAILQ_INSERT_TAIL(&_dispatch_sources[0], @@ -764,6 +918,7 @@ _dispatch_kevent_init() (uintptr_t)&_dispatch_kevent_data_or; _dispatch_kevent_data_add.dk_kevent.udata = (uintptr_t)&_dispatch_kevent_data_add; +#endif // !DISPATCH_USE_EV_UDATA_SPECIFIC } static inline uintptr_t @@ -797,6 +952,7 @@ _dispatch_kevent_find(uint64_t ident, short filter) static void _dispatch_kevent_insert(dispatch_kevent_t dk) { + if (dk->dk_kevent.flags & EV_UDATA_SPECIFIC) return; _dispatch_kevent_guard(dk); uintptr_t hash = _dispatch_kevent_hash(dk->dk_kevent.ident, dk->dk_kevent.filter); @@ -807,12 +963,14 @@ _dispatch_kevent_insert(dispatch_kevent_t dk) static bool _dispatch_kevent_register(dispatch_kevent_t *dkp, uint32_t *flgp) { - dispatch_kevent_t dk, ds_dkev = *dkp; + dispatch_kevent_t dk = NULL, ds_dkev = *dkp; uint32_t new_flags; bool do_resume = false; - dk = _dispatch_kevent_find(ds_dkev->dk_kevent.ident, - ds_dkev->dk_kevent.filter); + if (!(ds_dkev->dk_kevent.flags & EV_UDATA_SPECIFIC)) { + dk = _dispatch_kevent_find(ds_dkev->dk_kevent.ident, + ds_dkev->dk_kevent.filter); + } if (dk) { // If an existing dispatch kevent is found, check to see if new flags // need to be added to the existing kevent @@ -836,7 +994,7 @@ _dispatch_kevent_register(dispatch_kevent_t *dkp, uint32_t *flgp) return do_resume; } -static bool +static long _dispatch_kevent_resume(dispatch_kevent_t dk, uint32_t new_flags, uint32_t del_flags) { @@ -853,31 +1011,36 @@ _dispatch_kevent_resume(dispatch_kevent_t dk, uint32_t new_flags, case DISPATCH_EVFILT_MACH_NOTIFICATION: return _dispatch_kevent_mach_notify_resume(dk, new_flags, del_flags); #endif - case EVFILT_PROC: - if (dk->dk_kevent.flags & EV_ONESHOT) { + default: + if (dk->dk_kevent.flags & EV_DELETE) { return 0; } - // fall through - default: r = _dispatch_kq_update(&dk->dk_kevent); - if (dk->dk_kevent.flags & EV_DISPATCH) { + if (r && (dk->dk_kevent.flags & EV_ADD) && + (dk->dk_kevent.flags & EV_UDATA_SPECIFIC)) { + dk->dk_kevent.flags |= EV_DELETE; + dk->dk_kevent.flags &= ~(EV_ADD|EV_ENABLE); + } else if (dk->dk_kevent.flags & EV_DISPATCH) { dk->dk_kevent.flags &= ~EV_ADD; } return r; } } -static void -_dispatch_kevent_dispose(dispatch_kevent_t dk) +static long +_dispatch_kevent_dispose(dispatch_kevent_t dk, int options) { - uintptr_t hash; - + long r = 0; switch (dk->dk_kevent.filter) { case DISPATCH_EVFILT_TIMER: case DISPATCH_EVFILT_CUSTOM_ADD: case DISPATCH_EVFILT_CUSTOM_OR: - // these sources live on statically allocated lists - return; + if (dk->dk_kevent.flags & EV_UDATA_SPECIFIC) { + free(dk); + } else { + // these sources live on statically allocated lists + } + return r; #if HAVE_MACH case EVFILT_MACHPORT: _dispatch_kevent_machport_resume(dk, 0, dk->dk_kevent.fflags); @@ -886,35 +1049,56 @@ _dispatch_kevent_dispose(dispatch_kevent_t dk) _dispatch_kevent_mach_notify_resume(dk, 0, dk->dk_kevent.fflags); break; #endif - case EVFILT_PROC: - if (dk->dk_kevent.flags & EV_ONESHOT) { - break; // implicitly deleted - } - // fall through default: if (~dk->dk_kevent.flags & EV_DELETE) { dk->dk_kevent.flags |= EV_DELETE; dk->dk_kevent.flags &= ~(EV_ADD|EV_ENABLE); - _dispatch_kq_update(&dk->dk_kevent); + if (options & DKEV_DISPOSE_IMMEDIATE_DELETE) { + dk->dk_kevent.flags |= EV_ENABLE; + } + r = _dispatch_kq_update(&dk->dk_kevent); + if (r == ENOENT && (options & DKEV_DISPOSE_IGNORE_ENOENT)) { + r = 0; + } + if (options & DKEV_DISPOSE_IMMEDIATE_DELETE) { + dk->dk_kevent.flags &= ~EV_ENABLE; + } } break; } - - hash = _dispatch_kevent_hash(dk->dk_kevent.ident, - dk->dk_kevent.filter); - TAILQ_REMOVE(&_dispatch_sources[hash], dk, dk_list); - _dispatch_kevent_unguard(dk); - free(dk); + if ((r == EINPROGRESS || r == ENOENT) && + (dk->dk_kevent.flags & EV_UDATA_SPECIFIC)) { + // deferred EV_DELETE or concurrent concurrent EV_DELETE delivery + dk->dk_kevent.flags &= ~EV_DELETE; + dk->dk_kevent.flags |= EV_ENABLE; + } else { + if ((dk->dk_kevent.flags & EV_UDATA_SPECIFIC)) { +#if DISPATCH_DEBUG + // zero/trash dr linkage + dispatch_source_refs_t dr = TAILQ_FIRST(&dk->dk_sources); + TAILQ_REMOVE(&dk->dk_sources, dr, dr_list); +#endif + } else { + uintptr_t hash = _dispatch_kevent_hash(dk->dk_kevent.ident, + dk->dk_kevent.filter); + TAILQ_REMOVE(&_dispatch_sources[hash], dk, dk_list); + } + _dispatch_kevent_unguard(dk); + free(dk); + } + return r; } -static void -_dispatch_kevent_unregister(dispatch_kevent_t dk, uint32_t flg) +static long +_dispatch_kevent_unregister(dispatch_kevent_t dk, uint32_t flg, int options) { dispatch_source_refs_t dri; uint32_t del_flags, fflags = 0; + long r = 0; - if (TAILQ_EMPTY(&dk->dk_sources)) { - _dispatch_kevent_dispose(dk); + if (TAILQ_EMPTY(&dk->dk_sources) || + (dk->dk_kevent.flags & EV_UDATA_SPECIFIC)) { + r = _dispatch_kevent_dispose(dk, options); } else { TAILQ_FOREACH(dri, &dk->dk_sources, dr_list) { dispatch_source_t dsi = _dispatch_source_from_refs(dri); @@ -924,20 +1108,21 @@ _dispatch_kevent_unregister(dispatch_kevent_t dk, uint32_t flg) del_flags = flg & ~fflags; if (del_flags) { dk->dk_kevent.flags |= EV_ADD; - dk->dk_kevent.fflags = fflags; - _dispatch_kevent_resume(dk, 0, del_flags); + dk->dk_kevent.fflags &= ~del_flags; + r = _dispatch_kevent_resume(dk, 0, del_flags); } } + return r; } DISPATCH_NOINLINE static void -_dispatch_kevent_proc_exit(struct kevent64_s *ke) +_dispatch_kevent_proc_exit(_dispatch_kevent_qos_s *ke) { // EVFILT_PROC may fail with ESRCH when the process exists but is a zombie // . As a workaround, we simulate an exit event for // any EVFILT_PROC with an invalid pid . - struct kevent64_s fake; + _dispatch_kevent_qos_s fake; fake = *ke; fake.flags &= ~EV_ERROR; fake.fflags = NOTE_EXIT; @@ -947,7 +1132,7 @@ _dispatch_kevent_proc_exit(struct kevent64_s *ke) DISPATCH_NOINLINE static void -_dispatch_kevent_error(struct kevent64_s *ke) +_dispatch_kevent_error(_dispatch_kevent_qos_s *ke) { _dispatch_kevent_debug(ke, __func__); if (ke->data) { @@ -961,27 +1146,30 @@ _dispatch_kevent_error(struct kevent64_s *ke) } static void -_dispatch_kevent_drain(struct kevent64_s *ke) +_dispatch_kevent_drain(_dispatch_kevent_qos_s *ke) { #if DISPATCH_DEBUG static dispatch_once_t pred; dispatch_once_f(&pred, NULL, _dispatch_kevent_debugger); #endif if (ke->filter == EVFILT_USER) { + _dispatch_kevent_mgr_debug(ke, __func__); return; } if (slowpath(ke->flags & EV_ERROR)) { - if (ke->filter == EVFILT_PROC) { + if (ke->filter == EVFILT_PROC && ke->data == ESRCH) { + ke->data = 0; // don't return error from caller if (ke->flags & EV_DELETE) { - // Process exited while monitored + _dispatch_debug("kevent[0x%llx]: ignoring ESRCH from " + "EVFILT_PROC EV_DELETE", ke->udata); return; - } else if (ke->data == ESRCH) { - return _dispatch_kevent_proc_exit(ke); } + _dispatch_debug("kevent[0x%llx]: ESRCH from EVFILT_PROC: " + "generating fake NOTE_EXIT", ke->udata); + return _dispatch_kevent_proc_exit(ke); } return _dispatch_kevent_error(ke); } - _dispatch_kevent_debug(ke, __func__); if (ke->filter == EVFILT_TIMER) { return _dispatch_timers_kevent(ke); } @@ -995,18 +1183,16 @@ _dispatch_kevent_drain(struct kevent64_s *ke) DISPATCH_NOINLINE static void -_dispatch_kevent_merge(struct kevent64_s *ke) +_dispatch_kevent_merge(_dispatch_kevent_qos_s *ke) { + _dispatch_kevent_debug(ke, __func__); dispatch_kevent_t dk; - dispatch_source_refs_t dri; + dispatch_source_refs_t dri, dr_next; dk = (void*)ke->udata; dispatch_assert(dk); - if (ke->flags & EV_ONESHOT) { - dk->dk_kevent.flags |= EV_ONESHOT; - } - TAILQ_FOREACH(dri, &dk->dk_sources, dr_list) { + TAILQ_FOREACH_SAFE(dri, &dk->dk_sources, dr_list, dr_next) { _dispatch_source_merge_kevent(_dispatch_source_from_refs(dri), ke); } } @@ -1168,6 +1354,7 @@ _dispatch_source_set_timer3(void *context) ds->ds_pending_data = 0; // Re-arm in case we got disarmed because of pending set_timer suspension (void)dispatch_atomic_or2o(ds, ds_atomic_flags, DSF_ARMED, release); + _dispatch_debug("kevent-source[%p]: rearmed kevent[%p]", ds, ds->ds_dkev); dispatch_resume(ds); // Must happen after resume to avoid getting disarmed due to suspension _dispatch_timers_update(ds); @@ -1385,7 +1572,7 @@ struct dispatch_kevent_s _dispatch_kevent_timer[] = { #define DISPATCH_KEVENT_TIMEOUT_INIT(qos, note) \ DISPATCH_KEVENT_TIMEOUT_INITIALIZER(DISPATCH_TIMER_QOS_##qos, note) -struct kevent64_s _dispatch_kevent_timeout[] = { +_dispatch_kevent_qos_s _dispatch_kevent_timeout[] = { DISPATCH_KEVENT_TIMEOUT_INIT(NORMAL, 0), DISPATCH_KEVENT_TIMEOUT_INIT(CRITICAL, NOTE_CRITICAL), DISPATCH_KEVENT_TIMEOUT_INIT(BACKGROUND, NOTE_BACKGROUND), @@ -1516,6 +1703,8 @@ _dispatch_timers_update(dispatch_source_t ds) ds->ds_pending_data) { tidx = DISPATCH_TIMER_INDEX_DISARM; (void)dispatch_atomic_and2o(ds, ds_atomic_flags, ~DSF_ARMED, relaxed); + _dispatch_debug("kevent-source[%p]: disarmed kevent[%p]", ds, + ds->ds_dkev); } else { tidx = _dispatch_source_timer_idx(dr); } @@ -1526,6 +1715,8 @@ _dispatch_timers_update(dispatch_source_t ds) ds->ds_is_installed = true; if (tidx != DISPATCH_TIMER_INDEX_DISARM) { (void)dispatch_atomic_or2o(ds, ds_atomic_flags, DSF_ARMED, relaxed); + _dispatch_debug("kevent-source[%p]: rearmed kevent[%p]", ds, + ds->ds_dkev); } _dispatch_object_debug(ds, "%s", __func__); ds->ds_dkev = NULL; @@ -1668,7 +1859,7 @@ _dispatch_timers_get_delay(uint64_t nows[], struct dispatch_timer_s timer[], } static bool -_dispatch_timers_program2(uint64_t nows[], struct kevent64_s *ke, +_dispatch_timers_program2(uint64_t nows[], _dispatch_kevent_qos_s *ke, unsigned int qos) { unsigned int tidx; @@ -1739,8 +1930,9 @@ _dispatch_timers_calendar_change(void) } static void -_dispatch_timers_kevent(struct kevent64_s *ke) +_dispatch_timers_kevent(_dispatch_kevent_qos_s *ke) { + _dispatch_kevent_debug(ke, __func__); dispatch_assert(ke->data > 0); dispatch_assert((ke->ident & DISPATCH_KEVENT_TIMEOUT_IDENT_MASK) == DISPATCH_KEVENT_TIMEOUT_IDENT_MASK); @@ -1933,6 +2125,8 @@ _dispatch_timer_aggregates_unregister(dispatch_source_t ds, unsigned int tidx) static int _dispatch_kq; +#if DISPATCH_USE_SELECT_FALLBACK + static unsigned int _dispatch_select_workaround; static fd_set _dispatch_rfds; static fd_set _dispatch_wfds; @@ -1941,9 +2135,8 @@ static uint64_t*_dispatch_wfd_ptrs; DISPATCH_NOINLINE static bool -_dispatch_select_register(struct kevent64_s *kev) +_dispatch_select_register(const _dispatch_kevent_qos_s *kev) { - // Must execute on manager queue DISPATCH_ASSERT_ON_MANAGER_QUEUE(); @@ -1965,8 +2158,9 @@ _dispatch_select_register(struct kevent64_s *kev) _dispatch_debug("select workaround used to read fd %d: 0x%lx", (int)kev->ident, (long)kev->data); } + return true; } - return true; + break; case EVFILT_WRITE: if ((kev->data == EINVAL || kev->data == ENOENT) && dispatch_assume(kev->ident < FD_SETSIZE)) { @@ -1981,15 +2175,16 @@ _dispatch_select_register(struct kevent64_s *kev) _dispatch_debug("select workaround used to write fd %d: 0x%lx", (int)kev->ident, (long)kev->data); } + return true; } - return true; + break; } return false; } DISPATCH_NOINLINE static bool -_dispatch_select_unregister(const struct kevent64_s *kev) +_dispatch_select_unregister(const _dispatch_kevent_qos_s *kev) { // Must execute on manager queue DISPATCH_ASSERT_ON_MANAGER_QUEUE(); @@ -2023,7 +2218,6 @@ _dispatch_mgr_select(bool poll) { static const struct timeval timeout_immediately = { 0, 0 }; fd_set tmp_rfds, tmp_wfds; - struct kevent64_s kev; int err, i, r; bool kevent_avail = false; @@ -2073,16 +2267,24 @@ _dispatch_mgr_select(bool poll) continue; } FD_CLR(i, &_dispatch_rfds); // emulate EV_DISPATCH - EV_SET64(&kev, i, EVFILT_READ, - EV_ADD|EV_ENABLE|EV_DISPATCH, 0, 1, - _dispatch_rfd_ptrs[i], 0, 0); + _dispatch_kevent_qos_s kev = { + .ident = (uint64_t)i, + .filter = EVFILT_READ, + .flags = EV_ADD|EV_ENABLE|EV_DISPATCH, + .data = 1, + .udata = _dispatch_rfd_ptrs[i], + }; _dispatch_kevent_drain(&kev); } if (FD_ISSET(i, &tmp_wfds)) { FD_CLR(i, &_dispatch_wfds); // emulate EV_DISPATCH - EV_SET64(&kev, i, EVFILT_WRITE, - EV_ADD|EV_ENABLE|EV_DISPATCH, 0, 1, - _dispatch_wfd_ptrs[i], 0, 0); + _dispatch_kevent_qos_s kev = { + .ident = (uint64_t)i, + .filter = EVFILT_WRITE, + .flags = EV_ADD|EV_ENABLE|EV_DISPATCH, + .data = 1, + .udata = _dispatch_wfd_ptrs[i], + }; _dispatch_kevent_drain(&kev); } } @@ -2090,13 +2292,15 @@ _dispatch_mgr_select(bool poll) return kevent_avail; } +#endif // DISPATCH_USE_SELECT_FALLBACK + #pragma mark - #pragma mark dispatch_kqueue static void _dispatch_kq_init(void *context DISPATCH_UNUSED) { - static const struct kevent64_s kev = { + static const _dispatch_kevent_qos_s kev = { .ident = 1, .filter = EVFILT_USER, .flags = EV_ADD|EV_CLEAR, @@ -2129,13 +2333,16 @@ _dispatch_kq_init(void *context DISPATCH_UNUSED) DISPATCH_CRASH("kqueue() failure"); break; } - } else if (dispatch_assume(_dispatch_kq < FD_SETSIZE)) { + } +#if DISPATCH_USE_SELECT_FALLBACK + else if (dispatch_assume(_dispatch_kq < FD_SETSIZE)) { // in case we fall back to select() FD_SET(_dispatch_kq, &_dispatch_rfds); } +#endif // DISPATCH_USE_SELECT_FALLBACK - (void)dispatch_assume_zero(kevent64(_dispatch_kq, &kev, 1, NULL, 0, 0, - NULL)); + (void)dispatch_assume_zero(kevent_qos(_dispatch_kq, &kev, 1, NULL, 0, NULL, + NULL, 0)); _dispatch_queue_push(_dispatch_mgr_q.do_targetq, &_dispatch_mgr_q, 0); } @@ -2151,23 +2358,24 @@ _dispatch_get_kq(void) DISPATCH_NOINLINE static long -_dispatch_kq_update(const struct kevent64_s *kev) +_dispatch_kq_update(const _dispatch_kevent_qos_s *kev) { int r; - struct kevent64_s kev_copy; + _dispatch_kevent_qos_s kev_error; +#if DISPATCH_USE_SELECT_FALLBACK if (slowpath(_dispatch_select_workaround) && (kev->flags & EV_DELETE)) { if (_dispatch_select_unregister(kev)) { return 0; } } - kev_copy = *kev; - // This ensures we don't get a pending kevent back while registering - // a new kevent - kev_copy.flags |= EV_RECEIPT; +#endif // DISPATCH_USE_SELECT_FALLBACK + if (kev->filter != EVFILT_USER || DISPATCH_MGR_QUEUE_DEBUG) { + _dispatch_kevent_debug(kev, __func__); + } retry: - r = dispatch_assume(kevent64(_dispatch_get_kq(), &kev_copy, 1, - &kev_copy, 1, 0, NULL)); + r = kevent_qos(_dispatch_get_kq(), kev, 1, &kev_error, + 1, NULL, NULL, KEVENT_FLAG_ERROR_EVENTS); if (slowpath(r == -1)) { int err = errno; switch (err) { @@ -2182,34 +2390,59 @@ retry: } return err; } - switch (kev_copy.data) { - case 0: + if (r == 0) { return 0; - case EBADF: - case EPERM: - case EINVAL: + } + if (kev_error.flags & EV_ERROR && kev_error.data) { + _dispatch_kevent_debug(&kev_error, __func__); + } + r = (int)kev_error.data; + switch (r) { + case 0: + _dispatch_kevent_mgr_debug(&kev_error, __func__); + break; + case EINPROGRESS: + // deferred EV_DELETE + break; case ENOENT: + if ((kev->flags & EV_DELETE) && (kev->flags & EV_UDATA_SPECIFIC)) { + // potential concurrent EV_DELETE delivery + break; + } + // fall through + case EINVAL: if ((kev->flags & (EV_ADD|EV_ENABLE)) && !(kev->flags & EV_DELETE)) { - if (_dispatch_select_register(&kev_copy)) { - return 0; +#if DISPATCH_USE_SELECT_FALLBACK + if (_dispatch_select_register(&kev_error)) { + r = 0; + break; + } +#elif DISPATCH_DEBUG + if (kev->filter == EVFILT_READ || kev->filter == EVFILT_WRITE) { + DISPATCH_CRASH("Unsupported fd for EVFILT_READ or EVFILT_WRITE " + "kevent"); } +#endif // DISPATCH_USE_SELECT_FALLBACK } // fall through + case EBADF: + case EPERM: default: - kev_copy.flags |= kev->flags; - _dispatch_kevent_drain(&kev_copy); + kev_error.flags |= kev->flags; + _dispatch_kevent_drain(&kev_error); + r = (int)kev_error.data; break; } - return (long)kev_copy.data; + return r; } #pragma mark - #pragma mark dispatch_mgr -static struct kevent64_s *_dispatch_kevent_enable; +static _dispatch_kevent_qos_s *_dispatch_kevent_enable; static void inline -_dispatch_mgr_kevent_reenable(struct kevent64_s *ke) +_dispatch_mgr_kevent_reenable(_dispatch_kevent_qos_s *ke) { dispatch_assert(!_dispatch_kevent_enable || _dispatch_kevent_enable == ke); _dispatch_kevent_enable = ke; @@ -2222,7 +2455,7 @@ _dispatch_mgr_wakeup(dispatch_queue_t dq DISPATCH_UNUSED) return false; } - static const struct kevent64_s kev = { + static const _dispatch_kevent_qos_s kev = { .ident = 1, .filter = EVFILT_USER, .fflags = NOTE_TRIGGER, @@ -2255,22 +2488,23 @@ DISPATCH_NOINLINE DISPATCH_NORETURN static void _dispatch_mgr_invoke(void) { - static const struct timespec timeout_immediately = { 0, 0 }; - struct kevent64_s kev; + _dispatch_kevent_qos_s kev; bool poll; int r; for (;;) { _dispatch_mgr_queue_drain(); poll = _dispatch_mgr_timers(); +#if DISPATCH_USE_SELECT_FALLBACK if (slowpath(_dispatch_select_workaround)) { poll = _dispatch_mgr_select(poll); if (!poll) continue; } +#endif // DISPATCH_USE_SELECT_FALLBACK poll = poll || _dispatch_queue_class_probe(&_dispatch_mgr_q); - r = kevent64(_dispatch_kq, _dispatch_kevent_enable, - _dispatch_kevent_enable ? 1 : 0, &kev, 1, 0, - poll ? &timeout_immediately : NULL); + r = kevent_qos(_dispatch_kq, _dispatch_kevent_enable, + _dispatch_kevent_enable ? 1 : 0, &kev, 1, NULL, NULL, + poll ? KEVENT_FLAG_IMMEDIATE : KEVENT_FLAG_NONE); _dispatch_kevent_enable = NULL; if (slowpath(r == -1)) { int err = errno; @@ -2292,7 +2526,9 @@ _dispatch_mgr_invoke(void) DISPATCH_NORETURN void -_dispatch_mgr_thread(dispatch_queue_t dq DISPATCH_UNUSED) +_dispatch_mgr_thread(dispatch_queue_t dq DISPATCH_UNUSED, + dispatch_object_t dou DISPATCH_UNUSED, + dispatch_invoke_flags_t flags DISPATCH_UNUSED) { _dispatch_mgr_init(); // never returns, so burn bridges behind us & clear stack 2k ahead @@ -2395,8 +2631,8 @@ static inline void _dispatch_memorystatus_init(void) {} #define DISPATCH_MACH_KEVENT_ARMED(dk) ((dk)->dk_kevent.ext[0]) -static void _dispatch_kevent_machport_drain(struct kevent64_s *ke); -static void _dispatch_kevent_mach_msg_drain(struct kevent64_s *ke); +static void _dispatch_kevent_machport_drain(_dispatch_kevent_qos_s *ke); +static void _dispatch_kevent_mach_msg_drain(_dispatch_kevent_qos_s *ke); static void _dispatch_kevent_mach_msg_recv(mach_msg_header_t *hdr); static void _dispatch_kevent_mach_msg_destroy(mach_msg_header_t *hdr); static void _dispatch_source_merge_mach_msg(dispatch_source_t ds, @@ -2415,7 +2651,7 @@ static void _dispatch_mach_msg_recv(dispatch_mach_t dm, dispatch_mach_reply_refs_t dmr, mach_msg_header_t *hdr, mach_msg_size_t siz); static void _dispatch_mach_merge_kevent(dispatch_mach_t dm, - const struct kevent64_s *ke); + const _dispatch_kevent_qos_s *ke); static inline mach_msg_option_t _dispatch_mach_checkin_options(void); static const size_t _dispatch_mach_recv_msg_size = @@ -2425,7 +2661,7 @@ static const size_t dispatch_mach_trailer_size = static mach_msg_size_t _dispatch_mach_recv_msg_buf_size; static mach_port_t _dispatch_mach_portset, _dispatch_mach_recv_portset; static mach_port_t _dispatch_mach_notify_port; -static struct kevent64_s _dispatch_mach_recv_kevent = { +static _dispatch_kevent_qos_s _dispatch_mach_recv_kevent = { .filter = EVFILT_MACHPORT, .flags = EV_ADD|EV_ENABLE|EV_DISPATCH, .fflags = DISPATCH_MACH_RCV_OPTIONS, @@ -2517,7 +2753,7 @@ _dispatch_get_mach_recv_portset(void) static void _dispatch_mach_portset_init(void *context DISPATCH_UNUSED) { - struct kevent64_s kev = { + _dispatch_kevent_qos_s kev = { .filter = EVFILT_MACHPORT, .flags = EV_ADD, }; @@ -2575,14 +2811,14 @@ _dispatch_mach_portset_update(dispatch_kevent_t dk, mach_port_t mps) } static void -_dispatch_kevent_mach_recv_reenable(struct kevent64_s *ke DISPATCH_UNUSED) +_dispatch_kevent_mach_recv_reenable(_dispatch_kevent_qos_s *ke DISPATCH_UNUSED) { #if (TARGET_IPHONE_SIMULATOR && \ IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1090) || \ (!TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED < 1090) // delete and re-add kevent to workaround if (ke->ext[1] != _dispatch_mach_recv_kevent.ext[1]) { - struct kevent64_s kev = _dispatch_mach_recv_kevent; + _dispatch_kevent_qos_s kev = _dispatch_mach_recv_kevent; kev.flags = EV_DELETE; _dispatch_kq_update(&kev); } @@ -2636,7 +2872,7 @@ _dispatch_kevent_mach_notify_resume(dispatch_kevent_t dk, uint32_t new_flags, } static inline void -_dispatch_kevent_mach_portset(struct kevent64_s *ke) +_dispatch_kevent_mach_portset(_dispatch_kevent_qos_s *ke) { if (ke->ident == _dispatch_mach_recv_portset) { return _dispatch_kevent_mach_msg_drain(ke); @@ -2649,11 +2885,11 @@ _dispatch_kevent_mach_portset(struct kevent64_s *ke) DISPATCH_NOINLINE static void -_dispatch_kevent_machport_drain(struct kevent64_s *ke) +_dispatch_kevent_machport_drain(_dispatch_kevent_qos_s *ke) { + _dispatch_kevent_debug(ke, __func__); mach_port_t name = (mach_port_name_t)ke->data; dispatch_kevent_t dk; - struct kevent64_s kev; _dispatch_debug_machport(name); dk = _dispatch_kevent_find(name, EVFILT_MACHPORT); @@ -2662,16 +2898,22 @@ _dispatch_kevent_machport_drain(struct kevent64_s *ke) } _dispatch_mach_portset_update(dk, MACH_PORT_NULL); // emulate EV_DISPATCH - EV_SET64(&kev, name, EVFILT_MACHPORT, EV_ADD|EV_ENABLE|EV_DISPATCH, - DISPATCH_MACH_RECV_MESSAGE, 0, (uintptr_t)dk, 0, 0); + _dispatch_kevent_qos_s kev = { + .ident = name, + .filter = EVFILT_MACHPORT, + .flags = EV_ADD|EV_ENABLE|EV_DISPATCH, + .fflags = DISPATCH_MACH_RECV_MESSAGE, + .udata = (uintptr_t)dk, + }; _dispatch_kevent_debug(&kev, __func__); _dispatch_kevent_merge(&kev); } DISPATCH_NOINLINE static void -_dispatch_kevent_mach_msg_drain(struct kevent64_s *ke) +_dispatch_kevent_mach_msg_drain(_dispatch_kevent_qos_s *ke) { + _dispatch_kevent_debug(ke, __func__); mach_msg_header_t *hdr = (mach_msg_header_t*)ke->ext[0]; mach_msg_size_t siz, msgsiz; mach_msg_return_t kr = (mach_msg_return_t)ke->fflags; @@ -2822,13 +3064,12 @@ _dispatch_source_merge_mach_msg(dispatch_source_t ds, dispatch_source_refs_t dr, return _dispatch_mach_msg_recv((dispatch_mach_t)ds, dmr, hdr, siz); } -DISPATCH_ALWAYS_INLINE -static inline void +DISPATCH_NOINLINE +static void _dispatch_mach_notify_merge(mach_port_t name, uint32_t flag, bool final) { dispatch_source_refs_t dri, dr_next; dispatch_kevent_t dk; - struct kevent64_s kev; bool unreg; dk = _dispatch_kevent_find(name, DISPATCH_EVFILT_MACH_NOTIFICATION); @@ -2838,8 +3079,13 @@ _dispatch_mach_notify_merge(mach_port_t name, uint32_t flag, bool final) // Update notification registration state. dk->dk_kevent.data &= ~_DISPATCH_MACH_SP_FLAGS; - EV_SET64(&kev, name, DISPATCH_EVFILT_MACH_NOTIFICATION, EV_ADD|EV_ENABLE, - flag, 0, (uintptr_t)dk, 0, 0); + _dispatch_kevent_qos_s kev = { + .ident = name, + .filter = DISPATCH_EVFILT_MACH_NOTIFICATION, + .flags = EV_ADD|EV_ENABLE, + .fflags = flag, + .udata = (uintptr_t)dk, + }; if (final) { // This can never happen again unreg = true; @@ -3166,7 +3412,7 @@ _dispatch_mach_reply_kevent_unregister(dispatch_mach_t dm, } dispatch_kevent_t dk = dmr->dmr_dkev; TAILQ_REMOVE(&dk->dk_sources, (dispatch_source_refs_t)dmr, dr_list); - _dispatch_kevent_unregister(dk, DISPATCH_MACH_RECV_MESSAGE_DIRECT_ONCE); + _dispatch_kevent_unregister(dk, DISPATCH_MACH_RECV_MESSAGE_DIRECT_ONCE, 0); TAILQ_REMOVE(&dm->dm_refs->dm_replies, dmr, dmr_list); if (dmr->dmr_voucher) _voucher_release(dmr->dmr_voucher); free(dmr); @@ -3222,7 +3468,7 @@ _dispatch_mach_kevent_unregister(dispatch_mach_t dm) dm->ds_pending_data_mask &= ~(unsigned long) (DISPATCH_MACH_SEND_POSSIBLE|DISPATCH_MACH_SEND_DEAD); _dispatch_kevent_unregister(dk, - DISPATCH_MACH_SEND_POSSIBLE|DISPATCH_MACH_SEND_DEAD); + DISPATCH_MACH_SEND_POSSIBLE|DISPATCH_MACH_SEND_DEAD, 0); } DISPATCH_NOINLINE @@ -3619,9 +3865,9 @@ _dispatch_mach_send(dispatch_mach_t dm) _dispatch_mach_send_drain(dm); } -DISPATCH_NOINLINE static void -_dispatch_mach_merge_kevent(dispatch_mach_t dm, const struct kevent64_s *ke) +_dispatch_mach_merge_kevent(dispatch_mach_t dm, + const _dispatch_kevent_qos_s *ke) { if (!(ke->fflags & dm->ds_pending_data_mask)) { return; @@ -3647,6 +3893,18 @@ _dispatch_mach_send_options(void) return options; } +DISPATCH_ALWAYS_INLINE +static inline pthread_priority_t +_dispatch_mach_priority_propagate(mach_msg_option_t options) +{ +#if DISPATCH_USE_NOIMPORTANCE_QOS + if (options & MACH_SEND_NOIMPORTANCE) return 0; +#else + (void)options; +#endif + return _dispatch_priority_propagate(); +} + DISPATCH_NOINLINE void dispatch_mach_send(dispatch_mach_t dm, dispatch_mach_msg_t dmsg, @@ -3658,6 +3916,7 @@ dispatch_mach_send(dispatch_mach_t dm, dispatch_mach_msg_t dmsg, } dispatch_retain(dmsg); dispatch_assert_zero(options & DISPATCH_MACH_OPTIONS_MASK); + pthread_priority_t priority = _dispatch_mach_priority_propagate(options); options |= _dispatch_mach_send_options(); _dispatch_mach_msg_set_options(dmsg, options & ~DISPATCH_MACH_OPTIONS_MASK); mach_msg_header_t *msg = _dispatch_mach_msg_get_msg(dmsg); @@ -3667,7 +3926,7 @@ dispatch_mach_send(dispatch_mach_t dm, dispatch_mach_msg_t dmsg, MACH_PORT_NULL); bool is_reply = (MACH_MSGH_BITS_REMOTE(msg->msgh_bits) == MACH_MSG_TYPE_MOVE_SEND_ONCE); - dmsg->dmsg_priority = _dispatch_priority_propagate(); + dmsg->dmsg_priority = priority; dmsg->dmsg_voucher = _voucher_copy(); _dispatch_voucher_debug("mach-msg[%p] set", dmsg->dmsg_voucher, dmsg); if ((!is_reply && slowpath(dr->dm_tail)) || @@ -3816,7 +4075,9 @@ _dispatch_mach_connect_invoke(dispatch_mach_t dm) DISPATCH_NOINLINE void -_dispatch_mach_msg_invoke(dispatch_mach_msg_t dmsg) +_dispatch_mach_msg_invoke(dispatch_mach_msg_t dmsg, + dispatch_object_t dou DISPATCH_UNUSED, + dispatch_invoke_flags_t flags DISPATCH_UNUSED) { dispatch_mach_t dm = (dispatch_mach_t)_dispatch_queue_get_current(); dispatch_mach_refs_t dr = dm->ds_refs; @@ -4029,9 +4290,10 @@ _dispatch_mach_invoke2(dispatch_object_t dou, DISPATCH_NOINLINE void -_dispatch_mach_invoke(dispatch_mach_t dm) +_dispatch_mach_invoke(dispatch_mach_t dm, dispatch_object_t dou, + dispatch_invoke_flags_t flags) { - _dispatch_queue_class_invoke(dm, _dispatch_mach_invoke2); + _dispatch_queue_class_invoke(dm, dou._dc, flags, _dispatch_mach_invoke2); } unsigned long @@ -4362,18 +4624,21 @@ _evfiltstr(short filt) _evfilt2(EVFILT_PROC); _evfilt2(EVFILT_SIGNAL); _evfilt2(EVFILT_TIMER); -#ifdef EVFILT_VM - _evfilt2(EVFILT_VM); -#endif -#ifdef EVFILT_MEMORYSTATUS - _evfilt2(EVFILT_MEMORYSTATUS); -#endif #if HAVE_MACH _evfilt2(EVFILT_MACHPORT); _evfilt2(DISPATCH_EVFILT_MACH_NOTIFICATION); #endif _evfilt2(EVFILT_FS); _evfilt2(EVFILT_USER); +#ifdef EVFILT_VM + _evfilt2(EVFILT_VM); +#endif +#ifdef EVFILT_SOCK + _evfilt2(EVFILT_SOCK); +#endif +#ifdef EVFILT_MEMORYSTATUS + _evfilt2(EVFILT_MEMORYSTATUS); +#endif _evfilt2(DISPATCH_EVFILT_TIMER); _evfilt2(DISPATCH_EVFILT_CUSTOM_ADD); @@ -4383,14 +4648,60 @@ _evfiltstr(short filt) } } +#if DISPATCH_DEBUG +static const char * +_evflagstr2(uint16_t *flagsp) +{ +#define _evflag2(f) \ + if ((*flagsp & (f)) == (f) && (f)) { \ + *flagsp &= ~(f); \ + return #f "|"; \ + } + _evflag2(EV_ADD); + _evflag2(EV_DELETE); + _evflag2(EV_ENABLE); + _evflag2(EV_DISABLE); + _evflag2(EV_ONESHOT); + _evflag2(EV_CLEAR); + _evflag2(EV_RECEIPT); + _evflag2(EV_DISPATCH); + _evflag2(EV_UDATA_SPECIFIC); + _evflag2(EV_POLL); + _evflag2(EV_OOBAND); + _evflag2(EV_ERROR); + _evflag2(EV_EOF); + *flagsp = 0; + return "EV_UNKNOWN "; +} + +DISPATCH_NOINLINE +static const char * +_evflagstr(uint16_t flags, char *str, size_t strsize) +{ + str[0] = 0; + while (flags) { + strlcat(str, _evflagstr2(&flags), strsize); + } + size_t sz = strlen(str); + if (sz) str[sz-1] = 0; + return str; +} +#endif + static size_t _dispatch_source_debug_attr(dispatch_source_t ds, char* buf, size_t bufsiz) { dispatch_queue_t target = ds->do_targetq; return dsnprintf(buf, bufsiz, "target = %s[%p], ident = 0x%lx, " - "pending_data = 0x%lx, pending_data_mask = 0x%lx, ", + "mask = 0x%lx, pending_data = 0x%lx, registered = %d, " + "armed = %d, deleted = %d%s%s, canceled = %d, needs_mgr = %d, ", target && target->dq_label ? target->dq_label : "", target, - ds->ds_ident_hack, ds->ds_pending_data, ds->ds_pending_data_mask); + ds->ds_ident_hack, ds->ds_pending_data_mask, ds->ds_pending_data, + ds->ds_is_installed, (bool)(ds->ds_atomic_flags & DSF_ARMED), + (bool)(ds->ds_atomic_flags & DSF_DELETED), ds->ds_pending_delete ? + " (pending)" : "", (ds->ds_atomic_flags & DSF_ONESHOT) ? + " (oneshot)" : "", (bool)(ds->ds_atomic_flags & DSF_CANCELED), + ds->ds_needs_mgr); } static size_t @@ -4414,8 +4725,10 @@ _dispatch_source_debug(dispatch_source_t ds, char* buf, size_t bufsiz) if (ds->ds_is_timer) { offset += _dispatch_timer_debug_attr(ds, &buf[offset], bufsiz - offset); } - offset += dsnprintf(&buf[offset], bufsiz - offset, "filter = %s }", - ds->ds_dkev ? _evfiltstr(ds->ds_dkev->dk_kevent.filter) : "????"); + offset += dsnprintf(&buf[offset], bufsiz - offset, "kevent = %p%s, " + "filter = %s }", ds->ds_dkev, ds->ds_is_direct_kevent ? " (direct)" + : "", ds->ds_dkev ? _evfiltstr(ds->ds_dkev->dk_kevent.filter) : + "????"); return offset; } @@ -4450,14 +4763,17 @@ _dispatch_mach_debug(dispatch_mach_t dm, char* buf, size_t bufsiz) } #if DISPATCH_DEBUG +DISPATCH_NOINLINE static void -_dispatch_kevent_debug(struct kevent64_s* kev, const char* str) -{ - _dispatch_log("kevent[%p] = { ident = 0x%llx, filter = %s, flags = 0x%x, " - "fflags = 0x%x, data = 0x%llx, udata = 0x%llx, ext[0] = 0x%llx, " - "ext[1] = 0x%llx }: %s", kev, kev->ident, _evfiltstr(kev->filter), - kev->flags, kev->fflags, kev->data, kev->udata, kev->ext[0], - kev->ext[1], str); +_dispatch_kevent_debug(const _dispatch_kevent_qos_s* kev, const char* str) +{ + char flagstr[256]; + _dispatch_debug("kevent[%p] = { ident = 0x%llx, filter = %s, " + "flags = %s (0x%x), fflags = 0x%x, data = 0x%llx, udata = 0x%llx, " + "ext[0] = 0x%llx, ext[1] = 0x%llx }: %s", kev, kev->ident, + _evfiltstr(kev->filter), _evflagstr(kev->flags, flagstr, + sizeof(flagstr)), kev->flags, kev->fflags, kev->data, kev->udata, + kev->ext[0], kev->ext[1], str); } static void diff --git a/src/source_internal.h b/src/source_internal.h index 12ccdda..6e8f40f 100644 --- a/src/source_internal.h +++ b/src/source_internal.h @@ -101,13 +101,13 @@ enum { struct dispatch_kevent_s { TAILQ_ENTRY(dispatch_kevent_s) dk_list; TAILQ_HEAD(, dispatch_source_refs_s) dk_sources; - struct kevent64_s dk_kevent; + _dispatch_kevent_qos_s dk_kevent; }; typedef struct dispatch_kevent_s *dispatch_kevent_t; struct dispatch_source_type_s { - struct kevent64_s ke; + _dispatch_kevent_qos_s ke; uint64_t mask; void (*init)(dispatch_source_t ds, dispatch_source_type_t type, uintptr_t handle, unsigned long mask, dispatch_queue_t q); @@ -168,6 +168,8 @@ _dispatch_source_timer_idx(dispatch_source_refs_t dr) // ds_atomic_flags bits #define DSF_CANCELED 1u // cancellation has been requested #define DSF_ARMED 2u // source is armed +#define DSF_DELETED 4u // source received EV_DELETE event +#define DSF_ONESHOT 8u // source received EV_ONESHOT event #define DISPATCH_SOURCE_HEADER(refs) \ dispatch_kevent_t ds_dkev; \ @@ -177,7 +179,10 @@ _dispatch_source_timer_idx(dispatch_source_refs_t dr) ds_is_level:1, \ ds_is_adder:1, \ ds_is_installed:1, \ + ds_is_direct_kevent:1, \ ds_needs_rearm:1, \ + ds_pending_delete:1, \ + ds_needs_mgr:1, \ ds_is_timer:1, \ ds_vmpressure_override:1, \ ds_memorystatus_override:1, \ @@ -262,7 +267,8 @@ struct dispatch_mach_msg_s { void _dispatch_source_xref_dispose(dispatch_source_t ds); void _dispatch_source_dispose(dispatch_source_t ds); -void _dispatch_source_invoke(dispatch_source_t ds); +void _dispatch_source_invoke(dispatch_source_t ds, dispatch_object_t dou, + dispatch_invoke_flags_t flags); unsigned long _dispatch_source_probe(dispatch_source_t ds); size_t _dispatch_source_debug(dispatch_source_t ds, char* buf, size_t bufsiz); void _dispatch_source_set_interval(dispatch_source_t ds, uint64_t interval); @@ -270,17 +276,20 @@ void _dispatch_source_set_event_handler_with_context_f(dispatch_source_t ds, void *ctxt, dispatch_function_t handler); void _dispatch_mach_dispose(dispatch_mach_t dm); -void _dispatch_mach_invoke(dispatch_mach_t dm); +void _dispatch_mach_invoke(dispatch_mach_t dm, dispatch_object_t dou, + dispatch_invoke_flags_t flags); unsigned long _dispatch_mach_probe(dispatch_mach_t dm); size_t _dispatch_mach_debug(dispatch_mach_t dm, char* buf, size_t bufsiz); void _dispatch_mach_msg_dispose(dispatch_mach_msg_t dmsg); -void _dispatch_mach_msg_invoke(dispatch_mach_msg_t dmsg); +void _dispatch_mach_msg_invoke(dispatch_mach_msg_t dmsg, dispatch_object_t dou, + dispatch_invoke_flags_t flags); size_t _dispatch_mach_msg_debug(dispatch_mach_msg_t dmsg, char* buf, size_t bufsiz); void _dispatch_mach_barrier_invoke(void *ctxt); unsigned long _dispatch_mgr_wakeup(dispatch_queue_t dq); -void _dispatch_mgr_thread(dispatch_queue_t dq); +void _dispatch_mgr_thread(dispatch_queue_t dq, dispatch_object_t dou, + dispatch_invoke_flags_t flags); #endif /* __DISPATCH_SOURCE_INTERNAL__ */ diff --git a/src/trace.h b/src/trace.h index df27ca8..ebab27c 100644 --- a/src/trace.h +++ b/src/trace.h @@ -27,7 +27,7 @@ #ifndef __DISPATCH_TRACE__ #define __DISPATCH_TRACE__ -#if !__OBJC2__ +#if !__OBJC2__ && !defined(__cplusplus) #if DISPATCH_USE_DTRACE || DISPATCH_USE_DTRACE_INTROSPECTION typedef struct dispatch_trace_timer_params_s { @@ -98,8 +98,8 @@ _dispatch_trace_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t)) _DISPATCH_SOURCE_TYPE && (_dq) != &_dispatch_mgr_q) { \ dispatch_source_t _ds = (dispatch_source_t)_do; \ _dc = _ds->ds_refs->ds_handler[DS_EVENT_HANDLER]; \ - _func = _dc->dc_func; \ - _ctxt = _dc->dc_ctxt; \ + _func = _dc ? _dc->dc_func : NULL; \ + _ctxt = _dc ? _dc->dc_ctxt : NULL; \ } else { \ _func = (dispatch_function_t)_dispatch_queue_invoke; \ _ctxt = _do->do_ctxt; \ @@ -321,6 +321,6 @@ _dispatch_trace_timer_fire(dispatch_source_refs_t dr, unsigned long data, #endif // DISPATCH_USE_DTRACE -#endif // !__OBJC2__ +#endif // !__OBJC2__ && !defined(__cplusplus) #endif // __DISPATCH_TRACE__ diff --git a/src/voucher.c b/src/voucher.c index e886faf..6f28c24 100644 --- a/src/voucher.c +++ b/src/voucher.c @@ -33,23 +33,19 @@ #define VOUCHER_ATM_COLLECT_THRESHOLD 1 #endif #define VATM_COLLECT_THRESHOLD_VALUE(t) (((t) - 1) * 2) -static volatile long _voucher_atm_collect_level; -static long _voucher_atm_collect_threshold = - VATM_COLLECT_THRESHOLD_VALUE(VOUCHER_ATM_COLLECT_THRESHOLD); -static unsigned long _voucher_atm_subid_bits; +static uint64_t volatile _voucher_atm_generation; typedef struct _voucher_atm_s *_voucher_atm_t; static void _voucher_activity_atfork_child(void); -static inline mach_voucher_t _voucher_get_atm_mach_voucher(voucher_t voucher); -static inline mach_voucher_t _voucher_activity_get_atm_mach_voucher( - _voucher_activity_t act); -static inline _voucher_activity_t _voucher_activity_get(voucher_t voucher); static _voucher_activity_t _voucher_activity_copy_from_mach_voucher( mach_voucher_t kv, voucher_activity_id_t va_id); static inline _voucher_activity_t _voucher_activity_retain( _voucher_activity_t act); static inline void _voucher_activity_release(_voucher_activity_t act); +static void _voucher_activity_remove(_voucher_activity_t act); +static inline _voucher_atm_t _voucher_atm_retain(_voucher_atm_t vatm); +static inline void _voucher_atm_release(_voucher_atm_t vatm); #pragma mark - #pragma mark voucher_t @@ -156,6 +152,18 @@ voucher_copy_without_importance(void) return _voucher_copy_without_importance(); } +voucher_t +voucher_retain(voucher_t voucher) +{ + return _voucher_retain(voucher); +} + +void +voucher_release(voucher_t voucher) +{ + return _voucher_release(voucher); +} + void _voucher_thread_cleanup(void *voucher) { @@ -164,7 +172,7 @@ _voucher_thread_cleanup(void *voucher) DISPATCH_CACHELINE_ALIGN static TAILQ_HEAD(, voucher_s) _vouchers[VL_HASH_SIZE]; -#define _vouchers(kv) (&_vouchers[VL_HASH((kv))]) +#define _vouchers_head(kv) (&_vouchers[VL_HASH((kv))]) static os_lock_handoff_s _vouchers_lock = OS_LOCK_HANDOFF_INIT; #define _vouchers_lock_lock() os_lock_lock(&_vouchers_lock) #define _vouchers_lock_unlock() os_lock_unlock(&_vouchers_lock) @@ -175,7 +183,7 @@ _voucher_find_and_retain(mach_voucher_t kv) voucher_t v; if (!kv) return NULL; _vouchers_lock_lock(); - TAILQ_FOREACH(v, _vouchers(kv), v_list) { + TAILQ_FOREACH(v, _vouchers_head(kv), v_list) { if (v->v_ipc_kvoucher == kv) { int xref_cnt = dispatch_atomic_inc2o(v, os_obj_xref_cnt, relaxed); _dispatch_voucher_debug("retain -> %d", v, xref_cnt + 1); @@ -204,7 +212,7 @@ _voucher_insert(voucher_t v) _dispatch_voucher_debug("corruption", v); DISPATCH_CRASH("Voucher corruption"); } - TAILQ_INSERT_TAIL(_vouchers(kv), v, v_list); + TAILQ_INSERT_TAIL(_vouchers_head(kv), v, v_list); _vouchers_lock_unlock(); } @@ -221,7 +229,7 @@ _voucher_remove(voucher_t v) // check for resurrection race with _voucher_find_and_retain if (dispatch_atomic_load2o(v, os_obj_xref_cnt, seq_cst) < 0 && _TAILQ_IS_ENQUEUED(v, v_list)) { - TAILQ_REMOVE(_vouchers(kv), v, v_list); + TAILQ_REMOVE(_vouchers_head(kv), v, v_list); _TAILQ_MARK_NOT_ENQUEUED(v, v_list); v->v_list.tqe_next = (void*)~0ull; } @@ -315,11 +323,20 @@ voucher_replace_default_voucher(void) #endif } +static inline _voucher_atm_t +_voucher_get_atm(voucher_t voucher) +{ + _voucher_atm_t vatm; + vatm = voucher && voucher->v_atm ? voucher->v_atm : _voucher_task_atm; + return vatm; +} + static inline mach_voucher_t _voucher_get_atm_mach_voucher(voucher_t voucher) { - _voucher_activity_t act = _voucher_activity_get(voucher); - return _voucher_activity_get_atm_mach_voucher(act); + _voucher_atm_t vatm = _voucher_get_atm(voucher); + mach_voucher_t kv = vatm ? vatm->vatm_kvoucher : MACH_VOUCHER_NULL; + return kv; } mach_voucher_t @@ -463,6 +480,7 @@ _voucher_create_with_mach_voucher(mach_voucher_t kv) } voucher_activity_id_t va_id = 0, va_base_id = 0; _voucher_activity_t act = NULL; + _voucher_atm_t vatm = NULL; if (activities) { va_id = *(voucher_activity_id_t*)content; act = _voucher_activity_copy_from_mach_voucher(rkv, va_id); @@ -474,8 +492,12 @@ _voucher_create_with_mach_voucher(mach_voucher_t kv) activities++; va_base_id = act->va_id; } + if (act) { + vatm = _voucher_atm_retain(act->va_atm); + } } v = _voucher_alloc(activities, priority, 0); + v->v_atm = vatm; v->v_activity = act; voucher_activity_id_t *activity_ids = _voucher_activity_ids(v); if (activities && va_base_id) { @@ -514,6 +536,7 @@ _voucher_create_with_priority_and_mach_voucher(voucher_t ov, if (activities) { if (ov->v_activity) { v->v_activity = _voucher_activity_retain(ov->v_activity); + v->v_atm = _voucher_atm_retain(ov->v_atm); } memcpy(_voucher_activity_ids(v), _voucher_activity_ids(ov), activities * sizeof(voucher_activity_id_t)); @@ -584,6 +607,7 @@ _voucher_create_without_importance(voucher_t ov) if (activities) { if (ov->v_activity) { v->v_activity = _voucher_activity_retain(ov->v_activity); + v->v_atm = _voucher_atm_retain(ov->v_atm); } memcpy(_voucher_activity_ids(v), _voucher_activity_ids(ov), activities * sizeof(voucher_activity_id_t)); @@ -603,6 +627,46 @@ _voucher_create_without_importance(voucher_t ov) return v; } +voucher_t +_voucher_create_accounting_voucher(voucher_t ov) +{ + // Nothing to do unless the old voucher has a kernel voucher. If it does + // doesn't, it can't have any accounting attributes. + if (!ov || !ov->v_kvoucher) return NULL; + kern_return_t kr = KERN_SUCCESS; + mach_voucher_t okv, kv = MACH_VOUCHER_NULL; + okv = ov->v_ipc_kvoucher ? ov->v_ipc_kvoucher : ov->v_kvoucher; +#if VOUCHER_USE_ATTR_BANK + const mach_voucher_attr_recipe_data_t accounting_copy_recipe = { + .key = MACH_VOUCHER_ATTR_KEY_BANK, + .command = MACH_VOUCHER_ATTR_COPY, + .previous_voucher = okv, + }; + kr = _voucher_create_mach_voucher(&accounting_copy_recipe, + sizeof(accounting_copy_recipe), &kv); +#endif + if (dispatch_assume_zero(kr) || !kv){ + return NULL; + } + voucher_t v = _voucher_find_and_retain(kv); + if (v) { + _dispatch_voucher_debug("kvoucher[0x%08x] find accounting voucher " + "from voucher[%p]", v, kv, ov); + _voucher_dealloc_mach_voucher(kv); + return v; + } + v = _voucher_alloc(0, 0, 0); + v->v_ipc_kvoucher = v->v_kvoucher = kv; + if (kv == okv) { + v->v_kvbase = _voucher_retain(ov); + _voucher_dealloc_mach_voucher(kv); // borrow base reference + } + _voucher_insert(v); + _dispatch_voucher_debug("kvoucher[0x%08x] create accounting voucher " + "from voucher[%p]", v, kv, ov); + return v; +} + voucher_t voucher_create_with_mach_msg(mach_msg_header_t *msg) { @@ -692,7 +756,11 @@ _voucher_dispose(voucher_t voucher) _voucher_activity_release(voucher->v_activity); voucher->v_activity = NULL; } - voucher->v_has_priority= 0; + if (voucher->v_atm) { + _voucher_atm_release(voucher->v_atm); + voucher->v_atm = NULL; + } + voucher->v_has_priority = 0; voucher->v_activities = 0; #if VOUCHER_ENABLE_RECIPE_OBJECTS voucher->v_recipe_extra_size = 0; @@ -773,7 +841,7 @@ void _voucher_init(void) { _voucher_libkernel_init(); - char *e, *end; + char *e; unsigned int i; for (i = 0; i < VL_HASH_SIZE; i++) { TAILQ_INIT(&_vouchers[i]); @@ -796,21 +864,6 @@ _voucher_init(void) _voucher_activity_mode = mode; if (_voucher_activity_disabled()) return; - e = getenv("LIBDISPATCH_ACTIVITY_ATM_SUBID_BITS"); - if (e) { - unsigned long v = strtoul(e, &end, 0); - if (v && !*end) { - _voucher_atm_subid_bits = v; - } - } - e = getenv("LIBDISPATCH_ACTIVITY_ATM_COLLECT_THRESHOLD"); - if (e) { - unsigned long v = strtoul(e, &end, 0); - if (v && v < LONG_MAX/2 && !*end) { - _voucher_atm_collect_threshold = - VATM_COLLECT_THRESHOLD_VALUE((long)v); - } - } // default task activity bool default_task_activity = DISPATCH_DEBUG; e = getenv("LIBDISPATCH_DEFAULT_TASK_ACTIVITY"); @@ -851,24 +904,12 @@ _Static_assert(sizeof(struct _voucher_activity_tracepoint_s) == 64, _Static_assert(sizeof(struct _voucher_activity_buffer_header_s) <= sizeof(struct _voucher_activity_tracepoint_s), "Buffer header too large"); -_Static_assert(offsetof(struct _voucher_activity_s, va_flags2) == - sizeof(struct _voucher_activity_tracepoint_s), - "Extended activity object misaligned"); #if __LP64__ -_Static_assert(sizeof(struct _voucher_activity_s) == - 3 * sizeof(struct _voucher_activity_tracepoint_s), - "Activity object too large"); -_Static_assert(offsetof(struct _voucher_activity_s, va_flags3) == - 2 * sizeof(struct _voucher_activity_tracepoint_s), - "Extended activity object misaligned"); -_Static_assert(offsetof(struct _voucher_atm_s, vatm_activities_lock) % 64 == 0, - "Bad ATM padding"); +_Static_assert(offsetof(struct _voucher_activity_s, va_buffers_lock) % 64 == 0, + "Bad activity padding"); _Static_assert(sizeof(struct _voucher_atm_s) <= 128, "ATM too large"); #else -_Static_assert(sizeof(struct _voucher_activity_s) == - 2 * sizeof(struct _voucher_activity_tracepoint_s), - "Activity object too large"); _Static_assert(sizeof(struct _voucher_atm_s) <= 64, "ATM too large"); #endif @@ -880,41 +921,15 @@ _Static_assert(sizeof(struct _voucher_activity_metadata_s) <= "Metadata too large"); _Static_assert(sizeof(_voucher_activity_bitmap_t) % 64 == 0, "Bad metadata bitmap size"); -_Static_assert(offsetof(struct _voucher_activity_metadata_s, - vam_atm_mbox_bitmap) % 64 == 0, - "Bad metadata padding"); -_Static_assert(offsetof(struct _voucher_activity_metadata_s, - vam_base_atm_subid) % 64 == 0, - "Bad metadata padding"); -_Static_assert(offsetof(struct _voucher_activity_metadata_s, vam_base_atm_lock) - % 32 == 0, - "Bad metadata padding"); -_Static_assert(offsetof(struct _voucher_activity_metadata_s, vam_atms) % 64 ==0, - "Bad metadata padding"); -_Static_assert(sizeof(_voucher_activity_bitmap_t) * 8 * - sizeof(atm_mailbox_offset_t) <= - sizeof(((_voucher_activity_metadata_t)NULL)->vam_kernel_metadata), - "Bad kernel metadata bitmap"); -_Static_assert(sizeof(atm_mailbox_offset_t) == 2 * sizeof(atm_subaid32_t), - "Bad kernel ATM mailbox sizes"); #endif -static const size_t _voucher_atm_mailboxes = - sizeof(((_voucher_activity_metadata_t)NULL)->vam_kernel_metadata) / - sizeof(atm_mailbox_offset_t); - #define va_buffers_lock(va) (&(va)->va_buffers_lock) -#define vatm_activities_lock(vatm) (&(vatm)->vatm_activities_lock) #define vatm_activities(vatm) (&(vatm)->vatm_activities) -#define vatm_used_activities(vatm) (&(vatm)->vatm_used_activities) -#define vam_base_atm_lock() (&_voucher_activity_heap->vam_base_atm_lock) -#define vam_nested_atm_lock() (&_voucher_activity_heap->vam_nested_atm_lock) #define vam_atms_lock() (&_voucher_activity_heap->vam_atms_lock) #define vam_activities_lock() (&_voucher_activity_heap->vam_activities_lock) #define vam_atms(hash) (&_voucher_activity_heap->vam_atms[hash]) #define vam_activities(hash) (&_voucher_activity_heap->vam_activities[hash]) #define vam_buffer_bitmap() (_voucher_activity_heap->vam_buffer_bitmap) -#define vam_atm_mbox_bitmap() (_voucher_activity_heap->vam_atm_mbox_bitmap) #define vam_pressure_locked_bitmap() \ (_voucher_activity_heap->vam_pressure_locked_bitmap) #define vam_buffer(i) ((void*)((char*)_voucher_activity_heap + \ @@ -925,12 +940,14 @@ static _voucher_activity_t _voucher_activity_create_with_atm( voucher_activity_trace_id_t trace_id, uint64_t location, _voucher_activity_buffer_header_t buffer); static _voucher_atm_t _voucher_atm_create(mach_voucher_t kv, atm_aid_t atm_id); -static voucher_activity_id_t _voucher_atm_nested_atm_id_make(void); +static void _voucher_activity_firehose_wait(_voucher_activity_t act, + _voucher_activity_buffer_header_t buffer); DISPATCH_ALWAYS_INLINE static inline uint32_t _voucher_default_activity_buffer_limit() { +#if 0 // FIXME: tune buffer chain sizes switch (_voucher_activity_mode) { case voucher_activity_mode_debug: case voucher_activity_mode_stream: @@ -938,19 +955,16 @@ _voucher_default_activity_buffer_limit() // (twice as much as non-default activities) return MAX(_voucher_activity_buffers_per_heap / 32, 3) - 1; } -#if TARGET_OS_EMBEDDED - // Low-profile modes: Default activity can use a total of 3 buffers. - return 2; -#else - // Low-profile modes: Default activity can use a total of 8 buffers. - return 7; #endif + // Low-profile modes: Default activity can use a total of 4 buffers. + return 3; } DISPATCH_ALWAYS_INLINE static inline uint32_t _voucher_activity_buffer_limit() { +#if 0 // FIXME: tune buffer chain sizes switch (_voucher_activity_mode) { case voucher_activity_mode_debug: case voucher_activity_mode_stream: @@ -958,13 +972,9 @@ _voucher_activity_buffer_limit() // of the entire heap. return MAX(_voucher_activity_buffers_per_heap / 64, 2) - 1; } -#if TARGET_OS_EMBEDDED +#endif // Low-profile modes: Each activity can use a total of 2 buffers. return 1; -#else - // Low-profile modes: Each activity can use a total of 4 buffers. - return 3; -#endif } // The two functions above return the number of *additional* buffers activities @@ -1017,7 +1027,7 @@ _voucher_activity_bitmap_set_first_unset_bit_upto( return index; } -DISPATCH_ALWAYS_INLINE +DISPATCH_ALWAYS_INLINE DISPATCH_UNUSED static inline size_t _voucher_activity_bitmap_set_first_unset_bit( _voucher_activity_bitmap_t volatile bitmap) @@ -1025,7 +1035,6 @@ _voucher_activity_bitmap_set_first_unset_bit( return _voucher_activity_bitmap_set_first_unset_bit_upto(bitmap, UINT_MAX); } - DISPATCH_ALWAYS_INLINE static inline void _voucher_activity_bitmap_clear_bit( @@ -1067,7 +1076,6 @@ _voucher_activity_heap_init(void *ctxt DISPATCH_UNUSED) task_trace_memory_info_data_t trace_memory_info = { .user_memory_address = vm_addr, .buffer_size = vm_size, - .mailbox_array_size = sizeof(heap->vam_kernel_metadata), }; kr = task_set_info(mach_task_self(), TASK_TRACE_MEMORY_INFO, (task_info_t)&trace_memory_info, TASK_TRACE_MEMORY_INFO_COUNT); @@ -1080,38 +1088,29 @@ _voucher_activity_heap_init(void *ctxt DISPATCH_UNUSED) return; } heap = (void*)vm_addr; - heap->vam_self_metadata.vasm_baseaddr = (void*)vm_addr; - heap->vam_buffer_bitmap[0] = 0xf; // first four buffers are reserved + heap->vasm_baseaddr = (void*)vm_addr; + heap->vam_buffer_bitmap[0] = 0x7; // first three buffers are reserved uint32_t i; for (i = 0; i < _voucher_activity_hash_size; i++) { TAILQ_INIT(&heap->vam_activities[i]); TAILQ_INIT(&heap->vam_atms[i]); } - uint32_t subid_max = VATM_SUBID_MAX; - if (_voucher_atm_subid_bits && - _voucher_atm_subid_bits < VATM_SUBID_MAXBITS) { - subid_max = MIN(VATM_SUBID_BITS2MAX(_voucher_atm_subid_bits), - VATM_SUBID_MAX); - } - heap->vam_base_atm_subid_max = subid_max; - _voucher_activity_lock_init(&heap->vam_base_atm_lock); - _voucher_activity_lock_init(&heap->vam_nested_atm_lock); _voucher_activity_lock_init(&heap->vam_atms_lock); _voucher_activity_lock_init(&heap->vam_activities_lock); _voucher_activity_heap = heap; _voucher_atm_t vatm = _voucher_atm_create(0, 0); dispatch_assert(vatm->vatm_kvoucher); - heap->vam_default_activity_atm = vatm; - _voucher_activity_buffer_header_t buffer = vam_buffer(3); // reserved index + _voucher_atm_retain(vatm); + + _voucher_activity_buffer_header_t buffer = vam_buffer(2); // reserved index // consumes vatm reference: - _voucher_activity_t va = _voucher_activity_create_with_atm(vatm, - VATM_ACTID(vatm, _voucher_default_activity_subid), 0, 0, buffer); + _voucher_activity_t va = _voucher_activity_create_with_atm(vatm, 0, 0, 0, + buffer); dispatch_assert(va); va->va_buffer_limit = _voucher_default_activity_buffer_limit(); _voucher_activity_default = va; - heap->vam_base_atm = _voucher_atm_create(0, 0); - heap->vam_nested_atm_id = _voucher_atm_nested_atm_id_make(); + _voucher_task_atm = vatm; } static void @@ -1135,6 +1134,32 @@ voucher_activity_get_metadata_buffer(size_t *length) return _voucher_activity_heap->vam_client_metadata; } +static _voucher_activity_buffer_hook_t _voucher_activity_buffer_hook; + +void +voucher_activity_buffer_hook_install_4libtrace( + _voucher_activity_buffer_hook_t hook) +{ + if (dispatch_atomic_cmpxchg(&_voucher_activity_buffer_hook, NULL, + (void*)hook, release)) return; + DISPATCH_CLIENT_CRASH("_voucher_activity_buffer_hook_install_4libtrace " \ + "called more than once"); +} + +#if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG +#define VOUCHER_ACTIVITY_BUFFER_DEBUG(reason, buffer) \ + _dispatch_debug("activity buffer %s (%p)", #reason, buffer) +#else +#define VOUCHER_ACTIVITY_BUFFER_DEBUG(reason, buffer) +#endif + +#define VOUCHER_ACTIVITY_BUFFER_HOOK_CALLOUT(reason, buffer) \ + if (buffer) { VOUCHER_ACTIVITY_BUFFER_DEBUG(reason, buffer); \ + if (slowpath(_voucher_activity_buffer_hook)) { \ + _voucher_activity_buffer_hook( \ + _voucher_activity_buffer_hook_reason_##reason, (buffer)); \ + } } + DISPATCH_ALWAYS_INLINE static inline _voucher_activity_buffer_header_t _voucher_activity_heap_buffer_alloc(void) @@ -1266,16 +1291,17 @@ _voucher_activity_heap_pressure_normal(void) DISPATCH_ALWAYS_INLINE static inline void _voucher_activity_buffer_init(_voucher_activity_t act, - _voucher_activity_buffer_header_t buffer, bool reuse) + _voucher_activity_buffer_header_t buffer, bool initial) { - if (!reuse) { - buffer->vabh_flags = _voucher_activity_trace_flag_buffer_header; - buffer->vabh_activity_id = act->va_id; - } - buffer->vabh_timestamp = _voucher_activity_timestamp(); - buffer->vabh_next_tracepoint_idx = 1; - buffer->vabh_sequence_no = dispatch_atomic_inc2o(act, va_max_sequence_no, - relaxed); + _voucher_activity_tracepoint_t vat = (_voucher_activity_tracepoint_t)buffer; + _voucher_activity_tracepoint_init_with_id(vat, act->va_trace_id, + act->va_location, !initial); + buffer->vabh_flags = _voucher_activity_trace_flag_buffer_header | + _voucher_activity_trace_flag_activity | + (initial ? _voucher_activity_trace_flag_start : 0); + buffer->vabh_activity_id = act->va_id; + buffer->vabh_pos.vabp_atomic_pos = 0; + buffer->vabh_pos.vabp_pos.vabp_next_tracepoint_idx = 1; } static _voucher_activity_buffer_header_t @@ -1285,27 +1311,29 @@ _voucher_activity_buffer_alloc_slow(_voucher_activity_t act, _voucher_activity_buffer_header_t buffer; _voucher_activity_lock_lock(va_buffers_lock(act)); // TODO: revisit locking buffer = act->va_current_buffer; - if (buffer != current) goto out; + if (buffer != current) { + _voucher_activity_lock_unlock(va_buffers_lock(act)); + return buffer; + } buffer = TAILQ_FIRST(&act->va_buffers); - if (buffer) { - _voucher_activity_buffer_init(act, buffer, true); - if (buffer != TAILQ_LAST(&act->va_buffers, + if (buffer != TAILQ_LAST(&act->va_buffers, _voucher_activity_buffer_list_s)) { - TAILQ_REMOVE(&act->va_buffers, buffer, vabh_list); - TAILQ_INSERT_TAIL(&act->va_buffers, buffer, vabh_list); - } + TAILQ_REMOVE(&act->va_buffers, buffer, vabh_list); + TAILQ_INSERT_TAIL(&act->va_buffers, buffer, vabh_list); } - if (!dispatch_atomic_cmpxchgv2o(act, va_current_buffer, current, buffer, + _voucher_activity_lock_unlock(va_buffers_lock(act)); + if (_voucher_activity_buffer_is_full(buffer)) { + _voucher_activity_firehose_wait(act, buffer); + } + if (dispatch_atomic_cmpxchgv2o(act, va_current_buffer, current, buffer, ¤t, release)) { - if (buffer) { - TAILQ_REMOVE(&act->va_buffers, buffer, vabh_list); - _voucher_activity_heap_buffer_free(buffer); + if (_voucher_activity_buffer_mark_full(current)) { + _voucher_activity_firehose_push(act, current); } + _dispatch_voucher_activity_debug("buffer reuse %p", act, buffer); + } else { buffer = current; } -out: - _voucher_activity_lock_unlock(va_buffers_lock(act)); - _dispatch_voucher_activity_debug("buffer reuse %p", act, buffer); return buffer; } @@ -1314,8 +1342,14 @@ _voucher_activity_buffer_alloc(_voucher_activity_t act, _voucher_activity_buffer_header_t current) { _voucher_activity_buffer_header_t buffer = NULL; - if (act->va_max_sequence_no < act->va_buffer_limit) { + if (act->va_buffer_count < act->va_buffer_limit) { buffer = _voucher_activity_heap_buffer_alloc(); + if (buffer && dispatch_atomic_inc2o(act, va_buffer_count, relaxed) > + act->va_buffer_limit) { + dispatch_atomic_dec2o(act, va_buffer_count, relaxed); + _voucher_activity_heap_buffer_free(buffer); + buffer = NULL; + } } if (!buffer) return _voucher_activity_buffer_alloc_slow(act, current); _voucher_activity_buffer_init(act, buffer, false); @@ -1324,11 +1358,15 @@ _voucher_activity_buffer_alloc(_voucher_activity_t act, _voucher_activity_lock_lock(va_buffers_lock(act)); TAILQ_INSERT_TAIL(&act->va_buffers, buffer, vabh_list); _voucher_activity_lock_unlock(va_buffers_lock(act)); + if (_voucher_activity_buffer_mark_full(current)) { + _voucher_activity_firehose_push(act, current); + } + _dispatch_voucher_activity_debug("buffer alloc %p", act, buffer); } else { + dispatch_atomic_dec2o(act, va_buffer_count, relaxed); _voucher_activity_heap_buffer_free(buffer); buffer = current; } - _dispatch_voucher_activity_debug("buffer alloc %p", act, buffer); return buffer; } @@ -1347,36 +1385,32 @@ _voucher_activity_buffer_alloc(_voucher_activity_t act, } } while (0); static void _voucher_activity_dispose(_voucher_activity_t act); -static _voucher_activity_t _voucher_atm_activity_mark_used( - _voucher_activity_t act); -static void _voucher_atm_activity_mark_unused(_voucher_activity_t act); static _voucher_atm_t _voucher_atm_copy(atm_aid_t atm_id); static inline void _voucher_atm_release(_voucher_atm_t vatm); -static void _voucher_atm_activity_insert(_voucher_atm_t vatm, - _voucher_activity_t act); -static void _voucher_atm_activity_remove(_voucher_activity_t act); static atm_aid_t _voucher_mach_voucher_get_atm_id(mach_voucher_t kv); DISPATCH_ALWAYS_INLINE static inline bool -_voucher_activity_copy(_voucher_activity_t act) +_voucher_activity_try_retain(_voucher_activity_t act) { - int use_cnt = dispatch_atomic_inc2o(act, va_use_count, relaxed); + // not using _os_object_refcnt* because we don't need barriers: + // activities are immutable and are in a hash table with a lock + int use_cnt = dispatch_atomic_inc2o(act, va_refcnt, relaxed); _dispatch_voucher_activity_debug("retain -> %d", act, use_cnt + 1); if (slowpath(use_cnt < 0)) { _dispatch_voucher_activity_debug("overrelease", act); DISPATCH_CRASH("Activity overrelease"); } - return (use_cnt == 0); + return use_cnt > 0; } DISPATCH_ALWAYS_INLINE static inline _voucher_activity_t _voucher_activity_retain(_voucher_activity_t act) { - if (_voucher_activity_copy(act)) { - _dispatch_voucher_activity_debug("invalid resurrection", act); - DISPATCH_CRASH("Invalid activity resurrection"); + if (slowpath(!_voucher_activity_try_retain(act))) { + _dispatch_voucher_activity_debug("resurrection", act); + DISPATCH_CRASH("Activity resurrection"); } return act; } @@ -1385,7 +1419,9 @@ DISPATCH_ALWAYS_INLINE static inline void _voucher_activity_release(_voucher_activity_t act) { - int use_cnt = dispatch_atomic_dec2o(act, va_use_count, relaxed); + // not using _os_object_refcnt* because we don't need barriers: + // activities are immutable and are in a hash table with a lock + int use_cnt = dispatch_atomic_dec2o(act, va_refcnt, relaxed); _dispatch_voucher_activity_debug("release -> %d", act, use_cnt + 1); if (fastpath(use_cnt >= 0)) { return; @@ -1394,83 +1430,53 @@ _voucher_activity_release(_voucher_activity_t act) _dispatch_voucher_activity_debug("overrelease", act); DISPATCH_CRASH("Activity overrelease"); } - return _voucher_atm_activity_mark_unused(act); -} - -DISPATCH_ALWAYS_INLINE -static inline _voucher_activity_t -_voucher_activity_atm_retain(_voucher_activity_t act) -{ - int refcnt = dispatch_atomic_inc2o(act, va_refcnt, relaxed); - _dispatch_voucher_activity_debug("atm retain -> %d", act, refcnt + 1); - if (slowpath(refcnt <= 0)) { - _dispatch_voucher_activity_debug("atm resurrection", act); - DISPATCH_CRASH("Activity ATM resurrection"); - } - return act; -} - -DISPATCH_ALWAYS_INLINE -static inline void -_voucher_activity_atm_release(_voucher_activity_t act) -{ - int refcnt = dispatch_atomic_dec2o(act, va_refcnt, relaxed); - _dispatch_voucher_activity_debug("atm release -> %d", act, refcnt + 1); - if (fastpath(refcnt >= 0)) { - return; - } - if (slowpath(refcnt < -1)) { - _dispatch_voucher_activity_debug("atm overrelease", act); - DISPATCH_CRASH("Activity ATM overrelease"); - } - return _voucher_activity_dispose(act); -} - -static inline _voucher_activity_t -_voucher_activity_get(voucher_t v) -{ - _voucher_activity_t act; - act = v && v->v_activity ? v->v_activity : _voucher_activity_default; - return act; + _voucher_activity_remove(act); + _voucher_activity_dispose(act); } static _voucher_activity_t -_voucher_activity_find(voucher_activity_id_t va_id, uint32_t hash) +_voucher_activity_find_and_retain(voucher_activity_id_t va_id, uint32_t hash) { + // not using _os_object_refcnt* because we don't need barriers: + // activities are immutable and are in a hash table with a lock + // // assumes vam_activities_lock held _voucher_activity_t act; - TAILQ_FOREACH(act, vam_activities(hash), va_list){ - if (act->va_id == va_id) break; + TAILQ_FOREACH(act, vam_activities(hash), va_list) { + if (act->va_id == va_id) { + if (fastpath(_voucher_activity_try_retain(act))) { + return act; + } + + // disallow resurrection + dispatch_atomic_dec2o(act, va_refcnt, relaxed); + _dispatch_voucher_activity_debug("undo resurrection", act); + } } - return act; + return NULL; } static _voucher_activity_t _voucher_activity_copy_from_id(voucher_activity_id_t va_id) { - bool resurrect = false; uint32_t hash = VACTID_HASH(va_id); _voucher_activity_lock_lock(vam_activities_lock()); - _voucher_activity_t act = _voucher_activity_find(va_id, hash); + _voucher_activity_t act = _voucher_activity_find_and_retain(va_id, hash); if (act) { - resurrect = _voucher_activity_copy(act); _dispatch_voucher_activity_debug("copy from id 0x%llx", act, va_id); } _voucher_activity_lock_unlock(vam_activities_lock()); - if (resurrect) return _voucher_atm_activity_mark_used(act); return act; } static _voucher_activity_t _voucher_activity_try_insert(_voucher_activity_t act_new) { - bool resurrect = false; voucher_activity_id_t va_id = act_new->va_id; uint32_t hash = VACTID_HASH(va_id); _voucher_activity_lock_lock(vam_activities_lock()); - _voucher_activity_t act = _voucher_activity_find(va_id, hash); + _voucher_activity_t act = _voucher_activity_find_and_retain(va_id, hash); if (act) { - resurrect = _voucher_activity_copy(act); _dispatch_voucher_activity_debug("try insert: failed (%p)", act,act_new); } else { if (slowpath(_TAILQ_IS_ENQUEUED(act_new, va_list))) { @@ -1481,31 +1487,25 @@ _voucher_activity_try_insert(_voucher_activity_t act_new) _dispatch_voucher_activity_debug("try insert: succeeded", act_new); } _voucher_activity_lock_unlock(vam_activities_lock()); - if (resurrect) return _voucher_atm_activity_mark_used(act); return act; } -static bool -_voucher_activity_try_remove(_voucher_activity_t act) +static void +_voucher_activity_remove(_voucher_activity_t act) { - bool r; voucher_activity_id_t va_id = act->va_id; uint32_t hash = VACTID_HASH(va_id); + _voucher_activity_lock_lock(vam_activities_lock()); - if (slowpath(!va_id)) { + if (slowpath(!va_id || !_TAILQ_IS_ENQUEUED(act, va_list))) { _dispatch_voucher_activity_debug("corruption", act); DISPATCH_CRASH("Activity corruption"); } - if ((r = (dispatch_atomic_load2o(act, va_use_count, seq_cst) < 0 && - _TAILQ_IS_ENQUEUED(act, va_list)))) { - TAILQ_REMOVE(vam_activities(hash), act, va_list); - _TAILQ_MARK_NOT_ENQUEUED(act, va_list); - act->va_list.tqe_next = (void*)~0ull; - } - _dispatch_voucher_activity_debug("try remove: %s", act, r ? "succeeded" : - "failed"); + TAILQ_REMOVE(vam_activities(hash), act, va_list); + _TAILQ_MARK_NOT_ENQUEUED(act, va_list); + act->va_list.tqe_next = (void*)~0ull; + _dispatch_voucher_activity_debug("remove", act); _voucher_activity_lock_unlock(vam_activities_lock()); - return r; } static _voucher_activity_t @@ -1519,39 +1519,30 @@ _voucher_activity_create_with_atm(_voucher_atm_t vatm, _voucher_atm_release(vatm); // consume vatm reference return NULL; } - if (!trace_id) trace_id = _voucher_activity_trace_id_release; - _voucher_activity_tracepoint_t vat = (_voucher_activity_tracepoint_t)buffer; - _voucher_activity_tracepoint_init_with_id(vat, trace_id, ~1ull); - _voucher_activity_t act = (_voucher_activity_t)buffer; - act->va_flags = _voucher_activity_trace_flag_buffer_header | - _voucher_activity_trace_flag_activity | - _voucher_activity_trace_flag_start | - _voucher_activity_trace_flag_wide_first; - act->vabh_next_tracepoint_idx = sizeof(*act)/sizeof(*vat); - act->va_max_sequence_no = 0; - act->va_id = va_id ? va_id : VATM_ACTID(vatm, 0); - act->va_use_count = 0; + _voucher_activity_t act = _dispatch_calloc(1ul, + sizeof(struct _voucher_activity_s)); + act->va_id = va_id; + act->va_trace_id = trace_id ? trace_id : _voucher_activity_trace_id_release; + act->va_location = location; act->va_buffer_limit = _voucher_activity_buffer_limit(); TAILQ_INIT(&act->va_buffers); - act->va_flags2 = _voucher_activity_trace_flag_activity | - _voucher_activity_trace_flag_wide_second; -#if __LP64__ - act->va_flags3 = act->va_flags2; -#endif - act->va_refcnt = 0; - act->va_location = location; act->va_current_buffer = buffer; act->va_atm = vatm; // transfer vatm reference _voucher_activity_lock_init(va_buffers_lock(act)); + if (dispatch_assume_zero(pthread_mutex_init(&act->va_mutex, NULL)) || + dispatch_assume_zero(pthread_cond_init(&act->va_cond, NULL))) { + DISPATCH_CLIENT_CRASH("Could not initialize activity"); + } _TAILQ_MARK_NOT_ENQUEUED(act, va_list); _TAILQ_MARK_NOT_ENQUEUED(act, va_atm_list); _TAILQ_MARK_NOT_ENQUEUED(act, va_atm_used_list); + + _voucher_activity_buffer_init(act, buffer, true); + TAILQ_INSERT_TAIL(&act->va_buffers, buffer, vabh_list); _voucher_activity_t actx = _voucher_activity_try_insert(act); if (actx) { _voucher_activity_dispose(act); act = actx; - } else { - _voucher_atm_activity_insert(vatm, act); } _dispatch_voucher_activity_debug("create", act); return act; @@ -1571,20 +1562,56 @@ _voucher_activity_dispose(_voucher_activity_t act) dispatch_assert(!_TAILQ_IS_ENQUEUED(act, va_atm_used_list)); _voucher_activity_buffer_header_t buffer, tmp; TAILQ_FOREACH_SAFE(buffer, &act->va_buffers, vabh_list, tmp) { - _dispatch_voucher_activity_debug("buffer free %p", act, buffer); + if (buffer->vabh_pos.vabp_pos.vabp_next_tracepoint_idx > 1) { + dispatch_assert(_voucher_activity_buffer_mark_full(buffer)); + _voucher_activity_firehose_push(act, buffer); + } TAILQ_REMOVE(&act->va_buffers, buffer, vabh_list); + _dispatch_voucher_activity_debug("buffer free %p", act, buffer); _voucher_activity_heap_buffer_free(buffer); } - buffer = (_voucher_activity_buffer_header_t)act; - _voucher_activity_heap_buffer_free(buffer); + (void)dispatch_assume_zero(pthread_mutex_destroy(&act->va_mutex)); + (void)dispatch_assume_zero(pthread_cond_destroy(&act->va_cond)); + free(act); } +DISPATCH_NOINLINE +void +_voucher_activity_firehose_push(_voucher_activity_t act, + _voucher_activity_buffer_header_t buffer) +{ + if (dispatch_assume_zero(pthread_mutex_lock(&act->va_mutex))) { + DISPATCH_CLIENT_CRASH("Activity corruption: mutex_lock"); + } + _dispatch_voucher_activity_debug("firehose push %p", act, buffer); + // TODO: call firehose_push + VOUCHER_ACTIVITY_BUFFER_HOOK_CALLOUT(full, buffer); + _voucher_activity_buffer_init(act, buffer, false); + if (dispatch_assume_zero(pthread_cond_broadcast(&act->va_cond))) { + DISPATCH_CLIENT_CRASH("Activity corruption: cond_broadcast"); + } + if (dispatch_assume_zero(pthread_mutex_unlock(&act->va_mutex))) { + DISPATCH_CLIENT_CRASH("Activity corruption: mutex_unlock"); + } +} + +DISPATCH_NOINLINE static void -_voucher_activity_collect(_voucher_activity_t act) +_voucher_activity_firehose_wait(_voucher_activity_t act, + _voucher_activity_buffer_header_t buffer) { - _dispatch_voucher_activity_debug("collect", act); - if (_voucher_activity_try_remove(act)) { - _voucher_atm_activity_remove(act); + if (dispatch_assume_zero(pthread_mutex_lock(&act->va_mutex))) { + DISPATCH_CLIENT_CRASH("Activity corruption: mutex_lock"); + } + while (_voucher_activity_buffer_is_full(buffer)) { + _dispatch_voucher_activity_debug("firehose wait %p", act, buffer); + if (dispatch_assume_zero(pthread_cond_wait(&act->va_cond, + &act->va_mutex))){ + DISPATCH_CLIENT_CRASH("Activity corruption: cond_wait"); + } + } + if (dispatch_assume_zero(pthread_mutex_unlock(&act->va_mutex))) { + DISPATCH_CLIENT_CRASH("Activity corruption: mutex_unlock"); } } @@ -1611,76 +1638,43 @@ _voucher_activity_copy_from_mach_voucher(mach_voucher_t kv, _voucher_activity_heap_buffer_free(buffer); return NULL; } - if (VACTID_BASEID(va_id) != VATMID2ACTID(atm_id)) va_id = 0; // consumes vatm reference: act = _voucher_activity_create_with_atm(vatm, va_id, 0, 0, buffer); _dispatch_voucher_activity_debug("copy from kvoucher[0x%08x]", act, kv); return act; } -#pragma mark - -#pragma mark _voucher_atm_mailbox - -DISPATCH_ALWAYS_INLINE -static inline atm_mailbox_offset_t -_voucher_atm_mailbox_alloc(void) -{ - atm_mailbox_offset_t mailbox_offset = MAILBOX_OFFSET_UNSET; - size_t index; - index = _voucher_activity_bitmap_set_first_unset_bit(vam_atm_mbox_bitmap()); - if (index < NO_BITS_WERE_UNSET) { - mailbox_offset = index * sizeof(atm_mailbox_offset_t); -#if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG - _dispatch_debug("mailbox alloc %zd (%lld)", index, mailbox_offset); -#endif - } - return mailbox_offset; -} - -DISPATCH_ALWAYS_INLINE -static inline void -_voucher_atm_mailbox_free(atm_mailbox_offset_t mailbox_offset) -{ - if (mailbox_offset == MAILBOX_OFFSET_UNSET) return; - size_t index = (size_t)mailbox_offset / sizeof(atm_mailbox_offset_t); -#if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG - _dispatch_debug("mailbox free %zd (%lld)", index, mailbox_offset); -#endif - _voucher_activity_bitmap_clear_bit(vam_atm_mbox_bitmap(), index); -} - -DISPATCH_ALWAYS_INLINE -static inline bool -_voucher_atm_mailbox_set(atm_mailbox_offset_t mailbox_offset, - atm_subaid32_t subaid, bool max_present) -{ - if (mailbox_offset == MAILBOX_OFFSET_UNSET) return false; - char *mailbox_base = (char*)_voucher_activity_heap->vam_kernel_metadata; - atm_subaid32_t *mailbox = (atm_subaid32_t*)(mailbox_base + mailbox_offset); - if (max_present) mailbox++; // second atm_subaid32_t in atm_mailbox_offset_t - if (*mailbox == subaid) return false; - *mailbox = subaid; - return true; -} - #pragma mark - #pragma mark _voucher_atm_t -static bool _voucher_atm_try_remove(_voucher_atm_t vatm); +static void _voucher_atm_remove(_voucher_atm_t vatm); static void _voucher_atm_dispose(_voucher_atm_t vatm, bool unregister); -static inline void _voucher_atm_collect_if_needed(bool updated); DISPATCH_ALWAYS_INLINE -static inline _voucher_atm_t -_voucher_atm_retain(_voucher_atm_t vatm) +static inline bool +_voucher_atm_try_retain(_voucher_atm_t vatm) { - // assumes vam_atms_lock or vam_base_atm_lock held + // not using _os_object_refcnt* because we don't need barriers: + // vouchers atm are immutable and are in a hash table with a lock + // + // assumes vam_atms_lock held int refcnt = dispatch_atomic_inc2o(vatm, vatm_refcnt, relaxed); _dispatch_voucher_atm_debug("retain -> %d", vatm, refcnt + 1); if (slowpath(refcnt < 0)) { _dispatch_voucher_atm_debug("overrelease", vatm); DISPATCH_CRASH("ATM overrelease"); } + return refcnt > 0; +} + +DISPATCH_ALWAYS_INLINE +static inline _voucher_atm_t +_voucher_atm_retain(_voucher_atm_t vatm) +{ + if (slowpath(!_voucher_atm_try_retain(vatm))) { + _dispatch_voucher_atm_debug("resurrection", vatm); + DISPATCH_CRASH("ATM resurrection"); + } return vatm; } @@ -1688,6 +1682,8 @@ DISPATCH_ALWAYS_INLINE static inline void _voucher_atm_release(_voucher_atm_t vatm) { + // not using _os_object_refcnt* because we don't need barriers: + // vouchers atm are immutable are into a hash table with a lock int refcnt = dispatch_atomic_dec2o(vatm, vatm_refcnt, relaxed); _dispatch_voucher_atm_debug("release -> %d", vatm, refcnt + 1); if (fastpath(refcnt >= 0)) { @@ -1697,20 +1693,30 @@ _voucher_atm_release(_voucher_atm_t vatm) _dispatch_voucher_atm_debug("overrelease", vatm); DISPATCH_CRASH("ATM overrelease"); } - if (_voucher_atm_try_remove(vatm)) { - _voucher_atm_dispose(vatm, true); - } + _voucher_atm_remove(vatm); + _voucher_atm_dispose(vatm, true); } static _voucher_atm_t -_voucher_atm_find(atm_aid_t atm_id, uint32_t hash) +_voucher_atm_find_and_retain(atm_aid_t atm_id, uint32_t hash) { + // not using _os_object_refcnt* because we don't need barriers: + // vouchers atm are immutable are into a hash table with a lock + // // assumes vam_atms_lock held _voucher_atm_t vatm; TAILQ_FOREACH(vatm, vam_atms(hash), vatm_list){ - if (vatm->vatm_id == atm_id) break; + if (vatm->vatm_id == atm_id) { + if (fastpath(_voucher_atm_try_retain(vatm))) { + return vatm; + } + + // disallow resurrection + dispatch_atomic_dec2o(vatm, vatm_refcnt, relaxed); + _dispatch_voucher_atm_debug("undo resurrection", vatm); + } } - return vatm; + return NULL; } static _voucher_atm_t @@ -1718,9 +1724,8 @@ _voucher_atm_copy(atm_aid_t atm_id) { uint32_t hash = VATMID_HASH(atm_id); _voucher_activity_lock_lock(vam_atms_lock()); - _voucher_atm_t vatm = _voucher_atm_find(atm_id, hash); + _voucher_atm_t vatm = _voucher_atm_find_and_retain(atm_id, hash); if (vatm) { - _voucher_atm_retain(vatm); _dispatch_voucher_atm_debug("copy", vatm); } _voucher_activity_lock_unlock(vam_atms_lock()); @@ -1733,9 +1738,8 @@ _voucher_atm_try_insert(_voucher_atm_t vatm_new) atm_aid_t atm_id = vatm_new->vatm_id; uint32_t hash = VATMID_HASH(atm_id); _voucher_activity_lock_lock(vam_atms_lock()); - _voucher_atm_t vatm = _voucher_atm_find(atm_id, hash); + _voucher_atm_t vatm = _voucher_atm_find_and_retain(atm_id, hash); if (vatm) { - _voucher_atm_retain(vatm); _dispatch_voucher_atm_debug("try insert: failed (%p)", vatm, vatm_new); } else { if (slowpath(_TAILQ_IS_ENQUEUED(vatm_new, vatm_list))) { @@ -1749,240 +1753,42 @@ _voucher_atm_try_insert(_voucher_atm_t vatm_new) return vatm; } -static bool -_voucher_atm_try_remove(_voucher_atm_t vatm) +static void +_voucher_atm_remove(_voucher_atm_t vatm) { - bool r; atm_aid_t atm_id = vatm->vatm_id; uint32_t hash = VATMID_HASH(atm_id); + _voucher_activity_lock_lock(vam_atms_lock()); - if (slowpath(!atm_id)) { + if (slowpath(!atm_id || !_TAILQ_IS_ENQUEUED(vatm, vatm_list))) { _dispatch_voucher_atm_debug("corruption", vatm); DISPATCH_CRASH("ATM corruption"); } - if ((r = (dispatch_atomic_load2o(vatm, vatm_refcnt, seq_cst) < 0 && - _TAILQ_IS_ENQUEUED(vatm, vatm_list)))) { - TAILQ_REMOVE(vam_atms(hash), vatm, vatm_list); - _TAILQ_MARK_NOT_ENQUEUED(vatm, vatm_list); - vatm->vatm_list.tqe_next = (void*)~0ull; - } - _dispatch_voucher_atm_debug("try remove: %s", vatm, r ? "succeeded" : - "failed"); + TAILQ_REMOVE(vam_atms(hash), vatm, vatm_list); + _TAILQ_MARK_NOT_ENQUEUED(vatm, vatm_list); + vatm->vatm_list.tqe_next = (void*)~0ull; + _dispatch_voucher_atm_debug("remove", vatm); _voucher_activity_lock_unlock(vam_atms_lock()); - return r; -} - -static bool -_voucher_atm_update_mailbox(_voucher_atm_t vatm) -{ - // Update kernel mailbox with largest allocated subaid for this atm_id - // assumes atm_activities_lock held - _voucher_activity_t act = TAILQ_LAST(vatm_activities(vatm), - _voucher_atm_activities_s); - atm_subaid32_t subaid = act ? VACTID_SUBID(act->va_id) : 0; - bool r = _voucher_atm_mailbox_set(vatm->vatm_mailbox_offset, subaid, true); - if (r) { - _dispatch_voucher_atm_debug("update max-present subaid 0x%x", vatm, - subaid); - } - return r; -} - -static bool -_voucher_atm_update_used_mailbox(_voucher_atm_t vatm) -{ - // Update kernel mailbox with smallest in-use subaid for this atm_id - // assumes atm_activities_lock held - _voucher_activity_t act = TAILQ_FIRST(vatm_used_activities(vatm)); - atm_subaid32_t subaid = act ? VACTID_SUBID(act->va_id) : ATM_SUBAID32_MAX; - bool r = _voucher_atm_mailbox_set(vatm->vatm_mailbox_offset, subaid, false); - if (r) { - _dispatch_voucher_atm_debug("update min-used subaid 0x%x", vatm, - subaid); - } - return r; -} - -static void -_voucher_atm_activity_insert(_voucher_atm_t vatm, _voucher_activity_t act) -{ - _voucher_activity_lock_lock(vatm_activities_lock(vatm)); - if (!_TAILQ_IS_ENQUEUED(act, va_atm_list)) { - _voucher_activity_ordered_insert(act, vatm_activities(vatm), - va_atm_list); - _voucher_atm_update_mailbox(vatm); - } - if (!_TAILQ_IS_ENQUEUED(act, va_atm_used_list)) { - _voucher_activity_ordered_insert(act, vatm_used_activities(vatm), - va_atm_used_list); - _voucher_atm_update_used_mailbox(vatm); - } - _dispatch_voucher_activity_debug("atm insert", act); - _voucher_activity_lock_unlock(vatm_activities_lock(vatm)); -} - -static void -_voucher_atm_activity_remove(_voucher_activity_t act) -{ - _voucher_atm_t vatm = act->va_atm; - _voucher_activity_lock_lock(vatm_activities_lock(vatm)); - _dispatch_voucher_activity_debug("atm remove", act); - if (_TAILQ_IS_ENQUEUED(act, va_atm_used_list)) { - TAILQ_REMOVE(vatm_activities(vatm), act, va_atm_used_list); - _TAILQ_MARK_NOT_ENQUEUED(act, va_atm_used_list); - _voucher_atm_update_used_mailbox(vatm); - } - if (_TAILQ_IS_ENQUEUED(act, va_atm_list)) { - TAILQ_REMOVE(vatm_activities(vatm), act, va_atm_list); - _TAILQ_MARK_NOT_ENQUEUED(act, va_atm_list); - _voucher_atm_update_mailbox(vatm); - // Balance initial creation refcnt. Caller must hold additional - // reference to ensure this does not release vatm before the unlock, - // see _voucher_atm_activity_collect - _voucher_activity_atm_release(act); - } - _voucher_activity_lock_unlock(vatm_activities_lock(vatm)); -} - -static _voucher_activity_t -_voucher_atm_activity_mark_used(_voucher_activity_t act) -{ - _voucher_atm_t vatm = act->va_atm; - _voucher_activity_lock_lock(vatm_activities_lock(vatm)); - if (!_TAILQ_IS_ENQUEUED(act, va_atm_used_list)) { - _voucher_activity_ordered_insert(act, vatm_used_activities(vatm), - va_atm_used_list); - _voucher_atm_update_used_mailbox(vatm); - _dispatch_voucher_activity_debug("mark used", act); - } - _voucher_activity_lock_unlock(vatm_activities_lock(vatm)); - return act; -} - -static void -_voucher_atm_activity_mark_unused(_voucher_activity_t act) -{ - bool atm_collect = false, updated = false; - _voucher_atm_t vatm = act->va_atm; - _voucher_activity_lock_lock(vatm_activities_lock(vatm)); - if (_TAILQ_IS_ENQUEUED(act, va_atm_used_list)) { - _dispatch_voucher_activity_debug("mark unused", act); - TAILQ_REMOVE(&vatm->vatm_used_activities, act, va_atm_used_list); - _TAILQ_MARK_NOT_ENQUEUED(act, va_atm_used_list); - atm_collect = true; - _voucher_atm_retain(vatm); - updated = _voucher_atm_update_used_mailbox(vatm); - } - _voucher_activity_lock_unlock(vatm_activities_lock(vatm)); - if (atm_collect) { - _voucher_atm_release(vatm); - _voucher_atm_collect_if_needed(updated); - } -} - -static void -_voucher_atm_activity_collect(_voucher_atm_t vatm, atm_subaid32_t min_subaid) -{ - _dispatch_voucher_atm_debug("collect min subaid 0x%x", vatm, min_subaid); - voucher_activity_id_t min_va_id = VATM_ACTID(vatm, min_subaid); - _voucher_activity_t act; - do { - _voucher_activity_lock_lock(vatm_activities_lock(vatm)); - TAILQ_FOREACH(act, vatm_activities(vatm), va_atm_list) { - if (act->va_id >= min_va_id) { - act = NULL; - break; - } - if (!_TAILQ_IS_ENQUEUED(act, va_atm_used_list)) { - _voucher_activity_atm_retain(act); - break; - } - } - _voucher_activity_lock_unlock(vatm_activities_lock(vatm)); - if (act) { - _voucher_activity_collect(act); - _voucher_activity_atm_release(act); - } - } while (act); } DISPATCH_NOINLINE static void -_voucher_atm_collect(void) +_voucher_atm_fault(mach_voucher_attr_command_t kvc_cmd) { - _voucher_atm_t vatms[_voucher_atm_mailboxes], vatm; - atm_aid_t aids[_voucher_atm_mailboxes]; - mach_atm_subaid_t subaids[_voucher_atm_mailboxes]; - uint32_t i, a = 0, s; + mach_voucher_t kv = _voucher_get_atm_mach_voucher(_voucher_get()); + if (!kv) return; - _voucher_activity_lock_lock(vam_atms_lock()); - for (i = 0; i < _voucher_activity_hash_size; i++) { - TAILQ_FOREACH(vatm, vam_atms(i), vatm_list){ - if (vatm == _voucher_activity_heap->vam_default_activity_atm || - vatm->vatm_mailbox_offset == MAILBOX_OFFSET_UNSET) continue; - _dispatch_voucher_atm_debug("find min subaid", vatm); - vatms[a] = _voucher_atm_retain(vatm); - aids[a] = vatm->vatm_id; - if (++a == _voucher_atm_mailboxes) goto out; + mach_atm_subaid_t subaid = 0; + voucher_t v = _voucher_get(); + if (v) { + unsigned int activities = v->v_activities; + voucher_activity_id_t *activity_ids = _voucher_activity_ids(v); + if (activities) { + subaid = activity_ids[0]; } } -out: - _voucher_activity_lock_unlock(vam_atms_lock()); - if (!a) return; - kern_return_t kr; - mach_voucher_t kv = vatms[0]->vatm_kvoucher; - mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&aids; - mach_voucher_attr_content_size_t kvc_in_size = sizeof(atm_aid_t) * a; - mach_voucher_attr_content_t kvc_out = (mach_voucher_attr_content_t)&subaids; - mach_voucher_attr_content_size_t kvc_out_size = sizeof(mach_atm_subaid_t)*a; - kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_ATM, - ATM_FIND_MIN_SUB_AID, kvc_in, kvc_in_size, kvc_out, &kvc_out_size); - DISPATCH_VERIFY_MIG(kr); - (void)dispatch_assume_zero(kr); - s = kvc_out_size / sizeof(mach_atm_subaid_t); -#if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG - _dispatch_debug("found min subaids (%u out of %u)", s, a); -#endif - for (i = 0; i < a; i++) { - if (i < s) _voucher_atm_activity_collect(vatms[i], - (atm_subaid32_t)subaids[i]); - _voucher_atm_release(vatms[i]); - } -} - -static inline void -_voucher_atm_collect_if_needed(bool updated) -{ - long level; - if (updated) { - level = dispatch_atomic_add(&_voucher_atm_collect_level, 2ul, relaxed); - } else { - level = _voucher_atm_collect_level; - if (!level) return; - } - if (level & 1 || level <= _voucher_atm_collect_threshold) return; - if (!dispatch_atomic_cmpxchg(&_voucher_atm_collect_level, level, level + 1, - acquire)) return; -#if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG - _dispatch_debug("atm collect: reached level %ld", level/2); -#endif - if (slowpath(level < 0)) { - DISPATCH_CRASH("ATM collection level corruption"); - } - _voucher_atm_collect(); - dispatch_atomic_sub(&_voucher_atm_collect_level, level + 1, release); -} - -DISPATCH_NOINLINE -static void -_voucher_atm_fault(mach_voucher_attr_command_t kvc_cmd) -{ - _voucher_activity_t act = _voucher_activity_get(_voucher_get()); - mach_voucher_t kv = _voucher_activity_get_atm_mach_voucher(act); - if (!kv) return; kern_return_t kr; - mach_atm_subaid_t subaid = VACTID_SUBID(act->va_id); mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&subaid; mach_voucher_attr_content_size_t kvc_in_size = sizeof(mach_atm_subaid_t); mach_voucher_attr_content_t kvc_out = (mach_voucher_attr_content_t)&subaid; @@ -2030,88 +1836,61 @@ _voucher_atm_mach_voucher_create(atm_aid_t *atm_id_ptr) return kv; } -static void -_voucher_atm_mailbox_mach_voucher_register(_voucher_atm_t vatm, - mach_voucher_t kv) +static mach_voucher_t +_voucher_atm_mach_voucher_copy(mach_voucher_t akv) { - _dispatch_voucher_atm_debug("mailbox register %lld with kvoucher[0x%08x]", - vatm, vatm->vatm_mailbox_offset, kv); kern_return_t kr; - mach_voucher_t akv; - atm_mailbox_offset_t offset = vatm->vatm_mailbox_offset; - mach_voucher_attr_recipe_t vr; - size_t vr_size; - static const mach_voucher_attr_recipe_data_t atm_register_recipe = { + mach_voucher_t kv; + const mach_voucher_attr_recipe_data_t atm_copy_recipe = { .key = MACH_VOUCHER_ATTR_KEY_ATM, - .command = MACH_VOUCHER_ATTR_ATM_REGISTER, - .content_size = sizeof(offset), + .command = MACH_VOUCHER_ATTR_COPY, + .previous_voucher = akv, }; - vr_size = sizeof(atm_register_recipe) + atm_register_recipe.content_size; - vr = alloca(vr_size); - *vr = atm_register_recipe; - vr->previous_voucher = kv; - memcpy(&vr->content, &offset, sizeof(offset)); - kr = _voucher_create_mach_voucher(vr, vr_size, &akv); + kr = _voucher_create_mach_voucher(&atm_copy_recipe, + sizeof(atm_copy_recipe), &kv); if (dispatch_assume_zero(kr)) { - DISPATCH_CLIENT_CRASH("Could not register ATM ID"); + DISPATCH_CLIENT_CRASH("Could not copy ATM mach voucher"); } - if (!vatm->vatm_kvoucher) { - vatm->vatm_kvoucher = akv; - } else { -#if !RDAR_17510224 - if (akv != vatm->vatm_kvoucher) { - DISPATCH_CRASH("Unexpected mach voucher returned by ATM ID " - "registration"); - } - _voucher_dealloc_mach_voucher(akv); -#else - DISPATCH_CRASH("Registered invalid ATM object"); -#endif - } - _dispatch_voucher_atm_debug("mailbox registered %lld", vatm, - vatm->vatm_mailbox_offset); + _dispatch_kvoucher_debug("copy atm voucher from [0x%08x]", kv, akv); + return kv; } static void -_voucher_atm_mailbox_register(_voucher_atm_t vatm) +_voucher_atm_register(_voucher_atm_t vatm) { mach_voucher_t kv = vatm->vatm_kvoucher; if (!kv) return; -#if !RDAR_17510224 - _voucher_atm_mailbox_mach_voucher_register(vatm, kv); -#else // RDAR_17510224 - _dispatch_voucher_atm_debug("mailbox register %lld", vatm, - vatm->vatm_mailbox_offset); kern_return_t kr; - atm_mailbox_offset_t offset = vatm->vatm_mailbox_offset; - mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&offset; - mach_voucher_attr_content_size_t kvc_in_size = sizeof(offset); + atm_guard_t gen = + dispatch_atomic_inc(&_voucher_atm_generation, relaxed); + _dispatch_voucher_atm_debug("atm register %lld", vatm, gen); + mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&gen; + mach_voucher_attr_content_size_t kvc_in_size = sizeof(gen); mach_voucher_attr_content_t kvc_out = NULL; mach_voucher_attr_content_size_t kvc_out_size = 0; kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_ATM, ATM_ACTION_REGISTER, kvc_in, kvc_in_size, kvc_out, &kvc_out_size); DISPATCH_VERIFY_MIG(kr); - if (dispatch_assume_zero(kr)) { + if (kr) { DISPATCH_CLIENT_CRASH("Could not register ATM ID"); } - _dispatch_voucher_atm_debug("mailbox registered %lld", vatm, - vatm->vatm_mailbox_offset); -#endif // RDAR_17510224 + vatm->vatm_generation = gen; + _dispatch_voucher_atm_debug("atm registered %lld", vatm, + vatm->vatm_generation); } -static bool -_voucher_atm_mailbox_unregister(_voucher_atm_t vatm) +static void +_voucher_atm_unregister(_voucher_atm_t vatm) { - if (vatm->vatm_mailbox_offset == MAILBOX_OFFSET_UNSET) return false; - _dispatch_voucher_atm_debug("mailbox unregister %lld", vatm, - vatm->vatm_mailbox_offset); + _dispatch_voucher_atm_debug("atm unregister %lld", vatm, + vatm->vatm_generation); mach_voucher_t kv = vatm->vatm_kvoucher; dispatch_assert(kv); kern_return_t kr; - atm_mailbox_offset_t offset = vatm->vatm_mailbox_offset; - mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&offset; - mach_voucher_attr_content_size_t kvc_in_size = sizeof(offset); + atm_guard_t gen = vatm->vatm_generation; + mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&gen; + mach_voucher_attr_content_size_t kvc_in_size = sizeof(gen); mach_voucher_attr_content_t kvc_out = NULL; mach_voucher_attr_content_size_t kvc_out_size = 0; kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_ATM, @@ -2119,43 +1898,25 @@ _voucher_atm_mailbox_unregister(_voucher_atm_t vatm) DISPATCH_VERIFY_MIG(kr); if (kr && kr != KERN_INVALID_VALUE) { (void)dispatch_assume_zero(kr); - DISPATCH_CLIENT_CRASH("Could not unregister ATM ID"); } - _dispatch_voucher_atm_debug("mailbox unregistered %lld", vatm, - vatm->vatm_mailbox_offset); - return true; + _dispatch_voucher_atm_debug("atm unregistered %lld", vatm, + vatm->vatm_generation); } static _voucher_atm_t _voucher_atm_create(mach_voucher_t kv, atm_aid_t atm_id) { - atm_mailbox_offset_t mailbox_offset = _voucher_atm_mailbox_alloc(); - if (kv && mailbox_offset == MAILBOX_OFFSET_UNSET) return NULL; _voucher_atm_t vatm = _dispatch_calloc(1ul, sizeof(struct _voucher_atm_s)); - if (!kv) { - kv = _voucher_atm_mach_voucher_create(&atm_id); - if (mailbox_offset == MAILBOX_OFFSET_UNSET) { - _voucher_dealloc_mach_voucher(kv); - } else { - vatm->vatm_kvoucher = kv; - } - kv = MACH_VOUCHER_NULL; - } + kv = kv ? _voucher_atm_mach_voucher_copy(kv) : + _voucher_atm_mach_voucher_create(&atm_id); + vatm->vatm_kvoucher = kv; vatm->vatm_id = atm_id; - vatm->vatm_mailbox_offset = mailbox_offset; - _voucher_activity_lock_init(vatm_activities_lock(vatm)); - TAILQ_INIT(&vatm->vatm_activities); - TAILQ_INIT(&vatm->vatm_used_activities); - _voucher_atm_mailbox_set(mailbox_offset, 0, true); - _voucher_atm_mailbox_set(mailbox_offset, ATM_SUBAID32_MAX, false); _voucher_atm_t vatmx = _voucher_atm_try_insert(vatm); if (vatmx) { _voucher_atm_dispose(vatm, false); vatm = vatmx; - } else if (kv) { - _voucher_atm_mailbox_mach_voucher_register(vatm, kv); } else { - _voucher_atm_mailbox_register(vatm); + _voucher_atm_register(vatm); } _dispatch_voucher_atm_debug("create with kvoucher[0x%08x]", vatm, kv); return vatm; @@ -2165,114 +1926,48 @@ static void _voucher_atm_dispose(_voucher_atm_t vatm, bool unregister) { _dispatch_voucher_atm_debug("dispose", vatm); - dispatch_assert(TAILQ_EMPTY(&vatm->vatm_activities)); - dispatch_assert(TAILQ_EMPTY(&vatm->vatm_used_activities)); if (slowpath(_TAILQ_IS_ENQUEUED(vatm, vatm_list))) { _dispatch_voucher_atm_debug("corruption", vatm); DISPATCH_CRASH("ATM corruption"); } vatm->vatm_list.tqe_next = DISPATCH_OBJECT_LISTLESS; - bool free_mailbox = (vatm->vatm_mailbox_offset != MAILBOX_OFFSET_UNSET); if (vatm->vatm_kvoucher) { - if (unregister) free_mailbox = _voucher_atm_mailbox_unregister(vatm); + if (unregister) _voucher_atm_unregister(vatm); _voucher_dealloc_mach_voucher(vatm->vatm_kvoucher); vatm->vatm_kvoucher = MACH_VOUCHER_NULL; } - if (free_mailbox) { - _voucher_atm_mailbox_free(vatm->vatm_mailbox_offset); - vatm->vatm_mailbox_offset = MAILBOX_OFFSET_UNSET; - } free(vatm); } -static inline mach_voucher_t -_voucher_activity_get_atm_mach_voucher(_voucher_activity_t act) -{ - mach_voucher_t kv; - kv = act && act->va_atm ? act->va_atm->vatm_kvoucher : MACH_VOUCHER_NULL; - return kv; -} - DISPATCH_NOINLINE -static _voucher_atm_t -_voucher_atm_base_copy_and_activity_id_make(voucher_activity_id_t *va_id_ptr) -{ - _voucher_atm_subid_t subid; - _voucher_atm_t vatm, vatm_old = NULL, vatm_new = NULL; - if (_voucher_activity_heap->vam_base_atm_subid_max == 1) { - vatm = _voucher_atm_create(0, 0); - subid = 1; - goto out; - } - _voucher_activity_lock_lock(vam_base_atm_lock()); - vatm = _voucher_activity_heap->vam_base_atm; -retry: - _voucher_atm_retain(vatm); - subid = _voucher_activity_heap->vam_base_atm_subid; - if (subid++ >= _voucher_activity_heap->vam_base_atm_subid_max) { - _voucher_activity_lock_unlock(vam_base_atm_lock()); - if (!vatm_new) vatm_new = _voucher_atm_create(0, 0); - _voucher_activity_lock_lock(vam_base_atm_lock()); - _voucher_atm_release(vatm); - vatm_old = vatm; - vatm = _voucher_activity_heap->vam_base_atm; - if (vatm != vatm_old) { - vatm_old = NULL; - goto retry; - } - _voucher_activity_heap->vam_base_atm = vatm = vatm_new; - _voucher_activity_heap->vam_base_atm_subid = subid = 1; - vatm_new = NULL; - _voucher_atm_retain(vatm); - _dispatch_voucher_atm_debug("base replace", vatm); - } else { - _voucher_activity_heap->vam_base_atm_subid = subid; - _dispatch_voucher_atm_debug("base copy", vatm); - } - _voucher_activity_lock_unlock(vam_base_atm_lock()); - if (vatm_old) _voucher_atm_release(vatm_old); - if (vatm_new) _voucher_atm_release(vatm_new); -out: - *va_id_ptr = VATM_ACTID(vatm, subid); - return vatm; -} - static voucher_activity_id_t -_voucher_atm_nested_atm_id_make(void) +_voucher_atm_subid_make(_voucher_atm_t vatm, voucher_activity_flag_t flags) { - atm_aid_t atm_id; - mach_voucher_t kv = _voucher_atm_mach_voucher_create(&atm_id); - _voucher_dealloc_mach_voucher(kv); // just need the unique ID - return VATMID2ACTID(atm_id); -} - -static voucher_activity_id_t -_voucher_atm_nested_activity_id_make(void) -{ - voucher_activity_id_t va_id, va_id_old, va_id_new; - _voucher_atm_subid_t subid; - _voucher_activity_lock_lock(vam_nested_atm_lock()); - va_id = _voucher_activity_heap->vam_nested_atm_id; -retry: - subid = _voucher_activity_heap->vam_nested_atm_subid; - if (subid++ >= VATM_SUBID_MAX) { - _voucher_activity_lock_unlock(vam_nested_atm_lock()); - va_id_new = _voucher_atm_nested_atm_id_make(); - va_id_old = va_id; - _voucher_activity_lock_lock(vam_nested_atm_lock()); - va_id = _voucher_activity_heap->vam_nested_atm_id; - if (va_id != va_id_old) goto retry; - _voucher_activity_heap->vam_nested_atm_id = va_id = va_id_new; - subid = 1; - } - _voucher_activity_heap->vam_nested_atm_subid = subid; - _voucher_activity_lock_unlock(vam_nested_atm_lock()); - return va_id + subid; + mach_voucher_t kv = vatm->vatm_kvoucher; + _dispatch_voucher_atm_debug("create subid from atm", vatm); + kern_return_t kr; + mach_atm_subaid_t naid; + mach_voucher_attr_content_t kvc_in = NULL; + mach_voucher_attr_content_size_t kvc_in_size = 0; + mach_voucher_attr_content_t kvc_out = (mach_voucher_attr_content_t)&naid; + mach_voucher_attr_content_size_t kvc_out_size = sizeof(naid); + kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_ATM, + ATM_ACTION_GETSUBAID, kvc_in, kvc_in_size, kvc_out, &kvc_out_size); + DISPATCH_VERIFY_MIG(kr); + if (dispatch_assume_zero(kr)) { + DISPATCH_CLIENT_CRASH("Could not get next ATM ID"); + } + _dispatch_voucher_atm_debug("created subid from atm %lld", vatm, naid); + return VATMID2ACTID(naid, flags); } #pragma mark - #pragma mark voucher_activity_id_t +static const size_t _voucher_activity_maxsize = + _voucher_activity_buffer_size - _voucher_activity_buffer_header_size - + _voucher_activity_strings_header_size; + voucher_activity_id_t voucher_activity_start_with_location(voucher_activity_trace_id_t trace_id, uint64_t location, voucher_activity_flag_t flags) @@ -2280,36 +1975,29 @@ voucher_activity_start_with_location(voucher_activity_trace_id_t trace_id, dispatch_once_f(&_voucher_activity_heap_pred, NULL, _voucher_activity_heap_init); if (!_voucher_activity_trace_id_enabled(trace_id)) return 0; - voucher_activity_id_t va_id = 0, va_base_id = 0; + voucher_activity_id_t va_id = 0; _voucher_atm_t vatm = NULL; _voucher_activity_t act = NULL; _voucher_activity_tracepoint_t vat = NULL; unsigned int activities = 1, oactivities = 0; voucher_t ov = _voucher_get(); + vatm = _voucher_get_atm(ov); if (!(flags & voucher_activity_flag_force) && ov && ov->v_activities) { oactivities = ov->v_activities; activities += oactivities; if (activities > _voucher_max_activities) { - va_id = _voucher_atm_nested_activity_id_make(); + va_id = _voucher_atm_subid_make(vatm, flags); goto out; } } - if (activities == 1) { - vatm = _voucher_atm_base_copy_and_activity_id_make(&va_id); - if (vatm->vatm_kvoucher) { - // consumes vatm reference: - act = _voucher_activity_create_with_atm(vatm, va_id, trace_id, - location, NULL); - vat = (_voucher_activity_tracepoint_t)act; - } else { - _voucher_atm_release(vatm); - } - if (!act) { - activities++; - // default to _voucher_activity_default base activity - va_base_id = _voucher_activity_default->va_id; - } - } + _voucher_atm_retain(vatm); + // required for v->v_atm = vatm below + _voucher_atm_retain(vatm); + va_id = _voucher_atm_subid_make(vatm, flags); + // consumes vatm reference: + act = _voucher_activity_create_with_atm(vatm, va_id, trace_id, location, + NULL); + vat = (_voucher_activity_tracepoint_t)act; pthread_priority_t priority = _voucher_get_priority(ov); mach_voucher_attr_recipe_size_t extra = ov ? _voucher_extra_size(ov) : 0; voucher_t v = _voucher_alloc(activities, priority, extra); @@ -2326,24 +2014,13 @@ voucher_activity_start_with_location(voucher_activity_trace_id_t trace_id, memcpy(activity_ids, _voucher_activity_ids(ov), oactivities * sizeof(voucher_activity_id_t)); } - if (!va_id) { - va_id = _voucher_atm_nested_activity_id_make(); - if (ov && ov->v_activity) { - act = _voucher_activity_retain(ov->v_activity); - } - } - if (va_base_id) activity_ids[0] = va_base_id; activity_ids[activities-1] = va_id; + v->v_atm = vatm; v->v_activity = act; _voucher_swap(ov, v); - if (vat) return va_id; // new _voucher_activity_s contains trace info + return va_id; // new activity buffer contains trace info out: - vat = _voucher_activity_trace_with_id(trace_id); - if (vat) { - vat->vat_flags |= _voucher_activity_trace_flag_activity | - _voucher_activity_trace_flag_start; - vat->vat_data[0] = va_id; - } + _voucher_activity_trace_activity_event(trace_id, va_id, start); return va_id; } @@ -2358,13 +2035,8 @@ void voucher_activity_end(voucher_activity_id_t va_id) { if (!va_id) return; - _voucher_activity_tracepoint_t vat; - vat = _voucher_activity_trace_with_id(_voucher_activity_trace_id_release); - if (vat) { - vat->vat_flags |= _voucher_activity_trace_flag_activity | - _voucher_activity_trace_flag_end; - vat->vat_data[0] = va_id; - } + _voucher_activity_trace_activity_event(_voucher_activity_trace_id_release, + va_id, end); voucher_t v = _voucher_get(); if (!v) return; unsigned int activities = v->v_activities, act_idx = activities; @@ -2401,6 +2073,7 @@ voucher_activity_end(voucher_activity_id_t va_id) } else { if (v->v_activity) { nv->v_activity = _voucher_activity_retain(v->v_activity); + nv->v_atm = _voucher_atm_retain(v->v_atm); } memcpy(new_activity_ids, activity_ids, --act_idx * sizeof(voucher_activity_id_t)); @@ -2434,12 +2107,17 @@ voucher_activity_get_namespace(void) { voucher_t v = _voucher_get(); if (!v || !v->v_activity) return 0; - return v->v_activity->va_namespace; + voucher_activity_trace_id_t trace_id = v->v_activity->va_trace_id; + uint8_t cns = (uint8_t)(trace_id >> + _voucher_activity_trace_id_code_namespace_shift); + return cns; } DISPATCH_NOINLINE _voucher_activity_tracepoint_t -_voucher_activity_tracepoint_get_slow(unsigned int slots) +_voucher_activity_buffer_tracepoint_acquire_slow(_voucher_activity_t *vap, + _voucher_activity_buffer_header_t *vabp, unsigned int slots, + size_t strsize, uint16_t *stroffsetp) { _voucher_activity_t act; _voucher_activity_buffer_header_t vab; @@ -2454,16 +2132,18 @@ _voucher_activity_tracepoint_get_slow(unsigned int slots) act = _voucher_activity_default; } vab = act->va_current_buffer; - if (vab && vab->vabh_next_tracepoint_idx <= - _voucher_activity_tracepoints_per_buffer) { + if (act == *vap && vab != *vabp) { goto retry; // another slowpath raced us } do { vab = _voucher_activity_buffer_alloc(act, vab); if (!vab) break; retry: - vat = _voucher_activity_buffer_tracepoint_get(vab, slots); + vat = _voucher_activity_buffer_tracepoint_acquire(vab, slots, strsize, + stroffsetp); } while (!vat); + *vap = act; + *vabp = vab; return vat; } @@ -2485,13 +2165,20 @@ voucher_activity_trace(voucher_activity_trace_id_t trace_id, uint64_t location, void *buffer, size_t length) { if (!_voucher_activity_trace_id_enabled(trace_id)) return 0; + _voucher_activity_t act; + _voucher_activity_buffer_header_t vab; _voucher_activity_tracepoint_t vat; const unsigned int slots = length <= sizeof(vat->vat_data) ? 1 : 2; - vat = _voucher_activity_tracepoint_get(slots); - if (!vat) vat = _voucher_activity_tracepoint_get_slow(slots); + act = _voucher_activity_get(); + vab = _voucher_activity_buffer_get_from_activity(act); + vat = _voucher_activity_buffer_tracepoint_acquire(vab, slots, 0, NULL); + if (!vat) { + vat = _voucher_activity_buffer_tracepoint_acquire_slow(&act, &vab, + slots, 0, NULL); + } if (!vat) return 0; uint64_t timestamp = _voucher_activity_tracepoint_init_with_id(vat, - trace_id, location); + trace_id, location, true); void *tbuf = vat->vat_data; size_t tlen = sizeof(vat->vat_data); if (length < tlen) { @@ -2515,6 +2202,72 @@ voucher_activity_trace(voucher_activity_trace_id_t trace_id, uint64_t location, } } _voucher_activity_trace_fault(trace_id); + if (_voucher_activity_buffer_tracepoint_release(vab)) { + _voucher_activity_firehose_push(act, vab); + } + return timestamp; +} + +uint64_t +voucher_activity_trace_strings(voucher_activity_trace_id_t trace_id, + uint64_t location, void *buffer, size_t length, const char *strings[], + size_t string_lengths[], size_t strings_size) +{ + if (!_voucher_activity_trace_id_enabled(trace_id)) return 0; + _voucher_activity_t act; + _voucher_activity_buffer_header_t vab; + _voucher_activity_tracepoint_t vat; + uint16_t offset; + const unsigned int slots = length <= sizeof(vat->vat_data) ? 1 : 2; + strings_size = MIN(strings_size, _voucher_activity_maxsize - + slots * sizeof(struct _voucher_activity_tracepoint_s)); + act = _voucher_activity_get(); + vab = _voucher_activity_buffer_get_from_activity(act); + vat = _voucher_activity_buffer_tracepoint_acquire(vab, slots, strings_size, + &offset); + if (!vat) { + vat = _voucher_activity_buffer_tracepoint_acquire_slow(&act, &vab, + slots, strings_size, &offset); + } + if (!vat) return 0; + uint64_t timestamp = _voucher_activity_tracepoint_init_with_id(vat, + trace_id, location, false); + vat->vat_flags |= _voucher_activity_trace_flag_tracepoint_strings; + vat->vat_stroff.vats_offset = offset; + void *tbuf = vat->vat_stroff.vats_data; + size_t tlen = sizeof(vat->vat_stroff.vats_data); + if (length < tlen) { + memcpy(tbuf, buffer, length); + } else { + memcpy(tbuf, buffer, tlen); + } + if (length > tlen) { + vat->vat_flags |= _voucher_activity_trace_flag_wide_first; + buffer += tlen; + length -= tlen; + (++vat)->vat_flags = _voucher_activity_trace_flag_tracepoint | + _voucher_activity_trace_flag_wide_second; + vat->vat_type = 0; vat->vat_namespace = 0; + tbuf = (void*)vat + offsetof(typeof(*vat), vat_code); + tlen = sizeof(*vat) - offsetof(typeof(*vat), vat_code); + if (length < tlen) { + memcpy(tbuf, buffer, length); + } else { + memcpy(tbuf, buffer, tlen); + } + } + const uint16_t offsetend = offset - (uint16_t)strings_size; + char *b = (char*)vab + _voucher_activity_buffer_size; + int i = 0; + while (offset > offsetend && strings[i]) { + size_t maxsize = MIN(string_lengths[i] + 1, offset - offsetend); + size_t len = strlcpy(b - offset, strings[i++], maxsize); + offset -= MIN(len + 1, maxsize); + } + _voucher_activity_trace_fault(trace_id); + if (_voucher_activity_buffer_tracepoint_release(vab)) { + _voucher_activity_firehose_push(act, vab); + } return timestamp; } @@ -2524,18 +2277,28 @@ voucher_activity_trace_args(voucher_activity_trace_id_t trace_id, uintptr_t arg4) { if (!_voucher_activity_trace_id_enabled(trace_id)) return 0; + _voucher_activity_t act; + _voucher_activity_buffer_header_t vab; _voucher_activity_tracepoint_t vat; - vat = _voucher_activity_tracepoint_get(1); - if (!vat) vat = _voucher_activity_tracepoint_get_slow(1); + act = _voucher_activity_get(); + vab = _voucher_activity_buffer_get_from_activity(act); + vat = _voucher_activity_buffer_tracepoint_acquire(vab, 1, 0, NULL); + if (!vat) { + vat = _voucher_activity_buffer_tracepoint_acquire_slow(&act, &vab, 1, + 0, NULL); + } if (!vat) return 0; uint64_t timestamp = _voucher_activity_tracepoint_init_with_id(vat, - trace_id, location); + trace_id, location, true); vat->vat_flags |= _voucher_activity_trace_flag_tracepoint_args; vat->vat_data[0] = arg1; vat->vat_data[1] = arg2; vat->vat_data[2] = arg3; vat->vat_data[3] = arg4; _voucher_activity_trace_fault(trace_id); + if (_voucher_activity_buffer_tracepoint_release(vab)) { + _voucher_activity_firehose_push(act, vab); + } return timestamp; } @@ -2576,9 +2339,9 @@ _voucher_debug(voucher_t v, char* buf, size_t bufsiz) if (v->v_activity) { _voucher_activity_t va = v->v_activity; _voucher_atm_t vatm = va->va_atm; - bufprintf("activity[%p] = { ID 0x%llx, use %d, atm[%p] = { " + bufprintf("activity[%p] = { ID 0x%llx, ref %d, atm[%p] = { " "AID 0x%llx, ref %d, kvoucher 0x%x } }, ", va, va->va_id, - va->va_use_count + 1, va->va_atm, vatm->vatm_id, + va->va_refcnt + 1, va->va_atm, vatm->vatm_id, vatm->vatm_refcnt + 1, vatm->vatm_kvoucher); } bufprintf("}"); @@ -2617,6 +2380,18 @@ voucher_copy_without_importance(void) return NULL; } +voucher_t +voucher_retain(voucher_t voucher) +{ + return voucher; +} + +void +voucher_release(voucher_t voucher) +{ + (void)voucher; +} + void voucher_replace_default_voucher(void) { @@ -2656,6 +2431,13 @@ _voucher_create_with_priority_and_mach_voucher(voucher_t voucher, return NULL; } +voucher_t +_voucher_create_accounting_voucher(voucher_t voucher) +{ + (void)voucher; + return NULL; +} + voucher_t voucher_create_with_mach_msg(mach_msg_header_t *msg) { @@ -2701,6 +2483,13 @@ voucher_activity_get_metadata_buffer(size_t *length) return NULL; } +void +voucher_activity_buffer_hook_install_4libtrace( + _voucher_activity_buffer_hook_t hook) +{ + (void)hook; +} + void _voucher_activity_heap_pressure_normal(void) { @@ -2754,6 +2543,16 @@ voucher_activity_trace(voucher_activity_trace_id_t trace_id, uint64_t location, return 0; } +uint64_t +voucher_activity_trace_strings(voucher_activity_trace_id_t trace_id, + uint64_t location, void *buffer, size_t length, const char *strings[], + size_t string_lengths[], size_t strings_size) +{ + (void)trace_id; (void)location; (void)buffer; (void)length; (void)strings; + (void)string_lengths; (void)strings_size; + return 0; +} + uint64_t voucher_activity_trace_args(voucher_activity_trace_id_t trace_id, uint64_t location, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, diff --git a/src/voucher_internal.h b/src/voucher_internal.h index 6fa1538..cc5ae22 100644 --- a/src/voucher_internal.h +++ b/src/voucher_internal.h @@ -93,7 +93,7 @@ voucher_get_mach_voucher(voucher_t voucher); #pragma mark voucher_t #if TARGET_IPHONE_SIMULATOR && \ - IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101000 + IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101100 #undef VOUCHER_USE_MACH_VOUCHER #define VOUCHER_USE_MACH_VOUCHER 0 #endif @@ -121,6 +121,7 @@ size_t _voucher_debug(voucher_t v, char* buf, size_t bufsiz); void _voucher_thread_cleanup(void *voucher); mach_voucher_t _voucher_get_mach_voucher(voucher_t voucher); voucher_t _voucher_create_without_importance(voucher_t voucher); +voucher_t _voucher_create_accounting_voucher(voucher_t voucher); mach_voucher_t _voucher_create_mach_voucher_with_priority(voucher_t voucher, pthread_priority_t priority); voucher_t _voucher_create_with_priority_and_mach_voucher(voucher_t voucher, @@ -134,6 +135,9 @@ _OS_OBJECT_DECL_SUBCLASS_INTERFACE(voucher_recipe, object) #endif #endif +voucher_t voucher_retain(voucher_t voucher); +void voucher_release(voucher_t voucher); + #define _TAILQ_IS_ENQUEUED(elm, field) \ ((elm)->field.tqe_prev != NULL) #define _TAILQ_MARK_NOT_ENQUEUED(elm, field) \ @@ -156,7 +160,8 @@ typedef struct voucher_s { TAILQ_ENTRY(voucher_s) v_list; mach_voucher_t v_kvoucher, v_ipc_kvoucher; // if equal, only one reference voucher_t v_kvbase; // if non-NULL, v_kvoucher is a borrowed reference - _voucher_activity_t v_activity; + struct _voucher_atm_s *v_atm; + struct _voucher_activity_s *v_activity; #if VOUCHER_ENABLE_RECIPE_OBJECTS size_t v_recipe_extra_offset; mach_voucher_attr_recipe_size_t v_recipe_extra_size; @@ -230,13 +235,15 @@ typedef uint32_t _voucher_priority_t; #define _dispatch_voucher_debug_machport(name) ((void)(name)) #endif -#if !(USE_OBJC && __OBJC2__) +#if !(USE_OBJC && __OBJC2__) && !defined(__cplusplus) DISPATCH_ALWAYS_INLINE static inline voucher_t _voucher_retain(voucher_t voucher) { #if !DISPATCH_VOUCHER_OBJC_DEBUG + // not using _os_object_refcnt* because we don't need barriers: + // vouchers are immutable and are in a hash table with a lock int xref_cnt = dispatch_atomic_inc2o(voucher, os_obj_xref_cnt, relaxed); _dispatch_voucher_debug("retain -> %d", voucher, xref_cnt + 1); if (slowpath(xref_cnt <= 0)) { @@ -256,6 +263,8 @@ static inline void _voucher_release(voucher_t voucher) { #if !DISPATCH_VOUCHER_OBJC_DEBUG + // not using _os_object_refcnt* because we don't need barriers: + // vouchers are immutable and are in a hash table with a lock int xref_cnt = dispatch_atomic_dec2o(voucher, os_obj_xref_cnt, relaxed); _dispatch_voucher_debug("release -> %d", voucher, xref_cnt + 1); if (fastpath(xref_cnt >= 0)) { @@ -515,76 +524,81 @@ typedef uint32_t _voucher_atm_subid_t; static const size_t _voucher_activity_hash_bits = 6; static const size_t _voucher_activity_hash_size = 1 << _voucher_activity_hash_bits; -#define VACTID_HASH(x) ((((uint32_t)((x) >> 32) + (uint32_t)(x)) * \ - 2654435761u) >> (32-_voucher_activity_hash_bits)) +#define VACTID_HASH(x) \ + (((uint32_t)(x) * 2654435761u) >> (32-_voucher_activity_hash_bits)) #define VATMID_HASH(x) \ (((uint32_t)(x) * 2654435761u) >> (32-_voucher_activity_hash_bits)) -#define VATMID2ACTID(x) ((uint64_t)(x) << 32) -#define VACTID_BASEID(x) ((uint64_t)(x) & (((uint64_t)UINT32_MAX) << 32)) -#define VACTID_SUBID(x) ((uint32_t)(x)) -#define VATM_ACTID(vatm, subid) (VATMID2ACTID((vatm)->vatm_id) + (subid)) -#define VATM_SUBID_BITS2MAX(bits) ((1u << (bits)) - 1) -#define VATM_SUBID_MAXBITS (32) -#define VATM_SUBID_MAX (ATM_SUBAID32_MAX) -#define MAILBOX_OFFSET_UNSET UINT64_MAX - -static const size_t _voucher_activity_buffers_per_heap = 512; -typedef unsigned long _voucher_activity_bitmap_base_t; -static const size_t _voucher_activity_bits_per_bitmap_base_t = - 8 * sizeof(_voucher_activity_bitmap_base_t); -static const size_t _voucher_activity_bitmaps_per_heap = - _voucher_activity_buffers_per_heap / - _voucher_activity_bits_per_bitmap_base_t; -typedef _voucher_activity_bitmap_base_t - _voucher_activity_bitmap_t[_voucher_activity_bitmaps_per_heap]; +#define VATMID2ACTID(x, flags) \ + (((voucher_activity_id_t)(x) & 0xffffffffffffff) | \ + (((voucher_activity_id_t)(flags) & 0xfe) << 55)) typedef struct _voucher_activity_metadata_s { - _voucher_activity_buffer_t vam_kernel_metadata; _voucher_activity_buffer_t vam_client_metadata; - struct _voucher_activity_self_metadata_s vam_self_metadata; -#if __LP64__ - uintptr_t vam_pad0[7]; -#else - uintptr_t vam_pad0[15]; -#endif - // cacheline - _voucher_activity_bitmap_t volatile vam_atm_mbox_bitmap; + struct _voucher_activity_metadata_opaque_s *vasm_baseaddr; _voucher_activity_bitmap_t volatile vam_buffer_bitmap; _voucher_activity_bitmap_t volatile vam_pressure_locked_bitmap; - // cacheline - _voucher_atm_subid_t vam_base_atm_subid; - _voucher_atm_subid_t vam_base_atm_subid_max; - _voucher_atm_subid_t vam_nested_atm_subid; - _voucher_atm_t vam_default_activity_atm; - _voucher_atm_t volatile vam_base_atm; - voucher_activity_id_t volatile vam_nested_atm_id; -#if __LP64__ - uintptr_t vam_pad2[3]; -#else - uintptr_t vam_pad2[1]; -#endif - _voucher_activity_lock_s vam_base_atm_lock; - _voucher_activity_lock_s vam_nested_atm_lock; _voucher_activity_lock_s vam_atms_lock; _voucher_activity_lock_s vam_activities_lock; - // cacheline TAILQ_HEAD(, _voucher_atm_s) vam_atms[_voucher_activity_hash_size]; TAILQ_HEAD(, _voucher_activity_s) vam_activities[_voucher_activity_hash_size]; } *_voucher_activity_metadata_t; +#pragma mark - +#pragma mark _voucher_atm_t + +typedef struct _voucher_atm_s { + int32_t volatile vatm_refcnt; + mach_voucher_t vatm_kvoucher; + atm_aid_t vatm_id; + atm_guard_t vatm_generation; + TAILQ_ENTRY(_voucher_atm_s) vatm_list; +#if __LP64__ + uintptr_t vatm_pad[3]; + // cacheline +#endif +} *_voucher_atm_t; + +extern _voucher_atm_t _voucher_task_atm; + #pragma mark - #pragma mark _voucher_activity_t -_voucher_activity_tracepoint_t _voucher_activity_tracepoint_get_slow( - unsigned int slots); +typedef struct _voucher_activity_s { + voucher_activity_id_t va_id; + voucher_activity_trace_id_t va_trace_id; + uint64_t va_location; + int32_t volatile va_refcnt; + uint32_t volatile va_buffer_count; + uint32_t va_buffer_limit; + _voucher_activity_buffer_header_t volatile va_current_buffer; + _voucher_atm_t va_atm; +#if __LP64__ + uint64_t va_unused; +#endif + // cacheline + _voucher_activity_lock_s va_buffers_lock; + TAILQ_HEAD(_voucher_activity_buffer_list_s, + _voucher_activity_buffer_header_s) va_buffers; + TAILQ_ENTRY(_voucher_activity_s) va_list; + TAILQ_ENTRY(_voucher_activity_s) va_atm_list; + TAILQ_ENTRY(_voucher_activity_s) va_atm_used_list; + pthread_mutex_t va_mutex; + pthread_cond_t va_cond; +} *_voucher_activity_t; + +_voucher_activity_tracepoint_t _voucher_activity_buffer_tracepoint_acquire_slow( + _voucher_activity_t *vap, _voucher_activity_buffer_header_t *vabp, + unsigned int slots, size_t strsize, uint16_t *stroffsetp); +void _voucher_activity_firehose_push(_voucher_activity_t act, + _voucher_activity_buffer_header_t buffer); extern _voucher_activity_t _voucher_activity_default; extern voucher_activity_mode_t _voucher_activity_mode; #if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG #define _dispatch_voucher_activity_debug(msg, act, ...) \ - _dispatch_debug("activity[%p] <0x%x>: atm[%p] <%lld>: " msg, (act), \ - (act) ? VACTID_SUBID((act)->va_id) : 0, (act) ? (act)->va_atm : NULL, \ + _dispatch_debug("activity[%p] <0x%llx>: atm[%p] <%lld>: " msg, (act), \ + (act) ? (act)->va_id : 0, (act) ? (act)->va_atm : NULL, \ (act) && (act)->va_atm ? (act)->va_atm->vatm_id : 0, ##__VA_ARGS__) #define _dispatch_voucher_atm_debug(msg, atm, ...) \ _dispatch_debug("atm[%p] <%lld> kvoucher[0x%08x]: " msg, (atm), \ @@ -597,13 +611,14 @@ extern voucher_activity_mode_t _voucher_activity_mode; DISPATCH_ALWAYS_INLINE static inline uint64_t -_voucher_activity_timestamp(void) +_voucher_activity_timestamp(bool approx) { #if TARGET_IPHONE_SIMULATOR && \ IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101000 + (void)approx; return mach_absolute_time(); #else - return mach_approximate_time(); + return approx ? mach_approximate_time() : mach_absolute_time(); #endif } @@ -616,45 +631,116 @@ _voucher_activity_thread_id(void) return thread_id; } +#define _voucher_activity_buffer_pos2length(pos) \ + ({ _voucher_activity_buffer_position_u _pos = (pos); \ + _pos.vabp_pos.vabp_next_tracepoint_idx * \ + sizeof(struct _voucher_activity_tracepoint_s) + \ + _pos.vabp_pos.vabp_string_offset; }) + DISPATCH_ALWAYS_INLINE static inline _voucher_activity_tracepoint_t -_voucher_activity_buffer_tracepoint_get(_voucher_activity_buffer_header_t vab, - unsigned int slots) +_voucher_activity_buffer_tracepoint_acquire( + _voucher_activity_buffer_header_t vab, unsigned int slots, + size_t strsize, uint16_t *stroffsetp) { - uint32_t idx = dispatch_atomic_add2o(vab, vabh_next_tracepoint_idx, - slots, relaxed); - if (idx <= _voucher_activity_tracepoints_per_buffer) { - return (_voucher_activity_tracepoint_t)vab + (idx - slots); - } - return NULL; + if (!vab) return NULL; + _voucher_activity_buffer_position_u pos_orig, pos; + pos_orig.vabp_atomic_pos = vab->vabh_pos.vabp_atomic_pos; + do { + pos.vabp_atomic_pos = pos_orig.vabp_atomic_pos; + pos.vabp_pos.vabp_next_tracepoint_idx += slots; + pos.vabp_pos.vabp_string_offset += strsize; + size_t len = _voucher_activity_buffer_pos2length(pos); + if (len > _voucher_activity_buffer_size || pos.vabp_pos.vabp_flags) { + return NULL; + } + if (len == _voucher_activity_buffer_size) { + pos.vabp_pos.vabp_flags |= _voucher_activity_buffer_full; + } + pos.vabp_pos.vabp_refcnt++; + } while (!dispatch_atomic_cmpxchgvw2o(vab, vabh_pos.vabp_atomic_pos, + pos_orig.vabp_atomic_pos, pos.vabp_atomic_pos, + &pos_orig.vabp_atomic_pos, relaxed)); + if (stroffsetp) *stroffsetp = pos.vabp_pos.vabp_string_offset; + return (_voucher_activity_tracepoint_t)vab + + pos_orig.vabp_pos.vabp_next_tracepoint_idx; } DISPATCH_ALWAYS_INLINE -static inline _voucher_activity_tracepoint_t -_voucher_activity_tracepoint_get_from_activity(_voucher_activity_t va, - unsigned int slots) +static inline bool +_voucher_activity_buffer_tracepoint_release( + _voucher_activity_buffer_header_t vab) { - _voucher_activity_buffer_header_t vab = va ? va->va_current_buffer : NULL; - return vab ? _voucher_activity_buffer_tracepoint_get(vab, slots) : NULL; + _voucher_activity_buffer_position_u pos_orig, pos; + pos_orig.vabp_atomic_pos = vab->vabh_pos.vabp_atomic_pos; + do { + pos.vabp_atomic_pos = pos_orig.vabp_atomic_pos; + pos.vabp_pos.vabp_refcnt--; + if (!pos.vabp_pos.vabp_refcnt && + (pos.vabp_pos.vabp_flags & _voucher_activity_buffer_full)) { + pos.vabp_pos.vabp_flags |= _voucher_activity_buffer_pushing; + } + } while (!dispatch_atomic_cmpxchgvw2o(vab, vabh_pos.vabp_atomic_pos, + pos_orig.vabp_atomic_pos, pos.vabp_atomic_pos, + &pos_orig.vabp_atomic_pos, relaxed)); + return (pos.vabp_pos.vabp_flags & _voucher_activity_buffer_pushing); } DISPATCH_ALWAYS_INLINE -static inline _voucher_activity_tracepoint_t -_voucher_activity_tracepoint_get(unsigned int slots) +static inline bool +_voucher_activity_buffer_mark_full(_voucher_activity_buffer_header_t vab) +{ + _voucher_activity_buffer_position_u pos_orig, pos; + pos_orig.vabp_atomic_pos = vab->vabh_pos.vabp_atomic_pos; + do { + pos.vabp_atomic_pos = pos_orig.vabp_atomic_pos; + if (pos.vabp_pos.vabp_flags & _voucher_activity_buffer_full) { + return false; + } + pos.vabp_pos.vabp_flags |= _voucher_activity_buffer_full; + if (!pos.vabp_pos.vabp_refcnt) { + pos.vabp_pos.vabp_flags |= _voucher_activity_buffer_pushing; + } + } while (!dispatch_atomic_cmpxchgvw2o(vab, vabh_pos.vabp_atomic_pos, + pos_orig.vabp_atomic_pos, pos.vabp_atomic_pos, + &pos_orig.vabp_atomic_pos, relaxed)); + return (pos.vabp_pos.vabp_flags & _voucher_activity_buffer_pushing); +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_voucher_activity_buffer_is_full(_voucher_activity_buffer_header_t vab) +{ + _voucher_activity_buffer_position_u pos; + pos.vabp_atomic_pos = vab->vabh_pos.vabp_atomic_pos; + return (pos.vabp_pos.vabp_flags); +} + +DISPATCH_ALWAYS_INLINE +static inline _voucher_activity_buffer_header_t +_voucher_activity_buffer_get_from_activity(_voucher_activity_t va) +{ + return va ? va->va_current_buffer : NULL; +} + +DISPATCH_ALWAYS_INLINE +static inline _voucher_activity_t +_voucher_activity_get(void) { _voucher_activity_t va; voucher_t v = _voucher_get(); va = v && v->v_activity ? v->v_activity : _voucher_activity_default; - return _voucher_activity_tracepoint_get_from_activity(va, slots); + return va; } DISPATCH_ALWAYS_INLINE static inline uint64_t _voucher_activity_tracepoint_init(_voucher_activity_tracepoint_t vat, - uint8_t type, uint8_t code_namespace, uint32_t code, uint64_t location) + uint8_t type, uint8_t code_namespace, uint32_t code, uint64_t location, + bool approx) { if (!location) location = (uint64_t)__builtin_return_address(0); - uint64_t timestamp = _voucher_activity_timestamp(); + uint64_t timestamp = _voucher_activity_timestamp(approx); vat->vat_flags = _voucher_activity_trace_flag_tracepoint, vat->vat_type = type, vat->vat_namespace = code_namespace, @@ -668,13 +754,14 @@ _voucher_activity_tracepoint_init(_voucher_activity_tracepoint_t vat, DISPATCH_ALWAYS_INLINE static inline uint64_t _voucher_activity_tracepoint_init_with_id(_voucher_activity_tracepoint_t vat, - voucher_activity_trace_id_t trace_id, uint64_t location) + voucher_activity_trace_id_t trace_id, uint64_t location, bool approx) { uint8_t type = (uint8_t)(trace_id >> _voucher_activity_trace_id_type_shift); uint8_t cns = (uint8_t)(trace_id >> _voucher_activity_trace_id_code_namespace_shift); uint32_t code = (uint32_t)trace_id; - return _voucher_activity_tracepoint_init(vat, type, cns, code, location); + return _voucher_activity_tracepoint_init(vat, type, cns, code, location, + approx); } DISPATCH_ALWAYS_INLINE @@ -720,69 +807,82 @@ _voucher_activity_disabled(void) } DISPATCH_ALWAYS_INLINE -static inline _voucher_activity_tracepoint_t +static inline void _voucher_activity_trace_args_inline(uint8_t type, uint8_t code_namespace, uint32_t code, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4) { - if (!_voucher_activity_trace_type_enabled(type)) return NULL; + if (!_voucher_activity_trace_type_enabled(type)) return; + _voucher_activity_t act; + _voucher_activity_buffer_header_t vab; _voucher_activity_tracepoint_t vat; - vat = _voucher_activity_tracepoint_get(1); - if (!vat) return NULL; - _voucher_activity_tracepoint_init(vat, type, code_namespace, code, 0); + act = _voucher_activity_get(); + vab = _voucher_activity_buffer_get_from_activity(act); + vat = _voucher_activity_buffer_tracepoint_acquire(vab, 1, 0, NULL); + if (!vat) return; + _voucher_activity_tracepoint_init(vat, type, code_namespace, code, 0, true); vat->vat_flags |= _voucher_activity_trace_flag_tracepoint_args; vat->vat_data[0] = arg1; vat->vat_data[1] = arg2; vat->vat_data[2] = arg3; vat->vat_data[3] = arg4; - return vat; -} - -DISPATCH_ALWAYS_INLINE -static inline _voucher_activity_tracepoint_t -_voucher_activity_trace_with_id_inline(voucher_activity_trace_id_t trace_id) -{ - _voucher_activity_tracepoint_t vat = _voucher_activity_tracepoint_get(1); - if (!vat) return NULL; - _voucher_activity_tracepoint_init_with_id(vat, trace_id, 0); - return vat; + if (_voucher_activity_buffer_tracepoint_release(vab)) { + _voucher_activity_firehose_push(act, vab); + } } DISPATCH_ALWAYS_INLINE -static inline _voucher_activity_tracepoint_t -_voucher_activity_trace_with_id(voucher_activity_trace_id_t trace_id) +static inline void +_voucher_activity_trace_activity_event(voucher_activity_trace_id_t trace_id, + voucher_activity_id_t va_id, _voucher_activity_tracepoint_flag_t flags) { - _voucher_activity_tracepoint_t vat = _voucher_activity_tracepoint_get(1); - if (!vat) vat = _voucher_activity_tracepoint_get_slow(1); - if (!vat) return NULL; - _voucher_activity_tracepoint_init_with_id(vat, trace_id, 0); - return vat; + _voucher_activity_t act; + _voucher_activity_buffer_header_t vab; + _voucher_activity_tracepoint_t vat; + act = _voucher_activity_get(); + vab = _voucher_activity_buffer_get_from_activity(act); + vat = _voucher_activity_buffer_tracepoint_acquire(vab, 1, 0, NULL); + if (!vat) return; + _voucher_activity_tracepoint_init_with_id(vat, trace_id, 0, false); + vat->vat_flags |= _voucher_activity_trace_flag_activity | flags; + vat->vat_data[0] = va_id; + if (_voucher_activity_buffer_tracepoint_release(vab)) { + _voucher_activity_firehose_push(act, vab); + } } +#define _voucher_activity_trace_activity_event(trace_id, va_id, type) \ + _voucher_activity_trace_activity_event(trace_id, va_id, \ + _voucher_activity_trace_flag_ ## type) DISPATCH_ALWAYS_INLINE static inline void _voucher_activity_trace_msg(voucher_t v, mach_msg_header_t *msg, uint32_t code) { if (!v || !v->v_activity) return; // Don't use default activity for IPC - const uint8_t type = voucher_activity_tracepoint_type_release; + const uint8_t type = voucher_activity_tracepoint_type_debug; const uint8_t code_namespace = _voucher_activity_tracepoint_namespace_ipc; if (!_voucher_activity_trace_type_enabled(type)) return; + _voucher_activity_buffer_header_t vab; _voucher_activity_tracepoint_t vat; - vat = _voucher_activity_tracepoint_get_from_activity(v->v_activity, 1); + vab = _voucher_activity_buffer_get_from_activity(v->v_activity); + vat = _voucher_activity_buffer_tracepoint_acquire(vab, 1, 0, NULL); if (!vat) return; // TODO: slowpath ? - _voucher_activity_tracepoint_init(vat, type, code_namespace, code, 0); + _voucher_activity_tracepoint_init(vat, type, code_namespace, code, 0, true); vat->vat_flags |= _voucher_activity_trace_flag_libdispatch; #if __has_extension(c_static_assert) _Static_assert(sizeof(mach_msg_header_t) <= sizeof(vat->vat_data), "mach_msg_header_t too large"); #endif memcpy(vat->vat_data, msg, sizeof(mach_msg_header_t)); + if (_voucher_activity_buffer_tracepoint_release(vab)) { + _voucher_activity_firehose_push(v->v_activity, vab); + } } #define _voucher_activity_trace_msg(v, msg, type) \ _voucher_activity_trace_msg(v, msg, \ _voucher_activity_tracepoint_namespace_ipc_ ## type) -#endif // !(USE_OBJC && __OBJC2__) +#endif // !(USE_OBJC && __OBJC2__) && !defined(__cplusplus) #else // VOUCHER_USE_MACH_VOUCHER diff --git a/xcodeconfig/libdispatch-introspection.xcconfig b/xcodeconfig/libdispatch-introspection.xcconfig index 1644ea9..a2f98f9 100644 --- a/xcodeconfig/libdispatch-introspection.xcconfig +++ b/xcodeconfig/libdispatch-introspection.xcconfig @@ -19,7 +19,7 @@ // BUILD_VARIANTS = normal -INSTALL_PATH_ACTUAL = /usr/lib/system/introspection +INSTALL_PATH = /usr/lib/system/introspection GCC_PREPROCESSOR_DEFINITIONS = $(GCC_PREPROCESSOR_DEFINITIONS) DISPATCH_INTROSPECTION=1 CONFIGURATION_BUILD_DIR = $(BUILD_DIR)/introspection diff --git a/xcodeconfig/libdispatch-resolved.xcconfig b/xcodeconfig/libdispatch-resolved.xcconfig index 70e405f..a42add8 100644 --- a/xcodeconfig/libdispatch-resolved.xcconfig +++ b/xcodeconfig/libdispatch-resolved.xcconfig @@ -18,7 +18,7 @@ // @APPLE_APACHE_LICENSE_HEADER_END@ // -SUPPORTED_PLATFORMS = iphoneos +SUPPORTED_PLATFORMS = iphoneos appletvos watchos PRODUCT_NAME = libdispatch_$(DISPATCH_RESOLVED_VARIANT) OTHER_LDFLAGS = SKIP_INSTALL = YES diff --git a/xcodeconfig/libdispatch.xcconfig b/xcodeconfig/libdispatch.xcconfig index 1d22933..7fc525d 100644 --- a/xcodeconfig/libdispatch.xcconfig +++ b/xcodeconfig/libdispatch.xcconfig @@ -19,20 +19,17 @@ // #include "/Makefiles/CoreOS/Xcode/BSD.xcconfig" -#include "/AppleInternal/XcodeConfig/SimulatorSupport.xcconfig" +#include "/AppleInternal/XcodeConfig/PlatformSupport.xcconfig" -// Set INSTALL_PATH[sdk=macosx*] when SimulatorSupport.xcconfig is unavailable -INSTALL_PATH[sdk=macosx*] = $(INSTALL_PATH_ACTUAL) - -SUPPORTED_PLATFORMS = macosx iphoneos iphonesimulator iphoneosnano iphonesimulatornano -ARCHS[sdk=iphonesimulator*] = $(NATIVE_ARCH_32_BIT) // Override BSD.xcconfig ARCHS +SDKROOT = macosx.internal +SUPPORTED_PLATFORMS = macosx iphoneos iphonesimulator appletvos appletvsimulator watchos watchsimulator PRODUCT_NAME = libdispatch EXECUTABLE_PREFIX = -INSTALL_PATH_ACTUAL = /usr/lib/system -PUBLIC_HEADERS_FOLDER_PATH = $(INSTALL_PATH_PREFIX)/usr/include/dispatch -PRIVATE_HEADERS_FOLDER_PATH = $(INSTALL_PATH_PREFIX)/usr/local/include/dispatch -OS_PUBLIC_HEADERS_FOLDER_PATH = $(INSTALL_PATH_PREFIX)/usr/include/os -OS_PRIVATE_HEADERS_FOLDER_PATH = $(INSTALL_PATH_PREFIX)/usr/local/include/os +INSTALL_PATH = /usr/lib/system +PUBLIC_HEADERS_FOLDER_PATH = /usr/include/dispatch +PRIVATE_HEADERS_FOLDER_PATH = /usr/local/include/dispatch +OS_PUBLIC_HEADERS_FOLDER_PATH = /usr/include/os +OS_PRIVATE_HEADERS_FOLDER_PATH = /usr/local/include/os HEADER_SEARCH_PATHS = $(PROJECT_DIR) $(PROJECT_DIR)/private $(PROJECT_DIR)/os LIBRARY_SEARCH_PATHS = $(SDKROOT)/usr/lib/system INSTALLHDRS_SCRIPT_PHASE = YES @@ -42,6 +39,8 @@ BUILD_VARIANTS = normal debug profile ONLY_ACTIVE_ARCH = NO CLANG_LINK_OBJC_RUNTIME = NO GCC_C_LANGUAGE_STANDARD = gnu11 +CLANG_CXX_LANGUAGE_STANDARD = gnu++11 +GCC_ENABLE_CPP_EXCEPTIONS = NO GCC_STRICT_ALIASING = YES GCC_SYMBOLS_PRIVATE_EXTERN = YES GCC_ENABLE_OBJC_GC[sdk=macosx*] = supported @@ -71,13 +70,13 @@ OTHER_CFLAGS_profile = $(OTHER_CFLAGS_normal) -DDISPATCH_PROFILE=1 OTHER_CFLAGS_debug = -fstack-protector -fno-inline -O0 -DDISPATCH_DEBUG=1 GENERATE_PROFILING_CODE = NO DYLIB_CURRENT_VERSION = $(CURRENT_PROJECT_VERSION) -UMBRELLA_LDFLAGS = -umbrella System -nodefaultlibs -ldyld -lcompiler_rt -lsystem_kernel -lsystem_platform -lsystem_pthread -lsystem_malloc -lsystem_c -lsystem_blocks -lunwind -UMBRELLA_LDFLAGS[sdk=iphonesimulator*] = -umbrella System -nodefaultlibs -ldyld_sim -lcompiler_rt_sim -lsystem_sim_c -lsystem_sim_blocks -lunwind_sim -Wl,-upward-lSystem +SIM_SUFFIX[sdk=*simulator*] = _sim +DYLIB_LDFLAGS = -umbrella System -nodefaultlibs -ldyld -lcompiler_rt -lsystem$(SIM_SUFFIX)_kernel -lsystem$(SIM_SUFFIX)_platform -lsystem$(SIM_SUFFIX)_pthread -lsystem_malloc -lsystem_c -lsystem_blocks -lunwind OBJC_LDFLAGS = -Wl,-upward-lobjc -Wl,-order_file,$(SRCROOT)/xcodeconfig/libdispatch.order -Wl,-alias_list,$(SRCROOT)/xcodeconfig/libdispatch_objc.aliases -Wl,-unexported_symbols_list,$(SRCROOT)/xcodeconfig/libdispatch.unexport OBJC_LDFLAGS[sdk=macosx*] = $(OBJC_LDFLAGS) -Wl,-upward-lauto OBJC_LDFLAGS[arch=i386][sdk=macosx*] = OBJC_EXCLUDED_SOURCE_FILE_NAMES_i386_macosx = object.m data.m ALIASES_LDFLAGS = -Wl,-alias_list,$(SRCROOT)/xcodeconfig/libdispatch.aliases PLATFORM_LDFLAGS[sdk=macosx*] = -Wl,-alias_list,$(SRCROOT)/xcodeconfig/libdispatch_macosx.aliases -OTHER_LDFLAGS = $(OTHER_LDFLAGS) $(UMBRELLA_LDFLAGS) $(CR_LDFLAGS) $(OBJC_LDFLAGS) $(ALIASES_LDFLAGS) $(PLATFORM_LDFLAGS) +OTHER_LDFLAGS = $(OTHER_LDFLAGS) $(DYLIB_LDFLAGS) $(CR_LDFLAGS) $(OBJC_LDFLAGS) $(ALIASES_LDFLAGS) $(PLATFORM_LDFLAGS) OTHER_MIGFLAGS = -novouchers diff --git a/xcodescripts/install-dtrace.sh b/xcodescripts/install-dtrace.sh index c0eb364..9397c04 100644 --- a/xcodescripts/install-dtrace.sh +++ b/xcodescripts/install-dtrace.sh @@ -19,7 +19,8 @@ # @APPLE_APACHE_LICENSE_HEADER_END@ # -if [ "${PLATFORM_NAME}" = iphoneos ]; then exit 0; fi +# This check equates to "is macosx or a simulator platform" +if [ "${PLATFORM_NAME}" == "${DEVICE_PLATFORM_NAME}" ]; then exit 0; fi if [ "${DEPLOYMENT_LOCATION}" != YES ]; then DSTROOT="${CONFIGURATION_BUILD_DIR}" -- 2.45.2