From 98cf8cd208104c07d83445c24626e57ca11b5775 Mon Sep 17 00:00:00 2001 From: Apple Date: Fri, 24 Oct 2014 18:06:19 +0000 Subject: [PATCH] libdispatch-442.1.4.tar.gz --- config/config.h | 6 + dispatch/base.h | 26 +- dispatch/block.h | 422 +++ dispatch/dispatch.h | 5 +- dispatch/introspection.h | 21 + dispatch/io.h | 26 +- dispatch/object.h | 238 +- dispatch/queue.h | 228 +- dispatch/source.h | 10 +- libdispatch.xcodeproj/project.pbxproj | 78 +- os/object.h | 69 +- private/data_private.h | 2 +- private/introspection_private.h | 20 - private/io_private.h | 40 +- private/layout_private.h | 86 + private/mach_private.h | 3 +- private/private.h | 3 +- private/queue_private.h | 81 +- private/source_private.h | 28 +- private/voucher_activity_private.h | 544 ++++ private/voucher_private.h | 375 +++ src/allocator.c | 44 +- src/allocator_internal.h | 23 +- src/apply.c | 36 +- src/data.m | 6 +- src/init.c | 325 +- src/inline_internal.h | 996 ++++++ src/internal.h | 206 +- src/introspection.c | 28 +- src/io.c | 17 +- src/object.c | 44 +- src/object.m | 173 +- src/object_internal.h | 4 +- src/once.c | 46 +- src/queue.c | 2213 ++++++++++--- src/queue_internal.h | 511 ++- src/semaphore.c | 35 +- src/semaphore_internal.h | 9 + src/shims.h | 93 + src/shims/atomic.h | 27 +- src/shims/atomic_sfb.h | 13 +- src/shims/hw_config.h | 144 +- src/shims/perfmon.h | 28 +- src/shims/time.h | 16 - src/shims/tsd.h | 35 +- src/shims/yield.h | 156 + src/source.c | 812 +++-- src/source_internal.h | 37 +- src/time.c | 4 +- src/trace.h | 101 +- src/voucher.c | 2774 +++++++++++++++++ src/voucher_internal.h | 929 ++++++ .../libdispatch-introspection.xcconfig | 4 +- .../libdispatch-resolver_iphoneos.order | 20 + xcodeconfig/libdispatch.aliases | 19 +- xcodeconfig/libdispatch.order | 7 +- xcodeconfig/libdispatch.xcconfig | 34 +- xcodeconfig/libdispatch_iphoneos.order | 20 + xcodeconfig/libdispatch_macosx.aliases | 2 - xcodeconfig/libdispatch_objc.aliases | 34 + xcodescripts/install-headers.sh | 2 + xcodescripts/mig-headers.sh | 2 +- 62 files changed, 10541 insertions(+), 1799 deletions(-) create mode 100644 dispatch/block.h create mode 100644 private/layout_private.h create mode 100644 private/voucher_activity_private.h create mode 100644 private/voucher_private.h create mode 100644 src/inline_internal.h create mode 100644 src/shims/yield.h create mode 100644 src/voucher.c create mode 100644 src/voucher_internal.h create mode 100644 xcodeconfig/libdispatch-resolver_iphoneos.order create mode 100644 xcodeconfig/libdispatch_iphoneos.order create mode 100644 xcodeconfig/libdispatch_objc.aliases diff --git a/config/config.h b/config/config.h index d2ad0ff..8944281 100644 --- a/config/config.h +++ b/config/config.h @@ -108,6 +108,12 @@ /* Define to 1 if you have the `pthread_workqueue_setdispatch_np' function. */ #define HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP 1 +/* Define to 1 if you have the `_pthread_workqueue_init' function. */ +#define HAVE__PTHREAD_WORKQUEUE_INIT 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PTHREAD_QOS_H 1 + /* Define to 1 if you have the header file. */ #define HAVE_STDINT_H 1 diff --git a/dispatch/base.h b/dispatch/base.h index af17ccf..01d5ec5 100644 --- a/dispatch/base.h +++ b/dispatch/base.h @@ -47,6 +47,7 @@ #define DISPATCH_WARN_RESULT __attribute__((__warn_unused_result__)) #define DISPATCH_MALLOC __attribute__((__malloc__)) #define DISPATCH_ALWAYS_INLINE __attribute__((__always_inline__)) +#define DISPATCH_UNAVAILABLE __attribute__((__unavailable__)) #else /*! @parseOnly */ #define DISPATCH_NORETURN @@ -80,9 +81,10 @@ #define DISPATCH_MALLOC /*! @parseOnly */ #define DISPATCH_ALWAYS_INLINE +/*! @parseOnly */ +#define DISPATCH_UNAVAILABLE #endif - #if TARGET_OS_WIN32 && defined(__DISPATCH_BUILDING_DISPATCH__) && \ defined(__cplusplus) #define DISPATCH_EXPORT extern "C" extern __declspec(dllexport) @@ -110,17 +112,35 @@ #define DISPATCH_EXPECT(x, v) (x) #endif -#if defined(__has_feature) -#if __has_feature(objc_fixed_enum) +#ifndef DISPATCH_RETURNS_RETAINED_BLOCK +#if defined(__has_attribute) +#if __has_attribute(ns_returns_retained) +#define DISPATCH_RETURNS_RETAINED_BLOCK __attribute__((__ns_returns_retained__)) +#else +#define DISPATCH_RETURNS_RETAINED_BLOCK +#endif +#else +#define DISPATCH_RETURNS_RETAINED_BLOCK +#endif +#endif + +#if defined(__has_feature) && defined(__has_extension) +#if __has_feature(objc_fixed_enum) || __has_extension(cxx_strong_enums) #define DISPATCH_ENUM(name, type, ...) \ typedef enum : type { __VA_ARGS__ } name##_t #else #define DISPATCH_ENUM(name, type, ...) \ enum { __VA_ARGS__ }; typedef type name##_t #endif +#if __has_feature(enumerator_attributes) +#define DISPATCH_ENUM_AVAILABLE_STARTING __OSX_AVAILABLE_STARTING +#else +#define DISPATCH_ENUM_AVAILABLE_STARTING(...) +#endif #else #define DISPATCH_ENUM(name, type, ...) \ enum { __VA_ARGS__ }; typedef type name##_t +#define DISPATCH_ENUM_AVAILABLE_STARTING(...) #endif typedef void (*dispatch_function_t)(void *); diff --git a/dispatch/block.h b/dispatch/block.h new file mode 100644 index 0000000..e82f665 --- /dev/null +++ b/dispatch/block.h @@ -0,0 +1,422 @@ +/* + * Copyright (c) 2014 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +#ifndef __DISPATCH_BLOCK__ +#define __DISPATCH_BLOCK__ + +#ifndef __DISPATCH_INDIRECT__ +#error "Please #include instead of this file directly." +#include // for HeaderDoc +#endif + +#ifdef __BLOCKS__ + +/*! + * @group Dispatch block objects + */ + +__BEGIN_DECLS + +/*! + * @typedef dispatch_block_flags_t + * Flags to pass to the dispatch_block_create* functions. + * + * @const DISPATCH_BLOCK_BARRIER + * Flag indicating that a dispatch block object should act as a barrier block + * when submitted to a DISPATCH_QUEUE_CONCURRENT queue. + * See dispatch_barrier_async() for details. + * This flag has no effect when the dispatch block object is invoked directly. + * + * @const DISPATCH_BLOCK_DETACHED + * Flag indicating that a dispatch block object should execute disassociated + * from current execution context attributes such as QOS class, os_activity_t + * and properties of the current IPC request (if any). If invoked directly, the + * block object will remove these attributes from the calling thread for the + * duration of the block body (before applying attributes assigned to the block + * object, if any). If submitted to a queue, the block object will be executed + * with the attributes of the queue (or any attributes specifically assigned to + * the block object). + * + * @const DISPATCH_BLOCK_ASSIGN_CURRENT + * Flag indicating that a dispatch block object should be assigned the execution + * context attributes that are current at the time the block object is created. + * This applies to attributes such as QOS class, os_activity_t and properties of + * the current IPC request (if any). If invoked directly, the block object will + * apply these attributes to the calling thread for the duration of the block + * body. If the block object is submitted to a queue, this flag replaces the + * default behavior of associating the submitted block instance with the + * execution context attributes that are current at the time of submission. + * If a specific QOS class is assigned with DISPATCH_BLOCK_NO_QOS_CLASS or + * dispatch_block_create_with_qos_class(), that QOS class takes precedence over + * the QOS class assignment indicated by this flag. + * + * @const DISPATCH_BLOCK_NO_QOS_CLASS + * Flag indicating that a dispatch block object should be not be assigned a QOS + * class. If invoked directly, the block object will be executed with the QOS + * class of the calling thread. If the block object is submitted to a queue, + * this replaces the default behavior of associating the submitted block + * instance with the QOS class current at the time of submission. + * This flag is ignored if a specific QOS class is assigned with + * dispatch_block_create_with_qos_class(). + * + * @const DISPATCH_BLOCK_INHERIT_QOS_CLASS + * Flag indicating that execution of a dispatch block object submitted to a + * queue should prefer the QOS class assigned to the queue over the QOS class + * assigned to the block (resp. associated with the block at the time of + * submission). The latter will only be used if the queue in question does not + * have an assigned QOS class, as long as doing so does not result in a QOS + * class lower than the QOS class inherited from the queue's target queue. + * This flag is the default when a dispatch block object is submitted to a queue + * for asynchronous execution and has no effect when the dispatch block object + * is invoked directly. It is ignored if DISPATCH_BLOCK_ENFORCE_QOS_CLASS is + * also passed. + * + * @const DISPATCH_BLOCK_ENFORCE_QOS_CLASS + * Flag indicating that execution of a dispatch block object submitted to a + * queue should prefer the QOS class assigned to the block (resp. associated + * with the block at the time of submission) over the QOS class assigned to the + * queue, as long as doing so will not result in a lower QOS class. + * This flag is the default when a dispatch block object is submitted to a queue + * for synchronous execution or when the dispatch block object is invoked + * directly. + */ +DISPATCH_ENUM(dispatch_block_flags, unsigned long, + DISPATCH_BLOCK_BARRIER + DISPATCH_ENUM_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) = 0x1, + DISPATCH_BLOCK_DETACHED + DISPATCH_ENUM_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) = 0x2, + DISPATCH_BLOCK_ASSIGN_CURRENT + DISPATCH_ENUM_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) = 0x4, + DISPATCH_BLOCK_NO_QOS_CLASS + DISPATCH_ENUM_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) = 0x8, + DISPATCH_BLOCK_INHERIT_QOS_CLASS + DISPATCH_ENUM_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) = 0x10, + DISPATCH_BLOCK_ENFORCE_QOS_CLASS + DISPATCH_ENUM_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) = 0x20, +); + +/*! + * @function dispatch_block_create + * + * @abstract + * Create a new dispatch block object on the heap from an existing block and + * the given flags. + * + * @discussion + * The provided block is Block_copy'ed to the heap and retained by the newly + * created dispatch block object. + * + * The returned dispatch block object is intended to be submitted to a dispatch + * queue with dispatch_async() and related functions, but may also be invoked + * directly. Both operations can be performed an arbitrary number of times but + * only the first completed execution of a dispatch block object can be waited + * on with dispatch_block_wait() or observed with dispatch_block_notify(). + * + * If the returned dispatch block object is submitted to a dispatch queue, the + * submitted block instance will be associated with the QOS class current at the + * time of submission, unless one of the following flags assigned a specific QOS + * class (or no QOS class) at the time of block creation: + * - DISPATCH_BLOCK_ASSIGN_CURRENT + * - DISPATCH_BLOCK_NO_QOS_CLASS + * - DISPATCH_BLOCK_DETACHED + * The QOS class the block object will be executed with also depends on the QOS + * class assigned to the queue and which of the following flags was specified or + * defaulted to: + * - DISPATCH_BLOCK_INHERIT_QOS_CLASS (default for asynchronous execution) + * - DISPATCH_BLOCK_ENFORCE_QOS_CLASS (default for synchronous execution) + * See description of dispatch_block_flags_t for details. + * + * If the returned dispatch block object is submitted directly to a serial queue + * and is configured to execute with a specific QOS class, the system will make + * a best effort to apply the necessary QOS overrides to ensure that blocks + * submitted earlier to the serial queue are executed at that same QOS class or + * higher. + * + * @param flags + * Configuration flags for the block object. + * Passing a value that is not a bitwise OR of flags from dispatch_block_flags_t + * results in NULL being returned. + * + * @param block + * The block to create the dispatch block object from. + * + * @result + * The newly created dispatch block object, or NULL. + * When not building with Objective-C ARC, must be released with a -[release] + * message or the Block_release() function. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) +DISPATCH_EXPORT DISPATCH_NONNULL2 DISPATCH_RETURNS_RETAINED_BLOCK +DISPATCH_WARN_RESULT DISPATCH_NOTHROW +dispatch_block_t +dispatch_block_create(dispatch_block_flags_t flags, dispatch_block_t block); + +/*! + * @function dispatch_block_create_with_qos_class + * + * @abstract + * Create a new dispatch block object on the heap from an existing block and + * the given flags, and assign it the specified QOS class and relative priority. + * + * @discussion + * The provided block is Block_copy'ed to the heap and retained by the newly + * created dispatch block object. + * + * The returned dispatch block object is intended to be submitted to a dispatch + * queue with dispatch_async() and related functions, but may also be invoked + * directly. Both operations can be performed an arbitrary number of times but + * only the first completed execution of a dispatch block object can be waited + * on with dispatch_block_wait() or observed with dispatch_block_notify(). + * + * If invoked directly, the returned dispatch block object will be executed with + * the assigned QOS class as long as that does not result in a lower QOS class + * than what is current on the calling thread. + * + * If the returned dispatch block object is submitted to a dispatch queue, the + * QOS class it will be executed with depends on the QOS class assigned to the + * block, the QOS class assigned to the queue and which of the following flags + * was specified or defaulted to: + * - DISPATCH_BLOCK_INHERIT_QOS_CLASS: default for asynchronous execution + * - DISPATCH_BLOCK_ENFORCE_QOS_CLASS: default for synchronous execution + * See description of dispatch_block_flags_t for details. + * + * If the returned dispatch block object is submitted directly to a serial queue + * and is configured to execute with a specific QOS class, the system will make + * a best effort to apply the necessary QOS overrides to ensure that blocks + * submitted earlier to the serial queue are executed at that same QOS class or + * higher. + * + * @param flags + * Configuration flags for the new block object. + * Passing a value that is not a bitwise OR of flags from dispatch_block_flags_t + * results in NULL being returned. + * + * @param qos_class + * A QOS class value: + * - QOS_CLASS_USER_INTERACTIVE + * - QOS_CLASS_USER_INITIATED + * - QOS_CLASS_DEFAULT + * - QOS_CLASS_UTILITY + * - QOS_CLASS_BACKGROUND + * - QOS_CLASS_UNSPECIFIED + * Passing QOS_CLASS_UNSPECIFIED is equivalent to specifying the + * DISPATCH_BLOCK_NO_QOS_CLASS flag. Passing any other value results in NULL + * being returned. + * + * @param relative_priority + * A relative priority within the QOS class. This value is a negative + * offset from the maximum supported scheduler priority for the given class. + * Passing a value greater than zero or less than QOS_MIN_RELATIVE_PRIORITY + * results in NULL being returned. + * + * @param block + * The block to create the dispatch block object from. + * + * @result + * The newly created dispatch block object, or NULL. + * When not building with Objective-C ARC, must be released with a -[release] + * message or the Block_release() function. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) +DISPATCH_EXPORT DISPATCH_NONNULL4 DISPATCH_RETURNS_RETAINED_BLOCK +DISPATCH_WARN_RESULT DISPATCH_NOTHROW +dispatch_block_t +dispatch_block_create_with_qos_class(dispatch_block_flags_t flags, + dispatch_qos_class_t qos_class, int relative_priority, + dispatch_block_t block); + +/*! + * @function dispatch_block_perform + * + * @abstract + * Create, synchronously execute and release a dispatch block object from the + * specified block and flags. + * + * @discussion + * Behaves identically to the sequence + * + * dispatch_block_t b = dispatch_block_create(flags, block); + * b(); + * Block_release(b); + * + * but may be implemented more efficiently internally by not requiring a copy + * to the heap of the specified block or the allocation of a new block object. + * + * @param flags + * Configuration flags for the temporary block object. + * The result of passing a value that is not a bitwise OR of flags from + * dispatch_block_flags_t is undefined. + * + * @param block + * The block to create the temporary block object from. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) +DISPATCH_EXPORT DISPATCH_NONNULL2 DISPATCH_NOTHROW +void +dispatch_block_perform(dispatch_block_flags_t flags, dispatch_block_t block); + +/*! + * @function dispatch_block_wait + * + * @abstract + * Wait synchronously until execution of the specified dispatch block object has + * completed or until the specified timeout has elapsed. + * + * @discussion + * This function will return immediately if execution of the block object has + * already completed. + * + * It is not possible to wait for multiple executions of the same block object + * with this interface; use dispatch_group_wait() for that purpose. A single + * dispatch block object may either be waited on once and executed once, + * or it may be executed any number of times. The behavior of any other + * combination is undefined. Submission to a dispatch queue counts as an + * execution, even if cancelation (dispatch_block_cancel) means the block's + * code never runs. + * + * The result of calling this function from multiple threads simultaneously + * with the same dispatch block object is undefined, but note that doing so + * would violate the rules described in the previous paragraph. + * + * If this function returns indicating that the specified timeout has elapsed, + * then that invocation does not count as the one allowed wait. + * + * If at the time this function is called, the specified dispatch block object + * has been submitted directly to a serial queue, the system will make a best + * effort to apply the necessary QOS overrides to ensure that the block and any + * blocks submitted earlier to that serial queue are executed at the QOS class + * (or higher) of the thread calling dispatch_block_wait(). + * + * @param block + * The dispatch block object to wait on. + * The result of passing NULL or a block object not returned by one of the + * dispatch_block_create* functions is undefined. + * + * @param timeout + * When to timeout (see dispatch_time). As a convenience, there are the + * DISPATCH_TIME_NOW and DISPATCH_TIME_FOREVER constants. + * + * @result + * Returns zero on success (the dispatch block object completed within the + * specified timeout) or non-zero on error (i.e. timed out). + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) +DISPATCH_EXPORT DISPATCH_NONNULL1 DISPATCH_NOTHROW +long +dispatch_block_wait(dispatch_block_t block, dispatch_time_t timeout); + +/*! + * @function dispatch_block_notify + * + * @abstract + * Schedule a notification block to be submitted to a queue when the execution + * of a specified dispatch block object has completed. + * + * @discussion + * This function will submit the notification block immediately if execution of + * the observed block object has already completed. + * + * It is not possible to be notified of multiple executions of the same block + * object with this interface, use dispatch_group_notify() for that purpose. + * + * A single dispatch block object may either be observed one or more times + * and executed once, or it may be executed any number of times. The behavior + * of any other combination is undefined. Submission to a dispatch queue + * counts as an execution, even if cancellation (dispatch_block_cancel) means + * the block's code never runs. + * + * If multiple notification blocks are scheduled for a single block object, + * there is no defined order in which the notification blocks will be submitted + * to their associated queues. + * + * @param block + * The dispatch block object to observe. + * The result of passing NULL or a block object not returned by one of the + * dispatch_block_create* functions is undefined. + * + * @param queue + * The queue to which the supplied notification block will be submitted when + * the observed block completes. + * + * @param notification_block + * The notification block to submit when the observed block object completes. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) +DISPATCH_EXPORT DISPATCH_NONNULL_ALL DISPATCH_NOTHROW +void +dispatch_block_notify(dispatch_block_t block, dispatch_queue_t queue, + dispatch_block_t notification_block); + +/*! + * @function dispatch_block_cancel + * + * @abstract + * Asynchronously cancel the specified dispatch block object. + * + * @discussion + * Cancellation causes any future execution of the dispatch block object to + * return immediately, but does not affect any execution of the block object + * that is already in progress. + * + * Release of any resources associated with the block object will be delayed + * until execution of the block object is next attempted (or any execution + * already in progress completes). + * + * NOTE: care needs to be taken to ensure that a block object that may be + * canceled does not capture any resources that require execution of the + * block body in order to be released (e.g. memory allocated with + * malloc(3) that the block body calls free(3) on). Such resources will + * be leaked if the block body is never executed due to cancellation. + * + * @param block + * The dispatch block object to cancel. + * The result of passing NULL or a block object not returned by one of the + * dispatch_block_create* functions is undefined. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) +DISPATCH_EXPORT DISPATCH_NONNULL_ALL DISPATCH_NOTHROW +void +dispatch_block_cancel(dispatch_block_t block); + +/*! + * @function dispatch_block_testcancel + * + * @abstract + * Tests whether the given dispatch block object has been canceled. + * + * @param block + * The dispatch block object to test. + * The result of passing NULL or a block object not returned by one of the + * dispatch_block_create* functions is undefined. + * + * @result + * Non-zero if canceled and zero if not canceled. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) +DISPATCH_EXPORT DISPATCH_NONNULL_ALL DISPATCH_WARN_RESULT DISPATCH_PURE +DISPATCH_NOTHROW +long +dispatch_block_testcancel(dispatch_block_t block); + +__END_DECLS + +#endif // __BLOCKS__ + +#endif // __DISPATCH_BLOCK__ diff --git a/dispatch/dispatch.h b/dispatch/dispatch.h index cb5af23..722b0c9 100644 --- a/dispatch/dispatch.h +++ b/dispatch/dispatch.h @@ -37,7 +37,7 @@ #define __OSX_AVAILABLE_STARTING(x, y) #endif -#define DISPATCH_API_VERSION 20130520 +#define DISPATCH_API_VERSION 20140804 #ifndef __DISPATCH_BUILDING_DISPATCH__ @@ -47,9 +47,10 @@ #include #include -#include #include +#include #include +#include #include #include #include diff --git a/dispatch/introspection.h b/dispatch/introspection.h index 9e96341..d20d90a 100644 --- a/dispatch/introspection.h +++ b/dispatch/introspection.h @@ -108,6 +108,27 @@ void dispatch_introspection_hook_queue_item_dequeue(dispatch_queue_t queue, dispatch_object_t item); +/*! + * @function dispatch_introspection_hook_queue_item_complete + * + * @abstract + * Interposable hook function called when an item previously dequeued from a + * dispatch queue has completed processing. + * + * @discussion + * The object pointer value passed to this function must be treated as a value + * only. It is intended solely for matching up with an earlier call to a + * dequeue hook function and must NOT be dereferenced. + * + * @param item + * Opaque dentifier for completed item. Must NOT be dereferenced. + */ + +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_7_1) +DISPATCH_EXPORT +void +dispatch_introspection_hook_queue_item_complete(dispatch_object_t item); + /*! * @function dispatch_introspection_hook_queue_callout_begin * diff --git a/dispatch/io.h b/dispatch/io.h index 569dbdb..d53d488 100644 --- a/dispatch/io.h +++ b/dispatch/io.h @@ -96,8 +96,8 @@ typedef int dispatch_fd_t; * submitted. * @param handler The handler to enqueue when data is ready to be * delivered. - * @param data The data read from the file descriptor. - * @param error An errno condition for the read operation or + * param data The data read from the file descriptor. + * param error An errno condition for the read operation or * zero if the read was successful. */ __OSX_AVAILABLE_STARTING(__MAC_10_7,__IPHONE_5_0) @@ -133,9 +133,9 @@ dispatch_read(dispatch_fd_t fd, * @param queue The dispatch queue to which the handler should be * submitted. * @param handler The handler to enqueue when the data has been written. - * @param data The data that could not be written to the I/O + * param data The data that could not be written to the I/O * channel, or NULL. - * @param error An errno condition for the write operation or + * param error An errno condition for the write operation or * zero if the write was successful. */ __OSX_AVAILABLE_STARTING(__MAC_10_7,__IPHONE_5_0) @@ -204,7 +204,7 @@ typedef unsigned long dispatch_io_type_t; * @param queue The dispatch queue to which the handler should be submitted. * @param cleanup_handler The handler to enqueue when the system * relinquishes control over the file descriptor. - * @param error An errno condition if control is relinquished + * param error An errno condition if control is relinquished * because channel creation failed, zero otherwise. * @result The newly created dispatch I/O channel or NULL if an error * occurred (invalid type specified). @@ -239,7 +239,7 @@ dispatch_io_create(dispatch_io_type_t type, * submitted. * @param cleanup_handler The handler to enqueue when the system * has closed the file at path. - * @param error An errno condition if control is relinquished + * param error An errno condition if control is relinquished * because channel creation or opening of the * specified file failed, zero otherwise. * @result The newly created dispatch I/O channel or NULL if an error @@ -280,7 +280,7 @@ dispatch_io_create_with_path(dispatch_io_type_t type, * relinquishes control over the file descriptor * (resp. closes the file at path) associated with * the existing channel. - * @param error An errno condition if control is relinquished + * param error An errno condition if control is relinquished * because channel creation failed, zero otherwise. * @result The newly created dispatch I/O channel or NULL if an error * occurred (invalid type specified). @@ -341,10 +341,10 @@ typedef void (^dispatch_io_handler_t)(bool done, dispatch_data_t data, * submitted. * @param io_handler The I/O handler to enqueue when data is ready to be * delivered. - * @param done A flag indicating whether the operation is complete. - * @param data An object with the data most recently read from the + * param done A flag indicating whether the operation is complete. + * param data An object with the data most recently read from the * I/O channel as part of this read operation, or NULL. - * @param error An errno condition for the read operation or zero if + * param error An errno condition for the read operation or zero if * the read was successful. */ __OSX_AVAILABLE_STARTING(__MAC_10_7,__IPHONE_5_0) @@ -393,11 +393,11 @@ dispatch_io_read(dispatch_io_t channel, * @param queue The dispatch queue to which the I/O handler should be * submitted. * @param io_handler The I/O handler to enqueue when data has been delivered. - * @param done A flag indicating whether the operation is complete. - * @param data An object of the data remaining to be + * param done A flag indicating whether the operation is complete. + * param data An object of the data remaining to be * written to the I/O channel as part of this write * operation, or NULL. - * @param error An errno condition for the write operation or zero + * param error An errno condition for the write operation or zero * if the write was successful. */ __OSX_AVAILABLE_STARTING(__MAC_10_7,__IPHONE_5_0) diff --git a/dispatch/object.h b/dispatch/object.h index 4ae0ab6..c637189 100644 --- a/dispatch/object.h +++ b/dispatch/object.h @@ -99,41 +99,50 @@ typedef union { #define DISPATCH_RETURNS_RETAINED #endif -__BEGIN_DECLS - /*! - * @function dispatch_debug + * @typedef dispatch_block_t * * @abstract - * Programmatically log debug information about a dispatch object. + * The type of blocks submitted to dispatch queues, which take no arguments + * and have no return value. * * @discussion - * Programmatically log debug information about a dispatch object. By default, - * the log output is sent to syslog at notice level. In the debug version of - * the library, the log output is sent to a file in /var/tmp. - * The log output destination can be configured via the LIBDISPATCH_LOG - * environment variable, valid values are: YES, NO, syslog, stderr, file. - * - * This function is deprecated and will be removed in a future release. - * Objective-C callers may use -debugDescription instead. - * - * @param object - * The object to introspect. - * - * @param message - * The message to log above and beyond the introspection. + * When not building with Objective-C ARC, a block object allocated on or + * copied to the heap must be released with a -[release] message or the + * Block_release() function. + * + * The declaration of a block literal allocates storage on the stack. + * Therefore, this is an invalid construct: + * + * dispatch_block_t block; + * if (x) { + * block = ^{ printf("true\n"); }; + * } else { + * block = ^{ printf("false\n"); }; + * } + * block(); // unsafe!!! + * + * + * What is happening behind the scenes: + * + * if (x) { + * struct Block __tmp_1 = ...; // setup details + * block = &__tmp_1; + * } else { + * struct Block __tmp_2 = ...; // setup details + * block = &__tmp_2; + * } + * + * + * As the example demonstrates, the address of a stack variable is escaping the + * scope in which it is allocated. That is a classic C bug. + * + * Instead, the block literal must be copied to the heap with the Block_copy() + * function or by sending it a -[copy] message. */ -__OSX_AVAILABLE_BUT_DEPRECATED(__MAC_10_6,__MAC_10_9,__IPHONE_4_0,__IPHONE_6_0) -DISPATCH_EXPORT DISPATCH_NONNULL2 DISPATCH_NOTHROW -__attribute__((__format__(printf,2,3))) -void -dispatch_debug(dispatch_object_t object, const char *message, ...); +typedef void (^dispatch_block_t)(void); -__OSX_AVAILABLE_BUT_DEPRECATED(__MAC_10_6,__MAC_10_9,__IPHONE_4_0,__IPHONE_6_0) -DISPATCH_EXPORT DISPATCH_NONNULL2 DISPATCH_NOTHROW -__attribute__((__format__(printf,2,0))) -void -dispatch_debugv(dispatch_object_t object, const char *message, va_list ap); +__BEGIN_DECLS /*! * @function dispatch_retain @@ -227,11 +236,11 @@ dispatch_set_context(dispatch_object_t object, void *context); * @abstract * Set the finalizer function for a dispatch object. * - * @param + * @param object * The dispatch object to modify. * The result of passing NULL in this parameter is undefined. * - * @param + * @param finalizer * The finalizer function pointer. * * @discussion @@ -246,7 +255,7 @@ __OSX_AVAILABLE_STARTING(__MAC_10_6,__IPHONE_4_0) DISPATCH_EXPORT DISPATCH_NOTHROW //DISPATCH_NONNULL1 void dispatch_set_finalizer_f(dispatch_object_t object, - dispatch_function_t finalizer); + dispatch_function_t finalizer); /*! * @function dispatch_suspend @@ -286,6 +295,173 @@ DISPATCH_EXPORT DISPATCH_NONNULL_ALL DISPATCH_NOTHROW void dispatch_resume(dispatch_object_t object); +/*! + * @function dispatch_wait + * + * @abstract + * Wait synchronously for an object or until the specified timeout has elapsed. + * + * @discussion + * Type-generic macro that maps to dispatch_block_wait, dispatch_group_wait or + * dispatch_semaphore_wait, depending on the type of the first argument. + * See documentation for these functions for more details. + * This function is unavailable for any other object type. + * + * @param object + * The object to wait on. + * The result of passing NULL in this parameter is undefined. + * + * @param timeout + * When to timeout (see dispatch_time). As a convenience, there are the + * DISPATCH_TIME_NOW and DISPATCH_TIME_FOREVER constants. + * + * @result + * Returns zero on success or non-zero on error (i.e. timed out). + */ +DISPATCH_UNAVAILABLE +DISPATCH_EXPORT DISPATCH_NONNULL1 DISPATCH_NOTHROW +long +dispatch_wait(void *object, dispatch_time_t timeout); +#if __has_extension(c_generic_selections) +#define dispatch_wait(object, timeout) \ + _Generic((object), \ + dispatch_block_t:dispatch_block_wait, \ + dispatch_group_t:dispatch_group_wait, \ + dispatch_semaphore_t:dispatch_semaphore_wait \ + )((object),(timeout)) +#endif + +/*! + * @function dispatch_notify + * + * @abstract + * Schedule a notification block to be submitted to a queue when the execution + * of a specified object has completed. + * + * @discussion + * Type-generic macro that maps to dispatch_block_notify or + * dispatch_group_notify, depending on the type of the first argument. + * See documentation for these functions for more details. + * This function is unavailable for any other object type. + * + * @param object + * The object to observe. + * The result of passing NULL in this parameter is undefined. + * + * @param queue + * The queue to which the supplied notification block will be submitted when + * the observed object completes. + * + * @param notification_block + * The block to submit when the observed object completes. + */ +DISPATCH_UNAVAILABLE +DISPATCH_EXPORT DISPATCH_NONNULL_ALL DISPATCH_NOTHROW +void +dispatch_notify(void *object, dispatch_object_t queue, + dispatch_block_t notification_block); +#if __has_extension(c_generic_selections) +#define dispatch_notify(object, queue, notification_block) \ + _Generic((object), \ + dispatch_block_t:dispatch_block_notify, \ + dispatch_group_t:dispatch_group_notify \ + )((object),(queue), (notification_block)) +#endif + +/*! + * @function dispatch_cancel + * + * @abstract + * Cancel the specified object. + * + * @discussion + * Type-generic macro that maps to dispatch_block_cancel or + * dispatch_source_cancel, depending on the type of the first argument. + * See documentation for these functions for more details. + * This function is unavailable for any other object type. + * + * @param object + * The object to cancel. + * The result of passing NULL in this parameter is undefined. + */ +DISPATCH_UNAVAILABLE +DISPATCH_EXPORT DISPATCH_NONNULL_ALL DISPATCH_NOTHROW +void +dispatch_cancel(void *object); +#if __has_extension(c_generic_selections) +#define dispatch_cancel(object) \ + _Generic((object), \ + dispatch_block_t:dispatch_block_cancel, \ + dispatch_source_t:dispatch_source_cancel \ + )((object)) +#endif + +/*! + * @function dispatch_testcancel + * + * @abstract + * Test whether the specified object has been canceled + * + * @discussion + * Type-generic macro that maps to dispatch_block_testcancel or + * dispatch_source_testcancel, depending on the type of the first argument. + * See documentation for these functions for more details. + * This function is unavailable for any other object type. + * + * @param object + * The object to test. + * The result of passing NULL in this parameter is undefined. + * + * @result + * Non-zero if canceled and zero if not canceled. + */ +DISPATCH_UNAVAILABLE +DISPATCH_EXPORT DISPATCH_NONNULL_ALL DISPATCH_WARN_RESULT DISPATCH_PURE +DISPATCH_NOTHROW +long +dispatch_testcancel(void *object); +#if __has_extension(c_generic_selections) +#define dispatch_testcancel(object) \ + _Generic((object), \ + dispatch_block_t:dispatch_block_testcancel, \ + dispatch_source_t:dispatch_source_testcancel \ + )((object)) +#endif + +/*! + * @function dispatch_debug + * + * @abstract + * Programmatically log debug information about a dispatch object. + * + * @discussion + * Programmatically log debug information about a dispatch object. By default, + * the log output is sent to syslog at notice level. In the debug version of + * the library, the log output is sent to a file in /var/tmp. + * The log output destination can be configured via the LIBDISPATCH_LOG + * environment variable, valid values are: YES, NO, syslog, stderr, file. + * + * This function is deprecated and will be removed in a future release. + * Objective-C callers may use -debugDescription instead. + * + * @param object + * The object to introspect. + * + * @param message + * The message to log above and beyond the introspection. + */ +__OSX_AVAILABLE_BUT_DEPRECATED(__MAC_10_6,__MAC_10_9,__IPHONE_4_0,__IPHONE_6_0) +DISPATCH_EXPORT DISPATCH_NONNULL2 DISPATCH_NOTHROW +__attribute__((__format__(printf,2,3))) +void +dispatch_debug(dispatch_object_t object, const char *message, ...); + +__OSX_AVAILABLE_BUT_DEPRECATED(__MAC_10_6,__MAC_10_9,__IPHONE_4_0,__IPHONE_6_0) +DISPATCH_EXPORT DISPATCH_NONNULL2 DISPATCH_NOTHROW +__attribute__((__format__(printf,2,0))) +void +dispatch_debugv(dispatch_object_t object, const char *message, va_list ap); + __END_DECLS #endif diff --git a/dispatch/queue.h b/dispatch/queue.h index 9090676..cc3ca94 100644 --- a/dispatch/queue.h +++ b/dispatch/queue.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2012 Apple Inc. All rights reserved. + * Copyright (c) 2008-2014 Apple Inc. All rights reserved. * * @APPLE_APACHE_LICENSE_HEADER_START@ * @@ -66,51 +66,6 @@ */ DISPATCH_DECL(dispatch_queue); -/*! - * @typedef dispatch_queue_attr_t - * - * @abstract - * Attribute for dispatch queues. - */ -DISPATCH_DECL(dispatch_queue_attr); - -/*! - * @typedef dispatch_block_t - * - * @abstract - * The prototype of blocks submitted to dispatch queues, which take no - * arguments and have no return value. - * - * @discussion - * The declaration of a block allocates storage on the stack. Therefore, this - * is an invalid construct: - * - * dispatch_block_t block; - * - * if (x) { - * block = ^{ printf("true\n"); }; - * } else { - * block = ^{ printf("false\n"); }; - * } - * block(); // unsafe!!! - * - * What is happening behind the scenes: - * - * if (x) { - * struct Block __tmp_1 = ...; // setup details - * block = &__tmp_1; - * } else { - * struct Block __tmp_2 = ...; // setup details - * block = &__tmp_2; - * } - * - * As the example demonstrates, the address of a stack variable is escaping the - * scope in which it is allocated. That is a classic C bug. - */ -#ifdef __BLOCKS__ -typedef void (^dispatch_block_t)(void); -#endif - __BEGIN_DECLS /*! @@ -345,6 +300,9 @@ DISPATCH_EXPORT DISPATCH_PURE DISPATCH_WARN_RESULT DISPATCH_NOTHROW dispatch_queue_t dispatch_get_current_queue(void); +__OSX_AVAILABLE_STARTING(__MAC_10_6,__IPHONE_4_0) +DISPATCH_EXPORT struct dispatch_queue_s _dispatch_main_q; + /*! * @function dispatch_get_main_queue * @@ -360,10 +318,12 @@ dispatch_get_current_queue(void); * Returns the main queue. This queue is created automatically on behalf of * the main thread before main() is called. */ -__OSX_AVAILABLE_STARTING(__MAC_10_6,__IPHONE_4_0) -DISPATCH_EXPORT struct dispatch_queue_s _dispatch_main_q; -#define dispatch_get_main_queue() \ - DISPATCH_GLOBAL_OBJECT(dispatch_queue_t, _dispatch_main_q) +DISPATCH_INLINE DISPATCH_ALWAYS_INLINE DISPATCH_CONST DISPATCH_NOTHROW +dispatch_queue_t +dispatch_get_main_queue(void) +{ + return DISPATCH_GLOBAL_OBJECT(dispatch_queue_t, _dispatch_main_q); +} /*! * @typedef dispatch_queue_priority_t @@ -400,32 +360,68 @@ DISPATCH_EXPORT struct dispatch_queue_s _dispatch_main_q; typedef long dispatch_queue_priority_t; +/*! + * @typedef dispatch_qos_class_t + * Alias for qos_class_t type. + */ +#if __has_include() +#include +typedef qos_class_t dispatch_qos_class_t; +#else +typedef unsigned int dispatch_qos_class_t; +#endif + /*! * @function dispatch_get_global_queue * * @abstract - * Returns a well-known global concurrent queue of a given priority level. + * Returns a well-known global concurrent queue of a given quality of service + * class. * * @discussion * The well-known global concurrent queues may not be modified. Calls to * dispatch_suspend(), dispatch_resume(), dispatch_set_context(), etc., will * have no effect when used with queues returned by this function. * - * @param priority - * A priority defined in dispatch_queue_priority_t + * @param identifier + * A quality of service class defined in qos_class_t or a priority defined in + * dispatch_queue_priority_t. + * + * It is recommended to use quality of service class values to identify the + * well-known global concurrent queues: + * - QOS_CLASS_USER_INTERACTIVE + * - QOS_CLASS_USER_INITIATED + * - QOS_CLASS_DEFAULT + * - QOS_CLASS_UTILITY + * - QOS_CLASS_BACKGROUND + * + * The global concurrent queues may still be identified by their priority, + * which map to the following QOS classes: + * - DISPATCH_QUEUE_PRIORITY_HIGH: QOS_CLASS_USER_INITIATED + * - DISPATCH_QUEUE_PRIORITY_DEFAULT: QOS_CLASS_DEFAULT + * - DISPATCH_QUEUE_PRIORITY_LOW: QOS_CLASS_UTILITY + * - DISPATCH_QUEUE_PRIORITY_BACKGROUND: QOS_CLASS_BACKGROUND * * @param flags * Reserved for future use. Passing any value other than zero may result in * a NULL return value. * * @result - * Returns the requested global queue. + * Returns the requested global queue or NULL if the requested global queue + * does not exist. */ __OSX_AVAILABLE_STARTING(__MAC_10_6,__IPHONE_4_0) DISPATCH_EXPORT DISPATCH_CONST DISPATCH_WARN_RESULT DISPATCH_NOTHROW dispatch_queue_t -dispatch_get_global_queue(dispatch_queue_priority_t priority, - unsigned long flags); +dispatch_get_global_queue(long identifier, unsigned long flags); + +/*! + * @typedef dispatch_queue_attr_t + * + * @abstract + * Attribute for dispatch queues. + */ +DISPATCH_DECL(dispatch_queue_attr); /*! * @const DISPATCH_QUEUE_SERIAL @@ -445,6 +441,63 @@ __OSX_AVAILABLE_STARTING(__MAC_10_7,__IPHONE_4_3) DISPATCH_EXPORT struct dispatch_queue_attr_s _dispatch_queue_attr_concurrent; +/*! + * @function dispatch_queue_attr_make_with_qos_class + * + * @abstract + * Returns an attribute value which may be provided to dispatch_queue_create() + * in order to assign a QOS class and relative priority to the queue. + * + * @discussion + * When specified in this manner, the QOS class and relative priority take + * precedence over those inherited from the dispatch queue's target queue (if + * any) as long that does not result in a lower QOS class and relative priority. + * + * The global queue priorities map to the following QOS classes: + * - DISPATCH_QUEUE_PRIORITY_HIGH: QOS_CLASS_USER_INITIATED + * - DISPATCH_QUEUE_PRIORITY_DEFAULT: QOS_CLASS_DEFAULT + * - DISPATCH_QUEUE_PRIORITY_LOW: QOS_CLASS_UTILITY + * - DISPATCH_QUEUE_PRIORITY_BACKGROUND: QOS_CLASS_BACKGROUND + * + * Example: + * + * dispatch_queue_t queue; + * dispatch_queue_attr_t attr; + * attr = dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, + * QOS_CLASS_UTILITY, 0); + * queue = dispatch_queue_create("com.example.myqueue", attr); + * + * + * @param attr + * A queue attribute value to be combined with the QOS class, or NULL. + * + * @param qos_class + * A QOS class value: + * - QOS_CLASS_USER_INTERACTIVE + * - QOS_CLASS_USER_INITIATED + * - QOS_CLASS_DEFAULT + * - QOS_CLASS_UTILITY + * - QOS_CLASS_BACKGROUND + * Passing any other value results in NULL being returned. + * + * @param relative_priority + * A relative priority within the QOS class. This value is a negative + * offset from the maximum supported scheduler priority for the given class. + * Passing a value greater than zero or less than QOS_MIN_RELATIVE_PRIORITY + * results in NULL being returned. + * + * @return + * Returns an attribute value which may be provided to dispatch_queue_create(), + * or NULL if an invalid QOS class was requested. + * The new value combines the attributes specified by the 'attr' parameter and + * the new QOS class and relative priority. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) +DISPATCH_EXPORT DISPATCH_WARN_RESULT DISPATCH_PURE DISPATCH_NOTHROW +dispatch_queue_attr_t +dispatch_queue_attr_make_with_qos_class(dispatch_queue_attr_t attr, + dispatch_qos_class_t qos_class, int relative_priority); + /*! * @function dispatch_queue_create * @@ -466,15 +519,23 @@ struct dispatch_queue_attr_s _dispatch_queue_attr_concurrent; * hold a reference to that queue. Therefore a queue will not be deallocated * until all pending blocks have finished. * - * The target queue of a newly created dispatch queue is the default priority - * global concurrent queue. + * Passing the result of the dispatch_queue_attr_make_with_qos_class() function + * to the attr parameter of this function allows a quality of service class and + * relative priority to be specified for the newly created queue. + * The quality of service class so specified takes precedence over the quality + * of service class of the newly created dispatch queue's target queue (if any) + * as long that does not result in a lower QOS class and relative priority. + * + * When no quality of service class is specified, the target queue of a newly + * created dispatch queue is the default priority global concurrent queue. * * @param label * A string label to attach to the queue. * This parameter is optional and may be NULL. * * @param attr - * DISPATCH_QUEUE_SERIAL or DISPATCH_QUEUE_CONCURRENT. + * DISPATCH_QUEUE_SERIAL, DISPATCH_QUEUE_CONCURRENT, or the result of a call to + * the function dispatch_queue_attr_make_with_qos_class(). * * @result * The newly created dispatch queue. @@ -513,6 +574,46 @@ DISPATCH_EXPORT DISPATCH_PURE DISPATCH_WARN_RESULT DISPATCH_NOTHROW const char * dispatch_queue_get_label(dispatch_queue_t queue); +/*! + * @function dispatch_queue_get_qos_class + * + * @abstract + * Returns the QOS class and relative priority of the given queue. + * + * @discussion + * If the given queue was created with an attribute value returned from + * dispatch_queue_attr_make_with_qos_class(), this function returns the QOS + * class and relative priority specified at that time; for any other attribute + * value it returns a QOS class of QOS_CLASS_UNSPECIFIED and a relative + * priority of 0. + * + * If the given queue is one of the global queues, this function returns its + * assigned QOS class value as documented under dispatch_get_global_queue() and + * a relative priority of 0; in the case of the main queue it returns the QOS + * value provided by qos_class_main() and a relative priority of 0. + * + * @param queue + * The queue to query. + * + * @param relative_priority_ptr + * A pointer to an int variable to be filled with the relative priority offset + * within the QOS class, or NULL. + * + * @return + * A QOS class value: + * - QOS_CLASS_USER_INTERACTIVE + * - QOS_CLASS_USER_INITIATED + * - QOS_CLASS_DEFAULT + * - QOS_CLASS_UTILITY + * - QOS_CLASS_BACKGROUND + * - QOS_CLASS_UNSPECIFIED + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) +DISPATCH_EXPORT DISPATCH_WARN_RESULT DISPATCH_NONNULL1 DISPATCH_NOTHROW +dispatch_qos_class_t +dispatch_queue_get_qos_class(dispatch_queue_t queue, + int *relative_priority_ptr); + /*! * @const DISPATCH_TARGET_QUEUE_DEFAULT * @discussion Constant to pass to the dispatch_set_target_queue() and @@ -530,9 +631,12 @@ dispatch_queue_get_label(dispatch_queue_t queue); * @discussion * An object's target queue is responsible for processing the object. * - * A dispatch queue's priority is inherited from its target queue. Use the - * dispatch_get_global_queue() function to obtain suitable target queue - * of the desired priority. + * When no quality of service class and relative priority is specified for a + * dispatch queue at the time of creation, a dispatch queue's quality of service + * class is inherited from its target queue. The dispatch_get_global_queue() + * function may be used to obtain a target queue of a specific quality of + * service class, however the use of dispatch_queue_attr_make_with_qos_class() + * is recommended instead. * * Blocks submitted to a serial queue whose target queue is another serial * queue will not be invoked concurrently with blocks submitted to the target diff --git a/dispatch/source.h b/dispatch/source.h index ebbf8b9..411ed06 100644 --- a/dispatch/source.h +++ b/dispatch/source.h @@ -130,7 +130,7 @@ DISPATCH_SOURCE_TYPE_DECL(mach_recv); */ #define DISPATCH_SOURCE_TYPE_MEMORYPRESSURE \ (&_dispatch_source_type_memorypressure) -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_NA) +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_8_0) DISPATCH_SOURCE_TYPE_DECL(memorypressure); /*! @@ -436,7 +436,7 @@ __OSX_AVAILABLE_STARTING(__MAC_10_6,__IPHONE_4_0) DISPATCH_EXPORT DISPATCH_NONNULL1 DISPATCH_NOTHROW void dispatch_source_set_cancel_handler(dispatch_source_t source, - dispatch_block_t cancel_handler); + dispatch_block_t handler); #endif /* __BLOCKS__ */ /*! @@ -461,7 +461,7 @@ __OSX_AVAILABLE_STARTING(__MAC_10_6,__IPHONE_4_0) DISPATCH_EXPORT DISPATCH_NONNULL1 DISPATCH_NOTHROW void dispatch_source_set_cancel_handler_f(dispatch_source_t source, - dispatch_function_t cancel_handler); + dispatch_function_t handler); /*! * @function dispatch_source_cancel @@ -711,7 +711,7 @@ __OSX_AVAILABLE_STARTING(__MAC_10_7,__IPHONE_4_3) DISPATCH_EXPORT DISPATCH_NONNULL1 DISPATCH_NOTHROW void dispatch_source_set_registration_handler(dispatch_source_t source, - dispatch_block_t registration_handler); + dispatch_block_t handler); #endif /* __BLOCKS__ */ /*! @@ -736,7 +736,7 @@ __OSX_AVAILABLE_STARTING(__MAC_10_7,__IPHONE_4_3) DISPATCH_EXPORT DISPATCH_NONNULL1 DISPATCH_NOTHROW void dispatch_source_set_registration_handler_f(dispatch_source_t source, - dispatch_function_t registration_handler); + dispatch_function_t handler); __END_DECLS diff --git a/libdispatch.xcodeproj/project.pbxproj b/libdispatch.xcodeproj/project.pbxproj index b465ba7..ff12a47 100644 --- a/libdispatch.xcodeproj/project.pbxproj +++ b/libdispatch.xcodeproj/project.pbxproj @@ -39,6 +39,8 @@ 2BBF5A65154B64F5002B20F9 /* allocator.c in Sources */ = {isa = PBXBuildFile; fileRef = 2BBF5A62154B64F5002B20F9 /* allocator.c */; }; 2BBF5A66154B64F5002B20F9 /* allocator.c in Sources */ = {isa = PBXBuildFile; fileRef = 2BBF5A62154B64F5002B20F9 /* allocator.c */; }; 2BBF5A67154B64F5002B20F9 /* allocator.c in Sources */ = {isa = PBXBuildFile; fileRef = 2BBF5A62154B64F5002B20F9 /* allocator.c */; }; + 2BE17C6418EA305E002CA4E8 /* layout_private.h in Headers */ = {isa = PBXBuildFile; fileRef = 2BE17C6318EA305E002CA4E8 /* layout_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; + 2BE17C6518EA305E002CA4E8 /* layout_private.h in Headers */ = {isa = PBXBuildFile; fileRef = 2BE17C6318EA305E002CA4E8 /* layout_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; 5A0095A210F274B0000E2A31 /* io_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 5A0095A110F274B0000E2A31 /* io_internal.h */; }; 5A27262610F26F1900751FBC /* io.c in Sources */ = {isa = PBXBuildFile; fileRef = 5A27262510F26F1900751FBC /* io.c */; }; 5A5D13AC0F6B280500197CC3 /* semaphore_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 5A5D13AB0F6B280500197CC3 /* semaphore_internal.h */; }; @@ -82,6 +84,20 @@ E43570B9126E93380097AB9F /* provider.d in Sources */ = {isa = PBXBuildFile; fileRef = E43570B8126E93380097AB9F /* provider.d */; }; E43570BA126E93380097AB9F /* provider.d in Sources */ = {isa = PBXBuildFile; fileRef = E43570B8126E93380097AB9F /* provider.d */; }; E43A710615783F7E0012D38D /* data_private.h in Headers */ = {isa = PBXBuildFile; fileRef = C913AC0E143BD34800B78976 /* data_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; + E44757DA17F4572600B82CA1 /* inline_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = E44757D917F4572600B82CA1 /* inline_internal.h */; }; + E44757DB17F4573500B82CA1 /* inline_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = E44757D917F4572600B82CA1 /* inline_internal.h */; }; + E44757DC17F4573600B82CA1 /* inline_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = E44757D917F4572600B82CA1 /* inline_internal.h */; }; + E44A8E6B1805C3E0009FFDB6 /* voucher.c in Sources */ = {isa = PBXBuildFile; fileRef = E44A8E6A1805C3E0009FFDB6 /* voucher.c */; }; + E44A8E6C1805C3E0009FFDB6 /* voucher.c in Sources */ = {isa = PBXBuildFile; fileRef = E44A8E6A1805C3E0009FFDB6 /* voucher.c */; }; + E44A8E6D1805C3E0009FFDB6 /* voucher.c in Sources */ = {isa = PBXBuildFile; fileRef = E44A8E6A1805C3E0009FFDB6 /* voucher.c */; }; + E44A8E6E1805C3E0009FFDB6 /* voucher.c in Sources */ = {isa = PBXBuildFile; fileRef = E44A8E6A1805C3E0009FFDB6 /* voucher.c */; }; + E44A8E6F1805C3E0009FFDB6 /* voucher.c in Sources */ = {isa = PBXBuildFile; fileRef = E44A8E6A1805C3E0009FFDB6 /* voucher.c */; }; + E44A8E701805C3E0009FFDB6 /* voucher.c in Sources */ = {isa = PBXBuildFile; fileRef = E44A8E6A1805C3E0009FFDB6 /* voucher.c */; }; + E44A8E721805C473009FFDB6 /* voucher_private.h in Headers */ = {isa = PBXBuildFile; fileRef = E44A8E711805C473009FFDB6 /* voucher_private.h */; }; + E44A8E731805C473009FFDB6 /* voucher_private.h in Headers */ = {isa = PBXBuildFile; fileRef = E44A8E711805C473009FFDB6 /* voucher_private.h */; }; + E44A8E7518066276009FFDB6 /* voucher_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = E44A8E7418066276009FFDB6 /* voucher_internal.h */; }; + E44A8E7618066276009FFDB6 /* voucher_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = E44A8E7418066276009FFDB6 /* voucher_internal.h */; }; + E44A8E7718066276009FFDB6 /* voucher_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = E44A8E7418066276009FFDB6 /* voucher_internal.h */; }; E44EBE3E1251659900645D88 /* init.c in Sources */ = {isa = PBXBuildFile; fileRef = E44EBE3B1251659900645D88 /* init.c */; }; E44EBE5412517EBE00645D88 /* protocol.defs in Sources */ = {isa = PBXBuildFile; fileRef = FC7BED950E8361E600161930 /* protocol.defs */; settings = {ATTRIBUTES = (Client, Server, ); }; }; E44EBE5512517EBE00645D88 /* init.c in Sources */ = {isa = PBXBuildFile; fileRef = E44EBE3B1251659900645D88 /* init.c */; }; @@ -129,6 +145,9 @@ E46DBC4D14EE10C80001F9F6 /* transform.c in Sources */ = {isa = PBXBuildFile; fileRef = C9C5F80D143C1771006DC718 /* transform.c */; }; E48AF55A16E70FD9004105FF /* io_private.h in Headers */ = {isa = PBXBuildFile; fileRef = E48AF55916E70FD9004105FF /* io_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; E48AF55B16E72D44004105FF /* io_private.h in Headers */ = {isa = PBXBuildFile; fileRef = E48AF55916E70FD9004105FF /* io_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; + E48EC97C1835BADD00EAC4F1 /* yield.h in Headers */ = {isa = PBXBuildFile; fileRef = E48EC97B1835BADD00EAC4F1 /* yield.h */; }; + E48EC97D1835BADD00EAC4F1 /* yield.h in Headers */ = {isa = PBXBuildFile; fileRef = E48EC97B1835BADD00EAC4F1 /* yield.h */; }; + E48EC97E1835BADD00EAC4F1 /* yield.h in Headers */ = {isa = PBXBuildFile; fileRef = E48EC97B1835BADD00EAC4F1 /* yield.h */; }; E49F2423125D3C960057C971 /* resolver.c in Sources */ = {isa = PBXBuildFile; fileRef = E44EBE371251656400645D88 /* resolver.c */; }; E49F2424125D3C970057C971 /* resolver.c in Sources */ = {isa = PBXBuildFile; fileRef = E44EBE371251656400645D88 /* resolver.c */; }; E49F2499125D48D80057C971 /* resolver.c in Sources */ = {isa = PBXBuildFile; fileRef = E44EBE371251656400645D88 /* resolver.c */; }; @@ -175,6 +194,8 @@ E4A2C9C5176019820000F809 /* atomic_llsc.h in Headers */ = {isa = PBXBuildFile; fileRef = E4A2C9C4176019760000F809 /* atomic_llsc.h */; }; E4A2C9C6176019830000F809 /* atomic_llsc.h in Headers */ = {isa = PBXBuildFile; fileRef = E4A2C9C4176019760000F809 /* atomic_llsc.h */; }; E4A2C9C7176019840000F809 /* atomic_llsc.h in Headers */ = {isa = PBXBuildFile; fileRef = E4A2C9C4176019760000F809 /* atomic_llsc.h */; }; + E4B3C3FE18C50D000039F49F /* voucher_activity_private.h in Headers */ = {isa = PBXBuildFile; fileRef = E4B3C3FD18C50D000039F49F /* voucher_activity_private.h */; }; + E4B3C3FF18C50D0E0039F49F /* voucher_activity_private.h in Headers */ = {isa = PBXBuildFile; fileRef = E4B3C3FD18C50D000039F49F /* voucher_activity_private.h */; }; E4B515BD164B2DA300E003AF /* provider.d in Sources */ = {isa = PBXBuildFile; fileRef = E43570B8126E93380097AB9F /* provider.d */; }; E4B515BE164B2DA300E003AF /* protocol.defs in Sources */ = {isa = PBXBuildFile; fileRef = FC7BED950E8361E600161930 /* protocol.defs */; settings = {ATTRIBUTES = (Client, Server, ); }; }; E4B515BF164B2DA300E003AF /* resolver.c in Sources */ = {isa = PBXBuildFile; fileRef = E44EBE371251656400645D88 /* resolver.c */; }; @@ -199,6 +220,8 @@ E4BA743C13A8911B0095BDF1 /* getprogname.h in Headers */ = {isa = PBXBuildFile; fileRef = E4BA743913A8911B0095BDF1 /* getprogname.h */; }; E4C1ED6F1263E714000D3C8B /* data_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = E4C1ED6E1263E714000D3C8B /* data_internal.h */; }; E4C1ED701263E714000D3C8B /* data_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = E4C1ED6E1263E714000D3C8B /* data_internal.h */; }; + E4D76A9318E325D200B1F98B /* block.h in Headers */ = {isa = PBXBuildFile; fileRef = E4D76A9218E325D200B1F98B /* block.h */; settings = {ATTRIBUTES = (Public, ); }; }; + E4D76A9418E325D200B1F98B /* block.h in Headers */ = {isa = PBXBuildFile; fileRef = E4D76A9218E325D200B1F98B /* block.h */; settings = {ATTRIBUTES = (Public, ); }; }; E4EB4A2714C35ECE00AA0FA9 /* object.h in Headers */ = {isa = PBXBuildFile; fileRef = E4EB4A2614C35ECE00AA0FA9 /* object.h */; }; E4EB4A2814C35ECE00AA0FA9 /* object.h in Headers */ = {isa = PBXBuildFile; fileRef = E4EB4A2614C35ECE00AA0FA9 /* object.h */; }; E4EC11AE12514302000DDBD1 /* queue.c in Sources */ = {isa = PBXBuildFile; fileRef = FC7BED8A0E8361E600161930 /* queue.c */; }; @@ -298,6 +321,7 @@ /* Begin PBXFileReference section */ 2BBF5A5F154B64D8002B20F9 /* allocator_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = allocator_internal.h; sourceTree = ""; }; 2BBF5A62154B64F5002B20F9 /* allocator.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = allocator.c; sourceTree = ""; }; + 2BE17C6318EA305E002CA4E8 /* layout_private.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = layout_private.h; sourceTree = ""; }; 5A0095A110F274B0000E2A31 /* io_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = io_internal.h; sourceTree = ""; }; 5A27262510F26F1900751FBC /* io.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; lineEnding = 0; path = io.c; sourceTree = ""; xcLanguageSpecificationIdentifier = xcode.lang.c; }; 5A5D13AB0F6B280500197CC3 /* semaphore_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = semaphore_internal.h; sourceTree = ""; }; @@ -346,7 +370,11 @@ E422DA3714D2AE11003C6EE4 /* libdispatch.unexport */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = libdispatch.unexport; sourceTree = ""; }; E43570B8126E93380097AB9F /* provider.d */ = {isa = PBXFileReference; explicitFileType = sourcecode.dtrace; fileEncoding = 4; path = provider.d; sourceTree = ""; }; E43D93F11097917E004F6A62 /* libdispatch.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = libdispatch.xcconfig; sourceTree = ""; }; + E44757D917F4572600B82CA1 /* inline_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = inline_internal.h; sourceTree = ""; }; E448727914C6215D00BB45C2 /* libdispatch.order */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = libdispatch.order; sourceTree = ""; }; + E44A8E6A1805C3E0009FFDB6 /* voucher.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = voucher.c; sourceTree = ""; }; + E44A8E711805C473009FFDB6 /* voucher_private.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = voucher_private.h; sourceTree = ""; }; + E44A8E7418066276009FFDB6 /* voucher_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = voucher_internal.h; sourceTree = ""; }; E44EBE331251654000645D88 /* resolver.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = resolver.h; sourceTree = ""; }; E44EBE371251656400645D88 /* resolver.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = resolver.c; sourceTree = ""; }; E44EBE3B1251659900645D88 /* init.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = init.c; sourceTree = ""; }; @@ -358,10 +386,14 @@ E47D6BB5125F0F800070D91C /* resolved.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = resolved.h; sourceTree = ""; }; E482F1CD12DBAB590030614D /* postprocess-headers.sh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; path = "postprocess-headers.sh"; sourceTree = ""; }; E48AF55916E70FD9004105FF /* io_private.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = io_private.h; path = private/io_private.h; sourceTree = SOURCE_ROOT; tabWidth = 8; }; + E48EC97B1835BADD00EAC4F1 /* yield.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = yield.h; sourceTree = ""; }; E49F24DF125D57FA0057C971 /* libdispatch.dylib */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.dylib"; includeInIndex = 0; path = libdispatch.dylib; sourceTree = BUILT_PRODUCTS_DIR; }; E49F251D125D630A0057C971 /* install-manpages.sh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; path = "install-manpages.sh"; sourceTree = ""; }; E49F251E125D631D0057C971 /* mig-headers.sh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; path = "mig-headers.sh"; sourceTree = ""; }; E4A2C9C4176019760000F809 /* atomic_llsc.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = atomic_llsc.h; sourceTree = ""; }; + E4B2D42D17A7F0F90034A18F /* libdispatch-resolver_iphoneos.order */ = {isa = PBXFileReference; lastKnownFileType = text; path = "libdispatch-resolver_iphoneos.order"; sourceTree = ""; }; + E4B2D42E17A7F0F90034A18F /* libdispatch_iphoneos.order */ = {isa = PBXFileReference; lastKnownFileType = text; path = libdispatch_iphoneos.order; sourceTree = ""; }; + E4B3C3FD18C50D000039F49F /* voucher_activity_private.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = voucher_activity_private.h; sourceTree = ""; }; E4B515D6164B2DA300E003AF /* libdispatch.dylib */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.dylib"; includeInIndex = 0; path = libdispatch.dylib; sourceTree = BUILT_PRODUCTS_DIR; }; E4B515D7164B2DFB00E003AF /* introspection_private.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = introspection_private.h; sourceTree = ""; }; E4B515D9164B2E9B00E003AF /* libdispatch-introspection.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = "libdispatch-introspection.xcconfig"; sourceTree = ""; }; @@ -372,6 +404,8 @@ E4BA743813A8900B0095BDF1 /* dispatch_read.3 */ = {isa = PBXFileReference; explicitFileType = text.man; path = dispatch_read.3; sourceTree = ""; }; E4BA743913A8911B0095BDF1 /* getprogname.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = getprogname.h; sourceTree = ""; }; E4C1ED6E1263E714000D3C8B /* data_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = data_internal.h; sourceTree = ""; }; + E4D76A9218E325D200B1F98B /* block.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = block.h; sourceTree = ""; }; + E4DC8D45191053EE0005C6F4 /* libdispatch_objc.aliases */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = libdispatch_objc.aliases; sourceTree = ""; }; E4EB4A2614C35ECE00AA0FA9 /* object.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = object.h; sourceTree = ""; }; E4EB4A2A14C36F4E00AA0FA9 /* install-headers.sh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; path = "install-headers.sh"; sourceTree = ""; }; E4EC11C312514302000DDBD1 /* libdispatch_up.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libdispatch_up.a; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -455,6 +489,7 @@ 96A8AA860F41E7A400CD570B /* source.c */, 96032E4A0F5CC8C700241C5F /* time.c */, C9C5F80D143C1771006DC718 /* transform.c */, + E44A8E6A1805C3E0009FFDB6 /* voucher.c */, FC7BED950E8361E600161930 /* protocol.defs */, E43570B8126E93380097AB9F /* provider.d */, ); @@ -517,8 +552,11 @@ E46DBC5814EE11BC0001F9F6 /* libdispatch-static.xcconfig */, E4B515D9164B2E9B00E003AF /* libdispatch-introspection.xcconfig */, E422DA3614D2A7E7003C6EE4 /* libdispatch.aliases */, + E4DC8D45191053EE0005C6F4 /* libdispatch_objc.aliases */, E416F53F175D04B800B23711 /* libdispatch_macosx.aliases */, E448727914C6215D00BB45C2 /* libdispatch.order */, + E4B2D42E17A7F0F90034A18F /* libdispatch_iphoneos.order */, + E4B2D42D17A7F0F90034A18F /* libdispatch-resolver_iphoneos.order */, E422DA3714D2AE11003C6EE4 /* libdispatch.unexport */, E421E5FD1716BEA70090DC9B /* libdispatch.interposable */, ); @@ -588,6 +626,7 @@ FC1832A2109923C7003403D5 /* perfmon.h */, FC1832A3109923C7003403D5 /* time.h */, FC1832A4109923C7003403D5 /* tsd.h */, + E48EC97B1835BADD00EAC4F1 /* yield.h */, ); path = shims; sourceTree = ""; @@ -596,6 +635,7 @@ isa = PBXGroup; children = ( 72CC942F0ECCD8750031B751 /* base.h */, + E4D76A9218E325D200B1F98B /* block.h */, 5AAB45C510D30D0C004407EA /* data.h */, FC7BED960E8361E600161930 /* dispatch.h */, FC5C9C1D0EADABE3006E462D /* group.h */, @@ -621,8 +661,11 @@ 96BC39BC0F3EBAB100C59689 /* queue_private.h */, FCEF047F0F5661960067401F /* source_private.h */, E4ECBAA415253C25002C313C /* mach_private.h */, + E44A8E711805C473009FFDB6 /* voucher_private.h */, + E4B3C3FD18C50D000039F49F /* voucher_activity_private.h */, 961B99350F3E83980006BC96 /* benchmark.h */, E4B515D7164B2DFB00E003AF /* introspection_private.h */, + 2BE17C6318EA305E002CA4E8 /* layout_private.h */, ); name = "Private Headers"; path = private; @@ -633,12 +676,14 @@ children = ( 2BBF5A5F154B64D8002B20F9 /* allocator_internal.h */, FC7BED8F0E8361E600161930 /* internal.h */, + E44757D917F4572600B82CA1 /* inline_internal.h */, E4C1ED6E1263E714000D3C8B /* data_internal.h */, 5A0095A110F274B0000E2A31 /* io_internal.h */, 965ECC200F3EAB71004DDD89 /* object_internal.h */, 96929D950F3EA2170041FF5D /* queue_internal.h */, 5A5D13AB0F6B280500197CC3 /* semaphore_internal.h */, FC0B34780FA2851C0080FFA0 /* source_internal.h */, + E44A8E7418066276009FFDB6 /* voucher_internal.h */, E422A0D412A557B5005E5BDB /* trace.h */, E44F9DA816543F79001DCD38 /* introspection_internal.h */, 96929D830F3EA1020041FF5D /* shims.h */, @@ -658,16 +703,20 @@ FC7BEDA50E8361E600161930 /* dispatch.h in Headers */, 72CC94300ECCD8750031B751 /* base.h in Headers */, 961B99500F3E85C30006BC96 /* object.h in Headers */, + E44757DA17F4572600B82CA1 /* inline_internal.h in Headers */, FC7BED9A0E8361E600161930 /* queue.h in Headers */, FC7BED9C0E8361E600161930 /* source.h in Headers */, + E4B3C3FE18C50D000039F49F /* voucher_activity_private.h in Headers */, 721F5C5D0F15520500FF03A6 /* semaphore.h in Headers */, FC5C9C1E0EADABE3006E462D /* group.h in Headers */, 96C9553B0F3EAEDD000D2CA4 /* once.h in Headers */, 5AAB45C410D30CC7004407EA /* io.h in Headers */, + E44A8E7518066276009FFDB6 /* voucher_internal.h in Headers */, E4630253176162D400E11F4C /* atomic_sfb.h in Headers */, 5AAB45C610D30D0C004407EA /* data.h in Headers */, 96032E4D0F5CC8D100241C5F /* time.h in Headers */, FC7BEDA20E8361E600161930 /* private.h in Headers */, + E4D76A9318E325D200B1F98B /* block.h in Headers */, E4A2C9C7176019840000F809 /* atomic_llsc.h in Headers */, C913AC0F143BD34800B78976 /* data_private.h in Headers */, 96BC39BD0F3EBAB100C59689 /* queue_private.h in Headers */, @@ -679,11 +728,14 @@ FC0B34790FA2851C0080FFA0 /* source_internal.h in Headers */, 5A5D13AC0F6B280500197CC3 /* semaphore_internal.h in Headers */, E4C1ED6F1263E714000D3C8B /* data_internal.h in Headers */, + E44A8E721805C473009FFDB6 /* voucher_private.h in Headers */, 5A0095A210F274B0000E2A31 /* io_internal.h in Headers */, FC1832A8109923C7003403D5 /* tsd.h in Headers */, 96929D840F3EA1020041FF5D /* atomic.h in Headers */, 96929D850F3EA1020041FF5D /* shims.h in Headers */, FC1832A7109923C7003403D5 /* time.h in Headers */, + E48EC97C1835BADD00EAC4F1 /* yield.h in Headers */, + 2BE17C6418EA305E002CA4E8 /* layout_private.h in Headers */, FC1832A6109923C7003403D5 /* perfmon.h in Headers */, FC9C70E8105EC9620074F9CA /* config.h in Headers */, E422A0D512A557B5005E5BDB /* trace.h in Headers */, @@ -705,16 +757,20 @@ E49F24AB125D57FA0057C971 /* dispatch.h in Headers */, E49F24AC125D57FA0057C971 /* base.h in Headers */, E49F24AD125D57FA0057C971 /* object.h in Headers */, + E44757DC17F4573600B82CA1 /* inline_internal.h in Headers */, E49F24AE125D57FA0057C971 /* queue.h in Headers */, E49F24AF125D57FA0057C971 /* source.h in Headers */, + E4B3C3FF18C50D0E0039F49F /* voucher_activity_private.h in Headers */, E49F24B0125D57FA0057C971 /* semaphore.h in Headers */, E49F24B1125D57FA0057C971 /* group.h in Headers */, E49F24B2125D57FA0057C971 /* once.h in Headers */, E49F24B3125D57FA0057C971 /* io.h in Headers */, + E44A8E7618066276009FFDB6 /* voucher_internal.h in Headers */, E4630252176162D300E11F4C /* atomic_sfb.h in Headers */, E49F24B4125D57FA0057C971 /* data.h in Headers */, E49F24B5125D57FA0057C971 /* time.h in Headers */, E49F24B6125D57FA0057C971 /* private.h in Headers */, + E4D76A9418E325D200B1F98B /* block.h in Headers */, E4A2C9C6176019830000F809 /* atomic_llsc.h in Headers */, E49F24B7125D57FA0057C971 /* queue_private.h in Headers */, E49F24B8125D57FA0057C971 /* source_private.h in Headers */, @@ -726,11 +782,14 @@ E49F24BD125D57FA0057C971 /* semaphore_internal.h in Headers */, E4C1ED701263E714000D3C8B /* data_internal.h in Headers */, E49F24BF125D57FA0057C971 /* io_internal.h in Headers */, + E44A8E731805C473009FFDB6 /* voucher_private.h in Headers */, E49F24C1125D57FA0057C971 /* tsd.h in Headers */, E49F24C2125D57FA0057C971 /* atomic.h in Headers */, E49F24C3125D57FA0057C971 /* shims.h in Headers */, E49F24C4125D57FA0057C971 /* time.h in Headers */, E49F24C5125D57FA0057C971 /* perfmon.h in Headers */, + E48EC97D1835BADD00EAC4F1 /* yield.h in Headers */, + 2BE17C6518EA305E002CA4E8 /* layout_private.h in Headers */, E49F24C6125D57FA0057C971 /* config.h in Headers */, E422A0D612A557B5005E5BDB /* trace.h in Headers */, E4BA743C13A8911B0095BDF1 /* getprogname.h in Headers */, @@ -761,12 +820,15 @@ E4630251176162D200E11F4C /* atomic_sfb.h in Headers */, E44F9DBE1654405B001DCD38 /* tsd.h in Headers */, E44F9DB816544053001DCD38 /* atomic.h in Headers */, + E44757DB17F4573500B82CA1 /* inline_internal.h in Headers */, E44F9DB71654404F001DCD38 /* shims.h in Headers */, E44F9DBC1654405B001DCD38 /* perfmon.h in Headers */, E44F9DBF165440EF001DCD38 /* config.h in Headers */, + E44A8E7718066276009FFDB6 /* voucher_internal.h in Headers */, E4A2C9C5176019820000F809 /* atomic_llsc.h in Headers */, E44F9DB616544043001DCD38 /* trace.h in Headers */, E44F9DB916544056001DCD38 /* getprogname.h in Headers */, + E48EC97E1835BADD00EAC4F1 /* yield.h in Headers */, E44F9DBA1654405B001DCD38 /* hw_config.h in Headers */, E44F9DC116544115001DCD38 /* object_private.h in Headers */, E44F9DC016544115001DCD38 /* object.h in Headers */, @@ -895,7 +957,7 @@ isa = PBXProject; attributes = { BuildIndependentTargetsInParallel = YES; - LastUpgradeCheck = 0500; + LastUpgradeCheck = 0600; }; buildConfigurationList = 1DEB91EF08733DB70010E9CD /* Build configuration list for PBXProject "libdispatch" */; compatibilityVersion = "Xcode 3.2"; @@ -1031,11 +1093,15 @@ "$(SRCROOT)/xcodescripts/install-headers.sh", "$(SRCROOT)/os/object.h", "$(SRCROOT)/os/object_private.h", + "$(SRCROOT)/private/voucher_private.h", + "$(SRCROOT)/private/voucher_activity_private.h", ); name = "Install Headers"; outputPaths = ( "$(CONFIGURATION_BUILD_DIR)/$(OS_PUBLIC_HEADERS_FOLDER_PATH)/object.h", "$(CONFIGURATION_BUILD_DIR)/$(OS_PRIVATE_HEADERS_FOLDER_PATH)/object_private.h", + "$(CONFIGURATION_BUILD_DIR)/$(OS_PRIVATE_HEADERS_FOLDER_PATH)/voucher_private.h", + "$(CONFIGURATION_BUILD_DIR)/$(OS_PRIVATE_HEADERS_FOLDER_PATH)/voucher_activity_private.h", ); runOnlyForDeploymentPostprocessing = 0; shellPath = "/bin/bash -e"; @@ -1051,11 +1117,15 @@ "$(SRCROOT)/xcodescripts/install-headers.sh", "$(SRCROOT)/os/object.h", "$(SRCROOT)/os/object_private.h", + "$(SRCROOT)/private/voucher_private.h", + "$(SRCROOT)/private/voucher_activity_private.h", ); name = "Install Headers"; outputPaths = ( "$(CONFIGURATION_BUILD_DIR)/$(OS_PUBLIC_HEADERS_FOLDER_PATH)/object.h", "$(CONFIGURATION_BUILD_DIR)/$(OS_PRIVATE_HEADERS_FOLDER_PATH)/object_private.h", + "$(CONFIGURATION_BUILD_DIR)/$(OS_PRIVATE_HEADERS_FOLDER_PATH)/voucher_private.h", + "$(CONFIGURATION_BUILD_DIR)/$(OS_PRIVATE_HEADERS_FOLDER_PATH)/voucher_activity_private.h", ); runOnlyForDeploymentPostprocessing = 0; shellPath = "/bin/bash -e"; @@ -1157,6 +1227,7 @@ E4FC3264145F46C9002FBDDB /* object.m in Sources */, 2BBF5A63154B64F5002B20F9 /* allocator.c in Sources */, E420867016027AE500EEE210 /* data.m in Sources */, + E44A8E6B1805C3E0009FFDB6 /* voucher.c in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1170,6 +1241,7 @@ E46DBC4314EE10C80001F9F6 /* queue.c in Sources */, E46DBC4414EE10C80001F9F6 /* semaphore.c in Sources */, E46DBC4514EE10C80001F9F6 /* once.c in Sources */, + E44A8E701805C3E0009FFDB6 /* voucher.c in Sources */, E46DBC4614EE10C80001F9F6 /* apply.c in Sources */, E46DBC4714EE10C80001F9F6 /* object.c in Sources */, E46DBC4814EE10C80001F9F6 /* benchmark.c in Sources */, @@ -1204,6 +1276,7 @@ E4FC3265145F46C9002FBDDB /* object.m in Sources */, 2BBF5A64154B64F5002B20F9 /* allocator.c in Sources */, E420867116027AE500EEE210 /* data.m in Sources */, + E44A8E6C1805C3E0009FFDB6 /* voucher.c in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1225,6 +1298,7 @@ E4B515C8164B2DA300E003AF /* time.c in Sources */, E4B515C9164B2DA300E003AF /* data.c in Sources */, E4B515CA164B2DA300E003AF /* io.c in Sources */, + E44A8E6F1805C3E0009FFDB6 /* voucher.c in Sources */, E4B515CB164B2DA300E003AF /* transform.c in Sources */, E4B515CC164B2DA300E003AF /* object.m in Sources */, E4B515CD164B2DA300E003AF /* allocator.c in Sources */, @@ -1255,6 +1329,7 @@ E4FC3266145F46C9002FBDDB /* object.m in Sources */, 2BBF5A65154B64F5002B20F9 /* allocator.c in Sources */, E420867316027AE500EEE210 /* data.m in Sources */, + E44A8E6E1805C3E0009FFDB6 /* voucher.c in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1280,6 +1355,7 @@ E4FC3267145F46C9002FBDDB /* object.m in Sources */, 2BBF5A66154B64F5002B20F9 /* allocator.c in Sources */, E420867216027AE500EEE210 /* data.m in Sources */, + E44A8E6D1805C3E0009FFDB6 /* voucher.c in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/os/object.h b/os/object.h index f8d23a3..944d331 100644 --- a/os/object.h +++ b/os/object.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2014 Apple Inc. All rights reserved. * * @APPLE_APACHE_LICENSE_HEADER_START@ * @@ -24,6 +24,7 @@ #ifdef __APPLE__ #include #endif +#include /*! * @header @@ -85,17 +86,26 @@ #else #define OS_OBJECT_RETURNS_RETAINED #endif +#if __has_attribute(ns_consumed) +#define OS_OBJECT_CONSUMED __attribute__((__ns_consumed__)) +#else +#define OS_OBJECT_CONSUMED +#endif #else #define OS_OBJECT_RETURNS_RETAINED +#define OS_OBJECT_CONSUMED #endif #if defined(__has_feature) #if __has_feature(objc_arc) #define OS_OBJECT_BRIDGE __bridge +#define OS_WARN_RESULT_NEEDS_RELEASE #else #define OS_OBJECT_BRIDGE +#define OS_WARN_RESULT_NEEDS_RELEASE OS_WARN_RESULT #endif #else #define OS_OBJECT_BRIDGE +#define OS_WARN_RESULT_NEEDS_RELEASE OS_WARN_RESULT #endif #ifndef OS_OBJECT_USE_OBJC_RETAIN_RELEASE #if defined(__clang_analyzer__) @@ -114,8 +124,65 @@ /*! @parseOnly */ #define OS_OBJECT_RETURNS_RETAINED /*! @parseOnly */ +#define OS_OBJECT_CONSUMED +/*! @parseOnly */ #define OS_OBJECT_BRIDGE +/*! @parseOnly */ +#define OS_WARN_RESULT_NEEDS_RELEASE OS_WARN_RESULT #define OS_OBJECT_USE_OBJC_RETAIN_RELEASE 0 #endif +#define OS_OBJECT_GLOBAL_OBJECT(type, object) ((OS_OBJECT_BRIDGE type)&(object)) + +__BEGIN_DECLS + +/*! + * @function os_retain + * + * @abstract + * Increment the reference count of an os_object. + * + * @discussion + * On a platform with the modern Objective-C runtime this is exactly equivalent + * to sending the object the -[retain] message. + * + * @param object + * The object to retain. + * + * @result + * The retained object. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_EXPORT +void* +os_retain(void *object); +#if OS_OBJECT_USE_OBJC +#undef os_retain +#define os_retain(object) [object retain] +#endif + +/*! + * @function os_release + * + * @abstract + * Decrement the reference count of a os_object. + * + * @discussion + * On a platform with the modern Objective-C runtime this is exactly equivalent + * to sending the object the -[release] message. + * + * @param object + * The object to release. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_EXPORT +void +os_release(void *object); +#if OS_OBJECT_USE_OBJC +#undef os_release +#define os_release(object) [object release] +#endif + +__END_DECLS + #endif diff --git a/private/data_private.h b/private/data_private.h index df60d28..751b7ce 100644 --- a/private/data_private.h +++ b/private/data_private.h @@ -164,7 +164,7 @@ dispatch_data_apply_f(dispatch_data_t data, void *context, __OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_6_0) DISPATCH_EXPORT DISPATCH_NONNULL_ALL DISPATCH_NOTHROW mach_port_t -dispatch_data_make_memory_entry(dispatch_data_t dd); +dispatch_data_make_memory_entry(dispatch_data_t data); #endif /*! diff --git a/private/introspection_private.h b/private/introspection_private.h index dd18bee..7ac0e7e 100644 --- a/private/introspection_private.h +++ b/private/introspection_private.h @@ -800,26 +800,6 @@ extern void dispatch_introspection_hook_callout_queue_item_complete( dispatch_continuation_t object); -/*! - * @function dispatch_introspection_hook_queue_item_complete - * - * @abstract - * Interposable hook function called when an item previously dequeued from a - * dispatch queue has completed processing. - * - * @discussion - * The object pointer value passed to this function must be treated as a value - * only. It is intended solely for matching up with an earlier call to a - * dequeue hook function and must NOT be dereferenced. - * - * @param item - * Opaque dentifier for completed item. Must NOT be dereferenced. - */ - -DISPATCH_EXPORT -void -dispatch_introspection_hook_queue_item_complete(dispatch_object_t item); - __END_DECLS #endif diff --git a/private/io_private.h b/private/io_private.h index c35b41f..4a00ee0 100644 --- a/private/io_private.h +++ b/private/io_private.h @@ -72,9 +72,9 @@ __BEGIN_DECLS * the handler function. * @param handler The handler to enqueue when data is ready to be * delivered. - * @param context Application-defined context parameter. - * @param data The data read from the file descriptor. - * @param error An errno condition for the read operation or + * param context Application-defined context parameter. + * param data The data read from the file descriptor. + * param error An errno condition for the read operation or * zero if the read was successful. */ __OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) @@ -113,10 +113,10 @@ dispatch_read_f(dispatch_fd_t fd, * @param context The application-defined context parameter to pass to * the handler function. * @param handler The handler to enqueue when the data has been written. - * @param context Application-defined context parameter. - * @param data The data that could not be written to the I/O + * param context Application-defined context parameter. + * param data The data that could not be written to the I/O * channel, or NULL. - * @param error An errno condition for the write operation or + * param error An errno condition for the write operation or * zero if the write was successful. */ __OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) @@ -151,8 +151,8 @@ dispatch_write_f(dispatch_fd_t fd, * the cleanup handler function. * @param cleanup_handler The handler to enqueue when the system * relinquishes control over the file descriptor. - * @param context Application-defined context parameter. - * @param error An errno condition if control is relinquished + * param context Application-defined context parameter. + * param error An errno condition if control is relinquished * because channel creation failed, zero otherwise. * @result The newly created dispatch I/O channel or NULL if an error * occurred (invalid type specified). @@ -190,8 +190,8 @@ dispatch_io_create_f(dispatch_io_type_t type, * the cleanup handler function. * @param cleanup_handler The handler to enqueue when the system * has closed the file at path. - * @param context Application-defined context parameter. - * @param error An errno condition if control is relinquished + * param context Application-defined context parameter. + * param error An errno condition if control is relinquished * because channel creation or opening of the * specified file failed, zero otherwise. * @result The newly created dispatch I/O channel or NULL if an error @@ -235,8 +235,8 @@ dispatch_io_create_with_path_f(dispatch_io_type_t type, * relinquishes control over the file descriptor * (resp. closes the file at path) associated with * the existing channel. - * @param context Application-defined context parameter. - * @param error An errno condition if control is relinquished + * param context Application-defined context parameter. + * param error An errno condition if control is relinquished * because channel creation failed, zero otherwise. * @result The newly created dispatch I/O channel or NULL if an error * occurred (invalid type specified). @@ -301,11 +301,11 @@ typedef void (*dispatch_io_handler_function_t)(void *context, bool done, * the handler function. * @param io_handler The I/O handler to enqueue when data is ready to be * delivered. - * @param context Application-defined context parameter. - * @param done A flag indicating whether the operation is complete. - * @param data An object with the data most recently read from the + * param context Application-defined context parameter. + * param done A flag indicating whether the operation is complete. + * param data An object with the data most recently read from the * I/O channel as part of this read operation, or NULL. - * @param error An errno condition for the read operation or zero if + * param error An errno condition for the read operation or zero if * the read was successful. */ __OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) @@ -357,12 +357,12 @@ dispatch_io_read_f(dispatch_io_t channel, * @param context The application-defined context parameter to pass to * the handler function. * @param io_handler The I/O handler to enqueue when data has been delivered. - * @param context Application-defined context parameter. - * @param done A flag indicating whether the operation is complete. - * @param data An object of the data remaining to be + * param context Application-defined context parameter. + * param done A flag indicating whether the operation is complete. + * param data An object of the data remaining to be * written to the I/O channel as part of this write * operation, or NULL. - * @param error An errno condition for the write operation or zero + * param error An errno condition for the write operation or zero * if the write was successful. */ __OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) diff --git a/private/layout_private.h b/private/layout_private.h new file mode 100644 index 0000000..17e8ed8 --- /dev/null +++ b/private/layout_private.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2014 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +#ifndef __DISPATCH_LAYOUT_PRIVATE__ +#define __DISPATCH_LAYOUT_PRIVATE__ + +#ifndef __DISPATCH_INDIRECT__ +#error "Please #include instead of this file directly." +#include // for HeaderDoc +#endif + +__BEGIN_DECLS + +#if !TARGET_OS_WIN32 +__OSX_AVAILABLE_STARTING(__MAC_10_6,__IPHONE_4_0) +DISPATCH_EXPORT const struct dispatch_queue_offsets_s { + // always add new fields at the end + const uint16_t dqo_version; + const uint16_t dqo_label; + const uint16_t dqo_label_size; + const uint16_t dqo_flags; + const uint16_t dqo_flags_size; + const uint16_t dqo_serialnum; + const uint16_t dqo_serialnum_size; + const uint16_t dqo_width; + const uint16_t dqo_width_size; + const uint16_t dqo_running; + const uint16_t dqo_running_size; + // fields added in dqo_version 5: + const uint16_t dqo_suspend_cnt; + const uint16_t dqo_suspend_cnt_size; + const uint16_t dqo_target_queue; + const uint16_t dqo_target_queue_size; + const uint16_t dqo_priority; + const uint16_t dqo_priority_size; +} dispatch_queue_offsets; +#endif + +#if DISPATCH_LAYOUT_SPI + +/*! + * @group Data Structure Layout SPI + * SPI intended for CoreSymbolication only + */ + +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +DISPATCH_EXPORT const struct dispatch_tsd_indexes_s { + // always add new fields at the end + const uint16_t dti_version; + const uint16_t dti_queue_index; + const uint16_t dti_voucher_index; + const uint16_t dti_qos_class_index; +} dispatch_tsd_indexes; + +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +DISPATCH_EXPORT const struct voucher_offsets_s { + // always add new fields at the end + const uint16_t vo_version; + const uint16_t vo_activity_ids_count; + const uint16_t vo_activity_ids_count_size; + const uint16_t vo_activity_ids_array; + const uint16_t vo_activity_ids_array_entry_size; +} voucher_offsets; + +#endif // DISPATCH_LAYOUT_SPI + +__END_DECLS + +#endif // __DISPATCH_LAYOUT_PRIVATE__ diff --git a/private/mach_private.h b/private/mach_private.h index 603330d..93c1e81 100644 --- a/private/mach_private.h +++ b/private/mach_private.h @@ -119,8 +119,7 @@ typedef mach_msg_context_trailer_t dispatch_mach_trailer_t; */ #define DISPATCH_MACH_RECEIVE_MAX_INLINE_MESSAGE_SIZE \ - ((PAGE_SIZE > 0x1000 ? 1 : 3) * PAGE_SIZE - \ - sizeof(dispatch_mach_trailer_t)) + (0x4000 - sizeof(dispatch_mach_trailer_t)) /*! * @typedef dispatch_mach_msg_t diff --git a/private/private.h b/private/private.h index 4e32e73..8fd5abc 100644 --- a/private/private.h +++ b/private/private.h @@ -57,13 +57,14 @@ #include #include #include +#include #undef __DISPATCH_INDIRECT__ #endif /* !__DISPATCH_BUILDING_DISPATCH__ */ // Check that public and private dispatch headers match -#if DISPATCH_API_VERSION != 20130520 // Keep in sync with +#if DISPATCH_API_VERSION != 20140804 // Keep in sync with #error "Dispatch header mismatch between /usr/include and /usr/local/include" #endif diff --git a/private/queue_private.h b/private/queue_private.h index dfef785..e813036 100644 --- a/private/queue_private.h +++ b/private/queue_private.h @@ -47,6 +47,30 @@ enum { #define DISPATCH_QUEUE_FLAGS_MASK (DISPATCH_QUEUE_OVERCOMMIT) +/*! + * @function dispatch_queue_attr_make_with_overcommit + * + * @discussion + * Returns a dispatch queue attribute value with the overcommit flag set to the + * specified value. + * + * @param attr + * A queue attribute value to be combined with the overcommit flag, or NULL. + * + * @param overcommit + * Boolean overcommit flag. + * + * @return + * Returns an attribute value which may be provided to dispatch_queue_create(). + * This new value combines the attributes specified by the 'attr' parameter and + * the overcommit flag. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) +DISPATCH_EXPORT DISPATCH_WARN_RESULT DISPATCH_PURE DISPATCH_NOTHROW +dispatch_queue_attr_t +dispatch_queue_attr_make_with_overcommit(dispatch_queue_attr_t attr, + bool overcommit); + /*! * @typedef dispatch_queue_priority_t * @@ -88,10 +112,11 @@ enum { #define DISPATCH_QUEUE_WIDTH_MAX_PHYSICAL_CPUS -2 #define DISPATCH_QUEUE_WIDTH_MAX_LOGICAL_CPUS -3 -__OSX_AVAILABLE_STARTING(__MAC_10_6,__IPHONE_4_0) +__OSX_AVAILABLE_BUT_DEPRECATED_MSG(__MAC_10_6,__MAC_10_10,__IPHONE_4_0,__IPHONE_8_0, \ + "Use dispatch_queue_create(name, DISPATCH_QUEUE_CONCURRENT) instead") DISPATCH_EXPORT DISPATCH_NONNULL_ALL DISPATCH_NOTHROW void -dispatch_queue_set_width(dispatch_queue_t dq, long width); // DEPRECATED +dispatch_queue_set_width(dispatch_queue_t dq, long width); /*! * @function dispatch_queue_create_with_target @@ -119,7 +144,8 @@ dispatch_queue_set_width(dispatch_queue_t dq, long width); // DEPRECATED * This parameter is optional and may be NULL. * * @param attr - * DISPATCH_QUEUE_SERIAL or DISPATCH_QUEUE_CONCURRENT. + * DISPATCH_QUEUE_SERIAL, DISPATCH_QUEUE_CONCURRENT, or the result of a call to + * the function dispatch_queue_attr_make_with_qos_class(). * * @param target * The target queue for the newly created queue. The target queue is retained. @@ -174,8 +200,8 @@ dispatch_queue_create_with_target(const char *label, * This parameter is optional and may be NULL. * * @param flags - * Reserved for future use. Passing any value other than zero may result in - * a NULL return value. + * Pass flags value returned by dispatch_pthread_root_queue_flags_pool_size() + * or 0 if unused. * * @param attr * Attributes passed to pthread_create(3) when creating worker pthreads. This @@ -197,6 +223,33 @@ DISPATCH_NOTHROW dispatch_queue_t dispatch_pthread_root_queue_create(const char *label, unsigned long flags, const pthread_attr_t *attr, dispatch_block_t configure); + +/*! + * @function dispatch_pthread_root_queue_flags_pool_size + * + * @abstract + * Returns flags argument to pass to dispatch_pthread_root_queue_create() to + * specify the maximum size of the pthread pool to use for a pthread root queue. + * + * @param pool_size + * Maximum size of the pthread pool to use for the root queue. The number of + * pthreads created for this root queue will never exceed this number but there + * is no guarantee that the specified number will be reached. + * Pass 0 to specify that a default pool size determined by the system should + * be used. + * + * @result + * The flags argument to pass to dispatch_pthread_root_queue_create(). + */ +DISPATCH_INLINE DISPATCH_ALWAYS_INLINE +unsigned long +dispatch_pthread_root_queue_flags_pool_size(uint8_t pool_size) +{ + #define _DISPATCH_PTHREAD_ROOT_QUEUE_FLAG_POOL_SIZE (0x80000000ul) + return (_DISPATCH_PTHREAD_ROOT_QUEUE_FLAG_POOL_SIZE | + (unsigned long)pool_size); +} + #endif /* __BLOCKS__ */ /*! @@ -209,24 +262,6 @@ dispatch_pthread_root_queue_create(const char *label, unsigned long flags, */ #define DISPATCH_APPLY_CURRENT_ROOT_QUEUE NULL -#if !TARGET_OS_WIN32 -__OSX_AVAILABLE_STARTING(__MAC_10_6,__IPHONE_4_0) -DISPATCH_EXPORT const struct dispatch_queue_offsets_s { - // always add new fields at the end - const uint16_t dqo_version; - const uint16_t dqo_label; - const uint16_t dqo_label_size; - const uint16_t dqo_flags; - const uint16_t dqo_flags_size; - const uint16_t dqo_serialnum; - const uint16_t dqo_serialnum_size; - const uint16_t dqo_width; - const uint16_t dqo_width_size; - const uint16_t dqo_running; - const uint16_t dqo_running_size; -} dispatch_queue_offsets; -#endif - /*! * @function dispatch_assert_queue * diff --git a/private/source_private.h b/private/source_private.h index 0f44e27..c4ce1d4 100644 --- a/private/source_private.h +++ b/private/source_private.h @@ -82,9 +82,11 @@ DISPATCH_EXPORT const struct dispatch_source_type_s _dispatch_source_type_vfs; * @const DISPATCH_SOURCE_TYPE_VM * @discussion A dispatch source that monitors virtual memory * The mask is a mask of desired events from dispatch_source_vm_flags_t. + * This type is deprecated, use DISPATCH_SOURCE_TYPE_MEMORYSTATUS instead. */ #define DISPATCH_SOURCE_TYPE_VM (&_dispatch_source_type_vm) -__OSX_AVAILABLE_STARTING(__MAC_10_7,__IPHONE_4_3) +__OSX_AVAILABLE_BUT_DEPRECATED_MSG(__MAC_10_7, __MAC_10_10, __IPHONE_4_3, + __IPHONE_8_0, "Use DISPATCH_SOURCE_TYPE_MEMORYSTATUS instead") DISPATCH_EXPORT const struct dispatch_source_type_s _dispatch_source_type_vm; /*! @@ -262,7 +264,9 @@ enum { */ enum { - DISPATCH_VM_PRESSURE = 0x80000000, + DISPATCH_VM_PRESSURE __OSX_AVAILABLE_BUT_DEPRECATED_MSG( + __MAC_10_7, __MAC_10_10, __IPHONE_4_3, __IPHONE_8_0, + "Use DISPATCH_MEMORYSTATUS_PRESSURE_WARN instead") = 0x80000000, }; /*! @@ -274,20 +278,22 @@ enum { * The system's memory pressure state has changed to warning. * @constant DISPATCH_MEMORYSTATUS_PRESSURE_CRITICAL * The system's memory pressure state has changed to critical. + * @constant DISPATCH_MEMORYSTATUS_LOW_SWAP + * The system's memory pressure state has entered the "low swap" condition. + * Restricted to the root user. */ enum { - DISPATCH_MEMORYSTATUS_PRESSURE_NORMAL = 0x01, - DISPATCH_MEMORYSTATUS_PRESSURE_WARN = 0x02, -#if !TARGET_OS_EMBEDDED - DISPATCH_MEMORYSTATUS_PRESSURE_CRITICAL = 0x04, -#endif + DISPATCH_MEMORYSTATUS_PRESSURE_NORMAL + __OSX_AVAILABLE_STARTING(__MAC_10_9, __IPHONE_6_0) = 0x01, + DISPATCH_MEMORYSTATUS_PRESSURE_WARN + __OSX_AVAILABLE_STARTING(__MAC_10_9, __IPHONE_6_0) = 0x02, + DISPATCH_MEMORYSTATUS_PRESSURE_CRITICAL + __OSX_AVAILABLE_STARTING(__MAC_10_9, __IPHONE_8_0) = 0x04, + DISPATCH_MEMORYSTATUS_LOW_SWAP + __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) = 0x08, }; -#if TARGET_IPHONE_SIMULATOR // rdar://problem/9219483 -#define DISPATCH_VM_PRESSURE DISPATCH_VNODE_ATTRIB -#endif - __BEGIN_DECLS /*! diff --git a/private/voucher_activity_private.h b/private/voucher_activity_private.h new file mode 100644 index 0000000..c02b236 --- /dev/null +++ b/private/voucher_activity_private.h @@ -0,0 +1,544 @@ +/* + * Copyright (c) 2013-2014 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +#ifndef __OS_VOUCHER_ACTIVITY_PRIVATE__ +#define __OS_VOUCHER_ACTIVITY_PRIVATE__ + +#include +#include +#if !defined(__DISPATCH_BUILDING_DISPATCH__) +#include +#endif + +#define OS_VOUCHER_ACTIVITY_SPI_VERSION 20140708 + +#if OS_VOUCHER_WEAK_IMPORT +#define OS_VOUCHER_EXPORT OS_EXPORT OS_WEAK_IMPORT +#else +#define OS_VOUCHER_EXPORT OS_EXPORT +#endif + +__BEGIN_DECLS + +#if OS_VOUCHER_ACTIVITY_SPI + +/*! + * @group Voucher Activity SPI + * SPI intended for libtrace only + */ + +/*! + * @typedef voucher_activity_id_t + * + * @abstract + * Opaque activity identifier. + * + * @discussion + * Scalar value type, not reference counted. + */ +typedef uint64_t voucher_activity_id_t; + +/*! + * @enum voucher_activity_tracepoint_type_t + * + * @abstract + * Types of tracepoints. + */ +OS_ENUM(voucher_activity_tracepoint_type, uint8_t, + voucher_activity_tracepoint_type_release = (1u << 0), + voucher_activity_tracepoint_type_debug = (1u << 1), + voucher_activity_tracepoint_type_error = (1u << 6) | (1u << 0), + voucher_activity_tracepoint_type_fault = (1u << 7) | (1u << 6) | (1u << 0), +); + +/*! + * @enum voucher_activity_flag_t + * + * @abstract + * Flags to pass to voucher_activity_start/voucher_activity_start_with_location + */ +OS_ENUM(voucher_activity_flag, unsigned long, + voucher_activity_flag_default = 0, + voucher_activity_flag_force = 0x1, +); + +/*! + * @typedef voucher_activity_trace_id_t + * + * @abstract + * Opaque tracepoint identifier. + */ +typedef uint64_t voucher_activity_trace_id_t; +static const uint8_t _voucher_activity_trace_id_type_shift = 40; +static const uint8_t _voucher_activity_trace_id_code_namespace_shift = 32; + +/*! + * @function voucher_activity_trace_id + * + * @abstract + * Return tracepoint identifier for specified arguments. + * + * @param type + * Tracepoint type from voucher_activity_tracepoint_type_t. + * + * @param code_namespace + * Namespace of 'code' argument. + * + * @param code + * Tracepoint code. + * + * @result + * Tracepoint identifier. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_INLINE OS_ALWAYS_INLINE +voucher_activity_trace_id_t +voucher_activity_trace_id(uint8_t type, uint8_t code_namespace, uint32_t code) +{ + return ((voucher_activity_trace_id_t)type << + _voucher_activity_trace_id_type_shift) | + ((voucher_activity_trace_id_t)code_namespace << + _voucher_activity_trace_id_code_namespace_shift) | + (voucher_activity_trace_id_t)code; +} + +/*! + * @function voucher_activity_start + * + * @abstract + * Creates a new activity identifier and marks the current thread as + * participating in the activity. + * + * @discussion + * As part of voucher transport, activities are automatically propagated by the + * system to other threads and processes (across IPC). + * + * Activities persist as long as any threads in any process are marked as + * participating. There may be many calls to voucher_activity_end() + * corresponding to one call to voucher_activity_start(). + * + * @param trace_id + * Tracepoint identifier returned by voucher_activity_trace_id(), intended for + * identification of the automatic tracepoint generated as part of creating the + * new activity. + * + * @param flags + * Pass voucher_activity_flag_force to indicate that existing activities + * on the current thread should not be inherited and that a new toplevel + * activity should be created. + * + * @result + * A new activity identifier. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_WARN_RESULT OS_NOTHROW +voucher_activity_id_t +voucher_activity_start(voucher_activity_trace_id_t trace_id, + voucher_activity_flag_t flags); + +/*! + * @function voucher_activity_start_with_location + * + * @abstract + * Creates a new activity identifier and marks the current thread as + * participating in the activity. + * + * @discussion + * As part of voucher transport, activities are automatically propagated by the + * system to other threads and processes (across IPC). + * + * Activities persist as long as any threads in any process are marked as + * participating. There may be many calls to voucher_activity_end() + * corresponding to one call to voucher_activity_start_with_location(). + * + * @param trace_id + * Tracepoint identifier returned by voucher_activity_trace_id(), intended for + * identification of the automatic tracepoint generated as part of creating the + * new activity. + * + * @param location + * Location identifier for the automatic tracepoint generated as part of + * creating the new activity. + * + * @param flags + * Pass voucher_activity_flag_force to indicate that existing activities + * on the current thread should not be inherited and that a new toplevel + * activity should be created. + * + * @result + * A new activity identifier. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_WARN_RESULT OS_NOTHROW +voucher_activity_id_t +voucher_activity_start_with_location(voucher_activity_trace_id_t trace_id, + uint64_t location, voucher_activity_flag_t flags); + +/*! + * @function voucher_activity_end + * + * @abstract + * Unmarks the current thread if it is marked as particpating in the activity + * with the specified identifier. + * + * @discussion + * Activities persist as long as any threads in any process are marked as + * participating. There may be many calls to voucher_activity_end() + * corresponding to one call to voucher_activity_start() or + * voucher_activity_start_with_location(). + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_NOTHROW +void +voucher_activity_end(voucher_activity_id_t activity_id); + +/*! + * @function voucher_get_activities + * + * @abstract + * Returns the list of activity identifiers that the current thread is marked + * with. + * + * @param entries + * Pointer to an array of activity identifiers to be filled in. + * + * @param count + * Pointer to the requested number of activity identifiers. + * On output will be filled with the number of activities that are available. + * + * @result + * Number of activity identifiers written to 'entries' + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_NOTHROW +unsigned int +voucher_get_activities(voucher_activity_id_t *entries, unsigned int *count); + +/*! + * @group Voucher Activity Trace SPI + * SPI intended for libtrace only + */ + +/*! + * @function voucher_activity_get_namespace + * + * @abstract + * Returns the namespace of the current activity. + * + * @result + * The namespace of the current activity (if any). + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_NOTHROW +uint8_t +voucher_activity_get_namespace(void); + +/*! + * @function voucher_activity_trace + * + * @abstract + * Add a tracepoint to trace buffer of the current activity. + * + * @param trace_id + * Tracepoint identifier returned by voucher_activity_trace_id() + * + * @param location + * Tracepoint location. + * + * @param buffer + * Pointer to packed buffer of tracepoint data. + * + * @param length + * Length of data at 'buffer'. + * + * @result + * Timestamp recorded in tracepoint or 0 if no tracepoint was recorded. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_NOTHROW +uint64_t +voucher_activity_trace(voucher_activity_trace_id_t trace_id, uint64_t location, + void *buffer, size_t length); + +/*! + * @function voucher_activity_trace_args + * + * @abstract + * Add a tracepoint to trace buffer of the current activity, recording + * specified arguments passed in registers. + * + * @param trace_id + * Tracepoint identifier returned by voucher_activity_trace_id() + * + * @param location + * Tracepoint location. + * + * @param arg1 + * Argument to be recorded in tracepoint data. + * + * @param arg2 + * Argument to be recorded in tracepoint data. + * + * @param arg3 + * Argument to be recorded in tracepoint data. + * + * @param arg4 + * Argument to be recorded in tracepoint data. + * + * @result + * Timestamp recorded in tracepoint or 0 if no tracepoint was recorded. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_NOTHROW +uint64_t +voucher_activity_trace_args(voucher_activity_trace_id_t trace_id, + uint64_t location, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, + uintptr_t arg4); + +/*! + * @group Voucher Activity Mode SPI + * SPI intended for libtrace only + */ + +/*! + * @enum voucher_activity_mode_t + * + * @abstract + * Voucher activity mode. + * + * @discussion + * Configure at process start by setting the OS_ACTIVITY_MODE environment + * variable. + */ +OS_ENUM(voucher_activity_mode, unsigned long, + voucher_activity_mode_disable = 0, + voucher_activity_mode_release = (1u << 0), + voucher_activity_mode_debug = (1u << 1), + voucher_activity_mode_stream = (1u << 2), +); + +/*! + * @function voucher_activity_get_mode + * + * @abstract + * Return current mode of voucher activity subsystem. + * + * @result + * Value from voucher_activity_mode_t enum. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_WARN_RESULT OS_NOTHROW +voucher_activity_mode_t +voucher_activity_get_mode(void); + +/*! + * @function voucher_activity_set_mode_4libtrace(void) + * + * @abstract + * Set the current mode of voucher activity subsystem. + * + * @param mode + * The new mode. + * + * Note that the new mode will take effect soon, but not immediately. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_NOTHROW +void +voucher_activity_set_mode_4libtrace(voucher_activity_mode_t mode); + +/*! + * @group Voucher Activity Metadata SPI + * SPI intended for libtrace only + */ + +/*! + * @function voucher_activity_get_metadata_buffer + * + * @abstract + * Return address and length of buffer in the process trace memory area + * reserved for libtrace metadata. + * + * @param length + * Pointer to size_t variable, filled with length of metadata buffer. + * + * @result + * Address of metadata buffer. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_WARN_RESULT OS_NOTHROW OS_NONNULL_ALL +void* +voucher_activity_get_metadata_buffer(size_t *length); + +#endif // OS_VOUCHER_ACTIVITY_SPI + +#if OS_VOUCHER_ACTIVITY_BUFFER_SPI + +/*! + * @group Voucher Activity Tracepoint SPI + * SPI intended for diagnosticd only + */ + +OS_ENUM(_voucher_activity_tracepoint_flag, uint16_t, + _voucher_activity_trace_flag_buffer_empty = 0, + _voucher_activity_trace_flag_tracepoint = (1u << 0), + _voucher_activity_trace_flag_tracepoint_args = (1u << 1), + _voucher_activity_trace_flag_wide_first = (1u << 6), + _voucher_activity_trace_flag_wide_second = (1u << 6) | (1u << 7), + _voucher_activity_trace_flag_start = (1u << 8), + _voucher_activity_trace_flag_end = (1u << 8) | (1u << 9), + _voucher_activity_trace_flag_libdispatch = (1u << 13), + _voucher_activity_trace_flag_activity = (1u << 14), + _voucher_activity_trace_flag_buffer_header = (1u << 15), +); + +// for tracepoints with _voucher_activity_trace_flag_libdispatch +OS_ENUM(_voucher_activity_tracepoint_namespace, uint8_t, + _voucher_activity_tracepoint_namespace_ipc = 0x1 +); +OS_ENUM(_voucher_activity_tracepoint_code, uint32_t, + _voucher_activity_tracepoint_namespace_ipc_send = 0x1, + _voucher_activity_tracepoint_namespace_ipc_receive = 0x2, +); + +typedef struct _voucher_activity_tracepoint_s { + uint16_t vat_flags; // voucher_activity_tracepoint_flag_t + uint8_t vat_type; // voucher_activity_tracepoint_type_t + uint8_t vat_namespace; // namespace for tracepoint code + uint32_t vat_code; // tracepoint code + uint64_t vat_thread; // pthread_t + uint64_t vat_timestamp; // absolute time + uint64_t vat_location; // tracepoint PC + uint64_t vat_data[4]; // trace data +} *_voucher_activity_tracepoint_t; + +/*! + * @group Voucher Activity Buffer Internals + * SPI intended for diagnosticd only + * Layout of structs is subject to change without notice + */ + +#include +#include +#include + +static const atm_subaid32_t _voucher_default_activity_subid = + ATM_SUBAID32_MAX-1; + +static const size_t _voucher_activity_buffer_size = 4096; +static const size_t _voucher_activity_tracepoints_per_buffer = + _voucher_activity_buffer_size / + sizeof(struct _voucher_activity_tracepoint_s); +typedef uint8_t _voucher_activity_buffer_t[_voucher_activity_buffer_size]; + +struct _voucher_activity_self_metadata_s { + struct _voucher_activity_metadata_opaque_s *vasm_baseaddr; +}; +typedef struct _voucher_activity_metadata_opaque_s { + _voucher_activity_buffer_t vam_kernel_metadata; + _voucher_activity_buffer_t vam_client_metadata; + union { + struct _voucher_activity_self_metadata_s vam_self_metadata; + _voucher_activity_buffer_t vam_self_metadata_opaque; + }; +} *_voucher_activity_metadata_opaque_t; + +typedef os_lock_handoff_s _voucher_activity_lock_s; + +typedef struct _voucher_atm_s { + int32_t volatile vatm_refcnt; + mach_voucher_t vatm_kvoucher; + atm_aid_t vatm_id; + atm_mailbox_offset_t vatm_mailbox_offset; + TAILQ_ENTRY(_voucher_atm_s) vatm_list; +#if __LP64__ + uintptr_t vatm_pad[3]; + // cacheline +#endif + _voucher_activity_lock_s vatm_activities_lock; + TAILQ_HEAD(_voucher_atm_activities_s, _voucher_activity_s) vatm_activities; + TAILQ_HEAD(, _voucher_activity_s) vatm_used_activities; +} *_voucher_atm_t; + +// must match layout of _voucher_activity_tracepoint_s +typedef struct _voucher_activity_buffer_header_s { + uint16_t vabh_flags; // _voucher_activity_trace_flag_buffer_header + uint8_t vabh_unused[6]; + uint64_t vabh_thread; + uint64_t vabh_timestamp; + uint32_t volatile vabh_next_tracepoint_idx; + uint32_t vabh_sequence_no; + voucher_activity_id_t vabh_activity_id; + uint64_t vabh_reserved; + TAILQ_ENTRY(_voucher_activity_buffer_header_s) vabh_list; +} *_voucher_activity_buffer_header_t; + +// must match layout of _voucher_activity_buffer_header_s +typedef struct _voucher_activity_s { + // first tracepoint entry + // must match layout of _voucher_activity_tracepoint_s + uint16_t va_flags; // _voucher_activity_trace_flag_buffer_header | + // _voucher_activity_trace_flag_activity | + // _voucher_activity_trace_flag_start | + // _voucher_activity_trace_flag_wide_first + uint8_t va_type; + uint8_t va_namespace; + uint32_t va_code; + uint64_t va_thread; + uint64_t va_timestamp; + uint32_t volatile vabh_next_tracepoint_idx; + uint32_t volatile va_max_sequence_no; + voucher_activity_id_t va_id; + int32_t volatile va_use_count; + uint32_t va_buffer_limit; + TAILQ_HEAD(_voucher_activity_buffer_list_s, + _voucher_activity_buffer_header_s) va_buffers; +#if !__LP64__ + uint64_t va_pad; +#endif + + // second tracepoint entry + // must match layout of _voucher_activity_tracepoint_s + uint16_t va_flags2; + uint8_t va_unused2[2]; + int32_t volatile va_refcnt; + uint64_t va_location; + _voucher_activity_buffer_header_t volatile va_current_buffer; + _voucher_atm_t va_atm; + _voucher_activity_lock_s va_buffers_lock; + uintptr_t va_pad2[2]; + +#if __LP64__ + // third tracepoint entry + // must match layout of _voucher_activity_tracepoint_s + uint16_t va_flags3; + uint8_t va_unused3[6]; + uintptr_t va_pad3; +#endif + TAILQ_ENTRY(_voucher_activity_s) va_list; + TAILQ_ENTRY(_voucher_activity_s) va_atm_list; + TAILQ_ENTRY(_voucher_activity_s) va_atm_used_list; +} *_voucher_activity_t; + +#endif // OS_VOUCHER_ACTIVITY_BUFFER_SPI + +__END_DECLS + +#endif // __OS_VOUCHER_ACTIVITY_PRIVATE__ diff --git a/private/voucher_private.h b/private/voucher_private.h new file mode 100644 index 0000000..fcc28f1 --- /dev/null +++ b/private/voucher_private.h @@ -0,0 +1,375 @@ +/* + * Copyright (c) 2013-2014 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +#ifndef __OS_VOUCHER_PRIVATE__ +#define __OS_VOUCHER_PRIVATE__ + +#include +#include + +#define OS_VOUCHER_SPI_VERSION 20140425 + +#if OS_VOUCHER_WEAK_IMPORT +#define OS_VOUCHER_EXPORT OS_EXPORT OS_WEAK_IMPORT +#else +#define OS_VOUCHER_EXPORT OS_EXPORT +#endif + +__BEGIN_DECLS + +/*! + * @group Voucher Transport SPI + * SPI intended for clients that need to transport vouchers. + */ + +/*! + * @typedef voucher_t + * + * @abstract + * Vouchers are immutable sets of key/value attributes that can be adopted on a + * thread in the current process or sent to another process. + * + * @discussion + * Voucher objects are os_objects (c.f. ). They are memory-managed + * with the os_retain()/os_release() functions or -[retain]/-[release] methods. + */ +#if OS_OBJECT_USE_OBJC +OS_OBJECT_DECL(voucher); +#else +typedef struct voucher_s *voucher_t; +#endif + +/*! + * @function voucher_adopt + * + * @abstract + * Adopt the specified voucher on the current thread and return the voucher + * that had been adopted previously. + * + * @discussion + * Adopted vouchers are automatically carried forward by the system to other + * threads and processes (across IPC). + * + * Consumes a reference to the specified voucher. + * Returns a reference to the previous voucher. + * + * @param voucher + * The voucher object to adopt on the current thread. + * + * @result + * The previously adopted voucher object. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_OBJECT_RETURNS_RETAINED OS_WARN_RESULT_NEEDS_RELEASE +OS_NOTHROW +voucher_t +voucher_adopt(voucher_t voucher OS_OBJECT_CONSUMED); + +/*! + * @function voucher_copy + * + * @abstract + * Returns a reference to the voucher that had been adopted previously on the + * current thread (or carried forward by the system). + * + * @result + * The currently adopted voucher object. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_OBJECT_RETURNS_RETAINED OS_WARN_RESULT OS_NOTHROW +voucher_t +voucher_copy(void); + +/*! + * @function voucher_copy_without_importance + * + * @abstract + * Returns a reference to a voucher object with all the properties of the + * voucher that had been adopted previously on the current thread, but + * without the importance properties that are frequently attached to vouchers + * carried with IPC requests. Importance properties may elevate the scheduling + * of threads that adopt or retain the voucher while they service the request. + * See xpc_transaction_begin(3) for further details on importance. + * + * @result + * A copy of the currently adopted voucher object, with importance removed. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_OBJECT_RETURNS_RETAINED OS_WARN_RESULT OS_NOTHROW +voucher_t +voucher_copy_without_importance(void); + +/*! + * @function voucher_replace_default_voucher + * + * @abstract + * Replace process attributes of default voucher (used for IPC by this process + * when no voucher is adopted on the sending thread) with the process attributes + * of the voucher adopted on the current thread. + * + * @discussion + * This allows a daemon to indicate from the context of an incoming IPC request + * that all future outgoing IPC from the process should be marked as acting + * "on behalf of" the sending process of the current IPC request (as long as the + * thread sending that outgoing IPC is not itself in the direct context of an + * IPC request, i.e. no voucher is adopted). + * + * If no voucher is adopted on the current thread or the current voucher does + * not contain any process attributes, the default voucher is reset to the + * default process attributes for the current process. + * + * CAUTION: Do NOT use this SPI without contacting the Darwin Runtime team. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_NOTHROW +void +voucher_replace_default_voucher(void); + +/*! + * @function voucher_decrement_importance_count4CF + * + * @abstract + * Decrement external importance count of the mach voucher in the specified + * voucher object. + * + * @discussion + * This is only intended for use by CoreFoundation to explicitly manage the + * App Nap state of an application following receiption of a de-nap IPC message. + * + * CAUTION: Do NOT use this SPI without contacting the Darwin Runtime team. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_NOTHROW +void +voucher_decrement_importance_count4CF(voucher_t voucher); + +/*! + * @group Dispatch block objects + */ + +#ifndef __DISPATCH_BUILDING_DISPATCH__ +#include +#endif /* !__DISPATCH_BUILDING_DISPATCH__ */ + +/*! + * @typedef dispatch_block_flags_t + * SPI Flags to pass to the dispatch_block_create* functions. + * + * @const DISPATCH_BLOCK_NO_VOUCHER + * Flag indicating that a dispatch block object should not be assigned a voucher + * object. If invoked directly, the block object will be executed with the + * voucher adopted on the calling thread. If the block object is submitted to a + * queue, this replaces the default behavior of associating the submitted block + * instance with the voucher adopted at the time of submission. + * This flag is ignored if a specific voucher object is assigned with the + * dispatch_block_create_with_voucher* functions, and is equivalent to passing + * the NULL voucher to these functions. + */ +#define DISPATCH_BLOCK_NO_VOUCHER (0x40) + +/*! + * @function dispatch_block_create_with_voucher + * + * @abstract + * Create a new dispatch block object on the heap from an existing block and + * the given flags, and assign it the specified voucher object. + * + * @discussion + * The provided block is Block_copy'ed to the heap, it and the specified voucher + * object are retained by the newly created dispatch block object. + * + * The returned dispatch block object is intended to be submitted to a dispatch + * queue with dispatch_async() and related functions, but may also be invoked + * directly. Both operations can be performed an arbitrary number of times but + * only the first completed execution of a dispatch block object can be waited + * on with dispatch_block_wait() or observed with dispatch_block_notify(). + * + * The returned dispatch block will be executed with the specified voucher + * adopted for the duration of the block body. If the NULL voucher is passed, + * the block will be executed with the voucher adopted on the calling thread, or + * with no voucher if the DISPATCH_BLOCK_DETACHED flag was also provided. + * + * If the returned dispatch block object is submitted to a dispatch queue, the + * submitted block instance will be associated with the QOS class current at the + * time of submission, unless one of the following flags assigned a specific QOS + * class (or no QOS class) at the time of block creation: + * - DISPATCH_BLOCK_ASSIGN_CURRENT + * - DISPATCH_BLOCK_NO_QOS_CLASS + * - DISPATCH_BLOCK_DETACHED + * The QOS class the block object will be executed with also depends on the QOS + * class assigned to the queue and which of the following flags was specified or + * defaulted to: + * - DISPATCH_BLOCK_INHERIT_QOS_CLASS (default for asynchronous execution) + * - DISPATCH_BLOCK_ENFORCE_QOS_CLASS (default for synchronous execution) + * See description of dispatch_block_flags_t for details. + * + * If the returned dispatch block object is submitted directly to a serial queue + * and is configured to execute with a specific QOS class, the system will make + * a best effort to apply the necessary QOS overrides to ensure that blocks + * submitted earlier to the serial queue are executed at that same QOS class or + * higher. + * + * @param flags + * Configuration flags for the block object. + * Passing a value that is not a bitwise OR of flags from dispatch_block_flags_t + * results in NULL being returned. + * + * @param voucher + * A voucher object or NULL. Passing NULL is equivalent to specifying the + * DISPATCH_BLOCK_NO_VOUCHER flag. + * + * @param block + * The block to create the dispatch block object from. + * + * @result + * The newly created dispatch block object, or NULL. + * When not building with Objective-C ARC, must be released with a -[release] + * message or the Block_release() function. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) +DISPATCH_EXPORT DISPATCH_NONNULL3 DISPATCH_RETURNS_RETAINED_BLOCK +DISPATCH_WARN_RESULT DISPATCH_NOTHROW +dispatch_block_t +dispatch_block_create_with_voucher(dispatch_block_flags_t flags, + voucher_t voucher, dispatch_block_t block); + +/*! + * @function dispatch_block_create_with_voucher_and_qos_class + * + * @abstract + * Create a new dispatch block object on the heap from an existing block and + * the given flags, and assign it the specified voucher object, QOS class and + * relative priority. + * + * @discussion + * The provided block is Block_copy'ed to the heap, it and the specified voucher + * object are retained by the newly created dispatch block object. + * + * The returned dispatch block object is intended to be submitted to a dispatch + * queue with dispatch_async() and related functions, but may also be invoked + * directly. Both operations can be performed an arbitrary number of times but + * only the first completed execution of a dispatch block object can be waited + * on with dispatch_block_wait() or observed with dispatch_block_notify(). + * + * The returned dispatch block will be executed with the specified voucher + * adopted for the duration of the block body. If the NULL voucher is passed, + * the block will be executed with the voucher adopted on the calling thread, or + * with no voucher if the DISPATCH_BLOCK_DETACHED flag was also provided. + * + * If invoked directly, the returned dispatch block object will be executed with + * the assigned QOS class as long as that does not result in a lower QOS class + * than what is current on the calling thread. + * + * If the returned dispatch block object is submitted to a dispatch queue, the + * QOS class it will be executed with depends on the QOS class assigned to the + * block, the QOS class assigned to the queue and which of the following flags + * was specified or defaulted to: + * - DISPATCH_BLOCK_INHERIT_QOS_CLASS: default for asynchronous execution + * - DISPATCH_BLOCK_ENFORCE_QOS_CLASS: default for synchronous execution + * See description of dispatch_block_flags_t for details. + * + * If the returned dispatch block object is submitted directly to a serial queue + * and is configured to execute with a specific QOS class, the system will make + * a best effort to apply the necessary QOS overrides to ensure that blocks + * submitted earlier to the serial queue are executed at that same QOS class or + * higher. + * + * @param flags + * Configuration flags for the block object. + * Passing a value that is not a bitwise OR of flags from dispatch_block_flags_t + * results in NULL being returned. + * + * @param voucher + * A voucher object or NULL. Passing NULL is equivalent to specifying the + * DISPATCH_BLOCK_NO_VOUCHER flag. + * + * @param qos_class + * A QOS class value: + * - QOS_CLASS_USER_INTERACTIVE + * - QOS_CLASS_USER_INITIATED + * - QOS_CLASS_DEFAULT + * - QOS_CLASS_UTILITY + * - QOS_CLASS_BACKGROUND + * - QOS_CLASS_UNSPECIFIED + * Passing QOS_CLASS_UNSPECIFIED is equivalent to specifying the + * DISPATCH_BLOCK_NO_QOS_CLASS flag. Passing any other value results in NULL + * being returned. + * + * @param relative_priority + * A relative priority within the QOS class. This value is a negative + * offset from the maximum supported scheduler priority for the given class. + * Passing a value greater than zero or less than QOS_MIN_RELATIVE_PRIORITY + * results in NULL being returned. + * + * @param block + * The block to create the dispatch block object from. + * + * @result + * The newly created dispatch block object, or NULL. + * When not building with Objective-C ARC, must be released with a -[release] + * message or the Block_release() function. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0) +DISPATCH_EXPORT DISPATCH_NONNULL5 DISPATCH_RETURNS_RETAINED_BLOCK +DISPATCH_WARN_RESULT DISPATCH_NOTHROW +dispatch_block_t +dispatch_block_create_with_voucher_and_qos_class(dispatch_block_flags_t flags, + voucher_t voucher, dispatch_qos_class_t qos_class, + int relative_priority, dispatch_block_t block); + +/*! + * @group Voucher Mach SPI + * SPI intended for clients that need to interact with mach messages or mach + * voucher ports directly. + */ + +#include + +/*! + * @function voucher_create_with_mach_msg + * + * @abstract + * Creates a new voucher object from a mach message carrying a mach voucher port + * + * @discussion + * Ownership of the mach voucher port in the message is transfered to the new + * voucher object and the message header mach voucher field is cleared. + * + * @param msg + * The mach message to query. + * + * @result + * The newly created voucher object or NULL if the message was not carrying a + * mach voucher. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_OBJECT_RETURNS_RETAINED OS_WARN_RESULT OS_NOTHROW +voucher_t +voucher_create_with_mach_msg(mach_msg_header_t *msg); + +__END_DECLS + +#endif // __OS_VOUCHER_PRIVATE__ + +#if (OS_VOUCHER_ACTIVITY_SPI || OS_VOUCHER_ACTIVITY_BUFFER_SPI) && \ + !defined(__DISPATCH_BUILDING_DISPATCH__) && \ + !defined(__OS_VOUCHER_ACTIVITY_PRIVATE__) +#include +#endif diff --git a/src/allocator.c b/src/allocator.c index 7b4c165..af1f3c1 100644 --- a/src/allocator.c +++ b/src/allocator.c @@ -127,7 +127,7 @@ continuation_is_in_first_page(dispatch_continuation_t c) // (the base of c's magazine == the base of c's page) // => c is in first page of magazine return (((uintptr_t)c & MAGAZINE_MASK) == - ((uintptr_t)c & ~(uintptr_t)PAGE_MASK)); + ((uintptr_t)c & ~(uintptr_t)DISPATCH_ALLOCATOR_PAGE_MASK)); #else (void)c; return false; @@ -173,7 +173,8 @@ madvisable_page_base_for_continuation(dispatch_continuation_t c) if (fastpath(continuation_is_in_first_page(c))) { return NULL; } - void *page_base = (void *)((uintptr_t)c & ~(uintptr_t)PAGE_MASK); + void *page_base = (void *)((uintptr_t)c & + ~(uintptr_t)DISPATCH_ALLOCATOR_PAGE_MASK); #if DISPATCH_DEBUG struct dispatch_magazine_s *m = magazine_for_continuation(c); if (slowpath(page_base < (void *)&m->conts)) { @@ -226,13 +227,8 @@ bitmap_set_first_unset_bit_upto_index(volatile bitmap_t *bitmap, // continuation is "uninitialized", so the caller shouldn't // load from it before storing, so we don't need to guard // against reordering those loads. -#if defined(__x86_64__) // TODO rdar://problem/11477843 - dispatch_assert(sizeof(*bitmap) == sizeof(uint64_t)); - return dispatch_atomic_set_first_bit((volatile uint64_t *)bitmap,max_index); -#else - dispatch_assert(sizeof(*bitmap) == sizeof(uint32_t)); - return dispatch_atomic_set_first_bit((volatile uint32_t *)bitmap,max_index); -#endif + dispatch_assert(sizeof(*bitmap) == sizeof(unsigned long)); + return dispatch_atomic_set_first_bit(bitmap,max_index); } DISPATCH_ALWAYS_INLINE @@ -257,9 +253,8 @@ bitmap_clear_bit(volatile bitmap_t *bitmap, unsigned int index, const bitmap_t mask = BITMAP_C(1) << index; bitmap_t b; - b = *bitmap; if (exclusively == CLEAR_EXCLUSIVELY) { - if (slowpath((b & mask) == 0)) { + if (slowpath((*bitmap & mask) == 0)) { DISPATCH_CRASH("Corruption: failed to clear bit exclusively"); } } @@ -397,11 +392,13 @@ _dispatch_alloc_try_create_heap(dispatch_heap_t *heap_ptr) } #if DISPATCH_DEBUG // Double-check our math. - dispatch_assert(aligned_region % PAGE_SIZE == 0); - dispatch_assert(aligned_region_end % PAGE_SIZE == 0); + dispatch_assert(aligned_region % DISPATCH_ALLOCATOR_PAGE_SIZE == 0); + dispatch_assert(aligned_region % vm_kernel_page_size == 0); + dispatch_assert(aligned_region_end % DISPATCH_ALLOCATOR_PAGE_SIZE == 0); + dispatch_assert(aligned_region_end % vm_kernel_page_size == 0); dispatch_assert(aligned_region_end > aligned_region); - dispatch_assert(top_slop_len % PAGE_SIZE == 0); - dispatch_assert(bottom_slop_len % PAGE_SIZE == 0); + dispatch_assert(top_slop_len % DISPATCH_ALLOCATOR_PAGE_SIZE == 0); + dispatch_assert(bottom_slop_len % DISPATCH_ALLOCATOR_PAGE_SIZE == 0); dispatch_assert(aligned_region_end + top_slop_len == region_end); dispatch_assert(region + bottom_slop_len == aligned_region); dispatch_assert(region_sz == bottom_slop_len + top_slop_len + @@ -566,9 +563,10 @@ _dispatch_alloc_maybe_madvise_page(dispatch_continuation_t c) // last_locked-1, BITMAPS_PER_PAGE, &page_bitmaps[0]); // Scribble to expose use-after-free bugs // madvise (syscall) flushes these stores - memset(page, DISPATCH_ALLOCATOR_SCRIBBLE, PAGE_SIZE); + memset(page, DISPATCH_ALLOCATOR_SCRIBBLE, DISPATCH_ALLOCATOR_PAGE_SIZE); #endif - (void)dispatch_assume_zero(madvise(page, PAGE_SIZE, MADV_FREE)); + (void)dispatch_assume_zero(madvise(page, DISPATCH_ALLOCATOR_PAGE_SIZE, + MADV_FREE)); unlock: while (last_locked > 1) { @@ -631,19 +629,23 @@ _dispatch_alloc_init(void) // self-aligned. dispatch_assert(offsetof(struct dispatch_magazine_s, conts) % (CONTINUATIONS_PER_BITMAP * DISPATCH_CONTINUATION_SIZE) == 0); - dispatch_assert(offsetof(struct dispatch_magazine_s, conts) == PAGE_SIZE); + dispatch_assert(offsetof(struct dispatch_magazine_s, conts) == + DISPATCH_ALLOCATOR_PAGE_SIZE); #if PACK_FIRST_PAGE_WITH_CONTINUATIONS // The continuations in the first page should actually fit within the first // page. - dispatch_assert(offsetof(struct dispatch_magazine_s, fp_conts) < PAGE_SIZE); + dispatch_assert(offsetof(struct dispatch_magazine_s, fp_conts) < + DISPATCH_ALLOCATOR_PAGE_SIZE); dispatch_assert(offsetof(struct dispatch_magazine_s, fp_conts) % DISPATCH_CONTINUATION_SIZE == 0); dispatch_assert(offsetof(struct dispatch_magazine_s, fp_conts) + - sizeof(((struct dispatch_magazine_s *)0x0)->fp_conts) == PAGE_SIZE); + sizeof(((struct dispatch_magazine_s *)0x0)->fp_conts) == + DISPATCH_ALLOCATOR_PAGE_SIZE); #endif // PACK_FIRST_PAGE_WITH_CONTINUATIONS } -#else +#elif (DISPATCH_ALLOCATOR && DISPATCH_CONTINUATION_MALLOC) \ + || (DISPATCH_CONTINUATION_MALLOC && DISPATCH_USE_MALLOCZONE) static inline void _dispatch_alloc_init(void) {} #endif diff --git a/src/allocator_internal.h b/src/allocator_internal.h index 5f223f6..f4c8ba0 100644 --- a/src/allocator_internal.h +++ b/src/allocator_internal.h @@ -74,7 +74,7 @@ #if DISPATCH_ALLOCATOR // Configuration here! -#define NUM_CPU _dispatch_hw_config.cc_max_logical +#define NUM_CPU dispatch_hw_config(logical_cpus) #define MAGAZINES_PER_HEAP (NUM_CPU) // Do you care about compaction or performance? @@ -84,6 +84,16 @@ #define PACK_FIRST_PAGE_WITH_CONTINUATIONS 0 #endif +#ifndef PAGE_MAX_SIZE +#define PAGE_MAX_SIZE PAGE_SIZE +#endif +#ifndef PAGE_MAX_MASK +#define PAGE_MAX_MASK PAGE_MASK +#endif +#define DISPATCH_ALLOCATOR_PAGE_SIZE PAGE_MAX_SIZE +#define DISPATCH_ALLOCATOR_PAGE_MASK PAGE_MAX_MASK + + #if TARGET_OS_EMBEDDED #define PAGES_PER_MAGAZINE 64 #else @@ -91,11 +101,11 @@ #endif // Use the largest type your platform is comfortable doing atomic ops with. -#if defined(__x86_64__) // TODO: rdar://11477843 +// TODO: rdar://11477843 typedef unsigned long bitmap_t; +#if defined(__LP64__) #define BYTES_PER_BITMAP 8 #else -typedef uint32_t bitmap_t; #define BYTES_PER_BITMAP 4 #endif @@ -107,7 +117,7 @@ typedef uint32_t bitmap_t; #define CONTINUATIONS_PER_BITMAP (BYTES_PER_BITMAP * 8) #define BITMAPS_PER_SUPERMAP (BYTES_PER_SUPERMAP * 8) -#define BYTES_PER_MAGAZINE (PAGES_PER_MAGAZINE * PAGE_SIZE) +#define BYTES_PER_MAGAZINE (PAGES_PER_MAGAZINE * DISPATCH_ALLOCATOR_PAGE_SIZE) #define CONSUMED_BYTES_PER_BITMAP (BYTES_PER_BITMAP + \ (DISPATCH_CONTINUATION_SIZE * CONTINUATIONS_PER_BITMAP)) @@ -117,7 +127,7 @@ typedef uint32_t bitmap_t; #define BYTES_PER_HEAP (BYTES_PER_MAGAZINE * MAGAZINES_PER_HEAP) -#define BYTES_PER_PAGE PAGE_SIZE +#define BYTES_PER_PAGE DISPATCH_ALLOCATOR_PAGE_SIZE #define CONTINUATIONS_PER_PAGE (BYTES_PER_PAGE / DISPATCH_CONTINUATION_SIZE) #define BITMAPS_PER_PAGE (CONTINUATIONS_PER_PAGE / CONTINUATIONS_PER_BITMAP) @@ -159,7 +169,8 @@ typedef uint32_t bitmap_t; (BYTES_LEFT_IN_FIRST_PAGE / CONSUMED_BYTES_PER_BITMAP) #define REMAINDER_IN_FIRST_PAGE (BYTES_LEFT_IN_FIRST_PAGE - \ (FULL_BITMAPS_IN_FIRST_PAGE * CONSUMED_BYTES_PER_BITMAP) - \ - (FULL_BITMAPS_IN_FIRST_PAGE ? 0 : ROUND_UP_TO_CONTINUATION_SIZE(BYTES_PER_BITMAP))) + (FULL_BITMAPS_IN_FIRST_PAGE ? 0 : \ + ROUND_UP_TO_CONTINUATION_SIZE(BYTES_PER_BITMAP))) #define REMAINDERED_CONTINUATIONS_IN_FIRST_PAGE \ (REMAINDER_IN_FIRST_PAGE / DISPATCH_CONTINUATION_SIZE) diff --git a/src/apply.c b/src/apply.c index aa187a0..e0ab2c3 100644 --- a/src/apply.c +++ b/src/apply.c @@ -24,7 +24,7 @@ typedef void (*dispatch_apply_function_t)(void *, size_t); DISPATCH_ALWAYS_INLINE static inline void -_dispatch_apply_invoke2(void *ctxt) +_dispatch_apply_invoke2(void *ctxt, bool redirect) { dispatch_apply_t da = (dispatch_apply_t)ctxt; size_t const iter = da->da_iterations; @@ -36,6 +36,7 @@ _dispatch_apply_invoke2(void *ctxt) // da_dc is only safe to access once the 'index lock' has been acquired dispatch_apply_function_t const func = (void *)da->da_dc->dc_func; void *const da_ctxt = da->da_dc->dc_ctxt; + dispatch_queue_t dq = da->da_dc->dc_data; _dispatch_perfmon_workitem_dec(); // this unit executes many items @@ -43,6 +44,14 @@ _dispatch_apply_invoke2(void *ctxt) size_t nested = (size_t)_dispatch_thread_getspecific(dispatch_apply_key); _dispatch_thread_setspecific(dispatch_apply_key, (void*)da->da_nested); + dispatch_queue_t old_dq; + pthread_priority_t old_dp; + if (redirect) { + old_dq = _dispatch_thread_getspecific(dispatch_queue_key); + _dispatch_thread_setspecific(dispatch_queue_key, dq); + old_dp = _dispatch_set_defaultpriority(dq->dq_priority); + } + // Striding is the responsibility of the caller. do { _dispatch_client_callout2(da_ctxt, idx, func); @@ -50,6 +59,11 @@ _dispatch_apply_invoke2(void *ctxt) done++; idx = dispatch_atomic_inc_orig2o(da, da_index, relaxed); } while (fastpath(idx < iter)); + + if (redirect) { + _dispatch_reset_defaultpriority(old_dp); + _dispatch_thread_setspecific(dispatch_queue_key, old_dq); + } _dispatch_thread_setspecific(dispatch_apply_key, (void*)nested); // The thread that finished the last workitem wakes up the possibly waiting @@ -67,20 +81,14 @@ DISPATCH_NOINLINE void _dispatch_apply_invoke(void *ctxt) { - _dispatch_apply_invoke2(ctxt); + _dispatch_apply_invoke2(ctxt, false); } DISPATCH_NOINLINE void _dispatch_apply_redirect_invoke(void *ctxt) { - dispatch_apply_t da = (dispatch_apply_t)ctxt; - dispatch_queue_t old_dq; - old_dq = (dispatch_queue_t)_dispatch_thread_getspecific(dispatch_queue_key); - - _dispatch_thread_setspecific(dispatch_queue_key, da->da_dc->dc_data); - _dispatch_apply_invoke2(ctxt); - _dispatch_thread_setspecific(dispatch_queue_key, old_dq); + _dispatch_apply_invoke2(ctxt, true); } static void @@ -118,6 +126,8 @@ _dispatch_apply_f2(dispatch_queue_t dq, dispatch_apply_t da, next->do_vtable = (void *)DISPATCH_OBJ_ASYNC_BIT; next->dc_func = func; next->dc_ctxt = da; + _dispatch_continuation_voucher_set(next, 0); + _dispatch_continuation_priority_set(next, 0, 0); next->do_next = head; head = next; @@ -130,7 +140,8 @@ _dispatch_apply_f2(dispatch_queue_t dq, dispatch_apply_t da, _dispatch_thread_semaphore_t sema = _dispatch_get_thread_semaphore(); da->da_sema = sema; - _dispatch_queue_push_list(dq, head, tail, continuation_cnt); + _dispatch_queue_push_list(dq, head, tail, head->dc_priority, + continuation_cnt); // Call the first element directly _dispatch_apply_invoke(da); _dispatch_perfmon_workitem_inc(); @@ -183,7 +194,7 @@ dispatch_apply_f(size_t iterations, dispatch_queue_t dq, void *ctxt, if (slowpath(iterations == 0)) { return; } - uint32_t thr_cnt = _dispatch_hw_config.cc_max_active; + uint32_t thr_cnt = dispatch_hw_config(active_cpus); size_t nested = (size_t)_dispatch_thread_getspecific(dispatch_apply_key); if (!slowpath(nested)) { nested = iterations; @@ -210,7 +221,8 @@ dispatch_apply_f(size_t iterations, dispatch_queue_t dq, void *ctxt, dispatch_queue_t old_dq; old_dq = (dispatch_queue_t)_dispatch_thread_getspecific(dispatch_queue_key); if (slowpath(dq == DISPATCH_APPLY_CURRENT_ROOT_QUEUE)) { - dq = old_dq ? old_dq : _dispatch_get_root_queue(0, 0); + dq = old_dq ? old_dq : _dispatch_get_root_queue( + _DISPATCH_QOS_CLASS_DEFAULT, false); while (slowpath(dq->do_targetq)) { dq = dq->do_targetq; } diff --git a/src/data.m b/src/data.m index 3e3eee1..c76f26a 100644 --- a/src/data.m +++ b/src/data.m @@ -83,7 +83,7 @@ dispatch_async_f(tq, ctxt, func); \ } \ if (tq) { \ - _dispatch_release(tq); \ + _os_object_release_internal((_os_object_t)tq); \ } - (void)dealloc { @@ -116,10 +116,10 @@ - (void)_setTargetQueue:(dispatch_queue_t)queue { struct dispatch_data_s *dd = (void*)self; - _dispatch_retain(queue); + _os_object_retain_internal((_os_object_t)queue); dispatch_queue_t prev; prev = dispatch_atomic_xchg2o(dd, do_targetq, queue, release); - if (prev) _dispatch_release(prev); + if (prev) _os_object_release_internal((_os_object_t)prev); } - (NSString *)debugDescription { diff --git a/src/init.c b/src/init.c index 5a8b4bb..5cbf805 100644 --- a/src/init.c +++ b/src/init.c @@ -71,6 +71,7 @@ pthread_key_t dispatch_sema4_key; pthread_key_t dispatch_cache_key; pthread_key_t dispatch_io_key; pthread_key_t dispatch_apply_key; +pthread_key_t dispatch_defaultpriority_key; #if DISPATCH_INTROSPECTION pthread_key_t dispatch_introspection_key; #elif DISPATCH_PERF_MON @@ -78,7 +79,30 @@ pthread_key_t dispatch_bcounter_key; #endif #endif // !DISPATCH_USE_DIRECT_TSD -struct _dispatch_hw_config_s _dispatch_hw_config; +#if VOUCHER_USE_MACH_VOUCHER +dispatch_once_t _voucher_task_mach_voucher_pred; +mach_voucher_t _voucher_task_mach_voucher; +_voucher_activity_t _voucher_activity_default; +#endif +voucher_activity_mode_t _voucher_activity_mode; +int _dispatch_set_qos_class_enabled; + + +DISPATCH_NOINLINE +voucher_activity_mode_t +voucher_activity_get_mode(void) +{ + return _voucher_activity_mode; +} + +void +voucher_activity_set_mode_4libtrace(voucher_activity_mode_t mode) +{ + if (_voucher_activity_disabled()) return; + _voucher_activity_mode = mode; +} + +DISPATCH_HW_CONFIG(); bool _dispatch_safe_fork = true, _dispatch_child_of_unsafe_fork; DISPATCH_NOINLINE @@ -88,7 +112,6 @@ _dispatch_is_multithreaded(void) return !_dispatch_safe_fork; } - DISPATCH_NOINLINE bool _dispatch_is_fork_of_multithreaded_parent(void) @@ -97,19 +120,50 @@ _dispatch_is_fork_of_multithreaded_parent(void) } const struct dispatch_queue_offsets_s dispatch_queue_offsets = { - .dqo_version = 4, + .dqo_version = 5, .dqo_label = offsetof(struct dispatch_queue_s, dq_label), .dqo_label_size = sizeof(((dispatch_queue_t)NULL)->dq_label), .dqo_flags = 0, .dqo_flags_size = 0, - .dqo_width = offsetof(struct dispatch_queue_s, dq_width), - .dqo_width_size = sizeof(((dispatch_queue_t)NULL)->dq_width), .dqo_serialnum = offsetof(struct dispatch_queue_s, dq_serialnum), .dqo_serialnum_size = sizeof(((dispatch_queue_t)NULL)->dq_serialnum), + .dqo_width = offsetof(struct dispatch_queue_s, dq_width), + .dqo_width_size = sizeof(((dispatch_queue_t)NULL)->dq_width), .dqo_running = offsetof(struct dispatch_queue_s, dq_running), .dqo_running_size = sizeof(((dispatch_queue_t)NULL)->dq_running), + .dqo_suspend_cnt = offsetof(struct dispatch_queue_s, do_suspend_cnt), + .dqo_suspend_cnt_size = sizeof(((dispatch_queue_t)NULL)->do_suspend_cnt), + .dqo_target_queue = offsetof(struct dispatch_queue_s, do_targetq), + .dqo_target_queue_size = sizeof(((dispatch_queue_t)NULL)->do_targetq), + .dqo_priority = offsetof(struct dispatch_queue_s, dq_priority), + .dqo_priority_size = sizeof(((dispatch_queue_t)NULL)->dq_priority), }; +#if VOUCHER_USE_MACH_VOUCHER +const struct voucher_offsets_s voucher_offsets = { + .vo_version = 1, + .vo_activity_ids_count = offsetof(struct voucher_s, v_activities), + .vo_activity_ids_count_size = sizeof(((voucher_t)NULL)->v_activities), + .vo_activity_ids_array = (uint16_t)_voucher_activity_ids((voucher_t)(NULL)), + .vo_activity_ids_array_entry_size = sizeof(voucher_activity_id_t), +}; +#else // VOUCHER_USE_MACH_VOUCHER +const struct voucher_offsets_s voucher_offsets = { + .vo_version = 0, +}; +#endif // VOUCHER_USE_MACH_VOUCHER + +#if DISPATCH_USE_DIRECT_TSD +const struct dispatch_tsd_indexes_s dispatch_tsd_indexes = { + .dti_version = 2, + .dti_queue_index = dispatch_queue_key, + .dti_voucher_index = dispatch_voucher_key, + .dti_qos_class_index = dispatch_priority_key, +}; +#else // DISPATCH_USE_DIRECT_TSD +#error Not implemented on this platform +#endif // DISPATCH_USE_DIRECT_TSD + // 6618342 Contact the team that owns the Instrument DTrace probe before // renaming this symbol DISPATCH_CACHELINE_ALIGN @@ -117,7 +171,7 @@ struct dispatch_queue_s _dispatch_main_q = { .do_vtable = DISPATCH_VTABLE(queue), #if !DISPATCH_USE_RESOLVERS .do_targetq = &_dispatch_root_queues[ - DISPATCH_ROOT_QUEUE_IDX_DEFAULT_OVERCOMMIT_PRIORITY], + DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT], #endif .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, @@ -129,13 +183,72 @@ struct dispatch_queue_s _dispatch_main_q = { .dq_serialnum = 1, }; -struct dispatch_queue_attr_s _dispatch_queue_attr_concurrent = { - .do_vtable = DISPATCH_VTABLE(queue_attr), - .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - .do_next = DISPATCH_OBJECT_LISTLESS, +#pragma mark - +#pragma mark dispatch_queue_attr_t + +#define DISPATCH_QUEUE_ATTR_INITIALIZER(qos, prio, overcommit, concurrent) \ + { \ + .do_vtable = DISPATCH_VTABLE(queue_attr), \ + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, \ + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, \ + .do_next = DISPATCH_OBJECT_LISTLESS, \ + .dqa_qos_class = (qos), \ + .dqa_relative_priority = (qos) ? (prio) : 0, \ + .dqa_overcommit = (overcommit), \ + .dqa_concurrent = (concurrent), \ + } + +#define DISPATCH_QUEUE_ATTR_KIND_INIT(qos, prio) \ + { \ + [DQA_INDEX_NON_OVERCOMMIT][DQA_INDEX_CONCURRENT] = \ + DISPATCH_QUEUE_ATTR_INITIALIZER(qos, prio, 0, 1), \ + [DQA_INDEX_NON_OVERCOMMIT][DQA_INDEX_SERIAL] = \ + DISPATCH_QUEUE_ATTR_INITIALIZER(qos, prio, 0, 0), \ + [DQA_INDEX_OVERCOMMIT][DQA_INDEX_CONCURRENT] = \ + DISPATCH_QUEUE_ATTR_INITIALIZER(qos, prio, 1, 1), \ + [DQA_INDEX_OVERCOMMIT][DQA_INDEX_SERIAL] = \ + DISPATCH_QUEUE_ATTR_INITIALIZER(qos, prio, 1, 0), \ + } + +#define DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, prio) \ + [prio] = DISPATCH_QUEUE_ATTR_KIND_INIT(qos, -(prio)) + +#define DISPATCH_QUEUE_ATTR_PRIO_INIT(qos) \ + { \ + DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 0), \ + DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 1), \ + DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 2), \ + DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 3), \ + DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 4), \ + DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 5), \ + DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 6), \ + DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 7), \ + DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 8), \ + DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 9), \ + DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 10), \ + DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 11), \ + DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 12), \ + DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 13), \ + DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 14), \ + DISPATCH_QUEUE_ATTR_PRIO_INITIALIZER(qos, 15), \ + } + +#define DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(qos) \ + [DQA_INDEX_QOS_CLASS_##qos] = \ + DISPATCH_QUEUE_ATTR_PRIO_INIT(_DISPATCH_QOS_CLASS_##qos) + +const struct dispatch_queue_attr_s _dispatch_queue_attrs[] + [DISPATCH_QUEUE_ATTR_PRIO_COUNT][2][2] = { + DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(UNSPECIFIED), + DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(MAINTENANCE), + DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(BACKGROUND), + DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(UTILITY), + DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(DEFAULT), + DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(USER_INITIATED), + DISPATCH_QUEUE_ATTR_QOS_INITIALIZER(USER_INTERACTIVE), }; + #pragma mark - #pragma mark dispatch_vtables @@ -346,7 +459,7 @@ _dispatch_abort(size_t line, long val) abort(); } -#if !DISPATCH_USE_OS_TRACE +#if !DISPATCH_USE_OS_DEBUG_LOG #pragma mark - #pragma mark dispatch_log @@ -423,24 +536,55 @@ _dispatch_logv_file(const char *msg, va_list ap) _dispatch_log_file(buf, len); } +#if DISPATCH_USE_SIMPLE_ASL +static inline void +_dispatch_syslog(const char *msg) +{ + _simple_asl_log(ASL_LEVEL_NOTICE, "com.apple.libsystem.libdispatch", msg); +} + +static inline void +_dispatch_vsyslog(const char *msg, va_list ap) +{ + char *str; + vasprintf(&str, msg, ap); + if (str) { + _dispatch_syslog(str); + free(str); + } +} +#else // DISPATCH_USE_SIMPLE_ASL +static inline void +_dispatch_syslog(const char *msg) +{ + syslog(LOG_NOTICE, "%s", msg); +} + +static inline void +_dispatch_vsyslog(const char *msg, va_list ap) +{ + vsyslog(LOG_NOTICE, msg, *ap_ptr); +} +#endif // DISPATCH_USE_SIMPLE_ASL + DISPATCH_ALWAYS_INLINE static inline void -_dispatch_logv(const char *msg, size_t len, va_list ap) +_dispatch_logv(const char *msg, size_t len, va_list *ap_ptr) { dispatch_once_f(&_dispatch_logv_pred, NULL, _dispatch_logv_init); if (slowpath(dispatch_log_disabled)) { return; } if (slowpath(dispatch_logfile != -1)) { - if (!ap) { + if (!ap_ptr) { return _dispatch_log_file((char*)msg, len); } - return _dispatch_logv_file(msg, ap); + return _dispatch_logv_file(msg, *ap_ptr); } - if (!ap) { - return syslog(LOG_NOTICE, "%s", msg); + if (!ap_ptr) { + return _dispatch_syslog(msg); } - return vsyslog(LOG_NOTICE, msg, ap); + return _dispatch_vsyslog(msg, *ap_ptr); } DISPATCH_NOINLINE @@ -450,11 +594,11 @@ _dispatch_log(const char *msg, ...) va_list ap; va_start(ap, msg); - _dispatch_logv(msg, 0, ap); + _dispatch_logv(msg, 0, &ap); va_end(ap); } -#endif // DISPATCH_USE_OS_TRACE +#endif // DISPATCH_USE_OS_DEBUG_LOG #pragma mark - #pragma mark dispatch_debug @@ -486,7 +630,7 @@ _dispatch_debugv(dispatch_object_t dou, const char *msg, va_list ap) offs = strlcpy(buf, "NULL: ", sizeof(buf)); } r = vsnprintf(buf + offs, sizeof(buf) - offs, msg, ap); -#if !DISPATCH_USE_OS_TRACE +#if !DISPATCH_USE_OS_DEBUG_LOG size_t len = offs + (r < 0 ? 0 : (size_t)r); if (len > sizeof(buf) - 1) { len = sizeof(buf) - 1; @@ -576,13 +720,47 @@ _dispatch_call_block_and_release(void *block) Block_release(b); } +#pragma mark - +#pragma mark _dispatch_block_create no_objc + +#if !USE_OBJC + +// The compiler hides the name of the function it generates, and changes it if +// we try to reference it directly, but the linker still sees it. +extern void DISPATCH_BLOCK_SPECIAL_INVOKE(void *) + asm("____dispatch_block_create_block_invoke"); +void (*_dispatch_block_special_invoke)(void*) = DISPATCH_BLOCK_SPECIAL_INVOKE; + +dispatch_block_t +_dispatch_block_create(dispatch_block_flags_t flags, voucher_t voucher, + pthread_priority_t pri, dispatch_block_t block) +{ + dispatch_block_t copy_block = _dispatch_Block_copy(block); // 17094902 + (void)voucher; // No voucher capture! (requires ObjC runtime) + struct dispatch_block_private_data_s dbpds = + DISPATCH_BLOCK_PRIVATE_DATA_INITIALIZER(flags, NULL, pri, copy_block); + dispatch_block_t new_block = _dispatch_Block_copy(^{ + // Capture object references, which retains copy_block. + // All retained objects must be captured by the *block*. We + // cannot borrow any references, because the block might be + // called zero or several times, so Block_release() is the + // only place that can release retained objects. + (void)copy_block; + _dispatch_block_invoke(&dbpds); + }); + Block_release(copy_block); + return new_block; +} + +#endif // !USE_OBJC + #endif // __BLOCKS__ #pragma mark - #pragma mark dispatch_client_callout // Abort on uncaught exceptions thrown from client callouts rdar://8577499 -#if DISPATCH_USE_CLIENT_CALLOUT && (__arm__ || !USE_OBJC) +#if DISPATCH_USE_CLIENT_CALLOUT && (__USING_SJLJ_EXCEPTIONS__ || !USE_OBJC) // On platforms with SjLj exceptions, avoid the SjLj overhead on every callout // by clearing the unwinder's TSD pointer to the handler stack around callouts @@ -708,6 +886,24 @@ _os_object_dispose(_os_object_t obj) return _os_object_dealloc(obj); } +void* +os_retain(void *obj) +{ + if (fastpath(obj)) { + return _os_object_retain(obj); + } + return obj; +} + +#undef os_release +void +os_release(void *obj) +{ + if (fastpath(obj)) { + return _os_object_release(obj); + } +} + #pragma mark - #pragma mark dispatch_autorelease_pool no_objc @@ -821,65 +1017,87 @@ const struct dispatch_source_type_s _dispatch_source_type_write = { }, }; -#if DISPATCH_USE_VM_PRESSURE +#if DISPATCH_USE_MEMORYSTATUS + #if TARGET_IPHONE_SIMULATOR // rdar://problem/9219483 static int _dispatch_ios_simulator_memory_warnings_fd = -1; static void -_dispatch_ios_simulator_vm_source_init(void *context DISPATCH_UNUSED) +_dispatch_ios_simulator_memorypressure_init(void *context DISPATCH_UNUSED) { - char *e = getenv("IPHONE_SIMULATOR_MEMORY_WARNINGS"); + char *e = getenv("SIMULATOR_MEMORY_WARNINGS"); if (!e) return; _dispatch_ios_simulator_memory_warnings_fd = open(e, O_EVTONLY); if (_dispatch_ios_simulator_memory_warnings_fd == -1) { (void)dispatch_assume_zero(errno); } } +#endif + static void -dispatch_source_type_vm_init(dispatch_source_t ds, +dispatch_source_type_memorystatus_init(dispatch_source_t ds, dispatch_source_type_t type DISPATCH_UNUSED, uintptr_t handle DISPATCH_UNUSED, - unsigned long mask, + unsigned long mask DISPATCH_UNUSED, dispatch_queue_t q DISPATCH_UNUSED) { +#if TARGET_IPHONE_SIMULATOR static dispatch_once_t pred; - dispatch_once_f(&pred, NULL, _dispatch_ios_simulator_vm_source_init); - ds->ds_dkev->dk_kevent.ident = (uint64_t)(mask & DISPATCH_VM_PRESSURE ? - _dispatch_ios_simulator_memory_warnings_fd : -1); + dispatch_once_f(&pred, NULL, _dispatch_ios_simulator_memorypressure_init); + handle = (uintptr_t)_dispatch_ios_simulator_memory_warnings_fd; + mask = NOTE_ATTRIB; + ds->ds_dkev->dk_kevent.filter = EVFILT_VNODE; + ds->ds_dkev->dk_kevent.ident = handle; + ds->ds_dkev->dk_kevent.flags |= EV_CLEAR; + ds->ds_dkev->dk_kevent.fflags = (uint32_t)mask; + ds->ds_ident_hack = handle; + ds->ds_pending_data_mask = mask; + ds->ds_memorystatus_override = 1; +#endif + ds->ds_is_level = false; } -const struct dispatch_source_type_s _dispatch_source_type_vm = { +#ifndef NOTE_MEMORYSTATUS_LOW_SWAP +#define NOTE_MEMORYSTATUS_LOW_SWAP 0x8 +#endif + +const struct dispatch_source_type_s _dispatch_source_type_memorystatus = { .ke = { - .filter = EVFILT_VNODE, - .flags = EV_CLEAR, + .filter = EVFILT_MEMORYSTATUS, + .flags = EV_DISPATCH, }, - .mask = NOTE_ATTRIB, - .init = dispatch_source_type_vm_init, + .mask = NOTE_MEMORYSTATUS_PRESSURE_NORMAL|NOTE_MEMORYSTATUS_PRESSURE_WARN + |NOTE_MEMORYSTATUS_PRESSURE_CRITICAL|NOTE_MEMORYSTATUS_LOW_SWAP, + .init = dispatch_source_type_memorystatus_init, }; -#else + static void dispatch_source_type_vm_init(dispatch_source_t ds, - dispatch_source_type_t type DISPATCH_UNUSED, - uintptr_t handle DISPATCH_UNUSED, - unsigned long mask DISPATCH_UNUSED, - dispatch_queue_t q DISPATCH_UNUSED) + dispatch_source_type_t type, + uintptr_t handle, + unsigned long mask, + dispatch_queue_t q) { - ds->ds_is_level = false; + // Map legacy vm pressure to memorystatus warning rdar://problem/15907505 + mask = NOTE_MEMORYSTATUS_PRESSURE_WARN; + ds->ds_dkev->dk_kevent.fflags = (uint32_t)mask; + ds->ds_pending_data_mask = mask; + ds->ds_vmpressure_override = 1; + dispatch_source_type_memorystatus_init(ds, type, handle, mask, q); } const struct dispatch_source_type_s _dispatch_source_type_vm = { .ke = { - .filter = EVFILT_VM, + .filter = EVFILT_MEMORYSTATUS, .flags = EV_DISPATCH, }, .mask = NOTE_VM_PRESSURE, .init = dispatch_source_type_vm_init, }; -#endif -#endif -#ifdef DISPATCH_USE_MEMORYSTATUS +#elif DISPATCH_USE_VM_PRESSURE + static void -dispatch_source_type_memorystatus_init(dispatch_source_t ds, +dispatch_source_type_vm_init(dispatch_source_t ds, dispatch_source_type_t type DISPATCH_UNUSED, uintptr_t handle DISPATCH_UNUSED, unsigned long mask DISPATCH_UNUSED, @@ -888,19 +1106,16 @@ dispatch_source_type_memorystatus_init(dispatch_source_t ds, ds->ds_is_level = false; } -const struct dispatch_source_type_s _dispatch_source_type_memorystatus = { +const struct dispatch_source_type_s _dispatch_source_type_vm = { .ke = { - .filter = EVFILT_MEMORYSTATUS, + .filter = EVFILT_VM, .flags = EV_DISPATCH, }, - .mask = NOTE_MEMORYSTATUS_PRESSURE_NORMAL|NOTE_MEMORYSTATUS_PRESSURE_WARN -#ifdef NOTE_MEMORYSTATUS_PRESSURE_CRITICAL - |NOTE_MEMORYSTATUS_PRESSURE_CRITICAL -#endif - , - .init = dispatch_source_type_memorystatus_init, + .mask = NOTE_VM_PRESSURE, + .init = dispatch_source_type_vm_init, }; -#endif + +#endif // DISPATCH_USE_VM_PRESSURE const struct dispatch_source_type_s _dispatch_source_type_proc = { .ke = { diff --git a/src/inline_internal.h b/src/inline_internal.h new file mode 100644 index 0000000..ea6953c --- /dev/null +++ b/src/inline_internal.h @@ -0,0 +1,996 @@ +/* + * Copyright (c) 2008-2013 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +/* + * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch + * which are subject to change in future releases of Mac OS X. Any applications + * relying on these interfaces WILL break. + */ + +#ifndef __DISPATCH_INLINE_INTERNAL__ +#define __DISPATCH_INLINE_INTERNAL__ + +#ifndef __DISPATCH_INDIRECT__ +#error "Please #include instead of this file directly." +#include // for HeaderDoc +#endif + +#if DISPATCH_USE_CLIENT_CALLOUT + +DISPATCH_NOTHROW void +_dispatch_client_callout(void *ctxt, dispatch_function_t f); +DISPATCH_NOTHROW void +_dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t)); +DISPATCH_NOTHROW bool +_dispatch_client_callout3(void *ctxt, dispatch_data_t region, size_t offset, + const void *buffer, size_t size, dispatch_data_applier_function_t f); +DISPATCH_NOTHROW void +_dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason, + dispatch_mach_msg_t dmsg, mach_error_t error, + dispatch_mach_handler_function_t f); + +#else // !DISPATCH_USE_CLIENT_CALLOUT + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_client_callout(void *ctxt, dispatch_function_t f) +{ + return f(ctxt); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t)) +{ + return f(ctxt, i); +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_dispatch_client_callout3(void *ctxt, dispatch_data_t region, size_t offset, + const void *buffer, size_t size, dispatch_data_applier_function_t f) +{ + return f(ctxt, region, offset, buffer, size); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason, + dispatch_mach_msg_t dmsg, mach_error_t error, + dispatch_mach_handler_function_t f) +{ + return f(ctxt, reason, dmsg, error); +} + +#endif // !DISPATCH_USE_CLIENT_CALLOUT + +#if !(USE_OBJC && __OBJC2__) + +#pragma mark - +#pragma mark _os_object_t & dispatch_object_t + +DISPATCH_ALWAYS_INLINE +static inline _os_object_t +_os_object_retain_internal_inline(_os_object_t obj) +{ + int ref_cnt = obj->os_obj_ref_cnt; + if (slowpath(ref_cnt == _OS_OBJECT_GLOBAL_REFCNT)) { + return obj; // global object + } + ref_cnt = dispatch_atomic_inc2o(obj, os_obj_ref_cnt, relaxed); + if (slowpath(ref_cnt <= 0)) { + DISPATCH_CRASH("Resurrection of an object"); + } + return obj; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_os_object_release_internal_inline(_os_object_t obj) +{ + int ref_cnt = obj->os_obj_ref_cnt; + if (slowpath(ref_cnt == _OS_OBJECT_GLOBAL_REFCNT)) { + return; // global object + } + ref_cnt = dispatch_atomic_dec2o(obj, os_obj_ref_cnt, relaxed); + if (fastpath(ref_cnt >= 0)) { + return; + } + if (slowpath(ref_cnt < -1)) { + DISPATCH_CRASH("Over-release of an object"); + } +#if DISPATCH_DEBUG + if (slowpath(obj->os_obj_xref_cnt >= 0)) { + DISPATCH_CRASH("Release while external references exist"); + } +#endif + return _os_object_dispose(obj); +} + +DISPATCH_ALWAYS_INLINE_NDEBUG +static inline void +_dispatch_retain(dispatch_object_t dou) +{ + (void)_os_object_retain_internal_inline(dou._os_obj); +} + +DISPATCH_ALWAYS_INLINE_NDEBUG +static inline void +_dispatch_release(dispatch_object_t dou) +{ + _os_object_release_internal_inline(dou._os_obj); +} + +#pragma mark - +#pragma mark dispatch_thread + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_wqthread_override_start(mach_port_t thread, + pthread_priority_t priority) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + if (!_dispatch_set_qos_class_enabled) return; + (void)_pthread_workqueue_override_start_direct(thread, priority); +#else + (void)thread; (void)priority; +#endif +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_wqthread_override_reset(void) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + if (!_dispatch_set_qos_class_enabled) return; + (void)_pthread_workqueue_override_reset(); +#endif +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_thread_override_start(mach_port_t thread, pthread_priority_t priority) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + if (!_dispatch_set_qos_class_enabled) return; + (void)_pthread_override_qos_class_start_direct(thread, priority); +#else + (void)thread; (void)priority; +#endif +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_thread_override_end(mach_port_t thread) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + if (!_dispatch_set_qos_class_enabled) return; + (void)_pthread_override_qos_class_end_direct(thread); +#else + (void)thread; +#endif +} + +#pragma mark - +#pragma mark dispatch_queue_t + +static inline bool _dispatch_queue_need_override(dispatch_queue_t dq, + pthread_priority_t pp); +static inline bool _dispatch_queue_need_override_retain(dispatch_queue_t dq, + pthread_priority_t pp); +static inline bool _dispatch_queue_retain_if_override(dispatch_queue_t dq, + pthread_priority_t pp); +static inline pthread_priority_t _dispatch_queue_get_override_priority( + dispatch_queue_t dq); +static inline pthread_priority_t _dispatch_queue_reset_override_priority( + dispatch_queue_t dq); +static inline pthread_priority_t _dispatch_get_defaultpriority(void); +static inline void _dispatch_set_defaultpriority_override(void); +static inline void _dispatch_reset_defaultpriority(pthread_priority_t priority); +static inline void _dispatch_set_priority(pthread_priority_t priority); + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_queue_set_thread(dispatch_queue_t dq) +{ + // The manager queue uses dispatch_queue_drain but is thread bound + if (!dq->dq_is_thread_bound) { + dq->dq_thread = _dispatch_thread_port(); + } +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_queue_clear_thread(dispatch_queue_t dq) +{ + if (!dq->dq_is_thread_bound) { + dq->dq_thread = MACH_PORT_NULL; + } +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_dispatch_queue_push_list2(dispatch_queue_t dq, struct dispatch_object_s *head, + struct dispatch_object_s *tail) +{ + struct dispatch_object_s *prev; + tail->do_next = NULL; + prev = dispatch_atomic_xchg2o(dq, dq_items_tail, tail, release); + if (fastpath(prev)) { + // if we crash here with a value less than 0x1000, then we are at a + // known bug in client code for example, see _dispatch_queue_dispose + // or _dispatch_atfork_child + prev->do_next = head; + } + return (prev != NULL); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_queue_push_list(dispatch_queue_t dq, dispatch_object_t _head, + dispatch_object_t _tail, pthread_priority_t pp, unsigned int n) +{ + struct dispatch_object_s *head = _head._do, *tail = _tail._do; + bool override = _dispatch_queue_need_override_retain(dq, pp); + if (!fastpath(_dispatch_queue_push_list2(dq, head, tail))) { + _dispatch_queue_push_list_slow(dq, pp, head, n, override); + } else if (override) { + _dispatch_queue_wakeup_with_qos_and_release(dq, pp); + } +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_queue_push(dispatch_queue_t dq, dispatch_object_t _tail, + pthread_priority_t pp) +{ + struct dispatch_object_s *tail = _tail._do; + bool override = _dispatch_queue_need_override_retain(dq, pp); + if (!fastpath(_dispatch_queue_push_list2(dq, tail, tail))) { + _dispatch_queue_push_slow(dq, pp, tail, override); + } else if (override) { + _dispatch_queue_wakeup_with_qos_and_release(dq, pp); + } +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_queue_push_wakeup(dispatch_queue_t dq, dispatch_object_t _tail, + pthread_priority_t pp, bool wakeup) +{ + // caller assumed to have a reference on dq + struct dispatch_object_s *tail = _tail._do; + if (!fastpath(_dispatch_queue_push_list2(dq, tail, tail))) { + _dispatch_queue_push_slow(dq, pp, tail, false); + } else if (_dispatch_queue_need_override(dq, pp)) { + _dispatch_queue_wakeup_with_qos(dq, pp); + } else if (slowpath(wakeup)) { + _dispatch_queue_wakeup(dq); + } +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_queue_class_invoke(dispatch_object_t dou, + dispatch_queue_t (*invoke)(dispatch_object_t, + _dispatch_thread_semaphore_t*)) +{ + pthread_priority_t p = 0; + dispatch_queue_t dq = dou._dq; + if (!slowpath(DISPATCH_OBJECT_SUSPENDED(dq)) && + fastpath(dispatch_atomic_cmpxchg2o(dq, dq_running, 0, 1, acquire))){ + _dispatch_queue_set_thread(dq); + dispatch_queue_t tq = NULL; + _dispatch_thread_semaphore_t sema = 0; + tq = invoke(dq, &sema); + _dispatch_queue_clear_thread(dq); + p = _dispatch_queue_reset_override_priority(dq); + if (p > (dq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK)) { + // Ensure that the root queue sees that this thread was overridden. + _dispatch_set_defaultpriority_override(); + } + // We do not need to check the result. + // When the suspend-count lock is dropped, then the check will happen. + (void)dispatch_atomic_dec2o(dq, dq_running, release); + if (sema) { + _dispatch_thread_semaphore_signal(sema); + } else if (tq) { + _dispatch_introspection_queue_item_complete(dq); + return _dispatch_queue_push(tq, dq, p); + } + } + dq->do_next = DISPATCH_OBJECT_LISTLESS; + _dispatch_introspection_queue_item_complete(dq); + if (!dispatch_atomic_sub2o(dq, do_suspend_cnt, + DISPATCH_OBJECT_SUSPEND_LOCK, seq_cst)) { + // seq_cst with atomic store to suspend_cnt + if (dispatch_atomic_load2o(dq, dq_running, seq_cst) == 0) { + // verify that the queue is idle + return _dispatch_queue_wakeup_with_qos_and_release(dq, p); + } + } + _dispatch_release(dq); // added when the queue is put on the list +} + +DISPATCH_ALWAYS_INLINE +static inline unsigned long +_dispatch_queue_class_probe(dispatch_object_t dou) +{ + dispatch_queue_t dq = dou._dq; + struct dispatch_object_s *tail; + // seq_cst with atomic store to suspend_cnt + tail = dispatch_atomic_load2o(dq, dq_items_tail, seq_cst); + return (unsigned long)slowpath(tail != NULL); +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_dispatch_object_suspended(dispatch_object_t dou) +{ + struct dispatch_object_s *obj = dou._do; + unsigned int suspend_cnt; + // seq_cst with atomic store to tail + suspend_cnt = dispatch_atomic_load2o(obj, do_suspend_cnt, seq_cst); + return slowpath(suspend_cnt >= DISPATCH_OBJECT_SUSPEND_INTERVAL); +} + +DISPATCH_ALWAYS_INLINE +static inline dispatch_queue_t +_dispatch_queue_get_current(void) +{ + return (dispatch_queue_t)_dispatch_thread_getspecific(dispatch_queue_key); +} + +DISPATCH_ALWAYS_INLINE DISPATCH_CONST +static inline dispatch_queue_t +_dispatch_get_root_queue(qos_class_t priority, bool overcommit) +{ + if (overcommit) switch (priority) { + case _DISPATCH_QOS_CLASS_MAINTENANCE: + return &_dispatch_root_queues[ + DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS_OVERCOMMIT]; + case _DISPATCH_QOS_CLASS_BACKGROUND: + return &_dispatch_root_queues[ + DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS_OVERCOMMIT]; + case _DISPATCH_QOS_CLASS_UTILITY: + return &_dispatch_root_queues[ + DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS_OVERCOMMIT]; + case _DISPATCH_QOS_CLASS_DEFAULT: + return &_dispatch_root_queues[ + DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT]; + case _DISPATCH_QOS_CLASS_USER_INITIATED: + return &_dispatch_root_queues[ + DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS_OVERCOMMIT]; + case _DISPATCH_QOS_CLASS_USER_INTERACTIVE: + return &_dispatch_root_queues[ + DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS_OVERCOMMIT]; + } else switch (priority) { + case _DISPATCH_QOS_CLASS_MAINTENANCE: + return &_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS]; + case _DISPATCH_QOS_CLASS_BACKGROUND: + return &_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS]; + case _DISPATCH_QOS_CLASS_UTILITY: + return &_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS]; + case _DISPATCH_QOS_CLASS_DEFAULT: + return &_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS]; + case _DISPATCH_QOS_CLASS_USER_INITIATED: + return &_dispatch_root_queues[ + DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS]; + case _DISPATCH_QOS_CLASS_USER_INTERACTIVE: + return &_dispatch_root_queues[ + DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS]; + } + return NULL; +} + +// Note to later developers: ensure that any initialization changes are +// made for statically allocated queues (i.e. _dispatch_main_q). +static inline void +_dispatch_queue_init(dispatch_queue_t dq) +{ + dq->do_next = (struct dispatch_queue_s *)DISPATCH_OBJECT_LISTLESS; + + dq->dq_running = 0; + dq->dq_width = 1; + dq->dq_serialnum = dispatch_atomic_inc_orig(&_dispatch_queue_serial_numbers, + relaxed); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_queue_set_bound_thread(dispatch_queue_t dq) +{ + //Tag thread-bound queues with the owning thread + dispatch_assert(dq->dq_is_thread_bound); + dq->dq_thread = _dispatch_thread_port(); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_queue_clear_bound_thread(dispatch_queue_t dq) +{ + dispatch_assert(dq->dq_is_thread_bound); + dq->dq_thread = MACH_PORT_NULL; +} + +DISPATCH_ALWAYS_INLINE +static inline mach_port_t +_dispatch_queue_get_bound_thread(dispatch_queue_t dq) +{ + dispatch_assert(dq->dq_is_thread_bound); + return dq->dq_thread; +} + +#pragma mark - +#pragma mark dispatch_priority + +DISPATCH_ALWAYS_INLINE +static inline pthread_priority_t +_dispatch_get_defaultpriority(void) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + pthread_priority_t priority = (uintptr_t)_dispatch_thread_getspecific( + dispatch_defaultpriority_key); + return priority; +#else + return 0; +#endif +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_reset_defaultpriority(pthread_priority_t priority) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + pthread_priority_t old_priority = _dispatch_get_defaultpriority(); + // if an inner-loop or'd in the override flag to the per-thread priority, + // it needs to be propogated up the chain + priority |= old_priority & _PTHREAD_PRIORITY_OVERRIDE_FLAG; + + if (slowpath(priority != old_priority)) { + _dispatch_thread_setspecific(dispatch_defaultpriority_key, + (void*)priority); + } +#else + (void)priority; +#endif +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_set_defaultpriority_override(void) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + pthread_priority_t old_priority = _dispatch_get_defaultpriority(); + pthread_priority_t priority = old_priority | + _PTHREAD_PRIORITY_OVERRIDE_FLAG; + + if (slowpath(priority != old_priority)) { + _dispatch_thread_setspecific(dispatch_defaultpriority_key, + (void*)priority); + } +#endif +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_dispatch_reset_defaultpriority_override(void) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + pthread_priority_t old_priority = _dispatch_get_defaultpriority(); + pthread_priority_t priority = old_priority & + ~((pthread_priority_t)_PTHREAD_PRIORITY_OVERRIDE_FLAG); + + if (slowpath(priority != old_priority)) { + _dispatch_thread_setspecific(dispatch_defaultpriority_key, + (void*)priority); + return true; + } +#endif + return false; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_queue_priority_inherit_from_target(dispatch_queue_t dq, + dispatch_queue_t tq) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + const pthread_priority_t rootqueue_flag = _PTHREAD_PRIORITY_ROOTQUEUE_FLAG; + const pthread_priority_t inherited_flag = _PTHREAD_PRIORITY_INHERIT_FLAG; + pthread_priority_t dqp = dq->dq_priority, tqp = tq->dq_priority; + if ((!dqp || (dqp & inherited_flag)) && (tqp & rootqueue_flag)) { + dq->dq_priority = (tqp & ~rootqueue_flag) | inherited_flag; + } +#else + (void)dq; (void)tq; +#endif +} + +DISPATCH_ALWAYS_INLINE +static inline pthread_priority_t +_dispatch_set_defaultpriority(pthread_priority_t priority) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + pthread_priority_t old_priority = _dispatch_get_defaultpriority(); + if (old_priority) { + pthread_priority_t flags, defaultqueue, basepri; + flags = (priority & _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG); + defaultqueue = (old_priority & _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG); + basepri = (old_priority & ~_PTHREAD_PRIORITY_FLAGS_MASK); + priority &= ~_PTHREAD_PRIORITY_FLAGS_MASK; + if (!priority) { + flags = _PTHREAD_PRIORITY_INHERIT_FLAG | defaultqueue; + priority = basepri; + } else if (priority < basepri && !defaultqueue) { // rdar://16349734 + priority = basepri; + } + priority |= flags | (old_priority & _PTHREAD_PRIORITY_OVERRIDE_FLAG); + } + if (slowpath(priority != old_priority)) { + _dispatch_thread_setspecific(dispatch_defaultpriority_key, + (void*)priority); + } + return old_priority; +#else + (void)priority; + return 0; +#endif +} + +DISPATCH_ALWAYS_INLINE +static inline pthread_priority_t +_dispatch_priority_adopt(pthread_priority_t priority, unsigned long flags) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + pthread_priority_t defaultpri = _dispatch_get_defaultpriority(); + bool enforce, inherited, defaultqueue; + enforce = (flags & DISPATCH_PRIORITY_ENFORCE) || + (priority & _PTHREAD_PRIORITY_ENFORCE_FLAG); + inherited = (defaultpri & _PTHREAD_PRIORITY_INHERIT_FLAG); + defaultqueue = (defaultpri & _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG); + defaultpri &= ~_PTHREAD_PRIORITY_FLAGS_MASK; + priority &= ~_PTHREAD_PRIORITY_FLAGS_MASK; + if (!priority) { + enforce = false; + } else if (!enforce) { + if (priority < defaultpri) { + if (defaultqueue) enforce = true; // rdar://16349734 + } else if (inherited || defaultqueue) { + enforce = true; + } + } else if (priority < defaultpri && !defaultqueue) { // rdar://16349734 + enforce = false; + } + return enforce ? priority : defaultpri; +#else + (void)priority; (void)flags; + return 0; +#endif +} + +DISPATCH_ALWAYS_INLINE +static inline pthread_priority_t +_dispatch_get_priority(void) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + pthread_priority_t priority = (uintptr_t)_dispatch_thread_getspecific( + dispatch_priority_key); + return (priority & ~_PTHREAD_PRIORITY_FLAGS_MASK); +#else + return 0; +#endif +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_set_priority_and_mach_voucher(pthread_priority_t priority, + mach_voucher_t kv) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + _pthread_set_flags_t flags = 0; + if (priority && _dispatch_set_qos_class_enabled) { + pthread_priority_t old_priority = _dispatch_get_priority(); + if (priority != old_priority && old_priority) { + flags |= _PTHREAD_SET_SELF_QOS_FLAG; + } + } + if (kv != VOUCHER_NO_MACH_VOUCHER) { +#if VOUCHER_USE_MACH_VOUCHER + flags |= _PTHREAD_SET_SELF_VOUCHER_FLAG; +#endif + } + if (!flags) return; + int r = _pthread_set_properties_self(flags, priority, kv); + (void)dispatch_assume_zero(r); +#elif VOUCHER_USE_MACH_VOUCHER +#error Invalid build configuration +#else + (void)priority; (void)kv; +#endif +} + +DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT +static inline voucher_t +_dispatch_set_priority_and_adopt_voucher(pthread_priority_t priority, + voucher_t voucher) +{ + pthread_priority_t p = (priority != DISPATCH_NO_PRIORITY) ? priority : 0; + voucher_t ov = DISPATCH_NO_VOUCHER; + mach_voucher_t kv = VOUCHER_NO_MACH_VOUCHER; + if (voucher != DISPATCH_NO_VOUCHER) { + ov = _voucher_get(); + kv = _voucher_swap_and_get_mach_voucher(ov, voucher); + } + _dispatch_set_priority_and_mach_voucher(p, kv); + return ov; +} + +DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT +static inline voucher_t +_dispatch_adopt_priority_and_voucher(pthread_priority_t priority, + voucher_t voucher, unsigned long flags) +{ + pthread_priority_t p = 0; + if (priority != DISPATCH_NO_PRIORITY) { + p = _dispatch_priority_adopt(priority, flags); + } + return _dispatch_set_priority_and_adopt_voucher(p, voucher); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_adopt_priority_and_replace_voucher(pthread_priority_t priority, + voucher_t voucher, unsigned long flags) +{ + voucher_t ov; + ov = _dispatch_adopt_priority_and_voucher(priority, voucher, flags); + if (voucher != DISPATCH_NO_VOUCHER && ov) _voucher_release(ov); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_set_priority_and_replace_voucher(pthread_priority_t priority, + voucher_t voucher) +{ + voucher_t ov; + ov = _dispatch_set_priority_and_adopt_voucher(priority, voucher); + if (voucher != DISPATCH_NO_VOUCHER && ov) _voucher_release(ov); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_set_priority(pthread_priority_t priority) +{ + _dispatch_set_priority_and_mach_voucher(priority, VOUCHER_NO_MACH_VOUCHER); +} + +DISPATCH_ALWAYS_INLINE +static inline pthread_priority_t +_dispatch_priority_normalize(pthread_priority_t pp) +{ + dispatch_assert_zero(pp & ~(pthread_priority_t) + _PTHREAD_PRIORITY_QOS_CLASS_MASK); + unsigned int qosbits = (unsigned int)pp, idx; + if (!qosbits) return 0; + idx = (unsigned int)(sizeof(qosbits)*8) - + (unsigned int)__builtin_clz(qosbits) - 1; + return (1 << idx); +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_dispatch_queue_need_override(dispatch_queue_t dq, pthread_priority_t pp) +{ + if (!pp || dx_type(dq) == DISPATCH_QUEUE_ROOT_TYPE) return false; + uint32_t p = (pp & _PTHREAD_PRIORITY_QOS_CLASS_MASK); + uint32_t o = dq->dq_override; + return (o < p); +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_dispatch_queue_need_override_retain(dispatch_queue_t dq, pthread_priority_t pp) +{ + bool override = _dispatch_queue_need_override(dq, pp); + if (override) _dispatch_retain(dq); + return override; +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_dispatch_queue_override_priority(dispatch_queue_t dq, pthread_priority_t pp) +{ + uint32_t p = (pp & _PTHREAD_PRIORITY_QOS_CLASS_MASK); + uint32_t o = dq->dq_override; + if (o < p) o = dispatch_atomic_or_orig2o(dq, dq_override, p, relaxed); + return (o < p); +} + +DISPATCH_ALWAYS_INLINE +static inline pthread_priority_t +_dispatch_queue_get_override_priority(dispatch_queue_t dq) +{ + uint32_t p = (dq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK); + uint32_t o = dq->dq_override; + if (o == p) return o; + return _dispatch_priority_normalize(o); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_queue_set_override_priority(dispatch_queue_t dq) +{ + uint32_t p = 0; + if (!(dq->dq_priority & _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG)) { + p = dq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK; + } + dispatch_atomic_store2o(dq, dq_override, p, relaxed); +} + +DISPATCH_ALWAYS_INLINE +static inline pthread_priority_t +_dispatch_queue_reset_override_priority(dispatch_queue_t dq) +{ + uint32_t p = 0; + if (!(dq->dq_priority & _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG)) { + p = dq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK; + } + uint32_t o = dispatch_atomic_xchg2o(dq, dq_override, p, relaxed); + if (o == p) return o; + return _dispatch_priority_normalize(o); +} + +DISPATCH_ALWAYS_INLINE +static inline pthread_priority_t +_dispatch_priority_propagate(void) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + pthread_priority_t priority = _dispatch_get_priority(); + if (priority > _dispatch_user_initiated_priority) { + // Cap QOS for propagation at user-initiated + priority = _dispatch_user_initiated_priority; + } + return priority; +#else + return 0; +#endif +} + +// including maintenance +DISPATCH_ALWAYS_INLINE +static inline bool +_dispatch_is_background_thread(void) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + pthread_priority_t priority; + priority = _dispatch_get_priority(); + return priority && (priority <= _dispatch_background_priority); +#else + return false; +#endif +} + +#pragma mark - +#pragma mark dispatch_block_t + +#ifdef __BLOCKS__ + +DISPATCH_ALWAYS_INLINE +static inline bool +_dispatch_block_has_private_data(const dispatch_block_t block) +{ + extern void (*_dispatch_block_special_invoke)(void*); + return (_dispatch_Block_invoke(block) == _dispatch_block_special_invoke); +} + +DISPATCH_ALWAYS_INLINE +static inline dispatch_block_private_data_t +_dispatch_block_get_data(const dispatch_block_t db) +{ + if (!_dispatch_block_has_private_data(db)) { + return NULL; + } + // Keep in sync with _dispatch_block_create implementation + uint8_t *x = (uint8_t *)db; + // x points to base of struct Block_layout + x += sizeof(struct Block_layout); + // x points to addresss of captured block + x += sizeof(dispatch_block_t); +#if USE_OBJC + // x points to addresss of captured voucher + x += sizeof(voucher_t); +#endif + // x points to base of captured dispatch_block_private_data_s structure + dispatch_block_private_data_t dbpd = (dispatch_block_private_data_t)x; + if (dbpd->dbpd_magic != DISPATCH_BLOCK_PRIVATE_DATA_MAGIC) { + DISPATCH_CRASH("Corruption of dispatch block object"); + } + return dbpd; +} + +DISPATCH_ALWAYS_INLINE +static inline pthread_priority_t +_dispatch_block_get_priority(const dispatch_block_t db) +{ + dispatch_block_private_data_t dbpd = _dispatch_block_get_data(db); + return dbpd ? dbpd->dbpd_priority : 0; +} + +DISPATCH_ALWAYS_INLINE +static inline dispatch_block_flags_t +_dispatch_block_get_flags(const dispatch_block_t db) +{ + dispatch_block_private_data_t dbpd = _dispatch_block_get_data(db); + return dbpd ? dbpd->dbpd_flags : 0; +} + +#define DISPATCH_BLOCK_HAS(flag, db) \ + ((_dispatch_block_get_flags((db)) & DISPATCH_BLOCK_HAS_ ## flag) != 0) +#define DISPATCH_BLOCK_IS(flag, db) \ + ((_dispatch_block_get_flags((db)) & DISPATCH_BLOCK_ ## flag) != 0) + +#endif + +#pragma mark - +#pragma mark dispatch_continuation_t + +DISPATCH_ALWAYS_INLINE +static inline dispatch_continuation_t +_dispatch_continuation_alloc_cacheonly(void) +{ + dispatch_continuation_t dc = (dispatch_continuation_t) + fastpath(_dispatch_thread_getspecific(dispatch_cache_key)); + if (dc) { + _dispatch_thread_setspecific(dispatch_cache_key, dc->do_next); + } + return dc; +} + +DISPATCH_ALWAYS_INLINE +static inline dispatch_continuation_t +_dispatch_continuation_alloc(void) +{ + dispatch_continuation_t dc = + fastpath(_dispatch_continuation_alloc_cacheonly()); + if(!dc) { + return _dispatch_continuation_alloc_from_heap(); + } + return dc; +} + +DISPATCH_ALWAYS_INLINE +static inline dispatch_continuation_t +_dispatch_continuation_free_cacheonly(dispatch_continuation_t dc) +{ + dispatch_continuation_t prev_dc = (dispatch_continuation_t) + fastpath(_dispatch_thread_getspecific(dispatch_cache_key)); + int cnt = prev_dc ? prev_dc->dc_cache_cnt + 1 : 1; + // Cap continuation cache + if (slowpath(cnt > _dispatch_continuation_cache_limit)) { + return dc; + } + dc->do_next = prev_dc; + dc->dc_cache_cnt = cnt; + _dispatch_thread_setspecific(dispatch_cache_key, dc); + return NULL; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_continuation_free(dispatch_continuation_t dc) +{ + dc = _dispatch_continuation_free_cacheonly(dc); + if (slowpath(dc)) { + _dispatch_continuation_free_to_cache_limit(dc); + } +} + +#include "trace.h" + +DISPATCH_ALWAYS_INLINE_NDEBUG +static inline void +_dispatch_continuation_pop(dispatch_object_t dou) +{ + dispatch_continuation_t dc = dou._dc, dc1; + dispatch_group_t dg; + + _dispatch_trace_continuation_pop(_dispatch_queue_get_current(), dou); + if (DISPATCH_OBJ_IS_VTABLE(dou._do)) { + return dx_invoke(dou._do); + } + + // Add the item back to the cache before calling the function. This + // allows the 'hot' continuation to be used for a quick callback. + // + // The ccache version is per-thread. + // Therefore, the object has not been reused yet. + // This generates better assembly. + if ((long)dc->do_vtable & DISPATCH_OBJ_ASYNC_BIT) { + _dispatch_continuation_voucher_adopt(dc); + dc1 = _dispatch_continuation_free_cacheonly(dc); + } else { + dc1 = NULL; + } + if ((long)dc->do_vtable & DISPATCH_OBJ_GROUP_BIT) { + dg = dc->dc_data; + } else { + dg = NULL; + } + _dispatch_client_callout(dc->dc_ctxt, dc->dc_func); + if (dg) { + dispatch_group_leave(dg); + _dispatch_release(dg); + } + _dispatch_introspection_queue_item_complete(dou); + if (slowpath(dc1)) { + _dispatch_continuation_free_to_cache_limit(dc1); + } +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_continuation_priority_set(dispatch_continuation_t dc, + pthread_priority_t pp, dispatch_block_flags_t flags) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + pthread_priority_t prio = 0; + if (flags & DISPATCH_BLOCK_HAS_PRIORITY) { + prio = pp; + } else if (!(flags & DISPATCH_BLOCK_NO_QOS_CLASS)) { + prio = _dispatch_priority_propagate(); + } + if (flags & DISPATCH_BLOCK_ENFORCE_QOS_CLASS) { + prio |= _PTHREAD_PRIORITY_ENFORCE_FLAG; + } + dc->dc_priority = prio; +#else + (void)dc; (void)pp; (void)flags; +#endif +} + +DISPATCH_ALWAYS_INLINE +static inline pthread_priority_t +_dispatch_continuation_get_override_priority(dispatch_queue_t dq, + dispatch_continuation_t dc) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + pthread_priority_t p = dc->dc_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK; + bool enforce = dc->dc_priority & _PTHREAD_PRIORITY_ENFORCE_FLAG; + pthread_priority_t dqp = dq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK; + bool defaultqueue = dq->dq_priority & _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG; + if (!p) { + enforce = false; + } else if (!enforce && (!dqp || defaultqueue)) { + enforce = true; + } + if (!enforce) { + p = dqp; + } + return p; +#else + (void)dq; (void)dc; + return 0; +#endif +} + +#endif // !(USE_OBJC && __OBJC2__) + +#endif /* __DISPATCH_INLINE_INTERNAL__ */ diff --git a/src/internal.h b/src/internal.h index ed1a9c7..33fcedb 100644 --- a/src/internal.h +++ b/src/internal.h @@ -41,6 +41,19 @@ #if !defined(DISPATCH_MACH_SPI) && TARGET_OS_MAC #define DISPATCH_MACH_SPI 1 #endif +#if !defined(OS_VOUCHER_CREATION_SPI) && TARGET_OS_MAC +#define OS_VOUCHER_CREATION_SPI 1 +#endif +#if !defined(OS_VOUCHER_ACTIVITY_SPI) && TARGET_OS_MAC +#define OS_VOUCHER_ACTIVITY_SPI 1 +#endif +#if !defined(OS_VOUCHER_ACTIVITY_BUFFER_SPI) && TARGET_OS_MAC && \ + __has_include() +#define OS_VOUCHER_ACTIVITY_BUFFER_SPI 1 +#endif +#if !defined(DISPATCH_LAYOUT_SPI) && TARGET_OS_MAC +#define DISPATCH_LAYOUT_SPI 1 +#endif #if !defined(USE_OBJC) && HAVE_OBJC #define USE_OBJC 1 @@ -69,9 +82,10 @@ #include -#include #include +#include #include +#include #include #include #include @@ -120,6 +134,9 @@ struct type name = { 0 } #if !TARGET_OS_WIN32 #include "io_private.h" #endif +#include "voucher_private.h" +#include "voucher_activity_private.h" +#include "layout_private.h" #include "benchmark.h" #include "private.h" @@ -152,7 +169,7 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void); #define DISPATCH_USE_DTRACE 1 #endif -#if ((!TARGET_OS_EMBEDDED && DISPATCH_INTROSPECTION) || DISPATCH_DEBUG || \ +#if DISPATCH_USE_DTRACE && (DISPATCH_INTROSPECTION || DISPATCH_DEBUG || \ DISPATCH_PROFILE) && !defined(DISPATCH_USE_DTRACE_INTROSPECTION) #define DISPATCH_USE_DTRACE_INTROSPECTION 1 #endif @@ -176,6 +193,7 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void); #include #include #include +#include #include #include #include @@ -222,9 +240,6 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void); #include #include #include -#if !TARGET_OS_WIN32 -#include -#endif #if HAVE_UNISTD_H #include #endif @@ -303,19 +318,30 @@ void _dispatch_bug_kevent_client(const char* msg, const char* filter, DISPATCH_NOINLINE DISPATCH_NORETURN void _dispatch_abort(size_t line, long val); -#if !defined(DISPATCH_USE_OS_TRACE) && DISPATCH_DEBUG -#if __has_include() -#define DISPATCH_USE_OS_TRACE 1 -#include +#if !defined(DISPATCH_USE_OS_DEBUG_LOG) && DISPATCH_DEBUG +#if __has_include() +#define DISPATCH_USE_OS_DEBUG_LOG 1 +#include +#endif +#endif // DISPATCH_USE_OS_DEBUG_LOG + +#if !defined(DISPATCH_USE_SIMPLE_ASL) && !DISPATCH_USE_OS_DEBUG_LOG +#if __has_include(<_simple.h>) +#define DISPATCH_USE_SIMPLE_ASL 1 +#include <_simple.h> #endif -#endif // DISPATCH_USE_OS_TRACE +#endif // DISPATCH_USE_SIMPLE_ASL -#if DISPATCH_USE_OS_TRACE -#define _dispatch_log(msg, ...) os_trace("libdispatch", msg, ## __VA_ARGS__) +#if !DISPATCH_USE_SIMPLE_ASL && !DISPATCH_USE_OS_DEBUG_LOG && !TARGET_OS_WIN32 +#include +#endif + +#if DISPATCH_USE_OS_DEBUG_LOG +#define _dispatch_log(msg, ...) os_debug_log("libdispatch", msg, ## __VA_ARGS__) #else DISPATCH_NOINLINE __attribute__((__format__(__printf__,1,2))) void _dispatch_log(const char *msg, ...); -#endif // DISPATCH_USE_OS_TRACE +#endif // DISPATCH_USE_OS_DEBUG_LOG #define dsnprintf(...) \ ({ int _r = snprintf(__VA_ARGS__); _r < 0 ? 0u : (size_t)_r; }) @@ -464,72 +490,15 @@ _dispatch_object_debug(dispatch_object_t object, const char *message, ...); #define _dispatch_object_debug(object, message, ...) #endif // DISPATCH_DEBUG -#if DISPATCH_USE_CLIENT_CALLOUT - -DISPATCH_NOTHROW void -_dispatch_client_callout(void *ctxt, dispatch_function_t f); -DISPATCH_NOTHROW void -_dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t)); -DISPATCH_NOTHROW bool -_dispatch_client_callout3(void *ctxt, dispatch_data_t region, size_t offset, - const void *buffer, size_t size, dispatch_data_applier_function_t f); -DISPATCH_NOTHROW void -_dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason, - dispatch_mach_msg_t dmsg, mach_error_t error, - dispatch_mach_handler_function_t f); - -#else // !DISPATCH_USE_CLIENT_CALLOUT - -DISPATCH_ALWAYS_INLINE -static inline void -_dispatch_client_callout(void *ctxt, dispatch_function_t f) -{ - return f(ctxt); -} - -DISPATCH_ALWAYS_INLINE -static inline void -_dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t)) -{ - return f(ctxt, i); -} - -DISPATCH_ALWAYS_INLINE -static inline bool -_dispatch_client_callout3(void *ctxt, dispatch_data_t region, size_t offset, - const void *buffer, size_t size, dispatch_data_applier_function_t f) -{ - return f(ctxt, region, offset, buffer, size); -} - -DISPATCH_ALWAYS_INLINE -static inline void -_dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason, - dispatch_mach_msg_t dmsg, mach_error_t error, - dispatch_mach_handler_function_t f); -{ - return f(ctxt, reason, dmsg, error); -} - -#endif // !DISPATCH_USE_CLIENT_CALLOUT - #ifdef __BLOCKS__ #define _dispatch_Block_invoke(bb) \ ((dispatch_function_t)((struct Block_layout *)bb)->invoke) -DISPATCH_ALWAYS_INLINE -static inline void -_dispatch_client_callout_block(dispatch_block_t b) -{ - return _dispatch_client_callout(b, _dispatch_Block_invoke(b)); -} - #if __GNUC__ dispatch_block_t _dispatch_Block_copy(dispatch_block_t block); #define _dispatch_Block_copy(x) ((typeof(x))_dispatch_Block_copy(x)) #else dispatch_block_t _dispatch_Block_copy(const void *block); #endif - void _dispatch_call_block_and_release(void *block); #endif /* __BLOCKS__ */ @@ -542,14 +511,8 @@ uint64_t _dispatch_timeout(dispatch_time_t when); extern bool _dispatch_safe_fork, _dispatch_child_of_unsafe_fork; -extern struct _dispatch_hw_config_s { - uint32_t cc_max_active; - uint32_t cc_max_logical; - uint32_t cc_max_physical; -} _dispatch_hw_config; - -#if !defined(DISPATCH_USE_OS_SEMAPHORE_CACHE) && !(TARGET_IPHONE_SIMULATOR && \ - IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1090) +#if !defined(DISPATCH_USE_OS_SEMAPHORE_CACHE) && !(TARGET_IPHONE_SIMULATOR) +// rdar://problem/15492045 #if __has_include() #define DISPATCH_USE_OS_SEMAPHORE_CACHE 1 #include @@ -562,17 +525,9 @@ extern struct _dispatch_hw_config_s { // Older Mac OS X and iOS Simulator fallbacks #if HAVE_PTHREAD_WORKQUEUES -#ifndef WORKQ_BG_PRIOQUEUE -#define WORKQ_BG_PRIOQUEUE 3 -#endif #ifndef WORKQ_ADDTHREADS_OPTION_OVERCOMMIT #define WORKQ_ADDTHREADS_OPTION_OVERCOMMIT 0x00000001 #endif -#if TARGET_IPHONE_SIMULATOR && IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1070 -#ifndef DISPATCH_NO_BG_PRIORITY -#define DISPATCH_NO_BG_PRIORITY 1 -#endif -#endif #if TARGET_IPHONE_SIMULATOR && IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1080 #ifndef DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK #define DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK 1 @@ -582,7 +537,21 @@ extern struct _dispatch_hw_config_s { #undef HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP #define HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP 0 #endif +#if TARGET_IPHONE_SIMULATOR && \ + IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101000 +#ifndef DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK +#define DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK 1 +#endif +#endif +#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED < 101000 +#undef HAVE__PTHREAD_WORKQUEUE_INIT +#define HAVE__PTHREAD_WORKQUEUE_INIT 0 +#endif #endif // HAVE_PTHREAD_WORKQUEUES +#if HAVE__PTHREAD_WORKQUEUE_INIT && PTHREAD_WORKQUEUE_SPI_VERSION >= 20140213 \ + && !defined(HAVE_PTHREAD_WORKQUEUE_QOS) +#define HAVE_PTHREAD_WORKQUEUE_QOS 1 +#endif #if HAVE_MACH #if !defined(MACH_NOTIFY_SEND_POSSIBLE) || (TARGET_IPHONE_SIMULATOR && \ @@ -592,33 +561,29 @@ extern struct _dispatch_hw_config_s { #endif #endif // HAVE_MACH -#ifdef EVFILT_VM -#ifndef DISPATCH_USE_VM_PRESSURE -#define DISPATCH_USE_VM_PRESSURE 1 -#endif -#endif // EVFILT_VM - #ifdef EVFILT_MEMORYSTATUS #ifndef DISPATCH_USE_MEMORYSTATUS #define DISPATCH_USE_MEMORYSTATUS 1 #endif #endif // EVFILT_MEMORYSTATUS -#if TARGET_IPHONE_SIMULATOR && IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1070 +#if defined(EVFILT_VM) && !DISPATCH_USE_MEMORYSTATUS +#ifndef DISPATCH_USE_VM_PRESSURE +#define DISPATCH_USE_VM_PRESSURE 1 +#endif +#endif // EVFILT_VM + +#if TARGET_IPHONE_SIMULATOR +#undef DISPATCH_USE_MEMORYSTATUS_SOURCE +#define DISPATCH_USE_MEMORYSTATUS_SOURCE 0 #undef DISPATCH_USE_VM_PRESSURE_SOURCE #define DISPATCH_USE_VM_PRESSURE_SOURCE 0 #endif // TARGET_IPHONE_SIMULATOR -#if TARGET_OS_EMBEDDED -#if !defined(DISPATCH_USE_VM_PRESSURE_SOURCE) && DISPATCH_USE_VM_PRESSURE -#define DISPATCH_USE_VM_PRESSURE_SOURCE 1 -#endif -#else // !TARGET_OS_EMBEDDED #if !defined(DISPATCH_USE_MEMORYSTATUS_SOURCE) && DISPATCH_USE_MEMORYSTATUS #define DISPATCH_USE_MEMORYSTATUS_SOURCE 1 #elif !defined(DISPATCH_USE_VM_PRESSURE_SOURCE) && DISPATCH_USE_VM_PRESSURE #define DISPATCH_USE_VM_PRESSURE_SOURCE 1 #endif -#endif // TARGET_OS_EMBEDDED #if !defined(NOTE_LEEWAY) || (TARGET_IPHONE_SIMULATOR && \ IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1090) @@ -647,6 +612,12 @@ extern struct _dispatch_hw_config_s { #endif #endif // F_SETNOSIGPIPE +#if defined(MACH_SEND_NOIMPORTANCE) +#ifndef DISPATCH_USE_CHECKIN_NOIMPORTANCE +#define DISPATCH_USE_CHECKIN_NOIMPORTANCE 1 // rdar://problem/16996737 +#endif +#endif // MACH_SEND_NOIMPORTANCE + #if HAVE_LIBPROC_INTERNAL_H #include @@ -680,9 +651,26 @@ extern struct _dispatch_hw_config_s { #endif // HAVE_SYS_GUARDED_H -#define _dispatch_hardware_crash() __builtin_trap() +#ifndef MACH_MSGH_BITS_VOUCHER_MASK +#define MACH_MSGH_BITS_VOUCHER_MASK 0x001f0000 +#define MACH_MSGH_BITS_SET_PORTS(remote, local, voucher) \ + (((remote) & MACH_MSGH_BITS_REMOTE_MASK) | \ + (((local) << 8) & MACH_MSGH_BITS_LOCAL_MASK) | \ + (((voucher) << 16) & MACH_MSGH_BITS_VOUCHER_MASK)) +#define MACH_MSGH_BITS_VOUCHER(bits) \ + (((bits) & MACH_MSGH_BITS_VOUCHER_MASK) >> 16) +#define MACH_MSGH_BITS_HAS_VOUCHER(bits) \ + (MACH_MSGH_BITS_VOUCHER(bits) != MACH_MSGH_BITS_ZERO) +#define msgh_voucher_port msgh_reserved +#define mach_voucher_t mach_port_t +#define MACH_VOUCHER_NULL MACH_PORT_NULL +#define MACH_SEND_INVALID_VOUCHER 0x10000005 +#endif + +#define _dispatch_hardware_crash() \ + __asm__(""); __builtin_trap() // -#define _dispatch_set_crash_log_message(x) +#define _dispatch_set_crash_log_message(msg) #if HAVE_MACH // MIG_REPLY_MISMATCH means either: @@ -712,16 +700,30 @@ extern struct _dispatch_hw_config_s { _dispatch_hardware_crash(); \ } while (0) +extern int _dispatch_set_qos_class_enabled; +#define DISPATCH_NO_VOUCHER ((voucher_t)(void*)~0ul) +#define DISPATCH_NO_PRIORITY ((pthread_priority_t)~0ul) +#define DISPATCH_PRIORITY_ENFORCE 0x1 +static inline void _dispatch_adopt_priority_and_replace_voucher( + pthread_priority_t priority, voucher_t voucher, unsigned long flags); +#if HAVE_MACH +static inline void _dispatch_set_priority_and_mach_voucher( + pthread_priority_t priority, mach_voucher_t kv); +mach_port_t _dispatch_get_mach_host_port(void); +#endif + + /* #includes dependent on internal.h */ #include "object_internal.h" #include "semaphore_internal.h" #include "introspection_internal.h" #include "queue_internal.h" #include "source_internal.h" +#include "voucher_internal.h" #include "data_internal.h" #if !TARGET_OS_WIN32 #include "io_internal.h" #endif -#include "trace.h" +#include "inline_internal.h" #endif /* __DISPATCH_INTERNAL__ */ diff --git a/src/introspection.c b/src/introspection.c index 18c11df..e907f85 100644 --- a/src/introspection.c +++ b/src/introspection.c @@ -24,7 +24,7 @@ #if DISPATCH_INTROSPECTION #include "internal.h" -#include "introspection.h" +#include "dispatch/introspection.h" #include "introspection_private.h" typedef struct dispatch_introspection_thread_s { @@ -149,7 +149,7 @@ _dispatch_introspection_continuation_get_info(dispatch_queue_t dq, bool apply = false; long flags = (long)dc->do_vtable; if (flags & DISPATCH_OBJ_SYNC_SLOW_BIT) { - waiter = dc->dc_data; + waiter = pthread_from_mach_thread_np((mach_port_t)dc->dc_data); if (flags & DISPATCH_OBJ_BARRIER_BIT) { dc = dc->dc_ctxt; dq = dc->dc_data; @@ -224,8 +224,8 @@ dispatch_introspection_queue_get_info(dispatch_queue_t dq) { bool global = (dq->do_xref_cnt == DISPATCH_OBJECT_GLOBAL_REFCNT) || (dq->do_ref_cnt == DISPATCH_OBJECT_GLOBAL_REFCNT); - uint32_t width = dq->dq_width; - if (width > 1 && width != UINT32_MAX) width /= 2; + uint16_t width = dq->dq_width; + if (width > 1 && width != DISPATCH_QUEUE_WIDTH_MAX) width /= 2; dispatch_introspection_queue_s diq = { .queue = dq, .target_queue = dq->do_targetq, @@ -248,17 +248,23 @@ dispatch_introspection_source_s _dispatch_introspection_source_get_info(dispatch_source_t ds) { dispatch_source_refs_t dr = ds->ds_refs; - void *ctxt = dr->ds_handler_ctxt; - dispatch_function_t handler = dr->ds_handler_func; - bool handler_is_block = ds->ds_handler_is_block; + dispatch_continuation_t dc = dr->ds_handler[DS_EVENT_HANDLER]; + void *ctxt = NULL; + dispatch_function_t handler = NULL; + bool hdlr_is_block = false; + if (dc) { + ctxt = dc->dc_ctxt; + handler = dc->dc_func; + hdlr_is_block = ((long)dc->do_vtable & DISPATCH_OBJ_BLOCK_RELEASE_BIT); + } bool after = (handler == _dispatch_after_timer_callback); if (after && !(ds->ds_atomic_flags & DSF_CANCELED)) { - dispatch_continuation_t dc = ctxt; + dc = ctxt; ctxt = dc->dc_ctxt; handler = dc->dc_func; - if (handler == _dispatch_call_block_and_release) { + hdlr_is_block = (handler == _dispatch_call_block_and_release); + if (hdlr_is_block) { handler = _dispatch_Block_invoke(ctxt); - handler_is_block = 1; } } dispatch_introspection_source_s dis = { @@ -270,7 +276,7 @@ _dispatch_introspection_source_get_info(dispatch_source_t ds) .handler = handler, .suspend_count = ds->do_suspend_cnt / 2, .enqueued = (ds->do_suspend_cnt & 1), - .handler_is_block = handler_is_block, + .handler_is_block = hdlr_is_block, .timer = ds->ds_is_timer, .after = after, }; diff --git a/src/io.c b/src/io.c index 48683a6..d66873b 100644 --- a/src/io.c +++ b/src/io.c @@ -212,7 +212,8 @@ _dispatch_io_create(dispatch_io_type_t type) dispatch_io_t channel = _dispatch_alloc(DISPATCH_VTABLE(io), sizeof(struct dispatch_io_s)); channel->do_next = DISPATCH_OBJECT_LISTLESS; - channel->do_targetq = _dispatch_get_root_queue(0, true); + channel->do_targetq = _dispatch_get_root_queue(_DISPATCH_QOS_CLASS_DEFAULT, + true); channel->params.type = type; channel->params.high = SIZE_MAX; channel->params.low = dispatch_io_defaults.low_water_chunks * @@ -854,8 +855,8 @@ dispatch_read(dispatch_fd_t fd, size_t length, dispatch_queue_t queue, dispatch_operation_t op = _dispatch_operation_create(DOP_DIR_READ, channel, 0, length, dispatch_data_empty, - _dispatch_get_root_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, - false), ^(bool done, dispatch_data_t data, int error) { + _dispatch_get_root_queue(_DISPATCH_QOS_CLASS_DEFAULT,false), + ^(bool done, dispatch_data_t data, int error) { if (data) { data = dispatch_data_create_concat(deliver_data, data); _dispatch_io_data_release(deliver_data); @@ -925,8 +926,8 @@ dispatch_write(dispatch_fd_t fd, dispatch_data_t data, dispatch_queue_t queue, dispatch_operation_t op = _dispatch_operation_create(DOP_DIR_WRITE, channel, 0, dispatch_data_get_size(data), data, - _dispatch_get_root_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, - false), ^(bool done, dispatch_data_t d, int error) { + _dispatch_get_root_queue(_DISPATCH_QOS_CLASS_DEFAULT,false), + ^(bool done, dispatch_data_t d, int error) { if (done) { if (d) { _dispatch_io_data_retain(d); @@ -1352,7 +1353,7 @@ _dispatch_fd_entry_create_with_fd(dispatch_fd_t fd, uintptr_t hash) ); } _dispatch_stream_init(fd_entry, _dispatch_get_root_queue( - DISPATCH_QUEUE_PRIORITY_DEFAULT, false)); + _DISPATCH_QOS_CLASS_DEFAULT, false)); } fd_entry->orig_flags = orig_flags; fd_entry->orig_nosigpipe = orig_nosigpipe; @@ -1420,7 +1421,7 @@ _dispatch_fd_entry_create_with_path(dispatch_io_path_data_t path_data, _dispatch_disk_init(fd_entry, major(dev)); } else { _dispatch_stream_init(fd_entry, _dispatch_get_root_queue( - DISPATCH_QUEUE_PRIORITY_DEFAULT, false)); + _DISPATCH_QOS_CLASS_DEFAULT, false)); } fd_entry->fd = -1; fd_entry->orig_flags = -1; @@ -1599,7 +1600,7 @@ _dispatch_disk_init(dispatch_fd_entry_t fd_entry, dev_t dev) disk->do_next = DISPATCH_OBJECT_LISTLESS; disk->do_xref_cnt = -1; disk->advise_list_depth = pending_reqs_depth; - disk->do_targetq = _dispatch_get_root_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, + disk->do_targetq = _dispatch_get_root_queue(_DISPATCH_QOS_CLASS_DEFAULT, false); disk->dev = dev; TAILQ_INIT(&disk->operations); diff --git a/src/object.c b/src/object.c index a305039..5b09de7 100644 --- a/src/object.c +++ b/src/object.c @@ -37,38 +37,14 @@ DISPATCH_NOINLINE _os_object_t _os_object_retain_internal(_os_object_t obj) { - int ref_cnt = obj->os_obj_ref_cnt; - if (slowpath(ref_cnt == _OS_OBJECT_GLOBAL_REFCNT)) { - return obj; // global object - } - ref_cnt = dispatch_atomic_inc2o(obj, os_obj_ref_cnt, relaxed); - if (slowpath(ref_cnt <= 0)) { - DISPATCH_CRASH("Resurrection of an object"); - } - return obj; + return _os_object_retain_internal_inline(obj); } DISPATCH_NOINLINE void _os_object_release_internal(_os_object_t obj) { - int ref_cnt = obj->os_obj_ref_cnt; - if (slowpath(ref_cnt == _OS_OBJECT_GLOBAL_REFCNT)) { - return; // global object - } - ref_cnt = dispatch_atomic_dec2o(obj, os_obj_ref_cnt, relaxed); - if (fastpath(ref_cnt >= 0)) { - return; - } - if (slowpath(ref_cnt < -1)) { - DISPATCH_CRASH("Over-release of an object"); - } -#if DISPATCH_DEBUG - if (slowpath(obj->os_obj_xref_cnt >= 0)) { - DISPATCH_CRASH("Release while external references exist"); - } -#endif - return _os_object_dispose(obj); + return _os_object_release_internal_inline(obj); } DISPATCH_NOINLINE @@ -156,12 +132,6 @@ dispatch_retain(dispatch_object_t dou) (void)_os_object_retain(dou._os_obj); } -void -_dispatch_retain(dispatch_object_t dou) -{ - (void)_os_object_retain_internal(dou._os_obj); -} - void dispatch_release(dispatch_object_t dou) { @@ -169,12 +139,6 @@ dispatch_release(dispatch_object_t dou) _os_object_release(dou._os_obj); } -void -_dispatch_release(dispatch_object_t dou) -{ - _os_object_release_internal(dou._os_obj); -} - static void _dispatch_dealloc(dispatch_object_t dou) { @@ -261,7 +225,7 @@ dispatch_suspend(dispatch_object_t dou) // rdar://8181908 explains why we need to do an internal retain at every // suspension. (void)dispatch_atomic_add2o(dou._do, do_suspend_cnt, - DISPATCH_OBJECT_SUSPEND_INTERVAL, relaxed); + DISPATCH_OBJECT_SUSPEND_INTERVAL, acquire); _dispatch_retain(dou._do); } @@ -290,7 +254,7 @@ dispatch_resume(dispatch_object_t dou) // If the previous value was less than the suspend interval, the object // has been over-resumed. unsigned int suspend_cnt = dispatch_atomic_sub_orig2o(dou._do, - do_suspend_cnt, DISPATCH_OBJECT_SUSPEND_INTERVAL, relaxed); + do_suspend_cnt, DISPATCH_OBJECT_SUSPEND_INTERVAL, release); if (fastpath(suspend_cnt > DISPATCH_OBJECT_SUSPEND_INTERVAL)) { // Balancing the retain() done in suspend() for rdar://8181908 return _dispatch_release(dou._do); diff --git a/src/object.m b/src/object.m index e64a4fd..953cb0b 100644 --- a/src/object.m +++ b/src/object.m @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2013 Apple Inc. All rights reserved. + * Copyright (c) 2011-2014 Apple Inc. All rights reserved. * * @APPLE_APACHE_LICENSE_HEADER_START@ * @@ -48,6 +48,7 @@ _os_object_gc_init(void) _os_object_have_gc = objc_collectingEnabled(); if (slowpath(_os_object_have_gc)) { _os_object_gc_zone = objc_collectableZone(); + (void)[OS_OBJECT_CLASS(object) class]; // OS_object class realization } } @@ -69,22 +70,55 @@ _os_object_make_collectable(_os_object_t obj) return obj; } -#define _os_objc_gc_retain(obj) \ - if (slowpath(_os_object_have_gc)) { \ - return auto_zone_retain(_os_object_gc_zone, obj); \ +DISPATCH_NOINLINE +static id +_os_objc_gc_retain(id obj) +{ + if (fastpath(obj)) { + auto_zone_retain(_os_object_gc_zone, obj); + } + return obj; +} + +DISPATCH_NOINLINE +static void +_os_objc_gc_release(id obj) +{ + if (fastpath(obj)) { + (void)auto_zone_release(_os_object_gc_zone, obj); + } + asm(""); // prevent tailcall +} + +DISPATCH_NOINLINE +static id +_os_object_gc_retain(id obj) +{ + if ([obj isKindOfClass:OS_OBJECT_OBJC_CLASS(object)]) { + return _os_object_retain(obj); + } else { + return _os_objc_gc_retain(obj); } +} -#define _os_objc_gc_release(obj) \ - if (slowpath(_os_object_have_gc)) { \ - return (void)auto_zone_release(_os_object_gc_zone, obj); \ +DISPATCH_NOINLINE +static void +_os_object_gc_release(id obj) +{ + if ([obj isKindOfClass:OS_OBJECT_OBJC_CLASS(object)]) { + return _os_object_release(obj); + } else { + return _os_objc_gc_release(obj); } +} #else // __OBJC_GC__ #define _os_object_gc_init() #define _os_object_make_uncollectable(obj) (obj) #define _os_object_make_collectable(obj) (obj) -#define _os_objc_gc_retain(obj) -#define _os_objc_gc_release(obj) +#define _os_object_have_gc 0 +#define _os_object_gc_retain(obj) (obj) +#define _os_object_gc_release(obj) #endif // __OBJC_GC__ #pragma mark - @@ -101,11 +135,26 @@ _os_objc_alloc(Class cls, size_t size) return obj; } +static void* +_os_objc_destructInstance(id obj) +{ + // noop if only Libystem is loaded + return obj; +} + void _os_object_init(void) { _objc_init(); _os_object_gc_init(); + if (slowpath(_os_object_have_gc)) return; + Block_callbacks_RR callbacks = { + sizeof(Block_callbacks_RR), + (void (*)(const void *))&objc_retain, + (void (*)(const void *))&objc_release, + (void (*)(const void *))&_os_objc_destructInstance + }; + _Block_use_RR2(&callbacks); } _os_object_t @@ -141,6 +190,22 @@ _os_object_dispose(_os_object_t obj) [obj _dispose]; } +#undef os_retain +void* +os_retain(void *obj) +{ + if (slowpath(_os_object_have_gc)) return _os_object_gc_retain(obj); + return objc_retain(obj); +} + +#undef os_release +void +os_release(void *obj) +{ + if (slowpath(_os_object_have_gc)) return _os_object_gc_release(obj); + return objc_release(obj); +} + #pragma mark - #pragma mark _os_object @@ -190,15 +255,13 @@ _dispatch_objc_alloc(Class cls, size_t size) void _dispatch_objc_retain(dispatch_object_t dou) { - _os_objc_gc_retain(dou); - return (void)[dou retain]; + return (void)os_retain(dou); } void _dispatch_objc_release(dispatch_object_t dou) { - _os_objc_gc_release(dou); - return [dou release]; + return os_release(dou); } void @@ -341,6 +404,58 @@ DISPATCH_CLASS_IMPL(io) DISPATCH_CLASS_IMPL(operation) DISPATCH_CLASS_IMPL(disk) +@implementation OS_OBJECT_CLASS(voucher) +DISPATCH_OBJC_LOAD() + +- (id)init { + self = [super init]; + [self release]; + self = nil; + return self; +} + +- (void)_xref_dispose { + return _voucher_xref_dispose(self); // calls _os_object_release_internal() +} + +- (void)_dispose { + return _voucher_dispose(self); // calls _os_object_dealloc() +} + +- (NSString *)debugDescription { + Class nsstring = objc_lookUpClass("NSString"); + if (!nsstring) return nil; + char buf[2048]; + _voucher_debug(self, buf, sizeof(buf)); + return [nsstring stringWithFormat: + [nsstring stringWithUTF8String:"<%s: %s>"], + class_getName([self class]), buf]; +} + +@end + +#if VOUCHER_ENABLE_RECIPE_OBJECTS +@implementation OS_OBJECT_CLASS(voucher_recipe) +DISPATCH_OBJC_LOAD() + +- (id)init { + self = [super init]; + [self release]; + self = nil; + return self; +} + +- (void)_dispose { + +} + +- (NSString *)debugDescription { + return nil; // TODO: voucher_recipe debugDescription +} + +@end +#endif + #pragma mark - #pragma mark dispatch_autorelease_pool @@ -362,7 +477,7 @@ _dispatch_autorelease_pool_pop(void *context) { #pragma mark dispatch_client_callout // Abort on uncaught exceptions thrown from client callouts rdar://8577499 -#if DISPATCH_USE_CLIENT_CALLOUT && !__arm__ +#if DISPATCH_USE_CLIENT_CALLOUT && !__USING_SJLJ_EXCEPTIONS__ // On platforms with zero-cost exceptions, use a compiler-generated catch-all // exception handler. @@ -421,4 +536,34 @@ _dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason, #endif // DISPATCH_USE_CLIENT_CALLOUT +#pragma mark - +#pragma mark _dispatch_block_create + +// The compiler hides the name of the function it generates, and changes it if +// we try to reference it directly, but the linker still sees it. +extern void DISPATCH_BLOCK_SPECIAL_INVOKE(void *) + asm("____dispatch_block_create_block_invoke"); +void (*_dispatch_block_special_invoke)(void*) = DISPATCH_BLOCK_SPECIAL_INVOKE; + +dispatch_block_t +_dispatch_block_create(dispatch_block_flags_t flags, voucher_t voucher, + pthread_priority_t pri, dispatch_block_t block) +{ + dispatch_block_t copy_block = _dispatch_Block_copy(block); // 17094902 + struct dispatch_block_private_data_s dbpds = + DISPATCH_BLOCK_PRIVATE_DATA_INITIALIZER(flags, voucher, pri, copy_block); + dispatch_block_t new_block = _dispatch_Block_copy(^{ + // Capture object references, which retains copy_block and voucher. + // All retained objects must be captured by the *block*. We + // cannot borrow any references, because the block might be + // called zero or several times, so Block_release() is the + // only place that can release retained objects. + (void)copy_block; + (void)voucher; + _dispatch_block_invoke(&dbpds); + }); + Block_release(copy_block); + return new_block; +} + #endif // USE_OBJC diff --git a/src/object_internal.h b/src/object_internal.h index b369663..c0d17ae 100644 --- a/src/object_internal.h +++ b/src/object_internal.h @@ -115,7 +115,7 @@ struct dispatch_queue_s *do_targetq; \ void *do_ctxt; \ void *do_finalizer; \ - unsigned int do_suspend_cnt; + unsigned int volatile do_suspend_cnt; #define DISPATCH_OBJECT_GLOBAL_REFCNT _OS_OBJECT_GLOBAL_REFCNT // "word and bit" must be a power of two to be safely subtracted @@ -175,8 +175,6 @@ struct dispatch_object_s { size_t _dispatch_object_debug_attr(dispatch_object_t dou, char* buf, size_t bufsiz); void *_dispatch_alloc(const void *vtable, size_t size); -void _dispatch_retain(dispatch_object_t dou); -void _dispatch_release(dispatch_object_t dou); void _dispatch_xref_dispose(dispatch_object_t dou); void _dispatch_dispose(dispatch_object_t dou); #if DISPATCH_COCOA_COMPAT diff --git a/src/once.c b/src/once.c index ef57fc3..86b1a03 100644 --- a/src/once.c +++ b/src/once.c @@ -24,12 +24,13 @@ #undef dispatch_once_f -struct _dispatch_once_waiter_s { +typedef struct _dispatch_once_waiter_s { volatile struct _dispatch_once_waiter_s *volatile dow_next; _dispatch_thread_semaphore_t dow_sema; -}; + mach_port_t dow_thread; +} *_dispatch_once_waiter_t; -#define DISPATCH_ONCE_DONE ((struct _dispatch_once_waiter_s *)~0l) +#define DISPATCH_ONCE_DONE ((_dispatch_once_waiter_t)~0l) #ifdef __BLOCKS__ void @@ -43,13 +44,13 @@ DISPATCH_NOINLINE void dispatch_once_f(dispatch_once_t *val, void *ctxt, dispatch_function_t func) { - struct _dispatch_once_waiter_s * volatile *vval = - (struct _dispatch_once_waiter_s**)val; - struct _dispatch_once_waiter_s dow = { NULL, 0 }; - struct _dispatch_once_waiter_s *tail, *tmp; + _dispatch_once_waiter_t volatile *vval = (_dispatch_once_waiter_t*)val; + struct _dispatch_once_waiter_s dow = { NULL, 0, MACH_PORT_NULL }; + _dispatch_once_waiter_t tail = &dow, next, tmp; _dispatch_thread_semaphore_t sema; - if (dispatch_atomic_cmpxchg(vval, NULL, &dow, acquire)) { + if (dispatch_atomic_cmpxchg(vval, NULL, tail, acquire)) { + dow.dow_thread = _dispatch_thread_port(); _dispatch_client_callout(ctxt, func); // The next barrier must be long and strong. @@ -104,26 +105,31 @@ dispatch_once_f(dispatch_once_t *val, void *ctxt, dispatch_function_t func) dispatch_atomic_maximally_synchronizing_barrier(); // above assumed to contain release barrier - tmp = dispatch_atomic_xchg(vval, DISPATCH_ONCE_DONE, relaxed); - tail = &dow; - while (tail != tmp) { - while (!tmp->dow_next) { - dispatch_hardware_pause(); - } - sema = tmp->dow_sema; - tmp = (struct _dispatch_once_waiter_s*)tmp->dow_next; + next = dispatch_atomic_xchg(vval, DISPATCH_ONCE_DONE, relaxed); + while (next != tail) { + _dispatch_wait_until(tmp = (_dispatch_once_waiter_t)next->dow_next); + sema = next->dow_sema; + next = tmp; _dispatch_thread_semaphore_signal(sema); } } else { dow.dow_sema = _dispatch_get_thread_semaphore(); - tmp = *vval; + next = *vval; for (;;) { - if (tmp == DISPATCH_ONCE_DONE) { + if (next == DISPATCH_ONCE_DONE) { break; } - if (dispatch_atomic_cmpxchgvw(vval, tmp, &dow, &tmp, release)) { - dow.dow_next = tmp; + if (dispatch_atomic_cmpxchgvw(vval, next, tail, &next, release)) { + dow.dow_thread = next->dow_thread; + dow.dow_next = next; + if (dow.dow_thread) { + pthread_priority_t pp = _dispatch_get_priority(); + _dispatch_thread_override_start(dow.dow_thread, pp); + } _dispatch_thread_semaphore_wait(dow.dow_sema); + if (dow.dow_thread) { + _dispatch_thread_override_end(dow.dow_thread); + } break; } } diff --git a/src/queue.c b/src/queue.c index 0c05b0b..b8b4ad9 100644 --- a/src/queue.c +++ b/src/queue.c @@ -30,10 +30,19 @@ #if DISPATCH_ENABLE_PTHREAD_ROOT_QUEUES || DISPATCH_ENABLE_THREAD_POOL #define DISPATCH_USE_PTHREAD_POOL 1 #endif -#if HAVE_PTHREAD_WORKQUEUES && !HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP && \ +#if HAVE_PTHREAD_WORKQUEUES && (!HAVE_PTHREAD_WORKQUEUE_QOS || DISPATCH_DEBUG) \ + && !defined(DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK) +#define DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK 1 +#endif +#if HAVE_PTHREAD_WORKQUEUES && DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK && \ + !HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP && \ !defined(DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK) #define DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK 1 #endif +#if HAVE_PTHREAD_WORKQUEUE_QOS && !DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK +#undef HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP +#define HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP 0 +#endif #if HAVE_PTHREAD_WORKQUEUES && DISPATCH_USE_PTHREAD_POOL && \ !DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK #define pthread_workqueue_t void* @@ -41,15 +50,22 @@ static void _dispatch_cache_cleanup(void *value); static void _dispatch_async_f_redirect(dispatch_queue_t dq, - dispatch_continuation_t dc); + dispatch_continuation_t dc, pthread_priority_t pp); static void _dispatch_queue_cleanup(void *ctxt); static inline void _dispatch_queue_wakeup_global2(dispatch_queue_t dq, unsigned int n); static inline void _dispatch_queue_wakeup_global(dispatch_queue_t dq); static inline _dispatch_thread_semaphore_t _dispatch_queue_drain_one_barrier_sync(dispatch_queue_t dq); +static inline bool _dispatch_queue_prepare_override(dispatch_queue_t dq, + dispatch_queue_t tq, pthread_priority_t p); +static inline void _dispatch_queue_push_override(dispatch_queue_t dq, + dispatch_queue_t tq, pthread_priority_t p); #if HAVE_PTHREAD_WORKQUEUES -static void _dispatch_worker_thread3(void *context); +static void _dispatch_worker_thread4(void *context); +#if HAVE_PTHREAD_WORKQUEUE_QOS +static void _dispatch_worker_thread3(pthread_priority_t priority); +#endif #if HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP static void _dispatch_worker_thread2(int priority, int options, void *context); #endif @@ -67,51 +83,95 @@ static void _dispatch_runloop_queue_port_init(void *ctxt); static void _dispatch_runloop_queue_port_dispose(dispatch_queue_t dq); #endif +static void _dispatch_root_queues_init(void *context); +static dispatch_once_t _dispatch_root_queues_pred; + #pragma mark - #pragma mark dispatch_root_queue +struct dispatch_pthread_root_queue_context_s { + pthread_attr_t dpq_thread_attr; + dispatch_block_t dpq_thread_configure; + struct dispatch_semaphore_s dpq_thread_mediator; +}; +typedef struct dispatch_pthread_root_queue_context_s * + dispatch_pthread_root_queue_context_t; + #if DISPATCH_ENABLE_THREAD_POOL -static struct dispatch_semaphore_s _dispatch_thread_mediator[] = { - [DISPATCH_ROOT_QUEUE_IDX_LOW_PRIORITY] = { - .do_vtable = DISPATCH_VTABLE(semaphore), - .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - }, - [DISPATCH_ROOT_QUEUE_IDX_LOW_OVERCOMMIT_PRIORITY] = { - .do_vtable = DISPATCH_VTABLE(semaphore), - .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - }, - [DISPATCH_ROOT_QUEUE_IDX_DEFAULT_PRIORITY] = { - .do_vtable = DISPATCH_VTABLE(semaphore), - .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - }, - [DISPATCH_ROOT_QUEUE_IDX_DEFAULT_OVERCOMMIT_PRIORITY] = { - .do_vtable = DISPATCH_VTABLE(semaphore), - .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - }, - [DISPATCH_ROOT_QUEUE_IDX_HIGH_PRIORITY] = { - .do_vtable = DISPATCH_VTABLE(semaphore), - .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - }, - [DISPATCH_ROOT_QUEUE_IDX_HIGH_OVERCOMMIT_PRIORITY] = { - .do_vtable = DISPATCH_VTABLE(semaphore), - .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - }, - [DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_PRIORITY] = { - .do_vtable = DISPATCH_VTABLE(semaphore), - .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - }, - [DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_OVERCOMMIT_PRIORITY] = { - .do_vtable = DISPATCH_VTABLE(semaphore), - .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, - }, +static struct dispatch_pthread_root_queue_context_s + _dispatch_pthread_root_queue_contexts[] = { + [DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS] = { + .dpq_thread_mediator = { + .do_vtable = DISPATCH_VTABLE(semaphore), + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + }}, + [DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS_OVERCOMMIT] = { + .dpq_thread_mediator = { + .do_vtable = DISPATCH_VTABLE(semaphore), + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + }}, + [DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS] = { + .dpq_thread_mediator = { + .do_vtable = DISPATCH_VTABLE(semaphore), + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + }}, + [DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS_OVERCOMMIT] = { + .dpq_thread_mediator = { + .do_vtable = DISPATCH_VTABLE(semaphore), + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + }}, + [DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS] = { + .dpq_thread_mediator = { + .do_vtable = DISPATCH_VTABLE(semaphore), + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + }}, + [DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS_OVERCOMMIT] = { + .dpq_thread_mediator = { + .do_vtable = DISPATCH_VTABLE(semaphore), + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + }}, + [DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS] = { + .dpq_thread_mediator = { + .do_vtable = DISPATCH_VTABLE(semaphore), + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + }}, + [DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT] = { + .dpq_thread_mediator = { + .do_vtable = DISPATCH_VTABLE(semaphore), + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + }}, + [DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS] = { + .dpq_thread_mediator = { + .do_vtable = DISPATCH_VTABLE(semaphore), + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + }}, + [DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS_OVERCOMMIT] = { + .dpq_thread_mediator = { + .do_vtable = DISPATCH_VTABLE(semaphore), + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + }}, + [DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS] = { + .dpq_thread_mediator = { + .do_vtable = DISPATCH_VTABLE(semaphore), + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + }}, + [DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS_OVERCOMMIT] = { + .dpq_thread_mediator = { + .do_vtable = DISPATCH_VTABLE(semaphore), + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + }}, }; #endif @@ -122,6 +182,7 @@ struct dispatch_root_queue_context_s { struct { unsigned int volatile dgq_pending; #if HAVE_PTHREAD_WORKQUEUES + qos_class_t dgq_qos; int dgq_wq_priority, dgq_wq_options; #if DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK || DISPATCH_USE_PTHREAD_POOL pthread_workqueue_t dgq_kworkqueue; @@ -129,7 +190,6 @@ struct dispatch_root_queue_context_s { #endif // HAVE_PTHREAD_WORKQUEUES #if DISPATCH_USE_PTHREAD_POOL void *dgq_ctxt; - dispatch_semaphore_t dgq_thread_mediator; uint32_t volatile dgq_thread_pool_size; #endif }; @@ -140,84 +200,136 @@ typedef struct dispatch_root_queue_context_s *dispatch_root_queue_context_t; DISPATCH_CACHELINE_ALIGN static struct dispatch_root_queue_context_s _dispatch_root_queue_contexts[] = { - [DISPATCH_ROOT_QUEUE_IDX_LOW_PRIORITY] = {{{ + [DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS] = {{{ +#if HAVE_PTHREAD_WORKQUEUES + .dgq_qos = _DISPATCH_QOS_CLASS_MAINTENANCE, + .dgq_wq_priority = WORKQ_BG_PRIOQUEUE, + .dgq_wq_options = 0, +#endif +#if DISPATCH_ENABLE_THREAD_POOL + .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ + DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS], +#endif + }}}, + [DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS_OVERCOMMIT] = {{{ +#if HAVE_PTHREAD_WORKQUEUES + .dgq_qos = _DISPATCH_QOS_CLASS_MAINTENANCE, + .dgq_wq_priority = WORKQ_BG_PRIOQUEUE, + .dgq_wq_options = WORKQ_ADDTHREADS_OPTION_OVERCOMMIT, +#endif +#if DISPATCH_ENABLE_THREAD_POOL + .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ + DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS_OVERCOMMIT], +#endif + }}}, + [DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS] = {{{ +#if HAVE_PTHREAD_WORKQUEUES + .dgq_qos = _DISPATCH_QOS_CLASS_BACKGROUND, + .dgq_wq_priority = WORKQ_BG_PRIOQUEUE, + .dgq_wq_options = 0, +#endif +#if DISPATCH_ENABLE_THREAD_POOL + .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ + DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS], +#endif + }}}, + [DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS_OVERCOMMIT] = {{{ +#if HAVE_PTHREAD_WORKQUEUES + .dgq_qos = _DISPATCH_QOS_CLASS_BACKGROUND, + .dgq_wq_priority = WORKQ_BG_PRIOQUEUE, + .dgq_wq_options = WORKQ_ADDTHREADS_OPTION_OVERCOMMIT, +#endif +#if DISPATCH_ENABLE_THREAD_POOL + .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ + DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS_OVERCOMMIT], +#endif + }}}, + [DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS] = {{{ #if HAVE_PTHREAD_WORKQUEUES + .dgq_qos = _DISPATCH_QOS_CLASS_UTILITY, .dgq_wq_priority = WORKQ_LOW_PRIOQUEUE, .dgq_wq_options = 0, #endif #if DISPATCH_ENABLE_THREAD_POOL - .dgq_thread_mediator = &_dispatch_thread_mediator[ - DISPATCH_ROOT_QUEUE_IDX_LOW_PRIORITY], + .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ + DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS], #endif }}}, - [DISPATCH_ROOT_QUEUE_IDX_LOW_OVERCOMMIT_PRIORITY] = {{{ + [DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS_OVERCOMMIT] = {{{ #if HAVE_PTHREAD_WORKQUEUES + .dgq_qos = _DISPATCH_QOS_CLASS_UTILITY, .dgq_wq_priority = WORKQ_LOW_PRIOQUEUE, .dgq_wq_options = WORKQ_ADDTHREADS_OPTION_OVERCOMMIT, #endif #if DISPATCH_ENABLE_THREAD_POOL - .dgq_thread_mediator = &_dispatch_thread_mediator[ - DISPATCH_ROOT_QUEUE_IDX_LOW_OVERCOMMIT_PRIORITY], + .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ + DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS_OVERCOMMIT], #endif }}}, - [DISPATCH_ROOT_QUEUE_IDX_DEFAULT_PRIORITY] = {{{ + [DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS] = {{{ #if HAVE_PTHREAD_WORKQUEUES + .dgq_qos = _DISPATCH_QOS_CLASS_DEFAULT, .dgq_wq_priority = WORKQ_DEFAULT_PRIOQUEUE, .dgq_wq_options = 0, #endif #if DISPATCH_ENABLE_THREAD_POOL - .dgq_thread_mediator = &_dispatch_thread_mediator[ - DISPATCH_ROOT_QUEUE_IDX_DEFAULT_PRIORITY], + .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ + DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS], #endif }}}, - [DISPATCH_ROOT_QUEUE_IDX_DEFAULT_OVERCOMMIT_PRIORITY] = {{{ + [DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT] = {{{ #if HAVE_PTHREAD_WORKQUEUES + .dgq_qos = _DISPATCH_QOS_CLASS_DEFAULT, .dgq_wq_priority = WORKQ_DEFAULT_PRIOQUEUE, .dgq_wq_options = WORKQ_ADDTHREADS_OPTION_OVERCOMMIT, #endif #if DISPATCH_ENABLE_THREAD_POOL - .dgq_thread_mediator = &_dispatch_thread_mediator[ - DISPATCH_ROOT_QUEUE_IDX_DEFAULT_OVERCOMMIT_PRIORITY], + .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ + DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT], #endif }}}, - [DISPATCH_ROOT_QUEUE_IDX_HIGH_PRIORITY] = {{{ + [DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS] = {{{ #if HAVE_PTHREAD_WORKQUEUES + .dgq_qos = _DISPATCH_QOS_CLASS_USER_INITIATED, .dgq_wq_priority = WORKQ_HIGH_PRIOQUEUE, .dgq_wq_options = 0, #endif #if DISPATCH_ENABLE_THREAD_POOL - .dgq_thread_mediator = &_dispatch_thread_mediator[ - DISPATCH_ROOT_QUEUE_IDX_HIGH_PRIORITY], + .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ + DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS], #endif }}}, - [DISPATCH_ROOT_QUEUE_IDX_HIGH_OVERCOMMIT_PRIORITY] = {{{ + [DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS_OVERCOMMIT] = {{{ #if HAVE_PTHREAD_WORKQUEUES + .dgq_qos = _DISPATCH_QOS_CLASS_USER_INITIATED, .dgq_wq_priority = WORKQ_HIGH_PRIOQUEUE, .dgq_wq_options = WORKQ_ADDTHREADS_OPTION_OVERCOMMIT, #endif #if DISPATCH_ENABLE_THREAD_POOL - .dgq_thread_mediator = &_dispatch_thread_mediator[ - DISPATCH_ROOT_QUEUE_IDX_HIGH_OVERCOMMIT_PRIORITY], + .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ + DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS_OVERCOMMIT], #endif }}}, - [DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_PRIORITY] = {{{ + [DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS] = {{{ #if HAVE_PTHREAD_WORKQUEUES - .dgq_wq_priority = WORKQ_BG_PRIOQUEUE, + .dgq_qos = _DISPATCH_QOS_CLASS_USER_INTERACTIVE, + .dgq_wq_priority = WORKQ_HIGH_PRIOQUEUE, .dgq_wq_options = 0, #endif #if DISPATCH_ENABLE_THREAD_POOL - .dgq_thread_mediator = &_dispatch_thread_mediator[ - DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_PRIORITY], + .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ + DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS], #endif }}}, - [DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_OVERCOMMIT_PRIORITY] = {{{ + [DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS_OVERCOMMIT] = {{{ #if HAVE_PTHREAD_WORKQUEUES - .dgq_wq_priority = WORKQ_BG_PRIOQUEUE, + .dgq_qos = _DISPATCH_QOS_CLASS_USER_INTERACTIVE, + .dgq_wq_priority = WORKQ_HIGH_PRIOQUEUE, .dgq_wq_options = WORKQ_ADDTHREADS_OPTION_OVERCOMMIT, #endif #if DISPATCH_ENABLE_THREAD_POOL - .dgq_thread_mediator = &_dispatch_thread_mediator[ - DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_OVERCOMMIT_PRIORITY], + .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ + DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS_OVERCOMMIT], #endif }}}, }; @@ -227,128 +339,206 @@ static struct dispatch_root_queue_context_s _dispatch_root_queue_contexts[] = { // dq_running is set to 2 so that barrier operations go through the slow path DISPATCH_CACHELINE_ALIGN struct dispatch_queue_s _dispatch_root_queues[] = { - [DISPATCH_ROOT_QUEUE_IDX_LOW_PRIORITY] = { + [DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS] = { .do_vtable = DISPATCH_VTABLE(queue_root), .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_suspend_cnt = DISPATCH_OBJECT_SUSPEND_LOCK, .do_ctxt = &_dispatch_root_queue_contexts[ - DISPATCH_ROOT_QUEUE_IDX_LOW_PRIORITY], - .dq_label = "com.apple.root.low-priority", + DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS], + .dq_label = "com.apple.root.maintenance-qos", .dq_running = 2, - .dq_width = UINT32_MAX, + .dq_width = DISPATCH_QUEUE_WIDTH_MAX, .dq_serialnum = 4, }, - [DISPATCH_ROOT_QUEUE_IDX_LOW_OVERCOMMIT_PRIORITY] = { + [DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS_OVERCOMMIT] = { .do_vtable = DISPATCH_VTABLE(queue_root), .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_suspend_cnt = DISPATCH_OBJECT_SUSPEND_LOCK, .do_ctxt = &_dispatch_root_queue_contexts[ - DISPATCH_ROOT_QUEUE_IDX_LOW_OVERCOMMIT_PRIORITY], - .dq_label = "com.apple.root.low-overcommit-priority", + DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS_OVERCOMMIT], + .dq_label = "com.apple.root.maintenance-qos.overcommit", .dq_running = 2, - .dq_width = UINT32_MAX, + .dq_width = DISPATCH_QUEUE_WIDTH_MAX, .dq_serialnum = 5, }, - [DISPATCH_ROOT_QUEUE_IDX_DEFAULT_PRIORITY] = { + [DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS] = { .do_vtable = DISPATCH_VTABLE(queue_root), .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_suspend_cnt = DISPATCH_OBJECT_SUSPEND_LOCK, .do_ctxt = &_dispatch_root_queue_contexts[ - DISPATCH_ROOT_QUEUE_IDX_DEFAULT_PRIORITY], - .dq_label = "com.apple.root.default-priority", + DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS], + .dq_label = "com.apple.root.background-qos", .dq_running = 2, - .dq_width = UINT32_MAX, + .dq_width = DISPATCH_QUEUE_WIDTH_MAX, .dq_serialnum = 6, }, - [DISPATCH_ROOT_QUEUE_IDX_DEFAULT_OVERCOMMIT_PRIORITY] = { + [DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS_OVERCOMMIT] = { .do_vtable = DISPATCH_VTABLE(queue_root), .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_suspend_cnt = DISPATCH_OBJECT_SUSPEND_LOCK, .do_ctxt = &_dispatch_root_queue_contexts[ - DISPATCH_ROOT_QUEUE_IDX_DEFAULT_OVERCOMMIT_PRIORITY], - .dq_label = "com.apple.root.default-overcommit-priority", + DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS_OVERCOMMIT], + .dq_label = "com.apple.root.background-qos.overcommit", .dq_running = 2, - .dq_width = UINT32_MAX, + .dq_width = DISPATCH_QUEUE_WIDTH_MAX, .dq_serialnum = 7, }, - [DISPATCH_ROOT_QUEUE_IDX_HIGH_PRIORITY] = { + [DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS] = { .do_vtable = DISPATCH_VTABLE(queue_root), .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_suspend_cnt = DISPATCH_OBJECT_SUSPEND_LOCK, .do_ctxt = &_dispatch_root_queue_contexts[ - DISPATCH_ROOT_QUEUE_IDX_HIGH_PRIORITY], - .dq_label = "com.apple.root.high-priority", + DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS], + .dq_label = "com.apple.root.utility-qos", .dq_running = 2, - .dq_width = UINT32_MAX, + .dq_width = DISPATCH_QUEUE_WIDTH_MAX, .dq_serialnum = 8, }, - [DISPATCH_ROOT_QUEUE_IDX_HIGH_OVERCOMMIT_PRIORITY] = { + [DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS_OVERCOMMIT] = { .do_vtable = DISPATCH_VTABLE(queue_root), .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_suspend_cnt = DISPATCH_OBJECT_SUSPEND_LOCK, .do_ctxt = &_dispatch_root_queue_contexts[ - DISPATCH_ROOT_QUEUE_IDX_HIGH_OVERCOMMIT_PRIORITY], - .dq_label = "com.apple.root.high-overcommit-priority", + DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS_OVERCOMMIT], + .dq_label = "com.apple.root.utility-qos.overcommit", .dq_running = 2, - .dq_width = UINT32_MAX, + .dq_width = DISPATCH_QUEUE_WIDTH_MAX, .dq_serialnum = 9, }, - [DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_PRIORITY] = { + [DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS] = { .do_vtable = DISPATCH_VTABLE(queue_root), .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_suspend_cnt = DISPATCH_OBJECT_SUSPEND_LOCK, .do_ctxt = &_dispatch_root_queue_contexts[ - DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_PRIORITY], - .dq_label = "com.apple.root.background-priority", + DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS], + .dq_label = "com.apple.root.default-qos", .dq_running = 2, - .dq_width = UINT32_MAX, + .dq_width = DISPATCH_QUEUE_WIDTH_MAX, .dq_serialnum = 10, }, - [DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_OVERCOMMIT_PRIORITY] = { + [DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT] = { .do_vtable = DISPATCH_VTABLE(queue_root), .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, .do_suspend_cnt = DISPATCH_OBJECT_SUSPEND_LOCK, .do_ctxt = &_dispatch_root_queue_contexts[ - DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_OVERCOMMIT_PRIORITY], - .dq_label = "com.apple.root.background-overcommit-priority", + DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT], + .dq_label = "com.apple.root.default-qos.overcommit", .dq_running = 2, - .dq_width = UINT32_MAX, + .dq_width = DISPATCH_QUEUE_WIDTH_MAX, .dq_serialnum = 11, }, + [DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS] = { + .do_vtable = DISPATCH_VTABLE(queue_root), + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_suspend_cnt = DISPATCH_OBJECT_SUSPEND_LOCK, + .do_ctxt = &_dispatch_root_queue_contexts[ + DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS], + .dq_label = "com.apple.root.user-initiated-qos", + .dq_running = 2, + .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_serialnum = 12, + }, + [DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS_OVERCOMMIT] = { + .do_vtable = DISPATCH_VTABLE(queue_root), + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_suspend_cnt = DISPATCH_OBJECT_SUSPEND_LOCK, + .do_ctxt = &_dispatch_root_queue_contexts[ + DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS_OVERCOMMIT], + .dq_label = "com.apple.root.user-initiated-qos.overcommit", + .dq_running = 2, + .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_serialnum = 13, + }, + [DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS] = { + .do_vtable = DISPATCH_VTABLE(queue_root), + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_suspend_cnt = DISPATCH_OBJECT_SUSPEND_LOCK, + .do_ctxt = &_dispatch_root_queue_contexts[ + DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS], + .dq_label = "com.apple.root.user-interactive-qos", + .dq_running = 2, + .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_serialnum = 14, + }, + [DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS_OVERCOMMIT] = { + .do_vtable = DISPATCH_VTABLE(queue_root), + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, + .do_suspend_cnt = DISPATCH_OBJECT_SUSPEND_LOCK, + .do_ctxt = &_dispatch_root_queue_contexts[ + DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS_OVERCOMMIT], + .dq_label = "com.apple.root.user-interactive-qos.overcommit", + .dq_running = 2, + .dq_width = DISPATCH_QUEUE_WIDTH_MAX, + .dq_serialnum = 15, + }, }; -#if HAVE_PTHREAD_WORKQUEUES +#if HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP static const dispatch_queue_t _dispatch_wq2root_queues[][2] = { + [WORKQ_BG_PRIOQUEUE][0] = &_dispatch_root_queues[ + DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS], + [WORKQ_BG_PRIOQUEUE][WORKQ_ADDTHREADS_OPTION_OVERCOMMIT] = + &_dispatch_root_queues[ + DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS_OVERCOMMIT], [WORKQ_LOW_PRIOQUEUE][0] = &_dispatch_root_queues[ - DISPATCH_ROOT_QUEUE_IDX_LOW_PRIORITY], + DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS], [WORKQ_LOW_PRIOQUEUE][WORKQ_ADDTHREADS_OPTION_OVERCOMMIT] = &_dispatch_root_queues[ - DISPATCH_ROOT_QUEUE_IDX_LOW_OVERCOMMIT_PRIORITY], + DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS_OVERCOMMIT], [WORKQ_DEFAULT_PRIOQUEUE][0] = &_dispatch_root_queues[ - DISPATCH_ROOT_QUEUE_IDX_DEFAULT_PRIORITY], + DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS], [WORKQ_DEFAULT_PRIOQUEUE][WORKQ_ADDTHREADS_OPTION_OVERCOMMIT] = &_dispatch_root_queues[ - DISPATCH_ROOT_QUEUE_IDX_DEFAULT_OVERCOMMIT_PRIORITY], + DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT], [WORKQ_HIGH_PRIOQUEUE][0] = &_dispatch_root_queues[ - DISPATCH_ROOT_QUEUE_IDX_HIGH_PRIORITY], + DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS], [WORKQ_HIGH_PRIOQUEUE][WORKQ_ADDTHREADS_OPTION_OVERCOMMIT] = &_dispatch_root_queues[ - DISPATCH_ROOT_QUEUE_IDX_HIGH_OVERCOMMIT_PRIORITY], - [WORKQ_BG_PRIOQUEUE][0] = &_dispatch_root_queues[ - DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_PRIORITY], - [WORKQ_BG_PRIOQUEUE][WORKQ_ADDTHREADS_OPTION_OVERCOMMIT] = - &_dispatch_root_queues[ - DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_OVERCOMMIT_PRIORITY], + DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS_OVERCOMMIT], +}; +#endif // HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP + +#define DISPATCH_PRIORITY_COUNT 5 + +enum { + // No DISPATCH_PRIORITY_IDX_MAINTENANCE define because there is no legacy + // maintenance priority + DISPATCH_PRIORITY_IDX_BACKGROUND = 0, + DISPATCH_PRIORITY_IDX_NON_INTERACTIVE, + DISPATCH_PRIORITY_IDX_LOW, + DISPATCH_PRIORITY_IDX_DEFAULT, + DISPATCH_PRIORITY_IDX_HIGH, +}; + +static qos_class_t _dispatch_priority2qos[] = { + [DISPATCH_PRIORITY_IDX_BACKGROUND] = _DISPATCH_QOS_CLASS_BACKGROUND, + [DISPATCH_PRIORITY_IDX_NON_INTERACTIVE] = _DISPATCH_QOS_CLASS_UTILITY, + [DISPATCH_PRIORITY_IDX_LOW] = _DISPATCH_QOS_CLASS_UTILITY, + [DISPATCH_PRIORITY_IDX_DEFAULT] = _DISPATCH_QOS_CLASS_DEFAULT, + [DISPATCH_PRIORITY_IDX_HIGH] = _DISPATCH_QOS_CLASS_USER_INITIATED, }; -#endif // HAVE_PTHREAD_WORKQUEUES + +#if HAVE_PTHREAD_WORKQUEUE_QOS +static const int _dispatch_priority2wq[] = { + [DISPATCH_PRIORITY_IDX_BACKGROUND] = WORKQ_BG_PRIOQUEUE, + [DISPATCH_PRIORITY_IDX_NON_INTERACTIVE] = WORKQ_NON_INTERACTIVE_PRIOQUEUE, + [DISPATCH_PRIORITY_IDX_LOW] = WORKQ_LOW_PRIOQUEUE, + [DISPATCH_PRIORITY_IDX_DEFAULT] = WORKQ_DEFAULT_PRIOQUEUE, + [DISPATCH_PRIORITY_IDX_HIGH] = WORKQ_HIGH_PRIOQUEUE, +}; +#endif #if DISPATCH_ENABLE_PTHREAD_ROOT_QUEUES static struct dispatch_queue_s _dispatch_mgr_root_queue; @@ -378,15 +568,58 @@ dispatch_get_global_queue(long priority, unsigned long flags) if (flags & ~(unsigned long)DISPATCH_QUEUE_OVERCOMMIT) { return NULL; } - return _dispatch_get_root_queue(priority, - flags & DISPATCH_QUEUE_OVERCOMMIT); + dispatch_once_f(&_dispatch_root_queues_pred, NULL, + _dispatch_root_queues_init); + qos_class_t qos; + switch (priority) { +#if !RDAR_17878963 || DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK + case _DISPATCH_QOS_CLASS_MAINTENANCE: + if (!_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS] + .dq_priority) { + // map maintenance to background on old kernel + qos = _dispatch_priority2qos[DISPATCH_PRIORITY_IDX_BACKGROUND]; + } else { + qos = (qos_class_t)priority; + } + break; +#endif // RDAR_17878963 || DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK + case DISPATCH_QUEUE_PRIORITY_BACKGROUND: + qos = _dispatch_priority2qos[DISPATCH_PRIORITY_IDX_BACKGROUND]; + break; + case DISPATCH_QUEUE_PRIORITY_NON_INTERACTIVE: + qos = _dispatch_priority2qos[DISPATCH_PRIORITY_IDX_NON_INTERACTIVE]; + break; + case DISPATCH_QUEUE_PRIORITY_LOW: + qos = _dispatch_priority2qos[DISPATCH_PRIORITY_IDX_LOW]; + break; + case DISPATCH_QUEUE_PRIORITY_DEFAULT: + qos = _dispatch_priority2qos[DISPATCH_PRIORITY_IDX_DEFAULT]; + break; + case DISPATCH_QUEUE_PRIORITY_HIGH: + qos = _dispatch_priority2qos[DISPATCH_PRIORITY_IDX_HIGH]; + break; + case _DISPATCH_QOS_CLASS_USER_INTERACTIVE: +#if DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK + if (!_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS] + .dq_priority) { + qos = _dispatch_priority2qos[DISPATCH_PRIORITY_IDX_HIGH]; + break; + } +#endif + // fallthrough + default: + qos = (qos_class_t)priority; + break; + } + return _dispatch_get_root_queue(qos, flags & DISPATCH_QUEUE_OVERCOMMIT); } DISPATCH_ALWAYS_INLINE static inline dispatch_queue_t _dispatch_get_current_queue(void) { - return _dispatch_queue_get_current() ?: _dispatch_get_root_queue(0, true); + return _dispatch_queue_get_current() ?: + _dispatch_get_root_queue(_DISPATCH_QOS_CLASS_DEFAULT, true); } dispatch_queue_t @@ -463,13 +696,52 @@ dispatch_assert_queue_not(dispatch_queue_t dq) #pragma mark - #pragma mark dispatch_init +#if HAVE_PTHREAD_WORKQUEUE_QOS +int _dispatch_set_qos_class_enabled; +pthread_priority_t _dispatch_background_priority; +pthread_priority_t _dispatch_user_initiated_priority; + static void -_dispatch_hw_config_init(void) +_dispatch_root_queues_init_qos(int supported) { - _dispatch_hw_config.cc_max_active = _dispatch_get_activecpu(); - _dispatch_hw_config.cc_max_logical = _dispatch_get_logicalcpu_max(); - _dispatch_hw_config.cc_max_physical = _dispatch_get_physicalcpu_max(); + pthread_priority_t p; + qos_class_t qos; + unsigned int i; + for (i = 0; i < DISPATCH_PRIORITY_COUNT; i++) { + p = _pthread_qos_class_encode_workqueue(_dispatch_priority2wq[i], 0); + qos = _pthread_qos_class_decode(p, NULL, NULL); + dispatch_assert(qos != _DISPATCH_QOS_CLASS_UNSPECIFIED); + _dispatch_priority2qos[i] = qos; + } + for (i = 0; i < DISPATCH_ROOT_QUEUE_COUNT; i++) { + qos = _dispatch_root_queue_contexts[i].dgq_qos; + if (qos == _DISPATCH_QOS_CLASS_MAINTENANCE && + !(supported & WORKQ_FEATURE_MAINTENANCE)) { + continue; + } + unsigned long flags = i & 1 ? _PTHREAD_PRIORITY_OVERCOMMIT_FLAG : 0; + flags |= _PTHREAD_PRIORITY_ROOTQUEUE_FLAG; + if (i == DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS || + i == DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT) { + flags |= _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG; + } + p = _pthread_qos_class_encode(qos, 0, flags); + _dispatch_root_queues[i].dq_priority = p; + } + p = _pthread_qos_class_encode(qos_class_main(), 0, 0); + _dispatch_main_q.dq_priority = p; + _dispatch_queue_set_override_priority(&_dispatch_main_q); + _dispatch_background_priority = _dispatch_root_queues[ + DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS].dq_priority & + ~_PTHREAD_PRIORITY_FLAGS_MASK; + _dispatch_user_initiated_priority = _dispatch_root_queues[ + DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS].dq_priority & + ~_PTHREAD_PRIORITY_FLAGS_MASK; + if (!slowpath(getenv("LIBDISPATCH_DISABLE_SET_QOS"))) { + _dispatch_set_qos_class_enabled = 1; + } } +#endif static inline bool _dispatch_root_queues_init_workq(void) @@ -481,8 +753,24 @@ _dispatch_root_queues_init_workq(void) disable_wq = slowpath(getenv("LIBDISPATCH_DISABLE_KWQ")); #endif int r; +#if HAVE_PTHREAD_WORKQUEUE_QOS + bool disable_qos = false; +#if DISPATCH_DEBUG + disable_qos = slowpath(getenv("LIBDISPATCH_DISABLE_QOS")); +#endif + if (!disable_qos && !disable_wq) { + r = _pthread_workqueue_supported(); + int supported = r; + if (r & WORKQ_FEATURE_FINEPRIO) { + r = _pthread_workqueue_init(_dispatch_worker_thread3, + offsetof(struct dispatch_queue_s, dq_serialnum), 0); + result = !r; + if (result) _dispatch_root_queues_init_qos(supported); + } + } +#endif // HAVE_PTHREAD_WORKQUEUE_QOS #if HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP - if (!disable_wq) { + if (!result && !disable_wq) { #if PTHREAD_WORKQUEUE_SPI_VERSION >= 20121218 pthread_workqueue_setdispatchoffset_np( offsetof(struct dispatch_queue_s, dq_serialnum)); @@ -509,11 +797,7 @@ _dispatch_root_queues_init_workq(void) dispatch_root_queue_context_t qc; qc = &_dispatch_root_queue_contexts[i]; #if DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK - if (!disable_wq -#if DISPATCH_NO_BG_PRIORITY - && (qc->dgq_wq_priority != WORKQ_BG_PRIOQUEUE) -#endif - ) { + if (!disable_wq) { r = pthread_workqueue_attr_setqueuepriority_np(&pwq_attr, qc->dgq_wq_priority); (void)dispatch_assume_zero(r); @@ -543,25 +827,41 @@ _dispatch_root_queues_init_workq(void) #if DISPATCH_USE_PTHREAD_POOL static inline void _dispatch_root_queue_init_pthread_pool(dispatch_root_queue_context_t qc, - bool overcommit) + uint8_t pool_size, bool overcommit) { - qc->dgq_thread_pool_size = overcommit ? MAX_PTHREAD_COUNT : - _dispatch_hw_config.cc_max_active; + dispatch_pthread_root_queue_context_t pqc = qc->dgq_ctxt; + uint32_t thread_pool_size = overcommit ? MAX_PTHREAD_COUNT : + dispatch_hw_config(active_cpus); + if (slowpath(pool_size) && pool_size < thread_pool_size) { + thread_pool_size = pool_size; + } + qc->dgq_thread_pool_size = thread_pool_size; + if (qc->dgq_qos) { + (void)dispatch_assume_zero(pthread_attr_init(&pqc->dpq_thread_attr)); + (void)dispatch_assume_zero(pthread_attr_setdetachstate( + &pqc->dpq_thread_attr, PTHREAD_CREATE_DETACHED)); +#if HAVE_PTHREAD_WORKQUEUE_QOS + (void)dispatch_assume_zero(pthread_attr_set_qos_class_np( + &pqc->dpq_thread_attr, qc->dgq_qos, 0)); +#endif + } #if USE_MACH_SEM // override the default FIFO behavior for the pool semaphores kern_return_t kr = semaphore_create(mach_task_self(), - &qc->dgq_thread_mediator->dsema_port, SYNC_POLICY_LIFO, 0); + &pqc->dpq_thread_mediator.dsema_port, SYNC_POLICY_LIFO, 0); DISPATCH_VERIFY_MIG(kr); (void)dispatch_assume_zero(kr); - (void)dispatch_assume(qc->dgq_thread_mediator->dsema_port); + (void)dispatch_assume(pqc->dpq_thread_mediator.dsema_port); #elif USE_POSIX_SEM /* XXXRW: POSIX semaphores don't support LIFO? */ - int ret = sem_init(&qc->dgq_thread_mediator->dsema_sem, 0, 0); + int ret = sem_init(&pqc->dpq_thread_mediator.dsema_sem), 0, 0); (void)dispatch_assume_zero(ret); #endif } #endif // DISPATCH_USE_PTHREAD_POOL +static dispatch_once_t _dispatch_root_queues_pred; + static void _dispatch_root_queues_init(void *context DISPATCH_UNUSED) { @@ -580,13 +880,12 @@ _dispatch_root_queues_init(void *context DISPATCH_UNUSED) } #endif _dispatch_root_queue_init_pthread_pool( - &_dispatch_root_queue_contexts[i], overcommit); + &_dispatch_root_queue_contexts[i], 0, overcommit); } #else DISPATCH_CRASH("Root queue initialization failed"); #endif // DISPATCH_ENABLE_THREAD_POOL } - } #define countof(x) (sizeof(x) / sizeof(x[0])) @@ -595,8 +894,8 @@ DISPATCH_EXPORT DISPATCH_NOTHROW void libdispatch_init(void) { - dispatch_assert(DISPATCH_QUEUE_PRIORITY_COUNT == 4); - dispatch_assert(DISPATCH_ROOT_QUEUE_COUNT == 8); + dispatch_assert(DISPATCH_QUEUE_QOS_COUNT == 6); + dispatch_assert(DISPATCH_ROOT_QUEUE_COUNT == 12); dispatch_assert(DISPATCH_QUEUE_PRIORITY_LOW == -DISPATCH_QUEUE_PRIORITY_HIGH); @@ -604,16 +903,24 @@ libdispatch_init(void) DISPATCH_ROOT_QUEUE_COUNT); dispatch_assert(countof(_dispatch_root_queue_contexts) == DISPATCH_ROOT_QUEUE_COUNT); -#if HAVE_PTHREAD_WORKQUEUES + dispatch_assert(countof(_dispatch_priority2qos) == + DISPATCH_PRIORITY_COUNT); +#if HAVE_PTHREAD_WORKQUEUE_QOS + dispatch_assert(countof(_dispatch_priority2wq) == + DISPATCH_PRIORITY_COUNT); +#endif +#if HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP dispatch_assert(sizeof(_dispatch_wq2root_queues) / sizeof(_dispatch_wq2root_queues[0][0]) == - DISPATCH_ROOT_QUEUE_COUNT); + WORKQ_NUM_PRIOQUEUE * 2); #endif #if DISPATCH_ENABLE_THREAD_POOL - dispatch_assert(countof(_dispatch_thread_mediator) == + dispatch_assert(countof(_dispatch_pthread_root_queue_contexts) == DISPATCH_ROOT_QUEUE_COUNT); #endif + dispatch_assert(offsetof(struct dispatch_continuation_s, do_next) == + offsetof(struct dispatch_object_s, do_next)); dispatch_assert(sizeof(struct dispatch_apply_s) <= DISPATCH_CONTINUATION_SIZE); dispatch_assert(sizeof(struct dispatch_queue_s) % DISPATCH_CACHELINE_SIZE @@ -622,20 +929,22 @@ libdispatch_init(void) DISPATCH_CACHELINE_SIZE == 0); _dispatch_thread_key_create(&dispatch_queue_key, _dispatch_queue_cleanup); -#if !DISPATCH_USE_OS_SEMAPHORE_CACHE - _dispatch_thread_key_create(&dispatch_sema4_key, - (void (*)(void *))_dispatch_thread_semaphore_dispose); -#endif + _dispatch_thread_key_create(&dispatch_voucher_key, _voucher_thread_cleanup); _dispatch_thread_key_create(&dispatch_cache_key, _dispatch_cache_cleanup); _dispatch_thread_key_create(&dispatch_io_key, NULL); _dispatch_thread_key_create(&dispatch_apply_key, NULL); -#if DISPATCH_PERF_MON + _dispatch_thread_key_create(&dispatch_defaultpriority_key, NULL); +#if DISPATCH_PERF_MON && !DISPATCH_INTROSPECTION _dispatch_thread_key_create(&dispatch_bcounter_key, NULL); #endif +#if !DISPATCH_USE_OS_SEMAPHORE_CACHE + _dispatch_thread_key_create(&dispatch_sema4_key, + (void (*)(void *))_dispatch_thread_semaphore_dispose); +#endif #if DISPATCH_USE_RESOLVERS // rdar://problem/8541707 _dispatch_main_q.do_targetq = &_dispatch_root_queues[ - DISPATCH_ROOT_QUEUE_IDX_DEFAULT_OVERCOMMIT_PRIORITY]; + DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT]; #endif _dispatch_thread_setspecific(dispatch_queue_key, &_dispatch_main_q); @@ -649,9 +958,45 @@ libdispatch_init(void) _dispatch_hw_config_init(); _dispatch_vtable_init(); _os_object_init(); + _voucher_init(); _dispatch_introspection_init(); } +#if HAVE_MACH +static dispatch_once_t _dispatch_mach_host_port_pred; +static mach_port_t _dispatch_mach_host_port; + +static void +_dispatch_mach_host_port_init(void *ctxt DISPATCH_UNUSED) +{ + kern_return_t kr; + mach_port_t mp, mhp = mach_host_self(); + kr = host_get_host_port(mhp, &mp); + DISPATCH_VERIFY_MIG(kr); + if (!kr) { + // mach_host_self returned the HOST_PRIV port + kr = mach_port_deallocate(mach_task_self(), mhp); + DISPATCH_VERIFY_MIG(kr); + (void)dispatch_assume_zero(kr); + mhp = mp; + } else if (kr != KERN_INVALID_ARGUMENT) { + (void)dispatch_assume_zero(kr); + } + if (!dispatch_assume(mhp)) { + DISPATCH_CRASH("Could not get unprivileged host port"); + } + _dispatch_mach_host_port = mhp; +} + +mach_port_t +_dispatch_get_mach_host_port(void) +{ + dispatch_once_f(&_dispatch_mach_host_port_pred, NULL, + _dispatch_mach_host_port_init); + return _dispatch_mach_host_port; +} +#endif + DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void) @@ -659,6 +1004,11 @@ dispatch_atfork_child(void) void *crash = (void *)0x100; size_t i; +#if HAVE_MACH + _dispatch_mach_host_port_pred = 0; + _dispatch_mach_host_port = MACH_VOUCHER_NULL; +#endif + _voucher_atfork_child(); if (_dispatch_safe_fork) { return; } @@ -676,6 +1026,94 @@ dispatch_atfork_child(void) } } +#pragma mark - +#pragma mark dispatch_queue_attr_t + +DISPATCH_ALWAYS_INLINE +static inline bool +_dispatch_qos_class_valid(dispatch_qos_class_t qos_class, int relative_priority) +{ + qos_class_t qos = (qos_class_t)qos_class; + switch (qos) { + case _DISPATCH_QOS_CLASS_MAINTENANCE: + case _DISPATCH_QOS_CLASS_BACKGROUND: + case _DISPATCH_QOS_CLASS_UTILITY: + case _DISPATCH_QOS_CLASS_DEFAULT: + case _DISPATCH_QOS_CLASS_USER_INITIATED: + case _DISPATCH_QOS_CLASS_USER_INTERACTIVE: + case _DISPATCH_QOS_CLASS_UNSPECIFIED: + break; + default: + return false; + } + if (relative_priority > 0 || relative_priority < QOS_MIN_RELATIVE_PRIORITY){ + return false; + } + return true; +} + +#define DISPATCH_QUEUE_ATTR_QOS2IDX_INITIALIZER(qos) \ + [_DISPATCH_QOS_CLASS_##qos] = DQA_INDEX_QOS_CLASS_##qos + +static const +_dispatch_queue_attr_index_qos_class_t _dispatch_queue_attr_qos2idx[] = { + DISPATCH_QUEUE_ATTR_QOS2IDX_INITIALIZER(UNSPECIFIED), + DISPATCH_QUEUE_ATTR_QOS2IDX_INITIALIZER(MAINTENANCE), + DISPATCH_QUEUE_ATTR_QOS2IDX_INITIALIZER(BACKGROUND), + DISPATCH_QUEUE_ATTR_QOS2IDX_INITIALIZER(UTILITY), + DISPATCH_QUEUE_ATTR_QOS2IDX_INITIALIZER(DEFAULT), + DISPATCH_QUEUE_ATTR_QOS2IDX_INITIALIZER(USER_INITIATED), + DISPATCH_QUEUE_ATTR_QOS2IDX_INITIALIZER(USER_INTERACTIVE), +}; + +#define DISPATCH_QUEUE_ATTR_OVERCOMMIT2IDX(overcommit) \ + (overcommit ? DQA_INDEX_OVERCOMMIT : DQA_INDEX_NON_OVERCOMMIT) + +#define DISPATCH_QUEUE_ATTR_CONCURRENT2IDX(concurrent) \ + (concurrent ? DQA_INDEX_CONCURRENT : DQA_INDEX_SERIAL) + +#define DISPATCH_QUEUE_ATTR_PRIO2IDX(prio) (-(prio)) + +#define DISPATCH_QUEUE_ATTR_QOS2IDX(qos) (_dispatch_queue_attr_qos2idx[(qos)]) + +static inline dispatch_queue_attr_t +_dispatch_get_queue_attr(qos_class_t qos, int prio, bool overcommit, + bool concurrent) +{ + return (dispatch_queue_attr_t)&_dispatch_queue_attrs + [DISPATCH_QUEUE_ATTR_QOS2IDX(qos)] + [DISPATCH_QUEUE_ATTR_PRIO2IDX(prio)] + [DISPATCH_QUEUE_ATTR_OVERCOMMIT2IDX(overcommit)] + [DISPATCH_QUEUE_ATTR_CONCURRENT2IDX(concurrent)]; +} + +dispatch_queue_attr_t +dispatch_queue_attr_make_with_qos_class(dispatch_queue_attr_t dqa, + dispatch_qos_class_t qos_class, int relative_priority) +{ + if (!_dispatch_qos_class_valid(qos_class, relative_priority)) return NULL; + if (!slowpath(dqa)) { + dqa = _dispatch_get_queue_attr(0, 0, false, false); + } else if (dqa->do_vtable != DISPATCH_VTABLE(queue_attr)) { + DISPATCH_CLIENT_CRASH("Invalid queue attribute"); + } + return _dispatch_get_queue_attr(qos_class, relative_priority, + dqa->dqa_overcommit, dqa->dqa_concurrent); +} + +dispatch_queue_attr_t +dispatch_queue_attr_make_with_overcommit(dispatch_queue_attr_t dqa, + bool overcommit) +{ + if (!slowpath(dqa)) { + dqa = _dispatch_get_queue_attr(0, 0, false, false); + } else if (dqa->do_vtable != DISPATCH_VTABLE(queue_attr)) { + DISPATCH_CLIENT_CRASH("Invalid queue attribute"); + } + return _dispatch_get_queue_attr(dqa->dqa_qos_class, + dqa->dqa_relative_priority, overcommit, dqa->dqa_concurrent); +} + #pragma mark - #pragma mark dispatch_queue_t @@ -683,38 +1121,80 @@ dispatch_atfork_child(void) // 1 - main_q // 2 - mgr_q // 3 - mgr_root_q -// 4,5,6,7,8,9,10,11 - global queues +// 4,5,6,7,8,9,10,11,12,13,14,15 - global queues // we use 'xadd' on Intel, so the initial value == next assigned -unsigned long volatile _dispatch_queue_serial_numbers = 12; +unsigned long volatile _dispatch_queue_serial_numbers = 16; dispatch_queue_t -dispatch_queue_create_with_target(const char *label, - dispatch_queue_attr_t attr, dispatch_queue_t tq) +dispatch_queue_create_with_target(const char *label, dispatch_queue_attr_t dqa, + dispatch_queue_t tq) { - dispatch_queue_t dq; - - dq = _dispatch_alloc(DISPATCH_VTABLE(queue), +#if DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK + // Be sure the root queue priorities are set + dispatch_once_f(&_dispatch_root_queues_pred, NULL, + _dispatch_root_queues_init); +#endif + bool disallow_tq = (slowpath(dqa) && dqa != DISPATCH_QUEUE_CONCURRENT); + if (!slowpath(dqa)) { + dqa = _dispatch_get_queue_attr(0, 0, false, false); + } else if (dqa->do_vtable != DISPATCH_VTABLE(queue_attr)) { + DISPATCH_CLIENT_CRASH("Invalid queue attribute"); + } + dispatch_queue_t dq = _dispatch_alloc(DISPATCH_VTABLE(queue), sizeof(struct dispatch_queue_s) - DISPATCH_QUEUE_CACHELINE_PAD); - _dispatch_queue_init(dq); if (label) { dq->dq_label = strdup(label); } + qos_class_t qos = dqa->dqa_qos_class; + bool overcommit = dqa->dqa_overcommit; +#if HAVE_PTHREAD_WORKQUEUE_QOS + dq->dq_priority = _pthread_qos_class_encode(qos, dqa->dqa_relative_priority, + overcommit); +#endif + if (dqa->dqa_concurrent) { + dq->dq_width = DISPATCH_QUEUE_WIDTH_MAX; + } else { + // Default serial queue target queue is overcommit! + overcommit = true; + } + if (!tq) { + if (qos == _DISPATCH_QOS_CLASS_UNSPECIFIED) { + qos = _DISPATCH_QOS_CLASS_DEFAULT; + } +#if DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK + if (qos == _DISPATCH_QOS_CLASS_USER_INTERACTIVE && + !_dispatch_root_queues[ + DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS].dq_priority) { + qos = _DISPATCH_QOS_CLASS_USER_INITIATED; + } +#endif + bool maintenance_fallback = false; +#if DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK + maintenance_fallback = true; +#endif // DISPATCH_USE_NOQOS_WORKQUEUE_FALLBACK + if (maintenance_fallback) { + if (qos == _DISPATCH_QOS_CLASS_MAINTENANCE && + !_dispatch_root_queues[ + DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS].dq_priority) { + qos = _DISPATCH_QOS_CLASS_BACKGROUND; + } + } - if (attr == DISPATCH_QUEUE_CONCURRENT) { - dq->dq_width = UINT32_MAX; - if (!tq) { - tq = _dispatch_get_root_queue(0, false); + tq = _dispatch_get_root_queue(qos, overcommit); + if (slowpath(!tq)) { + DISPATCH_CLIENT_CRASH("Invalid queue attribute"); } } else { - if (!tq) { - // Default target queue is overcommit! - tq = _dispatch_get_root_queue(0, true); - } - if (slowpath(attr)) { - dispatch_debug_assert(!attr, "Invalid attribute"); + _dispatch_retain(tq); + if (disallow_tq) { + // TODO: override target queue's qos/overcommit ? + DISPATCH_CLIENT_CRASH("Invalid combination of target queue & " + "queue attribute"); } + _dispatch_queue_priority_inherit_from_target(dq, tq); } + _dispatch_queue_set_override_priority(dq); dq->do_targetq = tq; _dispatch_object_debug(dq, "%s", __func__); return _dispatch_introspection_queue_create(dq); @@ -770,6 +1250,22 @@ dispatch_queue_get_label(dispatch_queue_t dq) return dq->dq_label ? dq->dq_label : ""; } +qos_class_t +dispatch_queue_get_qos_class(dispatch_queue_t dq, int *relative_priority_ptr) +{ + qos_class_t qos = _DISPATCH_QOS_CLASS_UNSPECIFIED; + int relative_priority = 0; +#if HAVE_PTHREAD_WORKQUEUE_QOS + pthread_priority_t dqp = dq->dq_priority; + if (dqp & _PTHREAD_PRIORITY_INHERIT_FLAG) dqp = 0; + qos = _pthread_qos_class_decode(dqp, &relative_priority, NULL); +#else + (void)dq; +#endif + if (relative_priority_ptr) *relative_priority_ptr = relative_priority; + return qos; +} + static void _dispatch_queue_set_width2(void *ctxt) { @@ -786,20 +1282,23 @@ _dispatch_queue_set_width2(void *ctxt) tmp = (unsigned int)w; } else switch (w) { case DISPATCH_QUEUE_WIDTH_MAX_PHYSICAL_CPUS: - tmp = _dispatch_hw_config.cc_max_physical; + tmp = dispatch_hw_config(physical_cpus); break; case DISPATCH_QUEUE_WIDTH_ACTIVE_CPUS: - tmp = _dispatch_hw_config.cc_max_active; + tmp = dispatch_hw_config(active_cpus); break; default: // fall through case DISPATCH_QUEUE_WIDTH_MAX_LOGICAL_CPUS: - tmp = _dispatch_hw_config.cc_max_logical; + tmp = dispatch_hw_config(logical_cpus); break; } + if (tmp > DISPATCH_QUEUE_WIDTH_MAX / 2) { + tmp = DISPATCH_QUEUE_WIDTH_MAX / 2; + } // multiply by two since the running count is inc/dec by two // (the low bit == barrier) - dq->dq_width = tmp * 2; + dq->dq_width = (typeof(dq->dq_width))(tmp * 2); _dispatch_object_debug(dq, "%s", __func__); } @@ -819,12 +1318,20 @@ dispatch_queue_set_width(dispatch_queue_t dq, long width) static void _dispatch_set_target_queue2(void *ctxt) { - dispatch_queue_t prev_dq, dq = _dispatch_queue_get_current(); + dispatch_queue_t prev_dq, dq = _dispatch_queue_get_current(), tq = ctxt; + mach_port_t th; + while (!dispatch_atomic_cmpxchgv2o(dq, dq_tqthread, MACH_PORT_NULL, + _dispatch_thread_port(), &th, acquire)) { + _dispatch_thread_switch(th, DISPATCH_YIELD_THREAD_SWITCH_OPTION, + DISPATCH_CONTENTION_USLEEP_START); + } + _dispatch_queue_priority_inherit_from_target(dq, tq); prev_dq = dq->do_targetq; - dq->do_targetq = ctxt; + dq->do_targetq = tq; _dispatch_release(prev_dq); _dispatch_object_debug(dq, "%s", __func__); + dispatch_atomic_store2o(dq, dq_tqthread, MACH_PORT_NULL, release); } void @@ -839,7 +1346,8 @@ dispatch_set_target_queue(dispatch_object_t dou, dispatch_queue_t dq) if (slowpath(!dq)) { bool is_concurrent_q = (type == _DISPATCH_QUEUE_TYPE && slowpath(dou._dq->dq_width > 1)); - dq = _dispatch_get_root_queue(0, !is_concurrent_q); + dq = _dispatch_get_root_queue(_DISPATCH_QOS_CLASS_DEFAULT, + !is_concurrent_q); } // TODO: put into the vtable switch(type) { @@ -864,14 +1372,6 @@ dispatch_set_target_queue(dispatch_object_t dou, dispatch_queue_t dq) #pragma mark - #pragma mark dispatch_pthread_root_queue -struct dispatch_pthread_root_queue_context_s { - pthread_attr_t dpq_thread_attr; - dispatch_block_t dpq_thread_configure; - struct dispatch_semaphore_s dpq_thread_mediator; -}; -typedef struct dispatch_pthread_root_queue_context_s * - dispatch_pthread_root_queue_context_t; - #if DISPATCH_ENABLE_PTHREAD_ROOT_QUEUES static struct dispatch_pthread_root_queue_context_s _dispatch_mgr_root_queue_pthread_context; @@ -891,11 +1391,12 @@ static struct dispatch_queue_s _dispatch_mgr_root_queue = { .do_ctxt = &_dispatch_mgr_root_queue_context, .dq_label = "com.apple.root.libdispatch-manager", .dq_running = 2, - .dq_width = UINT32_MAX, + .dq_width = DISPATCH_QUEUE_WIDTH_MAX, .dq_serialnum = 3, }; static struct { volatile int prio; + int default_prio; int policy; pthread_t tid; } _dispatch_mgr_sched; @@ -911,8 +1412,9 @@ _dispatch_mgr_sched_init(void *ctxt DISPATCH_UNUSED) (void)dispatch_assume_zero(pthread_attr_getschedpolicy(attr, &_dispatch_mgr_sched.policy)); (void)dispatch_assume_zero(pthread_attr_getschedparam(attr, ¶m)); - // high-priority workq threads are at priority 2 above default - _dispatch_mgr_sched.prio = param.sched_priority + 2; + // legacy priority calls allowed when requesting above default priority + _dispatch_mgr_sched.default_prio = param.sched_priority; + _dispatch_mgr_sched.prio = _dispatch_mgr_sched.default_prio; } DISPATCH_NOINLINE @@ -927,9 +1429,19 @@ _dispatch_mgr_root_queue_init(void) PTHREAD_CREATE_DETACHED)); #if !DISPATCH_DEBUG (void)dispatch_assume_zero(pthread_attr_setstacksize(attr, 64 * 1024)); +#endif +#if HAVE_PTHREAD_WORKQUEUE_QOS + if (_dispatch_set_qos_class_enabled) { + qos_class_t qos = qos_class_main(); + (void)dispatch_assume_zero(pthread_attr_set_qos_class_np(attr, qos, 0)); + _dispatch_mgr_q.dq_priority = _pthread_qos_class_encode(qos, 0, 0); + _dispatch_queue_set_override_priority(&_dispatch_mgr_q); + } #endif param.sched_priority = _dispatch_mgr_sched.prio; - (void)dispatch_assume_zero(pthread_attr_setschedparam(attr, ¶m)); + if (param.sched_priority > _dispatch_mgr_sched.default_prio) { + (void)dispatch_assume_zero(pthread_attr_setschedparam(attr, ¶m)); + } return &_dispatch_mgr_sched.tid; } @@ -939,8 +1451,11 @@ _dispatch_mgr_priority_apply(void) struct sched_param param; do { param.sched_priority = _dispatch_mgr_sched.prio; - (void)dispatch_assume_zero(pthread_setschedparam( - _dispatch_mgr_sched.tid, _dispatch_mgr_sched.policy, ¶m)); + if (param.sched_priority > _dispatch_mgr_sched.default_prio) { + (void)dispatch_assume_zero(pthread_setschedparam( + _dispatch_mgr_sched.tid, _dispatch_mgr_sched.policy, + ¶m)); + } } while (_dispatch_mgr_sched.prio > param.sched_priority); } @@ -982,10 +1497,9 @@ dispatch_pthread_root_queue_create(const char *label, unsigned long flags, dispatch_root_queue_context_t qc; dispatch_pthread_root_queue_context_t pqc; size_t dqs; + uint8_t pool_size = flags & _DISPATCH_PTHREAD_ROOT_QUEUE_FLAG_POOL_SIZE ? + (uint8_t)(flags & ~_DISPATCH_PTHREAD_ROOT_QUEUE_FLAG_POOL_SIZE) : 0; - if (slowpath(flags)) { - return NULL; - } dqs = sizeof(struct dispatch_queue_s) - DISPATCH_QUEUE_CACHELINE_PAD; dq = _dispatch_alloc(DISPATCH_VTABLE(queue_root), dqs + sizeof(struct dispatch_root_queue_context_s) + @@ -1002,18 +1516,25 @@ dispatch_pthread_root_queue_create(const char *label, unsigned long flags, dq->do_ctxt = qc; dq->do_targetq = NULL; dq->dq_running = 2; - dq->dq_width = UINT32_MAX; + dq->dq_width = DISPATCH_QUEUE_WIDTH_MAX; pqc->dpq_thread_mediator.do_vtable = DISPATCH_VTABLE(semaphore); - qc->dgq_thread_mediator = &pqc->dpq_thread_mediator; qc->dgq_ctxt = pqc; #if HAVE_PTHREAD_WORKQUEUES qc->dgq_kworkqueue = (void*)(~0ul); #endif - _dispatch_root_queue_init_pthread_pool(qc, true); // rdar://11352331 + _dispatch_root_queue_init_pthread_pool(qc, pool_size, true); if (attr) { memcpy(&pqc->dpq_thread_attr, attr, sizeof(pthread_attr_t)); +#if HAVE_PTHREAD_WORKQUEUE_QOS + qos_class_t qos = 0; + if (!pthread_attr_get_qos_class_np(&pqc->dpq_thread_attr, &qos, NULL) + && qos > _DISPATCH_QOS_CLASS_DEFAULT) { + DISPATCH_CLIENT_CRASH("pthread root queues do not support " + "explicit QoS attributes"); + } +#endif _dispatch_mgr_priority_raise(&pqc->dpq_thread_attr); } else { (void)dispatch_assume_zero(pthread_attr_init(&pqc->dpq_thread_attr)); @@ -1040,11 +1561,13 @@ _dispatch_pthread_root_queue_dispose(dispatch_queue_t dq) dispatch_root_queue_context_t qc = dq->do_ctxt; dispatch_pthread_root_queue_context_t pqc = qc->dgq_ctxt; - _dispatch_semaphore_dispose(qc->dgq_thread_mediator); + pthread_attr_destroy(&pqc->dpq_thread_attr); + _dispatch_semaphore_dispose(&pqc->dpq_thread_mediator); if (pqc->dpq_thread_configure) { Block_release(pqc->dpq_thread_configure); } - dq->do_targetq = _dispatch_get_root_queue(0, false); + dq->do_targetq = _dispatch_get_root_queue(_DISPATCH_QOS_CLASS_DEFAULT, + false); #endif if (dq->dq_label) { free((void*)dq->dq_label); @@ -1078,7 +1601,7 @@ _dispatch_queue_specific_queue_dispose(dispatch_queue_specific_queue_t dqsq) TAILQ_FOREACH_SAFE(dqs, &dqsq->dqsq_contexts, dqs_list, tmp) { if (dqs->dqs_destructor) { dispatch_async_f(_dispatch_get_root_queue( - DISPATCH_QUEUE_PRIORITY_DEFAULT, false), dqs->dqs_ctxt, + _DISPATCH_QOS_CLASS_DEFAULT, false), dqs->dqs_ctxt, dqs->dqs_destructor); } free(dqs); @@ -1095,9 +1618,9 @@ _dispatch_queue_init_specific(dispatch_queue_t dq) sizeof(struct dispatch_queue_specific_queue_s)); _dispatch_queue_init((dispatch_queue_t)dqsq); dqsq->do_xref_cnt = -1; - dqsq->do_targetq = _dispatch_get_root_queue(DISPATCH_QUEUE_PRIORITY_HIGH, - true); - dqsq->dq_width = UINT32_MAX; + dqsq->do_targetq = _dispatch_get_root_queue( + _DISPATCH_QOS_CLASS_USER_INITIATED, true); + dqsq->dq_width = DISPATCH_QUEUE_WIDTH_MAX; dqsq->dq_label = "queue-specific"; TAILQ_INIT(&dqsq->dqsq_contexts); if (slowpath(!dispatch_atomic_cmpxchg2o(dq, dq_specific_q, NULL, @@ -1118,7 +1641,7 @@ _dispatch_queue_set_specific(void *ctxt) // Destroy previous context for existing key if (dqs->dqs_destructor) { dispatch_async_f(_dispatch_get_root_queue( - DISPATCH_QUEUE_PRIORITY_DEFAULT, false), dqs->dqs_ctxt, + _DISPATCH_QOS_CLASS_DEFAULT, false), dqs->dqs_ctxt, dqs->dqs_destructor); } if (dqsn->dqs_ctxt) { @@ -1227,7 +1750,7 @@ _dispatch_queue_debug_attr(dispatch_queue_t dq, char* buf, size_t bufsiz) target->dq_label : "", target, dq->dq_width / 2, dq->dq_running / 2, dq->dq_running & 1); if (dq->dq_is_thread_bound) { - offset += dsnprintf(buf, bufsiz, ", thread = %p ", + offset += dsnprintf(buf, bufsiz, ", thread = 0x%x ", _dispatch_queue_get_bound_thread(dq)); } return offset; @@ -1256,7 +1779,7 @@ dispatch_debug_queue(dispatch_queue_t dq, const char* str) { } #endif -#if DISPATCH_PERF_MON +#if DISPATCH_PERF_MON && !DISPATCH_INTROSPECTION static OSSpinLock _dispatch_stats_lock; static struct { uint64_t time_total; @@ -1267,18 +1790,13 @@ static struct { static void _dispatch_queue_merge_stats(uint64_t start) { - uint64_t avg, delta = _dispatch_absolute_time() - start; - unsigned long count, bucket; + uint64_t delta = _dispatch_absolute_time() - start; + unsigned long count; - count = (size_t)_dispatch_thread_getspecific(dispatch_bcounter_key); + count = (unsigned long)_dispatch_thread_getspecific(dispatch_bcounter_key); _dispatch_thread_setspecific(dispatch_bcounter_key, NULL); - if (count) { - avg = delta / count; - bucket = flsll(avg); - } else { - bucket = 0; - } + int bucket = flsl((long)count); // 64-bit counters on 32-bit require a lock or a queue OSSpinLockLock(&_dispatch_stats_lock); @@ -1328,7 +1846,8 @@ _dispatch_continuation_free_to_cache_limit(dispatch_continuation_t dc) dispatch_continuation_t next_dc; dc = _dispatch_thread_getspecific(dispatch_cache_key); int cnt; - if (!dc || (cnt = dc->do_ref_cnt-_dispatch_continuation_cache_limit) <= 0) { + if (!dc || (cnt = dc->dc_cache_cnt - + _dispatch_continuation_cache_limit) <= 0){ return; } do { @@ -1353,92 +1872,446 @@ _dispatch_continuation_redirect(dispatch_queue_t dq, dispatch_object_t dou) (_dispatch_thread_semaphore_t)dc->dc_other); _dispatch_introspection_queue_item_complete(dou); } else { - _dispatch_async_f_redirect(dq, dc); + _dispatch_async_f_redirect(dq, dc, + _dispatch_queue_get_override_priority(dq)); } _dispatch_perfmon_workitem_inc(); } -DISPATCH_ALWAYS_INLINE_NDEBUG -static inline void -_dispatch_continuation_pop(dispatch_object_t dou) -{ - dispatch_continuation_t dc = dou._dc, dc1; - dispatch_group_t dg; +#pragma mark - +#pragma mark dispatch_block_create - _dispatch_trace_continuation_pop(_dispatch_queue_get_current(), dou); - if (DISPATCH_OBJ_IS_VTABLE(dou._do)) { - return dx_invoke(dou._do); - } +#if __BLOCKS__ - // Add the item back to the cache before calling the function. This - // allows the 'hot' continuation to be used for a quick callback. - // - // The ccache version is per-thread. - // Therefore, the object has not been reused yet. - // This generates better assembly. - if ((long)dc->do_vtable & DISPATCH_OBJ_ASYNC_BIT) { - dc1 = _dispatch_continuation_free_cacheonly(dc); - } else { - dc1 = NULL; +DISPATCH_ALWAYS_INLINE +static inline bool +_dispatch_block_flags_valid(dispatch_block_flags_t flags) +{ + return ((flags & ~DISPATCH_BLOCK_API_MASK) == 0); +} + +DISPATCH_ALWAYS_INLINE +static inline dispatch_block_flags_t +_dispatch_block_normalize_flags(dispatch_block_flags_t flags) +{ + if (flags & (DISPATCH_BLOCK_NO_VOUCHER|DISPATCH_BLOCK_DETACHED)) { + flags |= DISPATCH_BLOCK_HAS_VOUCHER; + } + if (flags & (DISPATCH_BLOCK_NO_QOS_CLASS|DISPATCH_BLOCK_DETACHED)) { + flags |= DISPATCH_BLOCK_HAS_PRIORITY; + } + return flags; +} + +static inline dispatch_block_t +_dispatch_block_create_with_voucher_and_priority(dispatch_block_flags_t flags, + voucher_t voucher, pthread_priority_t pri, dispatch_block_t block) +{ + flags = _dispatch_block_normalize_flags(flags); + voucher_t cv = NULL; + bool assign = (flags & DISPATCH_BLOCK_ASSIGN_CURRENT); + if (assign && !(flags & DISPATCH_BLOCK_HAS_VOUCHER)) { + voucher = cv = voucher_copy(); + flags |= DISPATCH_BLOCK_HAS_VOUCHER; + } + if (assign && !(flags & DISPATCH_BLOCK_HAS_PRIORITY)) { + pri = _dispatch_priority_propagate(); + flags |= DISPATCH_BLOCK_HAS_PRIORITY; + } + dispatch_block_t db = _dispatch_block_create(flags, voucher, pri, block); + if (cv) _voucher_release(cv); +#if DISPATCH_DEBUG + dispatch_assert(_dispatch_block_get_data(db)); +#endif + return db; +} + +dispatch_block_t +dispatch_block_create(dispatch_block_flags_t flags, dispatch_block_t block) +{ + if (!_dispatch_block_flags_valid(flags)) return NULL; + return _dispatch_block_create_with_voucher_and_priority(flags, NULL, 0, + block); +} + +dispatch_block_t +dispatch_block_create_with_qos_class(dispatch_block_flags_t flags, + dispatch_qos_class_t qos_class, int relative_priority, + dispatch_block_t block) +{ + if (!_dispatch_block_flags_valid(flags)) return NULL; + if (!_dispatch_qos_class_valid(qos_class, relative_priority)) return NULL; + flags |= DISPATCH_BLOCK_HAS_PRIORITY; + pthread_priority_t pri = 0; +#if HAVE_PTHREAD_WORKQUEUE_QOS + pri = _pthread_qos_class_encode(qos_class, relative_priority, 0); +#endif + return _dispatch_block_create_with_voucher_and_priority(flags, NULL, + pri, block); +} + +dispatch_block_t +dispatch_block_create_with_voucher(dispatch_block_flags_t flags, + voucher_t voucher, dispatch_block_t block) +{ + if (!_dispatch_block_flags_valid(flags)) return NULL; + flags |= DISPATCH_BLOCK_HAS_VOUCHER; + return _dispatch_block_create_with_voucher_and_priority(flags, voucher, 0, + block); +} + +dispatch_block_t +dispatch_block_create_with_voucher_and_qos_class(dispatch_block_flags_t flags, + voucher_t voucher, dispatch_qos_class_t qos_class, + int relative_priority, dispatch_block_t block) +{ + if (!_dispatch_block_flags_valid(flags)) return NULL; + if (!_dispatch_qos_class_valid(qos_class, relative_priority)) return NULL; + flags |= (DISPATCH_BLOCK_HAS_VOUCHER|DISPATCH_BLOCK_HAS_PRIORITY); + pthread_priority_t pri = 0; +#if HAVE_PTHREAD_WORKQUEUE_QOS + pri = _pthread_qos_class_encode(qos_class, relative_priority, 0); +#endif + return _dispatch_block_create_with_voucher_and_priority(flags, voucher, + pri, block); +} + +void +dispatch_block_perform(dispatch_block_flags_t flags, dispatch_block_t block) +{ + if (!_dispatch_block_flags_valid(flags)) { + DISPATCH_CLIENT_CRASH("Invalid flags passed to " + "dispatch_block_perform()"); + } + flags = _dispatch_block_normalize_flags(flags); + struct dispatch_block_private_data_s dbpds = + DISPATCH_BLOCK_PRIVATE_DATA_INITIALIZER(flags, NULL, 0, block); + dbpds.dbpd_atomic_flags |= DBF_PERFORM; // no group_leave at end of invoke + return _dispatch_block_invoke(&dbpds); +} + +#define _dbpd_group(dbpd) ((dispatch_group_t)&(dbpd)->dbpd_group) + +void +_dispatch_block_invoke(const struct dispatch_block_private_data_s *dbcpd) +{ + dispatch_block_private_data_t dbpd = (dispatch_block_private_data_t)dbcpd; + dispatch_block_flags_t flags = dbpd->dbpd_flags; + unsigned int atomic_flags = dbpd->dbpd_atomic_flags; + if (slowpath(atomic_flags & DBF_WAITED)) { + DISPATCH_CLIENT_CRASH("A block object may not be both run more " + "than once and waited for"); + } + if (atomic_flags & DBF_CANCELED) goto out; + + pthread_priority_t op = DISPATCH_NO_PRIORITY, p = DISPATCH_NO_PRIORITY; + unsigned long override = 0; + if (flags & DISPATCH_BLOCK_HAS_PRIORITY) { + op = _dispatch_get_priority(); + p = dbpd->dbpd_priority; + override = (flags & DISPATCH_BLOCK_ENFORCE_QOS_CLASS) || + !(flags & DISPATCH_BLOCK_INHERIT_QOS_CLASS) ? + DISPATCH_PRIORITY_ENFORCE : 0; + } + voucher_t ov, v = DISPATCH_NO_VOUCHER; + if (flags & DISPATCH_BLOCK_HAS_VOUCHER) { + v = dbpd->dbpd_voucher; + if (v) _voucher_retain(v); + } + ov = _dispatch_adopt_priority_and_voucher(p, v, override); + dbpd->dbpd_thread = _dispatch_thread_port(); + dbpd->dbpd_block(); + _dispatch_set_priority_and_replace_voucher(op, ov); +out: + if ((atomic_flags & DBF_PERFORM) == 0) { + if (dispatch_atomic_inc2o(dbpd, dbpd_performed, acquire) == 1) { + dispatch_group_leave(_dbpd_group(dbpd)); + } + } +} + +static void +_dispatch_block_sync_invoke(void *block) +{ + dispatch_block_t b = block; + dispatch_block_private_data_t dbpd = _dispatch_block_get_data(b); + dispatch_block_flags_t flags = dbpd->dbpd_flags; + unsigned int atomic_flags = dbpd->dbpd_atomic_flags; + if (slowpath(atomic_flags & DBF_WAITED)) { + DISPATCH_CLIENT_CRASH("A block object may not be both run more " + "than once and waited for"); + } + if (atomic_flags & DBF_CANCELED) goto out; + + pthread_priority_t op = DISPATCH_NO_PRIORITY, p = DISPATCH_NO_PRIORITY; + unsigned long override = 0; + if (flags & DISPATCH_BLOCK_HAS_PRIORITY) { + op = _dispatch_get_priority(); + p = dbpd->dbpd_priority; + override = (flags & DISPATCH_BLOCK_ENFORCE_QOS_CLASS) || + !(flags & DISPATCH_BLOCK_INHERIT_QOS_CLASS) ? + DISPATCH_PRIORITY_ENFORCE : 0; + } + voucher_t ov, v = DISPATCH_NO_VOUCHER; + if (flags & DISPATCH_BLOCK_HAS_VOUCHER) { + v = dbpd->dbpd_voucher; + if (v) _voucher_retain(v); + } + ov = _dispatch_adopt_priority_and_voucher(p, v, override); + dbpd->dbpd_block(); + _dispatch_set_priority_and_replace_voucher(op, ov); +out: + if ((atomic_flags & DBF_PERFORM) == 0) { + if (dispatch_atomic_inc2o(dbpd, dbpd_performed, acquire) == 1) { + dispatch_group_leave(_dbpd_group(dbpd)); + } + } + + dispatch_queue_t dq = _dispatch_queue_get_current(); + if (dispatch_atomic_cmpxchg2o(dbpd, dbpd_queue, dq, NULL, acquire)) { + // balances dispatch_{,barrier_,}sync + _dispatch_release(dq); } - if ((long)dc->do_vtable & DISPATCH_OBJ_GROUP_BIT) { - dg = dc->dc_data; +} + +static void +_dispatch_block_async_invoke_and_release(void *block) +{ + dispatch_block_t b = block; + dispatch_block_private_data_t dbpd = _dispatch_block_get_data(b); + dispatch_block_flags_t flags = dbpd->dbpd_flags; + unsigned int atomic_flags = dbpd->dbpd_atomic_flags; + if (slowpath(atomic_flags & DBF_WAITED)) { + DISPATCH_CLIENT_CRASH("A block object may not be both run more " + "than once and waited for"); + } + if (atomic_flags & DBF_CANCELED) goto out; + + pthread_priority_t p = DISPATCH_NO_PRIORITY; + unsigned long override = 0; + if (flags & DISPATCH_BLOCK_HAS_PRIORITY) { + override = (flags & DISPATCH_BLOCK_ENFORCE_QOS_CLASS) ? + DISPATCH_PRIORITY_ENFORCE : 0; + p = dbpd->dbpd_priority; + } + voucher_t v = DISPATCH_NO_VOUCHER; + if (flags & DISPATCH_BLOCK_HAS_VOUCHER) { + v = dbpd->dbpd_voucher; + if (v) _voucher_retain(v); + } + _dispatch_adopt_priority_and_replace_voucher(p, v, override); + dbpd->dbpd_block(); +out: + if ((atomic_flags & DBF_PERFORM) == 0) { + if (dispatch_atomic_inc2o(dbpd, dbpd_performed, acquire) == 1) { + dispatch_group_leave(_dbpd_group(dbpd)); + } + } + dispatch_queue_t dq = _dispatch_queue_get_current(); + if (dispatch_atomic_cmpxchg2o(dbpd, dbpd_queue, dq, NULL, acquire)) { + // balances dispatch_{,barrier_,group_}async + _dispatch_release(dq); + } + Block_release(b); +} + +void +dispatch_block_cancel(dispatch_block_t db) +{ + dispatch_block_private_data_t dbpd = _dispatch_block_get_data(db); + if (!dbpd) { + DISPATCH_CLIENT_CRASH("Invalid block object passed to " + "dispatch_block_cancel()"); + } + (void)dispatch_atomic_or2o(dbpd, dbpd_atomic_flags, DBF_CANCELED, relaxed); +} + +long +dispatch_block_testcancel(dispatch_block_t db) +{ + dispatch_block_private_data_t dbpd = _dispatch_block_get_data(db); + if (!dbpd) { + DISPATCH_CLIENT_CRASH("Invalid block object passed to " + "dispatch_block_testcancel()"); + } + return (bool)(dbpd->dbpd_atomic_flags & DBF_CANCELED); +} + +long +dispatch_block_wait(dispatch_block_t db, dispatch_time_t timeout) +{ + dispatch_block_private_data_t dbpd = _dispatch_block_get_data(db); + if (!dbpd) { + DISPATCH_CLIENT_CRASH("Invalid block object passed to " + "dispatch_block_wait()"); + } + + unsigned int flags = dispatch_atomic_or_orig2o(dbpd, dbpd_atomic_flags, + DBF_WAITING, relaxed); + if (slowpath(flags & (DBF_WAITED | DBF_WAITING))) { + DISPATCH_CLIENT_CRASH("A block object may not be waited for " + "more than once"); + } + + // If we know the queue where this block is + // enqueued, or the thread that's executing it, then we should boost + // it here. + + pthread_priority_t pp = _dispatch_get_priority(); + + dispatch_queue_t boost_dq; + boost_dq = dispatch_atomic_xchg2o(dbpd, dbpd_queue, NULL, acquire); + if (boost_dq) { + // release balances dispatch_{,barrier_,group_}async. + // Can't put the queue back in the timeout case: the block might + // finish after we fell out of group_wait and see our NULL, so + // neither of us would ever release. Side effect: After a _wait + // that times out, subsequent waits will not boost the qos of the + // still-running block. + _dispatch_queue_wakeup_with_qos_and_release(boost_dq, pp); + } + + mach_port_t boost_th = dbpd->dbpd_thread; + if (boost_th) { + _dispatch_thread_override_start(boost_th, pp); + } + + int performed = dispatch_atomic_load2o(dbpd, dbpd_performed, relaxed); + if (slowpath(performed > 1 || (boost_th && boost_dq))) { + DISPATCH_CLIENT_CRASH("A block object may not be both run more " + "than once and waited for"); + } + + long ret = dispatch_group_wait(_dbpd_group(dbpd), timeout); + + if (boost_th) { + _dispatch_thread_override_end(boost_th); + } + + if (ret) { + // timed out: reverse our changes + (void)dispatch_atomic_and2o(dbpd, dbpd_atomic_flags, + ~DBF_WAITING, relaxed); } else { - dg = NULL; + (void)dispatch_atomic_or2o(dbpd, dbpd_atomic_flags, + DBF_WAITED, relaxed); + // don't need to re-test here: the second call would see + // the first call's WAITING } - _dispatch_client_callout(dc->dc_ctxt, dc->dc_func); - if (dg) { - dispatch_group_leave(dg); - _dispatch_release(dg); + + return ret; +} + +void +dispatch_block_notify(dispatch_block_t db, dispatch_queue_t queue, + dispatch_block_t notification_block) +{ + dispatch_block_private_data_t dbpd = _dispatch_block_get_data(db); + if (!dbpd) { + DISPATCH_CLIENT_CRASH("Invalid block object passed to " + "dispatch_block_notify()"); } - _dispatch_introspection_queue_item_complete(dou); - if (slowpath(dc1)) { - _dispatch_continuation_free_to_cache_limit(dc1); + int performed = dispatch_atomic_load2o(dbpd, dbpd_performed, relaxed); + if (slowpath(performed > 1)) { + DISPATCH_CLIENT_CRASH("A block object may not be both run more " + "than once and observed"); } + + return dispatch_group_notify(_dbpd_group(dbpd), queue, notification_block); } +#endif // __BLOCKS__ + #pragma mark - #pragma mark dispatch_barrier_async DISPATCH_NOINLINE static void _dispatch_barrier_async_f_slow(dispatch_queue_t dq, void *ctxt, - dispatch_function_t func) + dispatch_function_t func, pthread_priority_t pp, + dispatch_block_flags_t flags) { dispatch_continuation_t dc = _dispatch_continuation_alloc_from_heap(); dc->do_vtable = (void *)(DISPATCH_OBJ_ASYNC_BIT | DISPATCH_OBJ_BARRIER_BIT); dc->dc_func = func; dc->dc_ctxt = ctxt; + _dispatch_continuation_voucher_set(dc, flags); + _dispatch_continuation_priority_set(dc, pp, flags); + + pp = _dispatch_continuation_get_override_priority(dq, dc); - _dispatch_queue_push(dq, dc); + _dispatch_queue_push(dq, dc, pp); } -DISPATCH_NOINLINE -void -dispatch_barrier_async_f(dispatch_queue_t dq, void *ctxt, - dispatch_function_t func) +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_barrier_async_f2(dispatch_queue_t dq, void *ctxt, + dispatch_function_t func, pthread_priority_t pp, + dispatch_block_flags_t flags) { dispatch_continuation_t dc; dc = fastpath(_dispatch_continuation_alloc_cacheonly()); if (!dc) { - return _dispatch_barrier_async_f_slow(dq, ctxt, func); + return _dispatch_barrier_async_f_slow(dq, ctxt, func, pp, flags); } dc->do_vtable = (void *)(DISPATCH_OBJ_ASYNC_BIT | DISPATCH_OBJ_BARRIER_BIT); dc->dc_func = func; dc->dc_ctxt = ctxt; + _dispatch_continuation_voucher_set(dc, flags); + _dispatch_continuation_priority_set(dc, pp, flags); + + pp = _dispatch_continuation_get_override_priority(dq, dc); - _dispatch_queue_push(dq, dc); + _dispatch_queue_push(dq, dc, pp); +} + +DISPATCH_NOINLINE +static void +_dispatch_barrier_async_f(dispatch_queue_t dq, void *ctxt, + dispatch_function_t func, pthread_priority_t pp, + dispatch_block_flags_t flags) +{ + return _dispatch_barrier_async_f2(dq, ctxt, func, pp, flags); +} + +DISPATCH_NOINLINE +void +dispatch_barrier_async_f(dispatch_queue_t dq, void *ctxt, + dispatch_function_t func) +{ + return _dispatch_barrier_async_f2(dq, ctxt, func, 0, 0); +} + +DISPATCH_NOINLINE +void +_dispatch_barrier_async_detached_f(dispatch_queue_t dq, void *ctxt, + dispatch_function_t func) +{ + return _dispatch_barrier_async_f2(dq, ctxt, func, 0, + DISPATCH_BLOCK_NO_QOS_CLASS|DISPATCH_BLOCK_NO_VOUCHER); } #ifdef __BLOCKS__ void dispatch_barrier_async(dispatch_queue_t dq, void (^work)(void)) { - dispatch_barrier_async_f(dq, _dispatch_Block_copy(work), - _dispatch_call_block_and_release); + dispatch_function_t func = _dispatch_call_block_and_release; + pthread_priority_t pp = 0; + dispatch_block_flags_t flags = 0; + if (slowpath(_dispatch_block_has_private_data(work))) { + func = _dispatch_block_async_invoke_and_release; + pp = _dispatch_block_get_priority(work); + flags = _dispatch_block_get_flags(work); + // balanced in d_block_async_invoke_and_release or d_block_wait + if (dispatch_atomic_cmpxchg2o(_dispatch_block_get_data(work), + dbpd_queue, NULL, dq, release)) { + _dispatch_retain(dq); + } + } + _dispatch_barrier_async_f(dq, _dispatch_Block_copy(work), func, pp, flags); } #endif @@ -1454,25 +2327,28 @@ _dispatch_async_redirect_invoke(void *ctxt) old_dq = _dispatch_thread_getspecific(dispatch_queue_key); _dispatch_thread_setspecific(dispatch_queue_key, dq); + pthread_priority_t old_dp = _dispatch_set_defaultpriority(dq->dq_priority); _dispatch_continuation_pop(other_dc); + _dispatch_reset_defaultpriority(old_dp); _dispatch_thread_setspecific(dispatch_queue_key, old_dq); rq = dq->do_targetq; while (slowpath(rq->do_targetq) && rq != old_dq) { if (dispatch_atomic_sub2o(rq, dq_running, 2, relaxed) == 0) { - _dispatch_wakeup(rq); + _dispatch_queue_wakeup(rq); } rq = rq->do_targetq; } if (dispatch_atomic_sub2o(dq, dq_running, 2, relaxed) == 0) { - _dispatch_wakeup(dq); + _dispatch_queue_wakeup(dq); } _dispatch_release(dq); } static inline void -_dispatch_async_f_redirect2(dispatch_queue_t dq, dispatch_continuation_t dc) +_dispatch_async_f_redirect2(dispatch_queue_t dq, dispatch_continuation_t dc, + pthread_priority_t pp) { uint32_t running = 2; @@ -1491,13 +2367,13 @@ _dispatch_async_f_redirect2(dispatch_queue_t dq, dispatch_continuation_t dc) dq = dq->do_targetq; } while (slowpath(dq->do_targetq)); - _dispatch_queue_push_wakeup(dq, dc, running == 0); + _dispatch_queue_push_wakeup(dq, dc, pp, running == 0); } DISPATCH_NOINLINE static void _dispatch_async_f_redirect(dispatch_queue_t dq, - dispatch_continuation_t other_dc) + dispatch_continuation_t other_dc, pthread_priority_t pp) { dispatch_continuation_t dc = _dispatch_continuation_alloc(); @@ -1506,19 +2382,22 @@ _dispatch_async_f_redirect(dispatch_queue_t dq, dc->dc_ctxt = dc; dc->dc_data = dq; dc->dc_other = other_dc; + dc->dc_priority = 0; + dc->dc_voucher = NULL; _dispatch_retain(dq); dq = dq->do_targetq; if (slowpath(dq->do_targetq)) { - return _dispatch_async_f_redirect2(dq, dc); + return _dispatch_async_f_redirect2(dq, dc, pp); } - _dispatch_queue_push(dq, dc); + _dispatch_queue_push(dq, dc, pp); } DISPATCH_NOINLINE static void -_dispatch_async_f2(dispatch_queue_t dq, dispatch_continuation_t dc) +_dispatch_async_f2(dispatch_queue_t dq, dispatch_continuation_t dc, + pthread_priority_t pp) { uint32_t running = 2; @@ -1533,78 +2412,108 @@ _dispatch_async_f2(dispatch_queue_t dq, dispatch_continuation_t dc) break; } if (!slowpath(running & 1)) { - return _dispatch_async_f_redirect(dq, dc); + return _dispatch_async_f_redirect(dq, dc, pp); } running = dispatch_atomic_sub2o(dq, dq_running, 2, relaxed); // We might get lucky and find that the barrier has ended by now } while (!(running & 1)); - _dispatch_queue_push_wakeup(dq, dc, running == 0); + _dispatch_queue_push_wakeup(dq, dc, pp, running == 0); } DISPATCH_NOINLINE static void _dispatch_async_f_slow(dispatch_queue_t dq, void *ctxt, - dispatch_function_t func) + dispatch_function_t func, pthread_priority_t pp, + dispatch_block_flags_t flags) { dispatch_continuation_t dc = _dispatch_continuation_alloc_from_heap(); dc->do_vtable = (void *)DISPATCH_OBJ_ASYNC_BIT; dc->dc_func = func; dc->dc_ctxt = ctxt; + _dispatch_continuation_voucher_set(dc, flags); + _dispatch_continuation_priority_set(dc, pp, flags); + + pp = _dispatch_continuation_get_override_priority(dq, dc); // No fastpath/slowpath hint because we simply don't know if (dq->do_targetq) { - return _dispatch_async_f2(dq, dc); + return _dispatch_async_f2(dq, dc, pp); } - _dispatch_queue_push(dq, dc); + _dispatch_queue_push(dq, dc, pp); } -DISPATCH_NOINLINE -void -dispatch_async_f(dispatch_queue_t dq, void *ctxt, dispatch_function_t func) +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_async_f(dispatch_queue_t dq, void *ctxt, dispatch_function_t func, + pthread_priority_t pp, dispatch_block_flags_t flags) { dispatch_continuation_t dc; // No fastpath/slowpath hint because we simply don't know - if (dq->dq_width == 1) { - return dispatch_barrier_async_f(dq, ctxt, func); + if (dq->dq_width == 1 || flags & DISPATCH_BLOCK_BARRIER) { + return _dispatch_barrier_async_f(dq, ctxt, func, pp, flags); } dc = fastpath(_dispatch_continuation_alloc_cacheonly()); if (!dc) { - return _dispatch_async_f_slow(dq, ctxt, func); + return _dispatch_async_f_slow(dq, ctxt, func, pp, flags); } dc->do_vtable = (void *)DISPATCH_OBJ_ASYNC_BIT; dc->dc_func = func; dc->dc_ctxt = ctxt; + _dispatch_continuation_voucher_set(dc, flags); + _dispatch_continuation_priority_set(dc, pp, flags); + + pp = _dispatch_continuation_get_override_priority(dq, dc); // No fastpath/slowpath hint because we simply don't know if (dq->do_targetq) { - return _dispatch_async_f2(dq, dc); + return _dispatch_async_f2(dq, dc, pp); } - _dispatch_queue_push(dq, dc); + _dispatch_queue_push(dq, dc, pp); +} + +DISPATCH_NOINLINE +void +dispatch_async_f(dispatch_queue_t dq, void *ctxt, dispatch_function_t func) +{ + return _dispatch_async_f(dq, ctxt, func, 0, 0); } #ifdef __BLOCKS__ void dispatch_async(dispatch_queue_t dq, void (^work)(void)) { - dispatch_async_f(dq, _dispatch_Block_copy(work), - _dispatch_call_block_and_release); + dispatch_function_t func = _dispatch_call_block_and_release; + dispatch_block_flags_t flags = 0; + pthread_priority_t pp = 0; + if (slowpath(_dispatch_block_has_private_data(work))) { + func = _dispatch_block_async_invoke_and_release; + pp = _dispatch_block_get_priority(work); + flags = _dispatch_block_get_flags(work); + // balanced in d_block_async_invoke_and_release or d_block_wait + if (dispatch_atomic_cmpxchg2o(_dispatch_block_get_data(work), + dbpd_queue, NULL, dq, release)) { + _dispatch_retain(dq); + } + } + _dispatch_async_f(dq, _dispatch_Block_copy(work), func, pp, flags); } #endif #pragma mark - #pragma mark dispatch_group_async -DISPATCH_NOINLINE -void -dispatch_group_async_f(dispatch_group_t dg, dispatch_queue_t dq, void *ctxt, - dispatch_function_t func) +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_group_async_f(dispatch_group_t dg, dispatch_queue_t dq, void *ctxt, + dispatch_function_t func, pthread_priority_t pp, + dispatch_block_flags_t flags) { dispatch_continuation_t dc; @@ -1613,17 +2522,32 @@ dispatch_group_async_f(dispatch_group_t dg, dispatch_queue_t dq, void *ctxt, dc = _dispatch_continuation_alloc(); - dc->do_vtable = (void *)(DISPATCH_OBJ_ASYNC_BIT | DISPATCH_OBJ_GROUP_BIT); + unsigned long barrier = (flags & DISPATCH_BLOCK_BARRIER) ? + DISPATCH_OBJ_BARRIER_BIT : 0; + dc->do_vtable = (void *)(DISPATCH_OBJ_ASYNC_BIT | DISPATCH_OBJ_GROUP_BIT | + barrier); dc->dc_func = func; dc->dc_ctxt = ctxt; dc->dc_data = dg; + _dispatch_continuation_voucher_set(dc, flags); + _dispatch_continuation_priority_set(dc, pp, flags); + + pp = _dispatch_continuation_get_override_priority(dq, dc); // No fastpath/slowpath hint because we simply don't know - if (dq->dq_width != 1 && dq->do_targetq) { - return _dispatch_async_f2(dq, dc); + if (dq->dq_width != 1 && !barrier && dq->do_targetq) { + return _dispatch_async_f2(dq, dc, pp); } - _dispatch_queue_push(dq, dc); + _dispatch_queue_push(dq, dc, pp); +} + +DISPATCH_NOINLINE +void +dispatch_group_async_f(dispatch_group_t dg, dispatch_queue_t dq, void *ctxt, + dispatch_function_t func) +{ + return _dispatch_group_async_f(dg, dq, ctxt, func, 0, 0); } #ifdef __BLOCKS__ @@ -1631,14 +2555,29 @@ void dispatch_group_async(dispatch_group_t dg, dispatch_queue_t dq, dispatch_block_t db) { - dispatch_group_async_f(dg, dq, _dispatch_Block_copy(db), - _dispatch_call_block_and_release); + dispatch_function_t func = _dispatch_call_block_and_release; + dispatch_block_flags_t flags = 0; + pthread_priority_t pp = 0; + if (slowpath(_dispatch_block_has_private_data(db))) { + func = _dispatch_block_async_invoke_and_release; + pp = _dispatch_block_get_priority(db); + flags = _dispatch_block_get_flags(db); + // balanced in d_block_async_invoke_and_release or d_block_wait + if (dispatch_atomic_cmpxchg2o(_dispatch_block_get_data(db), + dbpd_queue, NULL, dq, release)) { + _dispatch_retain(dq); + } + } + _dispatch_group_async_f(dg, dq, _dispatch_Block_copy(db), func, pp, flags); } #endif #pragma mark - #pragma mark dispatch_function_invoke +static void _dispatch_sync_f(dispatch_queue_t dq, void *ctxt, + dispatch_function_t func, pthread_priority_t pp); + DISPATCH_ALWAYS_INLINE static inline void _dispatch_function_invoke(dispatch_queue_t dq, void *ctxt, @@ -1661,14 +2600,14 @@ _dispatch_sync_recurse_invoke(void *ctxt) DISPATCH_ALWAYS_INLINE static inline void _dispatch_function_recurse(dispatch_queue_t dq, void *ctxt, - dispatch_function_t func) + dispatch_function_t func, pthread_priority_t pp) { struct dispatch_continuation_s dc = { .dc_data = dq, .dc_func = func, .dc_ctxt = ctxt, }; - dispatch_sync_f(dq->do_targetq, &dc, _dispatch_sync_recurse_invoke); + _dispatch_sync_f(dq->do_targetq, &dc, _dispatch_sync_recurse_invoke, pp); } #pragma mark - @@ -1684,6 +2623,7 @@ _dispatch_barrier_sync_f_pop(dispatch_queue_t dq, dispatch_object_t dou, { _dispatch_thread_semaphore_t sema; dispatch_continuation_t dc = dou._dc; + mach_port_t th; if (DISPATCH_OBJ_IS_VTABLE(dc) || ((long)dc->do_vtable & (DISPATCH_OBJ_BARRIER_BIT | DISPATCH_OBJ_SYNC_SLOW_BIT)) != @@ -1693,6 +2633,7 @@ _dispatch_barrier_sync_f_pop(dispatch_queue_t dq, dispatch_object_t dou, _dispatch_trace_continuation_pop(dq, dc); _dispatch_perfmon_workitem_inc(); + th = (mach_port_t)dc->dc_data; dc = dc->dc_ctxt; dq = dc->dc_data; sema = (_dispatch_thread_semaphore_t)dc->dc_other; @@ -1704,6 +2645,8 @@ _dispatch_barrier_sync_f_pop(dispatch_queue_t dq, dispatch_object_t dou, (void)dispatch_atomic_add2o(dq, dq_running, 2, relaxed); } _dispatch_introspection_queue_item_complete(dou); + _dispatch_wqthread_override_start(th, + _dispatch_queue_get_override_priority(dq)); return sema ? sema : MACH_PORT_DEAD; } @@ -1719,6 +2662,7 @@ _dispatch_barrier_sync_f_slow_invoke(void *ctxt) #if DISPATCH_COCOA_COMPAT if (slowpath(dq->dq_is_thread_bound)) { // The queue is bound to a non-dispatch thread (e.g. main thread) + _dispatch_continuation_voucher_adopt(dc); _dispatch_client_callout(dc->dc_ctxt, dc->dc_func); dispatch_atomic_store2o(dc, dc_func, NULL, release); _dispatch_thread_semaphore_signal(sema); // release @@ -1735,17 +2679,14 @@ _dispatch_barrier_sync_f_slow_invoke(void *ctxt) DISPATCH_NOINLINE static void _dispatch_barrier_sync_f_slow(dispatch_queue_t dq, void *ctxt, - dispatch_function_t func) + dispatch_function_t func, pthread_priority_t pp) { if (slowpath(!dq->do_targetq)) { // the global concurrent queues do not need strict ordering (void)dispatch_atomic_add2o(dq, dq_running, 2, relaxed); return _dispatch_sync_f_invoke(dq, ctxt, func); } - // It's preferred to execute synchronous blocks on the current thread - // due to thread-local side effects, garbage collection, etc. However, - // blocks submitted to the main thread MUST be run on the main thread - + if (!pp) pp = (_dispatch_get_priority() | _PTHREAD_PRIORITY_ENFORCE_FLAG); _dispatch_thread_semaphore_t sema = _dispatch_get_thread_semaphore(); struct dispatch_continuation_s dc = { .dc_data = dq, @@ -1755,16 +2696,24 @@ _dispatch_barrier_sync_f_slow(dispatch_queue_t dq, void *ctxt, #endif .dc_other = (void*)sema, }; +#if DISPATCH_COCOA_COMPAT + // It's preferred to execute synchronous blocks on the current thread + // due to thread-local side effects, garbage collection, etc. However, + // blocks submitted to the main thread MUST be run on the main thread + if (slowpath(dq->dq_is_thread_bound)) { + _dispatch_continuation_voucher_set(&dc, 0); + } +#endif struct dispatch_continuation_s dbss = { .do_vtable = (void *)(DISPATCH_OBJ_BARRIER_BIT | DISPATCH_OBJ_SYNC_SLOW_BIT), .dc_func = _dispatch_barrier_sync_f_slow_invoke, .dc_ctxt = &dc, -#if DISPATCH_INTROSPECTION - .dc_data = (void*)_dispatch_thread_self(), -#endif + .dc_data = (void*)(uintptr_t)_dispatch_thread_port(), + .dc_priority = pp, }; - _dispatch_queue_push(dq, &dbss); + _dispatch_queue_push(dq, &dbss, + _dispatch_continuation_get_override_priority(dq, &dbss)); _dispatch_thread_semaphore_wait(sema); // acquire _dispatch_put_thread_semaphore(sema); @@ -1775,11 +2724,15 @@ _dispatch_barrier_sync_f_slow(dispatch_queue_t dq, void *ctxt, return; } #endif + + _dispatch_queue_set_thread(dq); if (slowpath(dq->do_targetq->do_targetq)) { - _dispatch_function_recurse(dq, ctxt, func); + _dispatch_function_recurse(dq, ctxt, func, pp); } else { _dispatch_function_invoke(dq, ctxt, func); } + _dispatch_queue_clear_thread(dq); + if (fastpath(dq->do_suspend_cnt < 2 * DISPATCH_OBJECT_SUSPEND_INTERVAL) && dq->dq_running == 2) { // rdar://problem/8290662 "lock transfer" @@ -1790,9 +2743,9 @@ _dispatch_barrier_sync_f_slow(dispatch_queue_t dq, void *ctxt, } } (void)dispatch_atomic_sub2o(dq, do_suspend_cnt, - DISPATCH_OBJECT_SUSPEND_INTERVAL, relaxed); - if (slowpath(dispatch_atomic_sub2o(dq, dq_running, 2, release) == 0)) { - _dispatch_wakeup(dq); + DISPATCH_OBJECT_SUSPEND_INTERVAL, release); + if (slowpath(dispatch_atomic_sub2o(dq, dq_running, 2, relaxed) == 0)) { + _dispatch_queue_wakeup(dq); } } @@ -1815,7 +2768,7 @@ _dispatch_barrier_sync_f2(dispatch_queue_t dq) } } if (slowpath(dispatch_atomic_dec2o(dq, dq_running, release) == 0)) { - _dispatch_wakeup(dq); + _dispatch_queue_wakeup(dq); } } @@ -1824,27 +2777,52 @@ static void _dispatch_barrier_sync_f_invoke(dispatch_queue_t dq, void *ctxt, dispatch_function_t func) { + _dispatch_queue_set_thread(dq); _dispatch_function_invoke(dq, ctxt, func); + _dispatch_queue_clear_thread(dq); if (slowpath(dq->dq_items_tail)) { return _dispatch_barrier_sync_f2(dq); } if (slowpath(dispatch_atomic_dec2o(dq, dq_running, release) == 0)) { - _dispatch_wakeup(dq); + _dispatch_queue_wakeup(dq); } } DISPATCH_NOINLINE static void _dispatch_barrier_sync_f_recurse(dispatch_queue_t dq, void *ctxt, - dispatch_function_t func) + dispatch_function_t func, pthread_priority_t pp) { - _dispatch_function_recurse(dq, ctxt, func); + _dispatch_queue_set_thread(dq); + _dispatch_function_recurse(dq, ctxt, func, pp); + _dispatch_queue_clear_thread(dq); if (slowpath(dq->dq_items_tail)) { return _dispatch_barrier_sync_f2(dq); } if (slowpath(dispatch_atomic_dec2o(dq, dq_running, release) == 0)) { - _dispatch_wakeup(dq); + _dispatch_queue_wakeup(dq); + } +} + +DISPATCH_NOINLINE +static void +_dispatch_barrier_sync_f(dispatch_queue_t dq, void *ctxt, + dispatch_function_t func, pthread_priority_t pp) +{ + // 1) ensure that this thread hasn't enqueued anything ahead of this call + // 2) the queue is not suspended + if (slowpath(dq->dq_items_tail) || slowpath(DISPATCH_OBJECT_SUSPENDED(dq))){ + return _dispatch_barrier_sync_f_slow(dq, ctxt, func, pp); + } + if (slowpath(!dispatch_atomic_cmpxchg2o(dq, dq_running, 0, 1, acquire))) { + // global concurrent queues and queues bound to non-dispatch threads + // always fall into the slow case + return _dispatch_barrier_sync_f_slow(dq, ctxt, func, pp); } + if (slowpath(dq->do_targetq->do_targetq)) { + return _dispatch_barrier_sync_f_recurse(dq, ctxt, func, pp); + } + _dispatch_barrier_sync_f_invoke(dq, ctxt, func); } DISPATCH_NOINLINE @@ -1855,46 +2833,64 @@ dispatch_barrier_sync_f(dispatch_queue_t dq, void *ctxt, // 1) ensure that this thread hasn't enqueued anything ahead of this call // 2) the queue is not suspended if (slowpath(dq->dq_items_tail) || slowpath(DISPATCH_OBJECT_SUSPENDED(dq))){ - return _dispatch_barrier_sync_f_slow(dq, ctxt, func); + return _dispatch_barrier_sync_f_slow(dq, ctxt, func, 0); } if (slowpath(!dispatch_atomic_cmpxchg2o(dq, dq_running, 0, 1, acquire))) { // global concurrent queues and queues bound to non-dispatch threads // always fall into the slow case - return _dispatch_barrier_sync_f_slow(dq, ctxt, func); + return _dispatch_barrier_sync_f_slow(dq, ctxt, func, 0); } if (slowpath(dq->do_targetq->do_targetq)) { - return _dispatch_barrier_sync_f_recurse(dq, ctxt, func); + return _dispatch_barrier_sync_f_recurse(dq, ctxt, func, 0); } _dispatch_barrier_sync_f_invoke(dq, ctxt, func); } #ifdef __BLOCKS__ -#if DISPATCH_COCOA_COMPAT DISPATCH_NOINLINE static void _dispatch_barrier_sync_slow(dispatch_queue_t dq, void (^work)(void)) { - // Blocks submitted to the main queue MUST be run on the main thread, - // therefore under GC we must Block_copy in order to notify the thread-local - // garbage collector that the objects are transferring to the main thread - // rdar://problem/7176237&7181849&7458685 - if (dispatch_begin_thread_4GC) { - dispatch_block_t block = _dispatch_Block_copy(work); - return dispatch_barrier_sync_f(dq, block, - _dispatch_call_block_and_release); + bool has_pd = _dispatch_block_has_private_data(work); + dispatch_function_t func = _dispatch_Block_invoke(work); + pthread_priority_t pp = 0; + if (has_pd) { + func = _dispatch_block_sync_invoke; + pp = _dispatch_block_get_priority(work); + dispatch_block_flags_t flags = _dispatch_block_get_flags(work); + if (flags & DISPATCH_BLOCK_HAS_PRIORITY) { + pthread_priority_t tp = _dispatch_get_priority(); + if (pp < tp) { + pp = tp | _PTHREAD_PRIORITY_ENFORCE_FLAG; + } else if (!(flags & DISPATCH_BLOCK_INHERIT_QOS_CLASS)) { + pp |= _PTHREAD_PRIORITY_ENFORCE_FLAG; + } + } + // balanced in d_block_sync_invoke or d_block_wait + if (dispatch_atomic_cmpxchg2o(_dispatch_block_get_data(work), + dbpd_queue, NULL, dq, release)) { + _dispatch_retain(dq); + } +#if DISPATCH_COCOA_COMPAT + } else if (dq->dq_is_thread_bound && dispatch_begin_thread_4GC) { + // Blocks submitted to the main queue MUST be run on the main thread, + // under GC we must Block_copy in order to notify the thread-local + // garbage collector that the objects are transferring to another thread + // rdar://problem/7176237&7181849&7458685 + work = _dispatch_Block_copy(work); + func = _dispatch_call_block_and_release; } - dispatch_barrier_sync_f(dq, work, _dispatch_Block_invoke(work)); -} #endif + _dispatch_barrier_sync_f(dq, work, func, pp); +} void dispatch_barrier_sync(dispatch_queue_t dq, void (^work)(void)) { -#if DISPATCH_COCOA_COMPAT - if (slowpath(dq->dq_is_thread_bound)) { + if (slowpath(dq->dq_is_thread_bound) || + slowpath(_dispatch_block_has_private_data(work))) { return _dispatch_barrier_sync_slow(dq, work); } -#endif dispatch_barrier_sync_f(dq, work, _dispatch_Block_invoke(work)); } #endif @@ -1904,9 +2900,11 @@ static void _dispatch_barrier_trysync_f_invoke(dispatch_queue_t dq, void *ctxt, dispatch_function_t func) { + _dispatch_queue_set_thread(dq); _dispatch_function_invoke(dq, ctxt, func); + _dispatch_queue_clear_thread(dq); if (slowpath(dispatch_atomic_dec2o(dq, dq_running, release) == 0)) { - _dispatch_wakeup(dq); + _dispatch_queue_wakeup(dq); } } @@ -1920,7 +2918,7 @@ _dispatch_barrier_trysync_f(dispatch_queue_t dq, void *ctxt, if (slowpath(dq->dq_items_tail) || slowpath(DISPATCH_OBJECT_SUSPENDED(dq)) || slowpath(!dispatch_atomic_cmpxchg2o(dq, dq_running, 0, 1, acquire))) { - return dispatch_barrier_async_f(dq, ctxt, func); + return _dispatch_barrier_async_detached_f(dq, ctxt, func); } _dispatch_barrier_trysync_f_invoke(dq, ctxt, func); } @@ -1931,30 +2929,34 @@ _dispatch_barrier_trysync_f(dispatch_queue_t dq, void *ctxt, DISPATCH_NOINLINE static void _dispatch_sync_f_slow(dispatch_queue_t dq, void *ctxt, dispatch_function_t func, - bool wakeup) + pthread_priority_t pp, bool wakeup) { + if (!pp) pp = (_dispatch_get_priority() | _PTHREAD_PRIORITY_ENFORCE_FLAG); _dispatch_thread_semaphore_t sema = _dispatch_get_thread_semaphore(); - struct dispatch_continuation_s dss = { + struct dispatch_continuation_s dc = { .do_vtable = (void*)DISPATCH_OBJ_SYNC_SLOW_BIT, #if DISPATCH_INTROSPECTION .dc_func = func, .dc_ctxt = ctxt, - .dc_data = (void*)_dispatch_thread_self(), + .dc_data = (void*)(uintptr_t)_dispatch_thread_port(), #endif .dc_other = (void*)sema, + .dc_priority = pp, }; - _dispatch_queue_push_wakeup(dq, &dss, wakeup); + _dispatch_queue_push_wakeup(dq, &dc, + _dispatch_continuation_get_override_priority(dq, &dc), wakeup); _dispatch_thread_semaphore_wait(sema); _dispatch_put_thread_semaphore(sema); if (slowpath(dq->do_targetq->do_targetq)) { - _dispatch_function_recurse(dq, ctxt, func); + _dispatch_function_recurse(dq, ctxt, func, pp); } else { _dispatch_function_invoke(dq, ctxt, func); } + if (slowpath(dispatch_atomic_sub2o(dq, dq_running, 2, relaxed) == 0)) { - _dispatch_wakeup(dq); + _dispatch_queue_wakeup(dq); } } @@ -1965,41 +2967,58 @@ _dispatch_sync_f_invoke(dispatch_queue_t dq, void *ctxt, { _dispatch_function_invoke(dq, ctxt, func); if (slowpath(dispatch_atomic_sub2o(dq, dq_running, 2, relaxed) == 0)) { - _dispatch_wakeup(dq); + _dispatch_queue_wakeup(dq); } } DISPATCH_NOINLINE static void _dispatch_sync_f_recurse(dispatch_queue_t dq, void *ctxt, - dispatch_function_t func) + dispatch_function_t func, pthread_priority_t pp) { - _dispatch_function_recurse(dq, ctxt, func); + _dispatch_function_recurse(dq, ctxt, func, pp); if (slowpath(dispatch_atomic_sub2o(dq, dq_running, 2, relaxed) == 0)) { - _dispatch_wakeup(dq); + _dispatch_queue_wakeup(dq); } } static inline void -_dispatch_sync_f2(dispatch_queue_t dq, void *ctxt, dispatch_function_t func) +_dispatch_sync_f2(dispatch_queue_t dq, void *ctxt, dispatch_function_t func, + pthread_priority_t pp) { // 1) ensure that this thread hasn't enqueued anything ahead of this call // 2) the queue is not suspended if (slowpath(dq->dq_items_tail) || slowpath(DISPATCH_OBJECT_SUSPENDED(dq))){ - return _dispatch_sync_f_slow(dq, ctxt, func, false); + return _dispatch_sync_f_slow(dq, ctxt, func, pp, false); } uint32_t running = dispatch_atomic_add2o(dq, dq_running, 2, relaxed); // re-check suspension after barrier check - if (slowpath(running & 1) || slowpath(DISPATCH_OBJECT_SUSPENDED(dq))) { + if (slowpath(running & 1) || _dispatch_object_suspended(dq)) { running = dispatch_atomic_sub2o(dq, dq_running, 2, relaxed); - return _dispatch_sync_f_slow(dq, ctxt, func, running == 0); + return _dispatch_sync_f_slow(dq, ctxt, func, pp, running == 0); } if (slowpath(dq->do_targetq->do_targetq)) { - return _dispatch_sync_f_recurse(dq, ctxt, func); + return _dispatch_sync_f_recurse(dq, ctxt, func, pp); } _dispatch_sync_f_invoke(dq, ctxt, func); } +DISPATCH_NOINLINE +static void +_dispatch_sync_f(dispatch_queue_t dq, void *ctxt, dispatch_function_t func, + pthread_priority_t pp) +{ + if (fastpath(dq->dq_width == 1)) { + return _dispatch_barrier_sync_f(dq, ctxt, func, pp); + } + if (slowpath(!dq->do_targetq)) { + // the global concurrent queues do not need strict ordering + (void)dispatch_atomic_add2o(dq, dq_running, 2, relaxed); + return _dispatch_sync_f_invoke(dq, ctxt, func); + } + _dispatch_sync_f2(dq, ctxt, func, pp); +} + DISPATCH_NOINLINE void dispatch_sync_f(dispatch_queue_t dq, void *ctxt, dispatch_function_t func) @@ -2012,35 +3031,65 @@ dispatch_sync_f(dispatch_queue_t dq, void *ctxt, dispatch_function_t func) (void)dispatch_atomic_add2o(dq, dq_running, 2, relaxed); return _dispatch_sync_f_invoke(dq, ctxt, func); } - _dispatch_sync_f2(dq, ctxt, func); + _dispatch_sync_f2(dq, ctxt, func, 0); } #ifdef __BLOCKS__ -#if DISPATCH_COCOA_COMPAT DISPATCH_NOINLINE static void _dispatch_sync_slow(dispatch_queue_t dq, void (^work)(void)) { - // Blocks submitted to the main queue MUST be run on the main thread, - // therefore under GC we must Block_copy in order to notify the thread-local - // garbage collector that the objects are transferring to the main thread - // rdar://problem/7176237&7181849&7458685 - if (dispatch_begin_thread_4GC) { - dispatch_block_t block = _dispatch_Block_copy(work); - return dispatch_sync_f(dq, block, _dispatch_call_block_and_release); + bool has_pd = _dispatch_block_has_private_data(work); + if (has_pd && (_dispatch_block_get_flags(work) & DISPATCH_BLOCK_BARRIER)) { + return _dispatch_barrier_sync_slow(dq, work); } - dispatch_sync_f(dq, work, _dispatch_Block_invoke(work)); -} + dispatch_function_t func = _dispatch_Block_invoke(work); + pthread_priority_t pp = 0; + if (has_pd) { + func = _dispatch_block_sync_invoke; + pp = _dispatch_block_get_priority(work); + dispatch_block_flags_t flags = _dispatch_block_get_flags(work); + if (flags & DISPATCH_BLOCK_HAS_PRIORITY) { + pthread_priority_t tp = _dispatch_get_priority(); + if (pp < tp) { + pp = tp | _PTHREAD_PRIORITY_ENFORCE_FLAG; + } else if (!(flags & DISPATCH_BLOCK_INHERIT_QOS_CLASS)) { + pp |= _PTHREAD_PRIORITY_ENFORCE_FLAG; + } + } + // balanced in d_block_sync_invoke or d_block_wait + if (dispatch_atomic_cmpxchg2o(_dispatch_block_get_data(work), + dbpd_queue, NULL, dq, release)) { + _dispatch_retain(dq); + } +#if DISPATCH_COCOA_COMPAT + } else if (dq->dq_is_thread_bound && dispatch_begin_thread_4GC) { + // Blocks submitted to the main queue MUST be run on the main thread, + // under GC we must Block_copy in order to notify the thread-local + // garbage collector that the objects are transferring to another thread + // rdar://problem/7176237&7181849&7458685 + work = _dispatch_Block_copy(work); + func = _dispatch_call_block_and_release; #endif + } + if (slowpath(!dq->do_targetq)) { + // the global concurrent queues do not need strict ordering + (void)dispatch_atomic_add2o(dq, dq_running, 2, relaxed); + return _dispatch_sync_f_invoke(dq, work, func); + } + _dispatch_sync_f2(dq, work, func, pp); +} void dispatch_sync(dispatch_queue_t dq, void (^work)(void)) { -#if DISPATCH_COCOA_COMPAT - if (slowpath(dq->dq_is_thread_bound)) { + if (fastpath(dq->dq_width == 1)) { + return dispatch_barrier_sync(dq, work); + } + if (slowpath(dq->dq_is_thread_bound) || + slowpath(_dispatch_block_has_private_data(work)) ) { return _dispatch_sync_slow(dq, work); } -#endif dispatch_sync_f(dq, work, _dispatch_Block_invoke(work)); } #endif @@ -2090,8 +3139,9 @@ dispatch_after_f(dispatch_time_t when, dispatch_queue_t queue, void *ctxt, ds = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, queue); dispatch_assert(ds); + // TODO: don't use a separate continuation & voucher dispatch_continuation_t dc = _dispatch_continuation_alloc(); - dc->do_vtable = (void *)(DISPATCH_OBJ_ASYNC_BIT | DISPATCH_OBJ_BARRIER_BIT); + dc->do_vtable = (void *)(DISPATCH_OBJ_ASYNC_BIT); dc->dc_func = func; dc->dc_ctxt = ctxt; dc->dc_data = ds; @@ -2123,10 +3173,10 @@ dispatch_after(dispatch_time_t when, dispatch_queue_t queue, #pragma mark - #pragma mark dispatch_queue_push -DISPATCH_NOINLINE -static void -_dispatch_queue_push_list_slow2(dispatch_queue_t dq, - struct dispatch_object_s *obj) +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_queue_push_list_slow2(dispatch_queue_t dq, pthread_priority_t pp, + struct dispatch_object_s *obj, bool retained) { // The queue must be retained before dq_items_head is written in order // to ensure that the reference is still valid when _dispatch_wakeup is @@ -2134,34 +3184,35 @@ _dispatch_queue_push_list_slow2(dispatch_queue_t dq, // dq_items_head and _dispatch_wakeup, the blocks submitted to the // queue may release the last reference to the queue when invoked by // _dispatch_queue_drain. - _dispatch_retain(dq); + if (!retained) _dispatch_retain(dq); dq->dq_items_head = obj; - _dispatch_wakeup(dq); - _dispatch_release(dq); + return _dispatch_queue_wakeup_with_qos_and_release(dq, pp); } DISPATCH_NOINLINE void -_dispatch_queue_push_list_slow(dispatch_queue_t dq, - struct dispatch_object_s *obj, unsigned int n) +_dispatch_queue_push_list_slow(dispatch_queue_t dq, pthread_priority_t pp, + struct dispatch_object_s *obj, unsigned int n, bool retained) { if (dx_type(dq) == DISPATCH_QUEUE_ROOT_TYPE && !dq->dq_is_thread_bound) { + dispatch_assert(!retained); dispatch_atomic_store2o(dq, dq_items_head, obj, relaxed); return _dispatch_queue_wakeup_global2(dq, n); } - _dispatch_queue_push_list_slow2(dq, obj); + _dispatch_queue_push_list_slow2(dq, pp, obj, retained); } DISPATCH_NOINLINE void -_dispatch_queue_push_slow(dispatch_queue_t dq, - struct dispatch_object_s *obj) +_dispatch_queue_push_slow(dispatch_queue_t dq, pthread_priority_t pp, + struct dispatch_object_s *obj, bool retained) { if (dx_type(dq) == DISPATCH_QUEUE_ROOT_TYPE && !dq->dq_is_thread_bound) { + dispatch_assert(!retained); dispatch_atomic_store2o(dq, dq_items_head, obj, relaxed); return _dispatch_queue_wakeup_global(dq); } - _dispatch_queue_push_list_slow2(dq, obj); + _dispatch_queue_push_list_slow2(dq, pp, obj, retained); } #pragma mark - @@ -2170,14 +3221,14 @@ _dispatch_queue_push_slow(dispatch_queue_t dq, unsigned long _dispatch_queue_probe(dispatch_queue_t dq) { - return (unsigned long)slowpath(dq->dq_items_tail != NULL); + return _dispatch_queue_class_probe(dq); } #if DISPATCH_COCOA_COMPAT unsigned long _dispatch_runloop_queue_probe(dispatch_queue_t dq) { - if (_dispatch_queue_probe(dq)) { + if (_dispatch_queue_class_probe(dq)) { if (dq->do_xref_cnt == -1) return true; // return _dispatch_runloop_queue_wakeup(dq); } @@ -2188,7 +3239,7 @@ _dispatch_runloop_queue_probe(dispatch_queue_t dq) unsigned long _dispatch_mgr_queue_probe(dispatch_queue_t dq) { - if (_dispatch_queue_probe(dq)) { + if (_dispatch_queue_class_probe(dq)) { return _dispatch_mgr_wakeup(dq); } return false; @@ -2209,24 +3260,23 @@ _dispatch_root_queue_probe(dispatch_queue_t dq) dispatch_queue_t _dispatch_wakeup(dispatch_object_t dou) { - if (slowpath(DISPATCH_OBJECT_SUSPENDED(dou._do))) { + unsigned long type = dx_metatype(dou._do); + if (type == _DISPATCH_QUEUE_TYPE || type == _DISPATCH_SOURCE_TYPE) { + return _dispatch_queue_wakeup(dou._dq); + } + if (_dispatch_object_suspended(dou)) { return NULL; } if (!dx_probe(dou._do)) { return NULL; } if (!dispatch_atomic_cmpxchg2o(dou._do, do_suspend_cnt, 0, - DISPATCH_OBJECT_SUSPEND_LOCK, release)) { -#if DISPATCH_COCOA_COMPAT - if (dou._dq == &_dispatch_main_q) { - return _dispatch_main_queue_wakeup(); - } -#endif + DISPATCH_OBJECT_SUSPEND_LOCK, acquire)) { return NULL; } _dispatch_retain(dou._do); dispatch_queue_t tq = dou._do->do_targetq; - _dispatch_queue_push(tq, dou._do); + _dispatch_queue_push(tq, dou._do, 0); return tq; // libdispatch does not need this, but the Instrument DTrace // probe does } @@ -2278,13 +3328,13 @@ DISPATCH_NOINLINE static void _dispatch_queue_wakeup_global_slow(dispatch_queue_t dq, unsigned int n) { - static dispatch_once_t pred; dispatch_root_queue_context_t qc = dq->do_ctxt; uint32_t i = n; int r; _dispatch_debug_root_queue(dq, __func__); - dispatch_once_f(&pred, NULL, _dispatch_root_queues_init); + dispatch_once_f(&_dispatch_root_queues_pred, NULL, + _dispatch_root_queues_init); #if HAVE_PTHREAD_WORKQUEUES #if DISPATCH_USE_PTHREAD_POOL @@ -2299,29 +3349,39 @@ _dispatch_queue_wakeup_global_slow(dispatch_queue_t dq, unsigned int n) unsigned int gen_cnt; do { r = pthread_workqueue_additem_np(qc->dgq_kworkqueue, - _dispatch_worker_thread3, dq, &wh, &gen_cnt); + _dispatch_worker_thread4, dq, &wh, &gen_cnt); (void)dispatch_assume_zero(r); } while (--i); return; } #endif // DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK #if HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP - r = pthread_workqueue_addthreads_np(qc->dgq_wq_priority, - qc->dgq_wq_options, (int)i); + if (!dq->dq_priority) { + r = pthread_workqueue_addthreads_np(qc->dgq_wq_priority, + qc->dgq_wq_options, (int)i); + (void)dispatch_assume_zero(r); + return; + } +#endif +#if HAVE_PTHREAD_WORKQUEUE_QOS + r = _pthread_workqueue_addthreads((int)i, dq->dq_priority); (void)dispatch_assume_zero(r); #endif return; } #endif // HAVE_PTHREAD_WORKQUEUES #if DISPATCH_USE_PTHREAD_POOL - if (fastpath(qc->dgq_thread_mediator)) { - while (dispatch_semaphore_signal(qc->dgq_thread_mediator)) { + dispatch_pthread_root_queue_context_t pqc = qc->dgq_ctxt; + if (fastpath(pqc->dpq_thread_mediator.do_vtable)) { + while (dispatch_semaphore_signal(&pqc->dpq_thread_mediator)) { if (!--i) { return; } } } - uint32_t j, t_count = qc->dgq_thread_pool_size; + uint32_t j, t_count; + // seq_cst with atomic store to tail + t_count = dispatch_atomic_load2o(qc, dgq_thread_pool_size, seq_cst); do { if (!t_count) { _dispatch_root_queue_debug("pthread pool is full for root queue: " @@ -2330,10 +3390,9 @@ _dispatch_queue_wakeup_global_slow(dispatch_queue_t dq, unsigned int n) } j = i > t_count ? t_count : i; } while (!dispatch_atomic_cmpxchgvw2o(qc, dgq_thread_pool_size, t_count, - t_count - j, &t_count, relaxed)); + t_count - j, &t_count, acquire)); - dispatch_pthread_root_queue_context_t pqc = qc->dgq_ctxt; - pthread_attr_t *attr = pqc ? &pqc->dpq_thread_attr : NULL; + pthread_attr_t *attr = &pqc->dpq_thread_attr; pthread_t tid, *pthr = &tid; #if DISPATCH_ENABLE_PTHREAD_ROOT_QUEUES if (slowpath(dq == &_dispatch_mgr_root_queue)) { @@ -2348,10 +3407,6 @@ _dispatch_queue_wakeup_global_slow(dispatch_queue_t dq, unsigned int n) } _dispatch_temporary_resource_shortage(); } - if (!attr) { - r = pthread_detach(*pthr); - (void)dispatch_assume_zero(r); - } } while (--j); #endif // DISPATCH_USE_PTHREAD_POOL } @@ -2359,7 +3414,7 @@ _dispatch_queue_wakeup_global_slow(dispatch_queue_t dq, unsigned int n) static inline void _dispatch_queue_wakeup_global2(dispatch_queue_t dq, unsigned int n) { - if (!dq->dq_items_tail) { + if (!_dispatch_queue_class_probe(dq)) { return; } #if HAVE_PTHREAD_WORKQUEUES @@ -2393,6 +3448,12 @@ dispatch_queue_invoke2(dispatch_object_t dou, { dispatch_queue_t dq = dou._dq; dispatch_queue_t otq = dq->do_targetq; + dispatch_queue_t cq = _dispatch_queue_get_current(); + + if (slowpath(cq != otq)) { + return otq; + } + *sema_ptr = _dispatch_queue_drain(dq); if (slowpath(otq != dq->do_targetq)) { @@ -2419,9 +3480,7 @@ static inline struct dispatch_object_s* _dispatch_queue_head(dispatch_queue_t dq) { struct dispatch_object_s *dc; - while (!(dc = fastpath(dq->dq_items_head))) { - dispatch_hardware_pause(); - } + _dispatch_wait_until(dc = fastpath(dq->dq_items_head)); return dc; } @@ -2434,10 +3493,7 @@ _dispatch_queue_next(dispatch_queue_t dq, struct dispatch_object_s *dc) dq->dq_items_head = next_dc; if (!next_dc && !dispatch_atomic_cmpxchg2o(dq, dq_items_tail, dc, NULL, relaxed)) { - // Enqueue is TIGHTLY controlled, we won't wait long. - while (!(next_dc = fastpath(dc->do_next))) { - dispatch_hardware_pause(); - } + _dispatch_wait_until(next_dc = fastpath(dc->do_next)); dq->dq_items_head = next_dc; } return next_dc; @@ -2457,6 +3513,15 @@ _dispatch_queue_drain(dispatch_object_t dou) orig_tq = dq->do_targetq; _dispatch_thread_setspecific(dispatch_queue_key, dq); + pthread_priority_t old_dp = _dispatch_set_defaultpriority(dq->dq_priority); + + pthread_priority_t op = _dispatch_queue_get_override_priority(dq); + pthread_priority_t dp = _dispatch_get_defaultpriority(); + dp &= _PTHREAD_PRIORITY_QOS_CLASS_MASK; + if (op > dp) { + _dispatch_wqthread_override_start(dq->dq_thread, op); + } + //dispatch_debug_queue(dq, __func__); while (dq->dq_items_tail) { @@ -2496,6 +3561,7 @@ _dispatch_queue_drain(dispatch_object_t dou) } out: + _dispatch_reset_defaultpriority(old_dp); _dispatch_thread_setspecific(dispatch_queue_key, old_dq); return sema; } @@ -2512,11 +3578,14 @@ _dispatch_main_queue_drain(void) .do_vtable = NULL, }; struct dispatch_object_s *dmarker = (void*)▮ - _dispatch_queue_push_notrace(dq, dmarker); + _dispatch_queue_push_notrace(dq, dmarker, 0); _dispatch_perfmon_start(); dispatch_queue_t old_dq = _dispatch_thread_getspecific(dispatch_queue_key); _dispatch_thread_setspecific(dispatch_queue_key, dq); + pthread_priority_t old_pri = _dispatch_get_priority(); + pthread_priority_t old_dp = _dispatch_set_defaultpriority(old_pri); + voucher_t voucher = _voucher_copy(); struct dispatch_object_s *dc, *next_dc; dc = _dispatch_queue_head(dq); @@ -2534,6 +3603,10 @@ out: if (next_dc) { _dispatch_main_queue_wakeup(); } + _dispatch_voucher_debug("main queue restore", voucher); + _dispatch_set_priority_and_replace_voucher(old_pri, voucher); + _dispatch_queue_reset_override_priority(dq); + _dispatch_reset_defaultpriority(old_dp); _dispatch_thread_setspecific(dispatch_queue_key, old_dq); _dispatch_perfmon_end(); _dispatch_force_cache_cleanup(); @@ -2548,6 +3621,9 @@ _dispatch_runloop_queue_drain_one(dispatch_queue_t dq) _dispatch_perfmon_start(); dispatch_queue_t old_dq = _dispatch_thread_getspecific(dispatch_queue_key); _dispatch_thread_setspecific(dispatch_queue_key, dq); + pthread_priority_t old_pri = _dispatch_get_priority(); + pthread_priority_t old_dp = _dispatch_set_defaultpriority(old_pri); + voucher_t voucher = _voucher_copy(); struct dispatch_object_s *dc, *next_dc; dc = _dispatch_queue_head(dq); @@ -2555,6 +3631,9 @@ _dispatch_runloop_queue_drain_one(dispatch_queue_t dq) _dispatch_continuation_pop(dc); _dispatch_perfmon_workitem_inc(); + _dispatch_voucher_debug("runloop queue restore", voucher); + _dispatch_set_priority_and_replace_voucher(old_pri, voucher); + _dispatch_reset_defaultpriority(old_dp); _dispatch_thread_setspecific(dispatch_queue_key, old_dq); _dispatch_perfmon_end(); _dispatch_force_cache_cleanup(); @@ -2591,28 +3670,219 @@ _dispatch_mgr_queue_drain(void) if (slowpath(_dispatch_queue_drain(dq))) { DISPATCH_CRASH("Sync onto manager queue"); } + _dispatch_voucher_debug("mgr queue clear", NULL); + _voucher_clear(); + _dispatch_queue_reset_override_priority(dq); + _dispatch_reset_defaultpriority_override(); _dispatch_perfmon_end(); _dispatch_force_cache_cleanup(); } #pragma mark - -#pragma mark dispatch_root_queue_drain +#pragma mark _dispatch_queue_wakeup_with_qos -#ifndef DISPATCH_CONTENTION_USE_RAND -#define DISPATCH_CONTENTION_USE_RAND (!TARGET_OS_EMBEDDED) -#endif -#ifndef DISPATCH_CONTENTION_SPINS_MAX -#define DISPATCH_CONTENTION_SPINS_MAX (128 - 1) +DISPATCH_NOINLINE +static dispatch_queue_t +_dispatch_queue_wakeup_with_qos_slow(dispatch_queue_t dq, pthread_priority_t pp, + bool retained) +{ + if (!dx_probe(dq) && (dq->dq_is_thread_bound || !dq->dq_thread)) { + if (retained) _dispatch_release(dq); + return NULL; + } + pp &= _PTHREAD_PRIORITY_QOS_CLASS_MASK; + bool override = _dispatch_queue_override_priority(dq, pp); + if (override && dq->dq_running > 1) { + override = false; + } + + if (!dispatch_atomic_cmpxchg2o(dq, do_suspend_cnt, 0, + DISPATCH_OBJECT_SUSPEND_LOCK, acquire)) { +#if DISPATCH_COCOA_COMPAT + if (dq == &_dispatch_main_q && dq->dq_is_thread_bound) { + return _dispatch_main_queue_wakeup(); + } #endif -#ifndef DISPATCH_CONTENTION_SPINS_MIN -#define DISPATCH_CONTENTION_SPINS_MIN (32 - 1) + if (override) { + mach_port_t th; + // to traverse the tq chain safely we must + // lock it to ensure it cannot change, unless the queue is running + // and we can just override the thread itself + if (dq->dq_thread) { + _dispatch_wqthread_override_start(dq->dq_thread, pp); + } else if (!dispatch_atomic_cmpxchgv2o(dq, dq_tqthread, + MACH_PORT_NULL, _dispatch_thread_port(), &th, acquire)) { + // already locked, override the owner, trysync will do a queue + // wakeup when it returns. + _dispatch_wqthread_override_start(th, pp); + } else { + dispatch_queue_t tq = dq->do_targetq; + if (_dispatch_queue_prepare_override(dq, tq, pp)) { + _dispatch_queue_push_override(dq, tq, pp); + } else { + _dispatch_queue_wakeup_with_qos(tq, pp); + } + dispatch_atomic_store2o(dq, dq_tqthread, MACH_PORT_NULL, + release); + } + } + if (retained) _dispatch_release(dq); + return NULL; + } + dispatch_queue_t tq = dq->do_targetq; + if (!retained) _dispatch_retain(dq); + if (override) { + override = _dispatch_queue_prepare_override(dq, tq, pp); + } + _dispatch_queue_push(tq, dq, pp); + if (override) { + _dispatch_queue_push_override(dq, tq, pp); + } + return tq; // libdispatch does not need this, but the Instrument DTrace + // probe does +} + +DISPATCH_ALWAYS_INLINE +static inline dispatch_queue_t +_dispatch_queue_wakeup_with_qos2(dispatch_queue_t dq, pthread_priority_t pp, + bool retained) +{ + if (_dispatch_object_suspended(dq)) { + _dispatch_queue_override_priority(dq, pp); + if (retained) _dispatch_release(dq); + return NULL; + } + return _dispatch_queue_wakeup_with_qos_slow(dq, pp, retained); +} + +DISPATCH_NOINLINE +void +_dispatch_queue_wakeup_with_qos_and_release(dispatch_queue_t dq, + pthread_priority_t pp) +{ + (void)_dispatch_queue_wakeup_with_qos2(dq, pp, true); +} + +DISPATCH_NOINLINE +void +_dispatch_queue_wakeup_with_qos(dispatch_queue_t dq, pthread_priority_t pp) +{ + (void)_dispatch_queue_wakeup_with_qos2(dq, pp, false); +} + +DISPATCH_NOINLINE +dispatch_queue_t +_dispatch_queue_wakeup(dispatch_queue_t dq) +{ + return _dispatch_queue_wakeup_with_qos2(dq, + _dispatch_queue_get_override_priority(dq), false); +} + +#if HAVE_PTHREAD_WORKQUEUE_QOS +static void +_dispatch_queue_override_invoke(void *ctxt) +{ + dispatch_continuation_t dc = (dispatch_continuation_t)ctxt; + dispatch_queue_t dq = dc->dc_data; + pthread_priority_t p = 0; + + if (!slowpath(DISPATCH_OBJECT_SUSPENDED(dq)) && + fastpath(dispatch_atomic_cmpxchg2o(dq, dq_running, 0, 1, acquire))) { + _dispatch_queue_set_thread(dq); + + _dispatch_object_debug(dq, "stolen onto thread 0x%x, 0x%lx", + dq->dq_thread, _dispatch_get_defaultpriority()); + + pthread_priority_t old_dp = _dispatch_get_defaultpriority(); + _dispatch_reset_defaultpriority(dc->dc_priority); + + dispatch_queue_t tq = NULL; + _dispatch_thread_semaphore_t sema = 0; + tq = dispatch_queue_invoke2(dq, &sema); + + _dispatch_queue_clear_thread(dq); + _dispatch_reset_defaultpriority(old_dp); + + uint32_t running = dispatch_atomic_dec2o(dq, dq_running, release); + if (sema) { + _dispatch_thread_semaphore_signal(sema); + } else if (!tq && running == 0) { + p = _dispatch_queue_reset_override_priority(dq); + if (p > (dq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK)) { + _dispatch_wqthread_override_reset(); + } + } + _dispatch_introspection_queue_item_complete(dq); + if (running == 0) { + return _dispatch_queue_wakeup_with_qos_and_release(dq, p); + } + } else { + mach_port_t th = dq->dq_thread; + if (th) { + p = _dispatch_queue_get_override_priority(dq); + _dispatch_object_debug(dq, "overriding thr 0x%x to priority 0x%lx", + th, p); + _dispatch_wqthread_override_start(th, p); + } + } + _dispatch_release(dq); // added when we pushed the override block +} #endif -#ifndef DISPATCH_CONTENTION_USLEEP_START -#define DISPATCH_CONTENTION_USLEEP_START 500 + +static inline bool +_dispatch_queue_prepare_override(dispatch_queue_t dq, dispatch_queue_t tq, + pthread_priority_t p) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + if (dx_type(tq) != DISPATCH_QUEUE_ROOT_TYPE || !tq->dq_priority) { + return false; + } + if (p <= (dq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK)) { + return false; + } + if (p <= (tq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK)) { + return false; + } + _dispatch_retain(dq); + return true; +#else + (void)dq; (void)tq; (void)p; + return false; #endif -#ifndef DISPATCH_CONTENTION_USLEEP_MAX -#define DISPATCH_CONTENTION_USLEEP_MAX 100000 +} + +static inline void +_dispatch_queue_push_override(dispatch_queue_t dq, dispatch_queue_t tq, + pthread_priority_t p) +{ +#if HAVE_PTHREAD_WORKQUEUE_QOS + unsigned int qosbit, idx, overcommit; + overcommit = (tq->dq_priority & _PTHREAD_PRIORITY_OVERCOMMIT_FLAG) ? 1 : 0; + qosbit = (p & _PTHREAD_PRIORITY_QOS_CLASS_MASK) >> + _PTHREAD_PRIORITY_QOS_CLASS_SHIFT; + idx = (unsigned int)__builtin_ffs((int)qosbit); + if (!idx || idx > DISPATCH_QUEUE_QOS_COUNT) { + DISPATCH_CRASH("Corrupted override priority"); + } + dispatch_queue_t rq = &_dispatch_root_queues[((idx-1) << 1) | overcommit]; + + dispatch_continuation_t dc = _dispatch_continuation_alloc(); + dc->do_vtable = (void *)(DISPATCH_OBJ_ASYNC_BIT | DISPATCH_OBJ_BARRIER_BIT); + dc->dc_func = _dispatch_queue_override_invoke; + dc->dc_ctxt = dc; + dc->dc_priority = tq->dq_priority; + dc->dc_voucher = NULL; + dc->dc_data = dq; + // dq retained by _dispatch_queue_prepare_override + + _dispatch_queue_push(rq, dc, 0); +#else + (void)dq; (void)tq; (void)p; #endif +} + +#pragma mark - +#pragma mark dispatch_root_queue_drain DISPATCH_NOINLINE static bool @@ -2621,27 +3891,15 @@ _dispatch_queue_concurrent_drain_one_slow(dispatch_queue_t dq) dispatch_root_queue_context_t qc = dq->do_ctxt; struct dispatch_object_s *const mediator = (void *)~0ul; bool pending = false, available = true; - unsigned int spins, sleep_time = DISPATCH_CONTENTION_USLEEP_START; + unsigned int sleep_time = DISPATCH_CONTENTION_USLEEP_START; do { // Spin for a short while in case the contention is temporary -- e.g. // when starting up after dispatch_apply, or when executing a few // short continuations in a row. -#if DISPATCH_CONTENTION_USE_RAND - // Use randomness to prevent threads from resonating at the same - // frequency and permanently contending. All threads sharing the same - // seed value is safe with the FreeBSD rand_r implementation. - static unsigned int seed; - spins = (rand_r(&seed) & DISPATCH_CONTENTION_SPINS_MAX) | - DISPATCH_CONTENTION_SPINS_MIN; -#else - spins = DISPATCH_CONTENTION_SPINS_MIN + - (DISPATCH_CONTENTION_SPINS_MAX-DISPATCH_CONTENTION_SPINS_MIN)/2; -#endif - while (spins--) { - dispatch_hardware_pause(); - if (fastpath(dq->dq_items_head != mediator)) goto out; - }; + if (_dispatch_contention_wait_until(dq->dq_items_head != mediator)) { + goto out; + } // Since we have serious contention, we need to back off. if (!pending) { // Mark this queue as pending to avoid requests for further threads @@ -2658,12 +3916,25 @@ _dispatch_queue_concurrent_drain_one_slow(dispatch_queue_t dq) // Create a new pending thread and then exit this thread. // The kernel will grant a new thread when the load subsides. _dispatch_debug("contention on global queue: %p", dq); - _dispatch_queue_wakeup_global(dq); available = false; out: if (pending) { (void)dispatch_atomic_dec2o(qc, dgq_pending, relaxed); } + if (!available) { + _dispatch_queue_wakeup_global(dq); + } + return available; +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_dispatch_queue_concurrent_drain_one2(dispatch_queue_t dq) +{ + // Wait for queue head and tail to be both non-empty or both empty + bool available; // + _dispatch_wait_until((dq->dq_items_head != NULL) == + (available = (dq->dq_items_tail != NULL))); return available; } @@ -2681,8 +3952,14 @@ start: // The first xchg on the tail will tell the enqueueing thread that it // is safe to blindly write out to the head pointer. A cmpxchg honors // the algorithm. - (void)dispatch_atomic_cmpxchg2o(dq, dq_items_head, mediator, NULL, - relaxed); + if (slowpath(!dispatch_atomic_cmpxchg2o(dq, dq_items_head, mediator, + NULL, relaxed))) { + goto start; + } + if (slowpath(dq->dq_items_tail) && // + _dispatch_queue_concurrent_drain_one2(dq)) { + goto start; + } _dispatch_root_queue_debug("no work on global queue: %p", dq); return NULL; } @@ -2706,11 +3983,8 @@ start: // both head and tail are NULL now goto out; } - - // There must be a next item now. This thread won't wait long. - while (!(next = head->do_next)) { - dispatch_hardware_pause(); - } + // There must be a next item now. + _dispatch_wait_until(next = head->do_next); } dispatch_atomic_store2o(dq, dq_items_head, next, relaxed); @@ -2728,6 +4002,9 @@ _dispatch_root_queue_drain(dispatch_queue_t dq) } #endif _dispatch_thread_setspecific(dispatch_queue_key, dq); + pthread_priority_t old_pri = _dispatch_get_priority(); + pthread_priority_t pri = dq->dq_priority ? dq->dq_priority : old_pri; + pthread_priority_t old_dp = _dispatch_set_defaultpriority(pri); #if DISPATCH_COCOA_COMPAT // ensure that high-level memory management techniques do not leak/crash @@ -2739,9 +4016,15 @@ _dispatch_root_queue_drain(dispatch_queue_t dq) _dispatch_perfmon_start(); struct dispatch_object_s *item; + bool reset = false; while ((item = fastpath(_dispatch_queue_concurrent_drain_one(dq)))) { + if (reset) _dispatch_wqthread_override_reset(); _dispatch_continuation_pop(item); + reset = _dispatch_reset_defaultpriority_override(); } + _dispatch_voucher_debug("root queue clear", NULL); + _dispatch_set_priority_and_replace_voucher(old_pri, NULL); + _dispatch_reset_defaultpriority(old_dp); _dispatch_perfmon_end(); #if DISPATCH_COCOA_COMPAT @@ -2759,18 +4042,41 @@ _dispatch_root_queue_drain(dispatch_queue_t dq) #if HAVE_PTHREAD_WORKQUEUES static void -_dispatch_worker_thread3(void *context) +_dispatch_worker_thread4(void *context) { dispatch_queue_t dq = context; dispatch_root_queue_context_t qc = dq->do_ctxt; _dispatch_introspection_thread_add(); - - (void)dispatch_atomic_dec2o(qc, dgq_pending, relaxed); + int pending = (int)dispatch_atomic_dec2o(qc, dgq_pending, relaxed); + dispatch_assert(pending >= 0); _dispatch_root_queue_drain(dq); __asm__(""); // prevent tailcall (for Instrument DTrace probe) +} +#if HAVE_PTHREAD_WORKQUEUE_QOS +static void +_dispatch_worker_thread3(pthread_priority_t priority) +{ + // Reset priority TSD to workaround + _dispatch_thread_setspecific(dispatch_priority_key, + (void*)(uintptr_t)(priority & ~_PTHREAD_PRIORITY_FLAGS_MASK)); + unsigned int overcommit, qosbit, idx; + overcommit = (priority & _PTHREAD_PRIORITY_OVERCOMMIT_FLAG) ? 1 : 0; + qosbit = (priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK) >> + _PTHREAD_PRIORITY_QOS_CLASS_SHIFT; + if (!_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS]. + dq_priority) { + // If kernel doesn't support maintenance, bottom bit is background. + // Shift to our idea of where background bit is. + qosbit <<= 1; + } + idx = (unsigned int)__builtin_ffs((int)qosbit); + dispatch_assert(idx > 0 && idx < DISPATCH_QUEUE_QOS_COUNT+1); + dispatch_queue_t dq = &_dispatch_root_queues[((idx-1) << 1) | overcommit]; + return _dispatch_worker_thread4(dq); } +#endif // HAVE_PTHREAD_WORKQUEUE_QOS #if HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP // 6618342 Contact the team that owns the Instrument DTrace probe before @@ -2783,7 +4089,7 @@ _dispatch_worker_thread2(int priority, int options, dispatch_assert(!(options & ~WORKQ_ADDTHREADS_OPTION_OVERCOMMIT)); dispatch_queue_t dq = _dispatch_wq2root_queues[priority][options]; - return _dispatch_worker_thread3(dq); + return _dispatch_worker_thread4(dq); } #endif // HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP #endif // HAVE_PTHREAD_WORKQUEUES @@ -2798,7 +4104,7 @@ _dispatch_worker_thread(void *context) dispatch_root_queue_context_t qc = dq->do_ctxt; dispatch_pthread_root_queue_context_t pqc = qc->dgq_ctxt; - if (pqc && pqc->dpq_thread_configure) { + if (pqc->dpq_thread_configure) { pqc->dpq_thread_configure(); } @@ -2811,16 +4117,13 @@ _dispatch_worker_thread(void *context) (void)dispatch_assume_zero(r); _dispatch_introspection_thread_add(); - // Non-pthread-root-queue pthreads use a 65 second timeout in case there - // are any timers that run once a minute - const int64_t timeout = (pqc ? 5ull : 65ull) * NSEC_PER_SEC; - + const int64_t timeout = 5ull * NSEC_PER_SEC; do { _dispatch_root_queue_drain(dq); - } while (dispatch_semaphore_wait(qc->dgq_thread_mediator, + } while (dispatch_semaphore_wait(&pqc->dpq_thread_mediator, dispatch_time(0, timeout)) == 0); - (void)dispatch_atomic_inc2o(qc, dgq_thread_pool_size, relaxed); + (void)dispatch_atomic_inc2o(qc, dgq_thread_pool_size, release); _dispatch_queue_wakeup_global(dq); _dispatch_release(dq); @@ -2876,7 +4179,7 @@ _dispatch_runloop_root_queue_create_4CF(const char *label, unsigned long flags) dqs = sizeof(struct dispatch_queue_s) - DISPATCH_QUEUE_CACHELINE_PAD; dq = _dispatch_alloc(DISPATCH_VTABLE(queue_runloop), dqs); _dispatch_queue_init(dq); - dq->do_targetq = _dispatch_get_root_queue(0, true); + dq->do_targetq = _dispatch_get_root_queue(_DISPATCH_QOS_CLASS_DEFAULT,true); dq->dq_label = label ? label : "runloop-queue"; // no-copy contract dq->do_suspend_cnt = DISPATCH_OBJECT_SUSPEND_LOCK; dq->dq_running = 1; @@ -2896,7 +4199,7 @@ _dispatch_runloop_queue_xref_dispose(dispatch_queue_t dq) DISPATCH_OBJECT_SUSPEND_LOCK, release); _dispatch_queue_clear_bound_thread(dq); if (suspend_cnt == 0) { - _dispatch_wakeup(dq); + _dispatch_queue_wakeup(dq); } } @@ -3067,15 +4370,15 @@ _dispatch_queue_cleanup2(void) DISPATCH_OBJECT_SUSPEND_LOCK, release); dq->dq_is_thread_bound = 0; if (suspend_cnt == 0) { - _dispatch_wakeup(dq); + _dispatch_queue_wakeup(dq); } // overload the "probably" variable to mean that dispatch_main() or // similar non-POSIX API was called // this has to run before the DISPATCH_COCOA_COMPAT below if (_dispatch_program_is_probably_callback_driven) { - dispatch_async_f(_dispatch_get_root_queue(0, true), NULL, - _dispatch_sig_thread); + _dispatch_barrier_async_detached_f(_dispatch_get_root_queue( + _DISPATCH_QOS_CLASS_DEFAULT, true), NULL, _dispatch_sig_thread); sleep(1); // workaround 6778970 } diff --git a/src/queue_internal.h b/src/queue_internal.h index 5f6476a..d76b15f 100644 --- a/src/queue_internal.h +++ b/src/queue_internal.h @@ -38,77 +38,15 @@ /* x86 & cortex-a8 have a 64 byte cacheline */ #define DISPATCH_CACHELINE_SIZE 64u -#define DISPATCH_CONTINUATION_SIZE DISPATCH_CACHELINE_SIZE #define ROUND_UP_TO_CACHELINE_SIZE(x) \ (((x) + (DISPATCH_CACHELINE_SIZE - 1u)) & \ ~(DISPATCH_CACHELINE_SIZE - 1u)) -#define ROUND_UP_TO_CONTINUATION_SIZE(x) \ - (((x) + (DISPATCH_CONTINUATION_SIZE - 1u)) & \ - ~(DISPATCH_CONTINUATION_SIZE - 1u)) -#define ROUND_UP_TO_VECTOR_SIZE(x) \ - (((x) + 15u) & ~15u) #define DISPATCH_CACHELINE_ALIGN \ __attribute__((__aligned__(DISPATCH_CACHELINE_SIZE))) -#define DISPATCH_QUEUE_CACHELINE_PADDING \ - char _dq_pad[DISPATCH_QUEUE_CACHELINE_PAD] -#ifdef __LP64__ -#define DISPATCH_QUEUE_CACHELINE_PAD (( \ - (3*sizeof(void*) - DISPATCH_INTROSPECTION_QUEUE_LIST_SIZE) \ - + DISPATCH_CACHELINE_SIZE) % DISPATCH_CACHELINE_SIZE) -#else -#define DISPATCH_QUEUE_CACHELINE_PAD (( \ - (0*sizeof(void*) - DISPATCH_INTROSPECTION_QUEUE_LIST_SIZE) \ - + DISPATCH_CACHELINE_SIZE) % DISPATCH_CACHELINE_SIZE) -#if !DISPATCH_INTROSPECTION -// No padding, DISPATCH_QUEUE_CACHELINE_PAD == 0 -#undef DISPATCH_QUEUE_CACHELINE_PADDING -#define DISPATCH_QUEUE_CACHELINE_PADDING -#endif -#endif - -// If dc_vtable is less than 127, then the object is a continuation. -// Otherwise, the object has a private layout and memory management rules. The -// layout until after 'do_next' must align with normal objects. -#define DISPATCH_CONTINUATION_HEADER(x) \ - _OS_OBJECT_HEADER( \ - const void *do_vtable, \ - do_ref_cnt, \ - do_xref_cnt); \ - struct dispatch_##x##_s *volatile do_next; \ - dispatch_function_t dc_func; \ - void *dc_ctxt; \ - void *dc_data; \ - void *dc_other; - -#define DISPATCH_OBJ_ASYNC_BIT 0x1 -#define DISPATCH_OBJ_BARRIER_BIT 0x2 -#define DISPATCH_OBJ_GROUP_BIT 0x4 -#define DISPATCH_OBJ_SYNC_SLOW_BIT 0x8 -// vtables are pointers far away from the low page in memory -#define DISPATCH_OBJ_IS_VTABLE(x) ((unsigned long)(x)->do_vtable > 127ul) - -struct dispatch_continuation_s { - DISPATCH_CONTINUATION_HEADER(continuation); -}; - -typedef struct dispatch_continuation_s *dispatch_continuation_t; - -struct dispatch_apply_s { - size_t volatile da_index, da_todo; - size_t da_iterations, da_nested; - dispatch_continuation_t da_dc; - _dispatch_thread_semaphore_t da_sema; - uint32_t da_thr_cnt; -}; - -typedef struct dispatch_apply_s *dispatch_apply_t; - -DISPATCH_CLASS_DECL(queue_attr); -struct dispatch_queue_attr_s { - DISPATCH_STRUCT_HEADER(queue_attr); -}; +#pragma mark - +#pragma mark dispatch_queue_t #define DISPATCH_QUEUE_HEADER \ uint32_t volatile dq_running; \ @@ -116,12 +54,30 @@ struct dispatch_queue_attr_s { /* LP64 global queue cacheline boundary */ \ struct dispatch_object_s *volatile dq_items_tail; \ dispatch_queue_t dq_specific_q; \ - uint32_t dq_width; \ - unsigned int dq_is_thread_bound:1; \ + uint16_t dq_width; \ + uint16_t dq_is_thread_bound:1; \ + pthread_priority_t dq_priority; \ + mach_port_t dq_thread; \ + mach_port_t volatile dq_tqthread; \ + uint32_t volatile dq_override; \ unsigned long dq_serialnum; \ const char *dq_label; \ DISPATCH_INTROSPECTION_QUEUE_LIST; +#define DISPATCH_QUEUE_WIDTH_MAX UINT16_MAX + +#define DISPATCH_QUEUE_CACHELINE_PADDING \ + char _dq_pad[DISPATCH_QUEUE_CACHELINE_PAD] +#ifdef __LP64__ +#define DISPATCH_QUEUE_CACHELINE_PAD (( \ + (0*sizeof(void*) - DISPATCH_INTROSPECTION_QUEUE_LIST_SIZE) \ + + DISPATCH_CACHELINE_SIZE) % DISPATCH_CACHELINE_SIZE) +#else +#define DISPATCH_QUEUE_CACHELINE_PAD (( \ + (13*sizeof(void*) - DISPATCH_INTROSPECTION_QUEUE_LIST_SIZE) \ + + DISPATCH_CACHELINE_SIZE) % DISPATCH_CACHELINE_SIZE) +#endif + DISPATCH_CLASS_DECL(queue); struct dispatch_queue_s { DISPATCH_STRUCT_HEADER(queue); @@ -136,17 +92,21 @@ DISPATCH_INTERNAL_SUBCLASS_DECL(queue_mgr, queue); DISPATCH_DECL_INTERNAL_SUBCLASS(dispatch_queue_specific_queue, dispatch_queue); DISPATCH_CLASS_DECL(queue_specific_queue); -extern struct dispatch_queue_s _dispatch_mgr_q; - void _dispatch_queue_destroy(dispatch_object_t dou); void _dispatch_queue_dispose(dispatch_queue_t dq); void _dispatch_queue_invoke(dispatch_queue_t dq); void _dispatch_queue_push_list_slow(dispatch_queue_t dq, - struct dispatch_object_s *obj, unsigned int n); + pthread_priority_t pp, struct dispatch_object_s *obj, unsigned int n, + bool retained); void _dispatch_queue_push_slow(dispatch_queue_t dq, - struct dispatch_object_s *obj); + pthread_priority_t pp, struct dispatch_object_s *obj, bool retained); unsigned long _dispatch_queue_probe(dispatch_queue_t dq); dispatch_queue_t _dispatch_wakeup(dispatch_object_t dou); +dispatch_queue_t _dispatch_queue_wakeup(dispatch_queue_t dq); +void _dispatch_queue_wakeup_with_qos(dispatch_queue_t dq, + pthread_priority_t pp); +void _dispatch_queue_wakeup_with_qos_and_release(dispatch_queue_t dq, + pthread_priority_t pp); _dispatch_thread_semaphore_t _dispatch_queue_drain(dispatch_object_t dou); void _dispatch_queue_specific_queue_dispose(dispatch_queue_specific_queue_t dqsq); @@ -167,6 +127,8 @@ void _dispatch_async_redirect_invoke(void *ctxt); void _dispatch_sync_recurse_invoke(void *ctxt); void _dispatch_apply_invoke(void *ctxt); void _dispatch_apply_redirect_invoke(void *ctxt); +void _dispatch_barrier_async_detached_f(dispatch_queue_t dq, void *ctxt, + dispatch_function_t func); void _dispatch_barrier_trysync_f(dispatch_queue_t dq, void *ctxt, dispatch_function_t func); @@ -181,194 +143,136 @@ size_t dispatch_queue_debug(dispatch_queue_t dq, char* buf, size_t bufsiz); size_t _dispatch_queue_debug_attr(dispatch_queue_t dq, char* buf, size_t bufsiz); -#define DISPATCH_QUEUE_PRIORITY_COUNT 4 -#define DISPATCH_ROOT_QUEUE_COUNT (DISPATCH_QUEUE_PRIORITY_COUNT * 2) +#define DISPATCH_QUEUE_QOS_COUNT 6 +#define DISPATCH_ROOT_QUEUE_COUNT (DISPATCH_QUEUE_QOS_COUNT * 2) -// overcommit priority index values need bit 1 set +// must be in lowest to highest qos order (as encoded in pthread_priority_t) +// overcommit qos index values need bit 1 set enum { - DISPATCH_ROOT_QUEUE_IDX_LOW_PRIORITY = 0, - DISPATCH_ROOT_QUEUE_IDX_LOW_OVERCOMMIT_PRIORITY, - DISPATCH_ROOT_QUEUE_IDX_DEFAULT_PRIORITY, - DISPATCH_ROOT_QUEUE_IDX_DEFAULT_OVERCOMMIT_PRIORITY, - DISPATCH_ROOT_QUEUE_IDX_HIGH_PRIORITY, - DISPATCH_ROOT_QUEUE_IDX_HIGH_OVERCOMMIT_PRIORITY, - DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_PRIORITY, - DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_OVERCOMMIT_PRIORITY, + DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS = 0, + DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS_OVERCOMMIT, + DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS, + DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS_OVERCOMMIT, + DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS, + DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS_OVERCOMMIT, + DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS, + DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT, + DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS, + DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS_OVERCOMMIT, + DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS, + DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS_OVERCOMMIT, }; extern unsigned long volatile _dispatch_queue_serial_numbers; extern struct dispatch_queue_s _dispatch_root_queues[]; +extern struct dispatch_queue_s _dispatch_mgr_q; -#if !(USE_OBJC && __OBJC2__) - -DISPATCH_ALWAYS_INLINE -static inline bool -_dispatch_queue_push_list2(dispatch_queue_t dq, struct dispatch_object_s *head, - struct dispatch_object_s *tail) -{ - struct dispatch_object_s *prev; - tail->do_next = NULL; - prev = dispatch_atomic_xchg2o(dq, dq_items_tail, tail, release); - if (fastpath(prev)) { - // if we crash here with a value less than 0x1000, then we are at a - // known bug in client code for example, see _dispatch_queue_dispose - // or _dispatch_atfork_child - prev->do_next = head; - } - return (prev != NULL); -} - -DISPATCH_ALWAYS_INLINE -static inline void -_dispatch_queue_push_list(dispatch_queue_t dq, dispatch_object_t _head, - dispatch_object_t _tail, unsigned int n) -{ - struct dispatch_object_s *head = _head._do, *tail = _tail._do; - if (!fastpath(_dispatch_queue_push_list2(dq, head, tail))) { - _dispatch_queue_push_list_slow(dq, head, n); - } -} - -DISPATCH_ALWAYS_INLINE -static inline void -_dispatch_queue_push(dispatch_queue_t dq, dispatch_object_t _tail) -{ - struct dispatch_object_s *tail = _tail._do; - if (!fastpath(_dispatch_queue_push_list2(dq, tail, tail))) { - _dispatch_queue_push_slow(dq, tail); - } -} - -DISPATCH_ALWAYS_INLINE -static inline void -_dispatch_queue_push_wakeup(dispatch_queue_t dq, dispatch_object_t _tail, - bool wakeup) -{ - struct dispatch_object_s *tail = _tail._do; - if (!fastpath(_dispatch_queue_push_list2(dq, tail, tail))) { - _dispatch_queue_push_slow(dq, tail); - } else if (slowpath(wakeup)) { - _dispatch_wakeup(dq); - } -} - -DISPATCH_ALWAYS_INLINE -static inline void -_dispatch_queue_class_invoke(dispatch_object_t dou, - dispatch_queue_t (*invoke)(dispatch_object_t, - _dispatch_thread_semaphore_t*)) -{ - dispatch_queue_t dq = dou._dq; - if (!slowpath(DISPATCH_OBJECT_SUSPENDED(dq)) && - fastpath(dispatch_atomic_cmpxchg2o(dq, dq_running, 0, 1, acquire))){ - dispatch_queue_t tq = NULL; - _dispatch_thread_semaphore_t sema = 0; - tq = invoke(dq, &sema); - // We do not need to check the result. - // When the suspend-count lock is dropped, then the check will happen. - (void)dispatch_atomic_dec2o(dq, dq_running, release); - if (sema) { - _dispatch_thread_semaphore_signal(sema); - } else if (tq) { - _dispatch_introspection_queue_item_complete(dq); - return _dispatch_queue_push(tq, dq); - } - } - dq->do_next = DISPATCH_OBJECT_LISTLESS; - if (!dispatch_atomic_sub2o(dq, do_suspend_cnt, - DISPATCH_OBJECT_SUSPEND_LOCK, release)) { - dispatch_atomic_barrier(seq_cst); // - if (dispatch_atomic_load2o(dq, dq_running, seq_cst) == 0) { - _dispatch_wakeup(dq); // verify that the queue is idle - } - } - _dispatch_introspection_queue_item_complete(dq); - _dispatch_release(dq); // added when the queue is put on the list -} - -DISPATCH_ALWAYS_INLINE -static inline dispatch_queue_t -_dispatch_queue_get_current(void) -{ - return (dispatch_queue_t)_dispatch_thread_getspecific(dispatch_queue_key); -} - -DISPATCH_ALWAYS_INLINE DISPATCH_CONST -static inline dispatch_queue_t -_dispatch_get_root_queue(long priority, bool overcommit) -{ - if (overcommit) switch (priority) { - case DISPATCH_QUEUE_PRIORITY_BACKGROUND: -#if !DISPATCH_NO_BG_PRIORITY - return &_dispatch_root_queues[ - DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_OVERCOMMIT_PRIORITY]; +#if HAVE_PTHREAD_WORKQUEUE_QOS +extern pthread_priority_t _dispatch_background_priority; +extern pthread_priority_t _dispatch_user_initiated_priority; +#endif + +#pragma mark - +#pragma mark dispatch_queue_attr_t + +DISPATCH_CLASS_DECL(queue_attr); +struct dispatch_queue_attr_s { + DISPATCH_STRUCT_HEADER(queue_attr); + qos_class_t dqa_qos_class; + int dqa_relative_priority; + unsigned int dqa_overcommit:1, dqa_concurrent:1; +}; + +enum { + DQA_INDEX_NON_OVERCOMMIT = 0, + DQA_INDEX_OVERCOMMIT, +}; + +enum { + DQA_INDEX_CONCURRENT = 0, + DQA_INDEX_SERIAL, +}; + +#define DISPATCH_QUEUE_ATTR_PRIO_COUNT (1 - QOS_MIN_RELATIVE_PRIORITY) + +typedef enum { + DQA_INDEX_QOS_CLASS_UNSPECIFIED = 0, + DQA_INDEX_QOS_CLASS_MAINTENANCE, + DQA_INDEX_QOS_CLASS_BACKGROUND, + DQA_INDEX_QOS_CLASS_UTILITY, + DQA_INDEX_QOS_CLASS_DEFAULT, + DQA_INDEX_QOS_CLASS_USER_INITIATED, + DQA_INDEX_QOS_CLASS_USER_INTERACTIVE, +} _dispatch_queue_attr_index_qos_class_t; + +extern const struct dispatch_queue_attr_s _dispatch_queue_attrs[] + [DISPATCH_QUEUE_ATTR_PRIO_COUNT][2][2]; + +#pragma mark - +#pragma mark dispatch_continuation_t + +// If dc_vtable is less than 127, then the object is a continuation. +// Otherwise, the object has a private layout and memory management rules. The +// layout until after 'do_next' must align with normal objects. +#if __LP64__ +#define DISPATCH_CONTINUATION_HEADER(x) \ + const void *do_vtable; \ + union { \ + pthread_priority_t dc_priority; \ + int dc_cache_cnt; \ + uintptr_t dc_pad; \ + }; \ + struct dispatch_##x##_s *volatile do_next; \ + struct voucher_s *dc_voucher; \ + dispatch_function_t dc_func; \ + void *dc_ctxt; \ + void *dc_data; \ + void *dc_other; +#define _DISPATCH_SIZEOF_PTR 8 +#else +#define DISPATCH_CONTINUATION_HEADER(x) \ + const void *do_vtable; \ + union { \ + pthread_priority_t dc_priority; \ + int dc_cache_cnt; \ + uintptr_t dc_pad; \ + }; \ + struct voucher_s *dc_voucher; \ + struct dispatch_##x##_s *volatile do_next; \ + dispatch_function_t dc_func; \ + void *dc_ctxt; \ + void *dc_data; \ + void *dc_other; +#define _DISPATCH_SIZEOF_PTR 4 #endif - case DISPATCH_QUEUE_PRIORITY_LOW: - case DISPATCH_QUEUE_PRIORITY_NON_INTERACTIVE: - return &_dispatch_root_queues[ - DISPATCH_ROOT_QUEUE_IDX_LOW_OVERCOMMIT_PRIORITY]; - case DISPATCH_QUEUE_PRIORITY_DEFAULT: - return &_dispatch_root_queues[ - DISPATCH_ROOT_QUEUE_IDX_DEFAULT_OVERCOMMIT_PRIORITY]; - case DISPATCH_QUEUE_PRIORITY_HIGH: - return &_dispatch_root_queues[ - DISPATCH_ROOT_QUEUE_IDX_HIGH_OVERCOMMIT_PRIORITY]; - } - switch (priority) { - case DISPATCH_QUEUE_PRIORITY_BACKGROUND: -#if !DISPATCH_NO_BG_PRIORITY - return &_dispatch_root_queues[ - DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_PRIORITY]; +#define _DISPATCH_CONTINUATION_PTRS 8 +#if DISPATCH_HW_CONFIG_UP +// UP devices don't contend on continuations so we don't need to force them to +// occupy a whole cacheline (which is intended to avoid contention) +#define DISPATCH_CONTINUATION_SIZE \ + (_DISPATCH_CONTINUATION_PTRS * _DISPATCH_SIZEOF_PTR) +#else +#define DISPATCH_CONTINUATION_SIZE ROUND_UP_TO_CACHELINE_SIZE( \ + (_DISPATCH_CONTINUATION_PTRS * _DISPATCH_SIZEOF_PTR)) #endif - case DISPATCH_QUEUE_PRIORITY_LOW: - case DISPATCH_QUEUE_PRIORITY_NON_INTERACTIVE: - return &_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_LOW_PRIORITY]; - case DISPATCH_QUEUE_PRIORITY_DEFAULT: - return &_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_DEFAULT_PRIORITY]; - case DISPATCH_QUEUE_PRIORITY_HIGH: - return &_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_HIGH_PRIORITY]; - default: - return NULL; - } -} - -// Note to later developers: ensure that any initialization changes are -// made for statically allocated queues (i.e. _dispatch_main_q). -static inline void -_dispatch_queue_init(dispatch_queue_t dq) -{ - dq->do_next = (struct dispatch_queue_s *)DISPATCH_OBJECT_LISTLESS; - - dq->dq_running = 0; - dq->dq_width = 1; - dq->dq_serialnum = dispatch_atomic_inc_orig(&_dispatch_queue_serial_numbers, - relaxed); -} - -DISPATCH_ALWAYS_INLINE -static inline void -_dispatch_queue_set_bound_thread(dispatch_queue_t dq) -{ - //Tag thread-bound queues with the owning thread - dispatch_assert(dq->dq_is_thread_bound); - dq->do_finalizer = (void*)_dispatch_thread_self(); -} - -DISPATCH_ALWAYS_INLINE -static inline void -_dispatch_queue_clear_bound_thread(dispatch_queue_t dq) -{ - dispatch_assert(dq->dq_is_thread_bound); - dq->do_finalizer = NULL; -} - -DISPATCH_ALWAYS_INLINE -static inline pthread_t -_dispatch_queue_get_bound_thread(dispatch_queue_t dq) -{ - dispatch_assert(dq->dq_is_thread_bound); - return (pthread_t)dq->do_finalizer; -} +#define ROUND_UP_TO_CONTINUATION_SIZE(x) \ + (((x) + (DISPATCH_CONTINUATION_SIZE - 1u)) & \ + ~(DISPATCH_CONTINUATION_SIZE - 1u)) + +#define DISPATCH_OBJ_ASYNC_BIT 0x1 +#define DISPATCH_OBJ_BARRIER_BIT 0x2 +#define DISPATCH_OBJ_GROUP_BIT 0x4 +#define DISPATCH_OBJ_SYNC_SLOW_BIT 0x8 +#define DISPATCH_OBJ_BLOCK_RELEASE_BIT 0x10 +#define DISPATCH_OBJ_CTXT_FETCH_BIT 0x20 +#define DISPATCH_OBJ_HAS_VOUCHER_BIT 0x80 +// vtables are pointers far away from the low page in memory +#define DISPATCH_OBJ_IS_VTABLE(x) ((unsigned long)(x)->do_vtable > 0xfful) + +struct dispatch_continuation_s { + DISPATCH_CONTINUATION_HEADER(continuation); +}; +typedef struct dispatch_continuation_s *dispatch_continuation_t; #ifndef DISPATCH_CONTINUATION_CACHE_LIMIT #if TARGET_OS_EMBEDDED @@ -392,56 +296,63 @@ void _dispatch_continuation_free_to_cache_limit(dispatch_continuation_t c); _dispatch_continuation_free_to_heap(c) #endif -DISPATCH_ALWAYS_INLINE -static inline dispatch_continuation_t -_dispatch_continuation_alloc_cacheonly(void) -{ - dispatch_continuation_t dc = (dispatch_continuation_t) - fastpath(_dispatch_thread_getspecific(dispatch_cache_key)); - if (dc) { - _dispatch_thread_setspecific(dispatch_cache_key, dc->do_next); - } - return dc; -} - -DISPATCH_ALWAYS_INLINE -static inline dispatch_continuation_t -_dispatch_continuation_alloc(void) -{ - dispatch_continuation_t dc = - fastpath(_dispatch_continuation_alloc_cacheonly()); - if(!dc) { - return _dispatch_continuation_alloc_from_heap(); - } - return dc; -} - -DISPATCH_ALWAYS_INLINE -static inline dispatch_continuation_t -_dispatch_continuation_free_cacheonly(dispatch_continuation_t dc) -{ - dispatch_continuation_t prev_dc = (dispatch_continuation_t) - fastpath(_dispatch_thread_getspecific(dispatch_cache_key)); - int cnt = prev_dc ? prev_dc->do_ref_cnt + 1 : 1; - // Cap continuation cache - if (slowpath(cnt > _dispatch_continuation_cache_limit)) { - return dc; - } - dc->do_next = prev_dc; - dc->do_ref_cnt = cnt; - _dispatch_thread_setspecific(dispatch_cache_key, dc); - return NULL; -} - -DISPATCH_ALWAYS_INLINE -static inline void -_dispatch_continuation_free(dispatch_continuation_t dc) -{ - dc = _dispatch_continuation_free_cacheonly(dc); - if (slowpath(dc)) { - _dispatch_continuation_free_to_cache_limit(dc); - } -} -#endif // !(USE_OBJC && __OBJC2__) +#pragma mark - +#pragma mark dispatch_apply_t + +struct dispatch_apply_s { + size_t volatile da_index, da_todo; + size_t da_iterations, da_nested; + dispatch_continuation_t da_dc; + _dispatch_thread_semaphore_t da_sema; + uint32_t da_thr_cnt; +}; +typedef struct dispatch_apply_s *dispatch_apply_t; + +#pragma mark - +#pragma mark dispatch_block_t + +#ifdef __BLOCKS__ + +#define DISPATCH_BLOCK_API_MASK (0x80u - 1) +#define DISPATCH_BLOCK_HAS_VOUCHER (1u << 31) +#define DISPATCH_BLOCK_HAS_PRIORITY (1u << 30) + +struct dispatch_block_private_data_s { + unsigned long dbpd_magic; + dispatch_block_flags_t dbpd_flags; + unsigned int volatile dbpd_atomic_flags; + int volatile dbpd_performed; + pthread_priority_t dbpd_priority; + voucher_t dbpd_voucher; + dispatch_block_t dbpd_block; + struct dispatch_semaphore_s dbpd_group; + dispatch_queue_t volatile dbpd_queue; + mach_port_t dbpd_thread; +}; +typedef struct dispatch_block_private_data_s *dispatch_block_private_data_t; + +// dbpd_atomic_flags bits +#define DBF_CANCELED 1u // block has been cancelled +#define DBF_WAITING 2u // dispatch_block_wait has begun +#define DBF_WAITED 4u // dispatch_block_wait has finished without timeout +#define DBF_PERFORM 8u // dispatch_block_perform: don't group_leave + +#define DISPATCH_BLOCK_PRIVATE_DATA_MAGIC 0xD159B10C // 0xDISPatch_BLOCk + +#define DISPATCH_BLOCK_PRIVATE_DATA_INITIALIZER(flags, voucher, prio, block) \ + { \ + .dbpd_magic = DISPATCH_BLOCK_PRIVATE_DATA_MAGIC, \ + .dbpd_flags = (flags), \ + .dbpd_priority = (prio), \ + .dbpd_voucher = (voucher), \ + .dbpd_block = (block), \ + .dbpd_group = DISPATCH_GROUP_INITIALIZER(1), \ + } + +dispatch_block_t _dispatch_block_create(dispatch_block_flags_t flags, + voucher_t voucher, pthread_priority_t priority, dispatch_block_t block); +void _dispatch_block_invoke(const struct dispatch_block_private_data_s *dbcpd); + +#endif /* __BLOCKS__ */ #endif diff --git a/src/semaphore.c b/src/semaphore.c index 20d9ae5..f9bfdbe 100644 --- a/src/semaphore.c +++ b/src/semaphore.c @@ -23,14 +23,23 @@ // semaphores are too fundamental to use the dispatch_assume*() macros #if USE_MACH_SEM #define DISPATCH_SEMAPHORE_VERIFY_KR(x) do { \ - if (slowpath(x)) { \ - DISPATCH_CRASH("flawed group/semaphore logic"); \ + if (slowpath((x) == KERN_INVALID_NAME)) { \ + DISPATCH_CLIENT_CRASH("Use-after-free of dispatch_semaphore_t"); \ + } else if (slowpath(x)) { \ + DISPATCH_CRASH("mach semaphore API failure"); \ + } \ + } while (0) +#define DISPATCH_GROUP_VERIFY_KR(x) do { \ + if (slowpath((x) == KERN_INVALID_NAME)) { \ + DISPATCH_CLIENT_CRASH("Use-after-free of dispatch_group_t"); \ + } else if (slowpath(x)) { \ + DISPATCH_CRASH("mach semaphore API failure"); \ } \ } while (0) #elif USE_POSIX_SEM #define DISPATCH_SEMAPHORE_VERIFY_RET(x) do { \ if (slowpath((x) == -1)) { \ - DISPATCH_CRASH("flawed group/semaphore logic"); \ + DISPATCH_CRASH("POSIX semaphore API failure"); \ } \ } while (0) #endif @@ -95,8 +104,8 @@ _dispatch_semaphore_init(long value, dispatch_object_t dou) dispatch_semaphore_t dsema = dou._dsema; dsema->do_next = (dispatch_semaphore_t)DISPATCH_OBJECT_LISTLESS; - dsema->do_targetq = dispatch_get_global_queue( - DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); + dsema->do_targetq = _dispatch_get_root_queue(_DISPATCH_QOS_CLASS_DEFAULT, + false); dsema->dsema_value = value; dsema->dsema_orig = value; #if USE_POSIX_SEM @@ -152,6 +161,7 @@ _dispatch_semaphore_create_port(semaphore_t *s4) if (!dispatch_atomic_cmpxchg(s4, 0, tmp, relaxed)) { kr = semaphore_destroy(mach_task_self(), tmp); + DISPATCH_VERIFY_MIG(kr); DISPATCH_SEMAPHORE_VERIFY_KR(kr); } } @@ -191,8 +201,10 @@ _dispatch_semaphore_dispose(dispatch_object_t dou) kern_return_t kr; if (dsema->dsema_port) { kr = semaphore_destroy(mach_task_self(), dsema->dsema_port); + DISPATCH_VERIFY_MIG(kr); DISPATCH_SEMAPHORE_VERIFY_KR(kr); } + dsema->dsema_port = MACH_PORT_DEAD; #elif USE_POSIX_SEM int ret = sem_destroy(&dsema->dsema_sem); DISPATCH_SEMAPHORE_VERIFY_RET(ret); @@ -442,7 +454,7 @@ _dispatch_group_wake(dispatch_semaphore_t dsema) _dispatch_semaphore_create_port(&dsema->dsema_port); do { kern_return_t kr = semaphore_signal(dsema->dsema_port); - DISPATCH_SEMAPHORE_VERIFY_KR(kr); + DISPATCH_GROUP_VERIFY_KR(kr); } while (--rval); #elif USE_POSIX_SEM do { @@ -463,9 +475,7 @@ _dispatch_group_wake(dispatch_semaphore_t dsema) do { next = fastpath(head->do_next); if (!next && head != tail) { - while (!(next = fastpath(head->do_next))) { - dispatch_hardware_pause(); - } + _dispatch_wait_until(next = fastpath(head->do_next)); } dispatch_queue_t dsn_queue = (dispatch_queue_t)head->dc_data; dc = _dispatch_continuation_free_cacheonly(head); @@ -552,7 +562,7 @@ again: } while (kr == KERN_ABORTED); if (kr != KERN_OPERATION_TIMED_OUT) { - DISPATCH_SEMAPHORE_VERIFY_KR(kr); + DISPATCH_GROUP_VERIFY_KR(kr); break; } #elif USE_POSIX_SEM @@ -599,7 +609,7 @@ again: do { kr = semaphore_wait(dsema->dsema_port); } while (kr == KERN_ABORTED); - DISPATCH_SEMAPHORE_VERIFY_KR(kr); + DISPATCH_GROUP_VERIFY_KR(kr); #elif USE_POSIX_SEM do { ret = sem_wait(&dsema->dsema_sem); @@ -651,7 +661,7 @@ dispatch_group_notify_f(dispatch_group_t dg, dispatch_queue_t dq, void *ctxt, } else { _dispatch_retain(dg); dispatch_atomic_store2o(dsema, dsema_notify_head, dsn, seq_cst); - dispatch_atomic_barrier(seq_cst); // + // seq_cst with atomic store to notify_head if (dispatch_atomic_load2o(dsema, dsema_value, seq_cst) == LONG_MAX) { _dispatch_group_wake(dsema); } @@ -710,6 +720,7 @@ _dispatch_thread_semaphore_dispose(_dispatch_thread_semaphore_t sema) #elif USE_MACH_SEM semaphore_t s4 = (semaphore_t)sema; kern_return_t kr = semaphore_destroy(mach_task_self(), s4); + DISPATCH_VERIFY_MIG(kr); DISPATCH_SEMAPHORE_VERIFY_KR(kr); #elif USE_POSIX_SEM sem_t s4 = (sem_t)sema; diff --git a/src/semaphore_internal.h b/src/semaphore_internal.h index c8174b6..01179cb 100644 --- a/src/semaphore_internal.h +++ b/src/semaphore_internal.h @@ -53,6 +53,15 @@ struct dispatch_semaphore_s { DISPATCH_CLASS_DECL(group); +#define DISPATCH_GROUP_INITIALIZER(s) \ + { \ + .do_vtable = (const void*)DISPATCH_VTABLE(group), \ + .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, \ + .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, \ + .dsema_value = LONG_MAX - (s), \ + .dsema_orig = LONG_MAX, \ + } + void _dispatch_semaphore_dispose(dispatch_object_t dou); size_t _dispatch_semaphore_debug(dispatch_object_t dou, char *buf, size_t bufsiz); diff --git a/src/shims.h b/src/shims.h index 32376ee..ae7f1c3 100644 --- a/src/shims.h +++ b/src/shims.h @@ -28,9 +28,73 @@ #define __DISPATCH_OS_SHIMS__ #include +#if HAVE_PTHREAD_QOS_H && __has_include() +#include +#if __has_include() +#include +#define _DISPATCH_QOS_CLASS_USER_INTERACTIVE QOS_CLASS_USER_INTERACTIVE +#define _DISPATCH_QOS_CLASS_USER_INITIATED QOS_CLASS_USER_INITIATED +#ifndef QOS_CLASS_LEGACY +#define _DISPATCH_QOS_CLASS_DEFAULT QOS_CLASS_LEGACY +#else +#define _DISPATCH_QOS_CLASS_DEFAULT QOS_CLASS_DEFAULT +#endif +#define _DISPATCH_QOS_CLASS_UTILITY QOS_CLASS_UTILITY +#define _DISPATCH_QOS_CLASS_BACKGROUND QOS_CLASS_BACKGROUND +#define _DISPATCH_QOS_CLASS_UNSPECIFIED QOS_CLASS_UNSPECIFIED +#else // pthread/qos_private.h +typedef unsigned long pthread_priority_t; +#endif // pthread/qos_private.h +#if __has_include() +#include +#define _DISPATCH_QOS_CLASS_MAINTENANCE QOS_CLASS_MAINTENANCE +#else // sys/qos_private.h +#define _DISPATCH_QOS_CLASS_MAINTENANCE 0x05 +#endif // sys/qos_private.h +#ifndef _PTHREAD_PRIORITY_ROOTQUEUE_FLAG +#define _PTHREAD_PRIORITY_ROOTQUEUE_FLAG 0x20000000 +#endif +#ifndef _PTHREAD_PRIORITY_ENFORCE_FLAG +#define _PTHREAD_PRIORITY_ENFORCE_FLAG 0x10000000 +#endif +#ifndef _PTHREAD_PRIORITY_OVERRIDE_FLAG +#define _PTHREAD_PRIORITY_OVERRIDE_FLAG 0x08000000 +#endif +#ifndef _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG +#define _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG 0x04000000 +#endif +#else // HAVE_PTHREAD_QOS_H +typedef unsigned int qos_class_t; +typedef unsigned long pthread_priority_t; +#define QOS_MIN_RELATIVE_PRIORITY (-15) +#define _PTHREAD_PRIORITY_QOS_CLASS_MASK 0x00ffff00 +#define _PTHREAD_PRIORITY_ROOTQUEUE_FLAG 0x20000000 +#define _PTHREAD_PRIORITY_ENFORCE_FLAG 0x10000000 +#define _PTHREAD_PRIORITY_OVERRIDE_FLAG 0x08000000 +#define _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG 0x04000000 +#endif // HAVE_PTHREAD_QOS_H +#ifndef _DISPATCH_QOS_CLASS_USER_INTERACTIVE +enum { + _DISPATCH_QOS_CLASS_USER_INTERACTIVE = 0x21, + _DISPATCH_QOS_CLASS_USER_INITIATED = 0x19, + _DISPATCH_QOS_CLASS_DEFAULT = 0x15, + _DISPATCH_QOS_CLASS_UTILITY = 0x11, + _DISPATCH_QOS_CLASS_BACKGROUND = 0x09, + _DISPATCH_QOS_CLASS_MAINTENANCE = 0x05, + _DISPATCH_QOS_CLASS_UNSPECIFIED = 0x00, +}; +#endif // _DISPATCH_QOS_CLASS_USER_INTERACTIVE #if HAVE_PTHREAD_WORKQUEUES +#if __has_include() +#include +#else #include #endif +#ifndef WORKQ_FEATURE_MAINTENANCE +#define WORKQ_FEATURE_MAINTENANCE 0x10 +#endif +#endif // HAVE_PTHREAD_WORKQUEUES + #if HAVE_PTHREAD_NP_H #include #endif @@ -54,6 +118,33 @@ inline size_t strlcpy(char *dst, const char *src, size_t size) { } #endif // TARGET_OS_WIN32 +#if PTHREAD_WORKQUEUE_SPI_VERSION < 20140716 +static inline int +_pthread_workqueue_override_start_direct(mach_port_t thread, + pthread_priority_t priority) +{ + (void)thread; (void)priority; + return 0; +} +#endif // PTHREAD_WORKQUEUE_SPI_VERSION < 20140716 + +#if PTHREAD_WORKQUEUE_SPI_VERSION < 20140707 +static inline int +_pthread_override_qos_class_start_direct(pthread_t thread, + pthread_priority_t priority) +{ + (void)thread; (void)priority; + return 0; +} + +static inline int +_pthread_override_qos_class_end_direct(mach_port_t thread) +{ + (void)thread; + return 0; +} +#endif // PTHREAD_WORKQUEUE_SPI_VERSION < 20140707 + #if !HAVE_NORETURN_BUILTIN_TRAP /* * XXXRW: Work-around for possible clang bug in which __builtin_trap() is not @@ -71,6 +162,8 @@ void __builtin_trap(void); #include "shims/atomic.h" #include "shims/atomic_sfb.h" #include "shims/tsd.h" +#include "shims/yield.h" + #include "shims/hw_config.h" #include "shims/perfmon.h" diff --git a/src/shims/atomic.h b/src/shims/atomic.h index 2f44775..24c113b 100644 --- a/src/shims/atomic.h +++ b/src/shims/atomic.h @@ -183,23 +183,25 @@ typedef enum _dispatch_atomic_memory_order default: \ _dispatch_atomic_full_barrier(); break; \ } }) -// Only emulate store seq_cst -> load seq_cst +// seq_cst: only emulate explicit store(seq_cst) -> load(seq_cst) #define dispatch_atomic_load(p, m) \ - ({ switch(dispatch_atomic_memory_order_##m) { \ - case _dispatch_atomic_memory_order_relaxed: \ + ({ typeof(*(p)) _r = *(p); \ + switch(dispatch_atomic_memory_order_##m) { \ case _dispatch_atomic_memory_order_seq_cst: \ + _dispatch_atomic_barrier(m); /* fallthrough */ \ + case _dispatch_atomic_memory_order_relaxed: \ break; \ default: \ _dispatch_atomic_unimplemented(); break; \ - }; *(p); }) + } _r; }) #define dispatch_atomic_store(p, v, m) \ ({ switch(dispatch_atomic_memory_order_##m) { \ case _dispatch_atomic_memory_order_release: \ + case _dispatch_atomic_memory_order_seq_cst: \ _dispatch_atomic_barrier(m); /* fallthrough */ \ case _dispatch_atomic_memory_order_relaxed: \ - case _dispatch_atomic_memory_order_seq_cst: \ *(p) = (v); break; \ - default:\ + default: \ _dispatch_atomic_unimplemented(); break; \ } switch(dispatch_atomic_memory_order_##m) { \ case _dispatch_atomic_memory_order_seq_cst: \ @@ -251,6 +253,15 @@ typedef enum _dispatch_atomic_memory_order ({ __asm__ __volatile__( \ "mfence" \ : : : "memory"); }) +#undef dispatch_atomic_load +#define dispatch_atomic_load(p, m) \ + ({ switch(dispatch_atomic_memory_order_##m) { \ + case _dispatch_atomic_memory_order_seq_cst: \ + case _dispatch_atomic_memory_order_relaxed: \ + break; \ + default: \ + _dispatch_atomic_unimplemented(); break; \ + } *(p); }) // xchg is faster than store + mfence #undef dispatch_atomic_store #define dispatch_atomic_store(p, v, m) \ @@ -272,7 +283,6 @@ typedef enum _dispatch_atomic_memory_order #pragma mark - #pragma mark generic -#define dispatch_hardware_pause() ({ __asm__(""); }) // assume atomic builtins provide barriers #define dispatch_atomic_barrier(m) // see comment in dispatch_once.c @@ -342,9 +352,6 @@ typedef enum _dispatch_atomic_memory_order #pragma mark - #pragma mark x86 -#undef dispatch_hardware_pause -#define dispatch_hardware_pause() ({ __asm__("pause"); }) - #undef dispatch_atomic_maximally_synchronizing_barrier #ifdef __LP64__ #define dispatch_atomic_maximally_synchronizing_barrier() \ diff --git a/src/shims/atomic_sfb.h b/src/shims/atomic_sfb.h index c5e7be3..087d98c 100644 --- a/src/shims/atomic_sfb.h +++ b/src/shims/atomic_sfb.h @@ -34,18 +34,18 @@ // Returns UINT_MAX if all the bits in p were already set. #define dispatch_atomic_set_first_bit(p,m) _dispatch_atomic_set_first_bit(p,m) -// TODO: rdar://11477843 DISPATCH_ALWAYS_INLINE static inline unsigned int -_dispatch_atomic_set_first_bit(volatile uint32_t *p, unsigned int max_index) +_dispatch_atomic_set_first_bit(volatile unsigned long *p, + unsigned int max_index) { unsigned int index; - typeof(*p) b, mask, b_masked; + unsigned long b, mask, b_masked; for (;;) { b = *p; // ffs returns 1 + index, or 0 if none set. - index = (unsigned int)__builtin_ffs((int)~b); + index = (unsigned int)__builtin_ffsl((long)~b); if (slowpath(index == 0)) { return UINT_MAX; } @@ -64,12 +64,11 @@ _dispatch_atomic_set_first_bit(volatile uint32_t *p, unsigned int max_index) #if defined(__x86_64__) || defined(__i386__) #undef dispatch_atomic_set_first_bit -// TODO: rdar://11477843 uint64_t -> long DISPATCH_ALWAYS_INLINE static inline unsigned int -dispatch_atomic_set_first_bit(volatile uint64_t *p, unsigned int max) +dispatch_atomic_set_first_bit(volatile unsigned long *p, unsigned int max) { - typeof(*p) val, bit; + unsigned long val, bit; if (max > (sizeof(val) * 8)) { __asm__ ( "1: \n\t" diff --git a/src/shims/hw_config.h b/src/shims/hw_config.h index ede0d48..2b85d4a 100644 --- a/src/shims/hw_config.h +++ b/src/shims/hw_config.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2011-2013 Apple Inc. All rights reserved. * * @APPLE_APACHE_LICENSE_HEADER_START@ * @@ -27,84 +27,103 @@ #ifndef __DISPATCH_SHIMS_HW_CONFIG__ #define __DISPATCH_SHIMS_HW_CONFIG__ -#if defined(__APPLE__) -#define DISPATCH_SYSCTL_LOGICAL_CPUS "hw.logicalcpu_max" -#define DISPATCH_SYSCTL_PHYSICAL_CPUS "hw.physicalcpu_max" -#define DISPATCH_SYSCTL_ACTIVE_CPUS "hw.activecpu" -#elif defined(__FreeBSD__) -#define DISPATCH_SYSCTL_LOGICAL_CPUS "kern.smp.cpus" -#define DISPATCH_SYSCTL_PHYSICAL_CPUS "kern.smp.cpus" -#define DISPATCH_SYSCTL_ACTIVE_CPUS "kern.smp.cpus" +#if !TARGET_OS_WIN32 + +typedef enum { + _dispatch_hw_config_logical_cpus, + _dispatch_hw_config_physical_cpus, + _dispatch_hw_config_active_cpus, +} _dispatch_hw_config_t; + +#if !defined(DISPATCH_HAVE_HW_CONFIG_COMMPAGE) && \ + defined(_COMM_PAGE_LOGICAL_CPUS) && \ + defined(_COMM_PAGE_PHYSICAL_CPUS) && defined(_COMM_PAGE_ACTIVE_CPUS) +#define DISPATCH_HAVE_HW_CONFIG_COMMPAGE 1 #endif -#if !TARGET_OS_WIN32 +#if DISPATCH_HAVE_HW_CONFIG_COMMPAGE +DISPATCH_ALWAYS_INLINE static inline uint32_t -_dispatch_get_logicalcpu_max(void) +_dispatch_hw_get_config(_dispatch_hw_config_t c) { - uint32_t val = 1; -#if defined(_COMM_PAGE_LOGICAL_CPUS) - uint8_t* u8val = (uint8_t*)(uintptr_t)_COMM_PAGE_LOGICAL_CPUS; - val = (uint32_t)*u8val; -#elif defined(DISPATCH_SYSCTL_LOGICAL_CPUS) - size_t valsz = sizeof(val); - int ret = sysctlbyname(DISPATCH_SYSCTL_LOGICAL_CPUS, - &val, &valsz, NULL, 0); - (void)dispatch_assume_zero(ret); - (void)dispatch_assume(valsz == sizeof(uint32_t)); -#elif HAVE_SYSCONF && defined(_SC_NPROCESSORS_ONLN) - int ret = (int)sysconf(_SC_NPROCESSORS_ONLN); - val = ret < 0 ? 1 : ret; -#else -#warning "no supported way to query logical CPU count" -#endif - return val; + uintptr_t p; + switch (c) { + case _dispatch_hw_config_logical_cpus: + p = _COMM_PAGE_LOGICAL_CPUS; break; + case _dispatch_hw_config_physical_cpus: + p = _COMM_PAGE_PHYSICAL_CPUS; break; + case _dispatch_hw_config_active_cpus: + p = _COMM_PAGE_ACTIVE_CPUS; break; + } + return *(uint8_t*)p; } +#define dispatch_hw_config(c) \ + _dispatch_hw_get_config(_dispatch_hw_config_##c) + +#define DISPATCH_HW_CONFIG() +#define _dispatch_hw_config_init() + +#else // DISPATCH_HAVE_HW_CONFIG_COMMPAGE + +extern struct _dispatch_hw_configs_s { + uint32_t logical_cpus; + uint32_t physical_cpus; + uint32_t active_cpus; +} _dispatch_hw_config; + +#define DISPATCH_HW_CONFIG() struct _dispatch_hw_configs_s _dispatch_hw_config +#define dispatch_hw_config(c) (_dispatch_hw_config.c) + +DISPATCH_ALWAYS_INLINE static inline uint32_t -_dispatch_get_physicalcpu_max(void) +_dispatch_hw_get_config(_dispatch_hw_config_t c) { uint32_t val = 1; -#if defined(_COMM_PAGE_PHYSICAL_CPUS) - uint8_t* u8val = (uint8_t*)(uintptr_t)_COMM_PAGE_PHYSICAL_CPUS; - val = (uint32_t)*u8val; -#elif defined(DISPATCH_SYSCTL_PHYSICAL_CPUS) - size_t valsz = sizeof(val); - int ret = sysctlbyname(DISPATCH_SYSCTL_LOGICAL_CPUS, - &val, &valsz, NULL, 0); - (void)dispatch_assume_zero(ret); - (void)dispatch_assume(valsz == sizeof(uint32_t)); -#elif HAVE_SYSCONF && defined(_SC_NPROCESSORS_ONLN) - int ret = (int)sysconf(_SC_NPROCESSORS_ONLN); - val = ret < 0 ? 1 : ret; -#else -#warning "no supported way to query physical CPU count" + const char *name = NULL; + int r; +#if defined(__APPLE__) + switch (c) { + case _dispatch_hw_config_logical_cpus: + name = "hw.logicalcpu_max"; break; + case _dispatch_hw_config_physical_cpus: + name = "hw.physicalcpu_max"; break; + case _dispatch_hw_config_active_cpus: + name = "hw.activecpu"; break; + } +#elif defined(__FreeBSD__) + (void)c; name = "kern.smp.cpus"; #endif + if (name) { + size_t valsz = sizeof(val); + r = sysctlbyname(name, &val, &valsz, NULL, 0); + (void)dispatch_assume_zero(r); + dispatch_assert(valsz == sizeof(uint32_t)); + } else { +#if HAVE_SYSCONF && defined(_SC_NPROCESSORS_ONLN) + r = (int)sysconf(_SC_NPROCESSORS_ONLN); + if (r > 0) val = (uint32_t)r; +#endif + } return val; } -static inline uint32_t -_dispatch_get_activecpu(void) +#define dispatch_hw_config_init(c) \ + _dispatch_hw_get_config(_dispatch_hw_config_##c) + +static inline void +_dispatch_hw_config_init(void) { - uint32_t val = 1; -#if defined(_COMM_PAGE_ACTIVE_CPUS) - uint8_t* u8val = (uint8_t*)(uintptr_t)_COMM_PAGE_ACTIVE_CPUS; - val = (uint32_t)*u8val; -#elif defined(DISPATCH_SYSCTL_ACTIVE_CPUS) - size_t valsz = sizeof(val); - int ret = sysctlbyname(DISPATCH_SYSCTL_ACTIVE_CPUS, - &val, &valsz, NULL, 0); - (void)dispatch_assume_zero(ret); - (void)dispatch_assume(valsz == sizeof(uint32_t)); -#elif HAVE_SYSCONF && defined(_SC_NPROCESSORS_ONLN) - int ret = (int)sysconf(_SC_NPROCESSORS_ONLN); - val = ret < 0 ? 1 : ret; -#else -#warning "no supported way to query active CPU count" -#endif - return val; + dispatch_hw_config(logical_cpus) = dispatch_hw_config_init(logical_cpus); + dispatch_hw_config(physical_cpus) = dispatch_hw_config_init(physical_cpus); + dispatch_hw_config(active_cpus) = dispatch_hw_config_init(active_cpus); } +#undef dispatch_hw_config_init + +#endif // DISPATCH_HAVE_HW_CONFIG_COMMPAGE + #else // TARGET_OS_WIN32 static inline long @@ -118,7 +137,6 @@ _dispatch_count_bits(unsigned long value) return bits; } - static inline uint32_t _dispatch_get_ncpus(void) { diff --git a/src/shims/perfmon.h b/src/shims/perfmon.h index f739006..8af33ea 100644 --- a/src/shims/perfmon.h +++ b/src/shims/perfmon.h @@ -27,7 +27,7 @@ #ifndef __DISPATCH_SHIMS_PERFMON__ #define __DISPATCH_SHIMS_PERFMON__ -#if DISPATCH_PERF_MON +#if DISPATCH_PERF_MON && !DISPATCH_INTROSPECTION #if defined (USE_APPLE_TSD_OPTIMIZATIONS) && defined(SIMULATE_5491082) && \ (defined(__i386__) || defined(__x86_64__)) @@ -63,32 +63,6 @@ _dispatch_perfmon_workitem_dec(void) } #endif /* USE_APPLE_TSD_OPTIMIZATIONS */ -// C99 doesn't define flsll() or ffsll() -#ifdef __LP64__ -#define flsll(x) flsl(x) -#else -static inline unsigned int -flsll(uint64_t val) -{ - union { - struct { -#ifdef __BIG_ENDIAN__ - unsigned int hi, low; -#else - unsigned int low, hi; -#endif - } words; - uint64_t word; - } _bucket = { - .word = val, - }; - if (_bucket.words.hi) { - return fls(_bucket.words.hi) + 32; - } - return fls(_bucket.words.low); -} -#endif - #define _dispatch_perfmon_start() \ uint64_t start = _dispatch_absolute_time() #define _dispatch_perfmon_end() \ diff --git a/src/shims/time.h b/src/shims/time.h index b30b989..7b34bc7 100644 --- a/src/shims/time.h +++ b/src/shims/time.h @@ -31,22 +31,6 @@ #error "Please #include instead of this file directly." #endif -DISPATCH_ALWAYS_INLINE_NDEBUG -static inline void -_dispatch_contention_usleep(unsigned int us) -{ -#if HAVE_MACH -#if defined(SWITCH_OPTION_DISPATCH_CONTENTION) && !(TARGET_IPHONE_SIMULATOR && \ - IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1090) - thread_switch(MACH_PORT_NULL, SWITCH_OPTION_DISPATCH_CONTENTION, us); -#else - thread_switch(MACH_PORT_NULL, SWITCH_OPTION_WAIT, ((us-1)/1000)+1); -#endif -#else - usleep(us); -#endif -} - #if TARGET_OS_WIN32 static inline unsigned int sleep(unsigned int seconds) diff --git a/src/shims/tsd.h b/src/shims/tsd.h index c7e387c..25de7e4 100644 --- a/src/shims/tsd.h +++ b/src/shims/tsd.h @@ -43,28 +43,37 @@ #if DISPATCH_USE_DIRECT_TSD static const unsigned long dispatch_queue_key = __PTK_LIBDISPATCH_KEY0; -#if DISPATCH_USE_OS_SEMAPHORE_CACHE -static const unsigned long dispatch_sema4_key = __TSD_SEMAPHORE_CACHE; -#else -static const unsigned long dispatch_sema4_key = __PTK_LIBDISPATCH_KEY1; -#endif +static const unsigned long dispatch_voucher_key = __PTK_LIBDISPATCH_KEY1; static const unsigned long dispatch_cache_key = __PTK_LIBDISPATCH_KEY2; static const unsigned long dispatch_io_key = __PTK_LIBDISPATCH_KEY3; static const unsigned long dispatch_apply_key = __PTK_LIBDISPATCH_KEY4; +static const unsigned long dispatch_defaultpriority_key =__PTK_LIBDISPATCH_KEY5; #if DISPATCH_INTROSPECTION -static const unsigned long dispatch_introspection_key = __PTK_LIBDISPATCH_KEY5; +static const unsigned long dispatch_introspection_key =__PTK_LIBDISPATCH_KEY5+1; #elif DISPATCH_PERF_MON -static const unsigned long dispatch_bcounter_key = __PTK_LIBDISPATCH_KEY5; +static const unsigned long dispatch_bcounter_key = __PTK_LIBDISPATCH_KEY5+1; +#endif +#if DISPATCH_USE_OS_SEMAPHORE_CACHE +static const unsigned long dispatch_sema4_key = __TSD_SEMAPHORE_CACHE; +#else +static const unsigned long dispatch_sema4_key = __PTK_LIBDISPATCH_KEY5+2; #endif +#ifndef __TSD_THREAD_QOS_CLASS +#define __TSD_THREAD_QOS_CLASS 4 +#endif +static const unsigned long dispatch_priority_key = __TSD_THREAD_QOS_CLASS; + DISPATCH_TSD_INLINE static inline void _dispatch_thread_key_create(const unsigned long *k, void (*d)(void *)) { + if (!*k || !d) return; dispatch_assert_zero(pthread_key_init_np((int)*k, d)); } #else extern pthread_key_t dispatch_queue_key; +extern pthread_key_t dispatch_voucher_key; #if DISPATCH_USE_OS_SEMAPHORE_CACHE #error "Invalid DISPATCH_USE_OS_SEMAPHORE_CACHE configuration" #else @@ -73,6 +82,7 @@ extern pthread_key_t dispatch_sema4_key; extern pthread_key_t dispatch_cache_key; extern pthread_key_t dispatch_io_key; extern pthread_key_t dispatch_apply_key; +extern pthread_key_t dispatch_defaultpriority_key; #if DISPATCH_INTROSPECTION extern pthread_key_t dispatch_introspection_key; #elif DISPATCH_PERF_MON @@ -127,6 +137,17 @@ _dispatch_thread_getspecific(pthread_key_t k) #endif #endif +#if TARGET_OS_WIN32 +#define _dispatch_thread_port() ((mach_port_t)0) +#else +#if DISPATCH_USE_DIRECT_TSD +#define _dispatch_thread_port() ((mach_port_t)_dispatch_thread_getspecific(\ + _PTHREAD_TSD_SLOT_MACH_THREAD_SELF)) +#else +#define _dispatch_thread_port() (pthread_mach_thread_np(_dispatch_thread_self())) +#endif +#endif + DISPATCH_TSD_INLINE DISPATCH_CONST static inline unsigned int _dispatch_cpu_number(void) diff --git a/src/shims/yield.h b/src/shims/yield.h new file mode 100644 index 0000000..2a884d6 --- /dev/null +++ b/src/shims/yield.h @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2013 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +/* + * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch + * which are subject to change in future releases of Mac OS X. Any applications + * relying on these interfaces WILL break. + */ + +#ifndef __DISPATCH_SHIMS_YIELD__ +#define __DISPATCH_SHIMS_YIELD__ + +#pragma mark - +#pragma mark _dispatch_wait_until + +#if DISPATCH_HW_CONFIG_UP +#define _dispatch_wait_until(c) do { \ + int _spins = 0; \ + while (!(c)) { \ + _spins++; \ + _dispatch_preemption_yield(_spins); \ + } } while (0) +#elif TARGET_OS_EMBEDDED +// +#ifndef DISPATCH_WAIT_SPINS +#define DISPATCH_WAIT_SPINS 1024 +#endif +#define _dispatch_wait_until(c) do { \ + int _spins = -(DISPATCH_WAIT_SPINS); \ + while (!(c)) { \ + if (slowpath(_spins++ >= 0)) { \ + _dispatch_preemption_yield(_spins); \ + } else { \ + dispatch_hardware_pause(); \ + } \ + } } while (0) +#else +#define _dispatch_wait_until(c) do { \ + while (!(c)) { \ + dispatch_hardware_pause(); \ + } } while (0) +#endif + +#pragma mark - +#pragma mark _dispatch_contention_wait_until + +#if DISPATCH_HW_CONFIG_UP +#define _dispatch_contention_wait_until(c) false +#else +#ifndef DISPATCH_CONTENTION_SPINS_MAX +#define DISPATCH_CONTENTION_SPINS_MAX (128 - 1) +#endif +#ifndef DISPATCH_CONTENTION_SPINS_MIN +#define DISPATCH_CONTENTION_SPINS_MIN (32 - 1) +#endif +#if TARGET_OS_EMBEDDED +#define _dispatch_contention_spins() \ + ((DISPATCH_CONTENTION_SPINS_MIN) + ((DISPATCH_CONTENTION_SPINS_MAX) - \ + (DISPATCH_CONTENTION_SPINS_MIN)) / 2) +#else +// Use randomness to prevent threads from resonating at the same +// frequency and permanently contending. All threads sharing the same +// seed value is safe with the FreeBSD rand_r implementation. +#define _dispatch_contention_spins() ({ \ + static unsigned int _seed; \ + ((unsigned int)rand_r(&_seed) & (DISPATCH_CONTENTION_SPINS_MAX)) | \ + (DISPATCH_CONTENTION_SPINS_MIN); }) +#endif +#define _dispatch_contention_wait_until(c) ({ \ + bool _out = false; \ + unsigned int _spins = _dispatch_contention_spins(); \ + while (_spins--) { \ + dispatch_hardware_pause(); \ + if ((_out = fastpath(c))) break; \ + }; _out; }) +#endif + +#pragma mark - +#pragma mark dispatch_hardware_pause + +#if defined(__x86_64__) || defined(__i386__) +#define dispatch_hardware_pause() __asm__("pause") +#elif (defined(__arm__) && defined(_ARM_ARCH_7) && defined(__thumb__)) || \ + defined(__arm64__) +#define dispatch_hardware_pause() __asm__("yield") +#define dispatch_hardware_wfe() __asm__("wfe") +#else +#define dispatch_hardware_pause() __asm__("") +#endif + +#pragma mark - +#pragma mark _dispatch_preemption_yield + +#if HAVE_MACH +#if defined(SWITCH_OPTION_OSLOCK_DEPRESS) && !(TARGET_IPHONE_SIMULATOR && \ + IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1090) +#define DISPATCH_YIELD_THREAD_SWITCH_OPTION SWITCH_OPTION_OSLOCK_DEPRESS +#else +#define DISPATCH_YIELD_THREAD_SWITCH_OPTION SWITCH_OPTION_DEPRESS +#endif +#define _dispatch_preemption_yield(n) _dispatch_thread_switch(MACH_PORT_NULL, \ + DISPATCH_YIELD_THREAD_SWITCH_OPTION, (mach_msg_timeout_t)(n)) +#else +#define _dispatch_preemption_yield(n) pthread_yield_np() +#endif // HAVE_MACH + +#pragma mark - +#pragma mark _dispatch_contention_usleep + +#ifndef DISPATCH_CONTENTION_USLEEP_START +#define DISPATCH_CONTENTION_USLEEP_START 500 +#endif +#ifndef DISPATCH_CONTENTION_USLEEP_MAX +#define DISPATCH_CONTENTION_USLEEP_MAX 100000 +#endif + +#if HAVE_MACH +#if defined(SWITCH_OPTION_DISPATCH_CONTENTION) && !(TARGET_IPHONE_SIMULATOR && \ + IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 1090) +#define _dispatch_contention_usleep(u) _dispatch_thread_switch(MACH_PORT_NULL, \ + SWITCH_OPTION_DISPATCH_CONTENTION, (u)) +#else +#define _dispatch_contention_usleep(u) _dispatch_thread_switch(MACH_PORT_NULL, \ + SWITCH_OPTION_WAIT, (((u)-1)/1000)+1) +#endif +#else +#define _dispatch_contention_usleep(u) usleep((u)) +#endif // HAVE_MACH + +#pragma mark - +#pragma mark _dispatch_thread_switch + +#if HAVE_MACH +#define _dispatch_thread_switch(thread_name, option, option_time) \ + thread_switch((thread_name), (option), (option_time)) + +#endif // HAVE_MACH + +#endif // __DISPATCH_SHIMS_YIELD__ diff --git a/src/source.c b/src/source.c index c6aaae3..b593ae0 100644 --- a/src/source.c +++ b/src/source.c @@ -202,23 +202,46 @@ dispatch_source_testcancel(dispatch_source_t ds) return (bool)(ds->ds_atomic_flags & DSF_CANCELED); } - unsigned long dispatch_source_get_mask(dispatch_source_t ds) { - return ds->ds_pending_data_mask; + unsigned long mask = ds->ds_pending_data_mask; + if (ds->ds_vmpressure_override) { + mask = NOTE_VM_PRESSURE; + } +#if TARGET_IPHONE_SIMULATOR + else if (ds->ds_memorystatus_override) { + mask = NOTE_MEMORYSTATUS_PRESSURE_WARN; + } +#endif + return mask; } uintptr_t dispatch_source_get_handle(dispatch_source_t ds) { - return (unsigned int)ds->ds_ident_hack; + unsigned int handle = (unsigned int)ds->ds_ident_hack; +#if TARGET_IPHONE_SIMULATOR + if (ds->ds_memorystatus_override) { + handle = 0; + } +#endif + return handle; } unsigned long dispatch_source_get_data(dispatch_source_t ds) { - return ds->ds_data; + unsigned long data = ds->ds_data; + if (ds->ds_vmpressure_override) { + data = NOTE_VM_PRESSURE; + } +#if TARGET_IPHONE_SIMULATOR + else if (ds->ds_memorystatus_override) { + data = NOTE_MEMORYSTATUS_PRESSURE_WARN; + } +#endif + return data; } void @@ -239,157 +262,172 @@ dispatch_source_merge_data(dispatch_source_t ds, unsigned long val) #pragma mark - #pragma mark dispatch_source_handler -#ifdef __BLOCKS__ -// 6618342 Contact the team that owns the Instrument DTrace probe before -// renaming this symbol -static void -_dispatch_source_set_event_handler2(void *context) +DISPATCH_ALWAYS_INLINE +static inline dispatch_continuation_t +_dispatch_source_handler_alloc(dispatch_source_t ds, void *handler, long kind, + bool block) { - dispatch_source_t ds = (dispatch_source_t)_dispatch_queue_get_current(); - dispatch_assert(dx_type(ds) == DISPATCH_SOURCE_KEVENT_TYPE); - dispatch_source_refs_t dr = ds->ds_refs; - - if (ds->ds_handler_is_block && dr->ds_handler_ctxt) { - Block_release(dr->ds_handler_ctxt); + dispatch_continuation_t dc = _dispatch_continuation_alloc(); + if (handler) { + dc->do_vtable = (void *)((block ? DISPATCH_OBJ_BLOCK_RELEASE_BIT : + DISPATCH_OBJ_CTXT_FETCH_BIT) | (kind != DS_EVENT_HANDLER ? + DISPATCH_OBJ_ASYNC_BIT : 0l)); + dc->dc_priority = 0; + dc->dc_voucher = NULL; + if (block) { +#ifdef __BLOCKS__ + if (slowpath(_dispatch_block_has_private_data(handler))) { + // sources don't propagate priority by default + dispatch_block_flags_t flags = DISPATCH_BLOCK_NO_QOS_CLASS; + flags |= _dispatch_block_get_flags(handler); + _dispatch_continuation_priority_set(dc, + _dispatch_block_get_priority(handler), flags); + } + if (kind != DS_EVENT_HANDLER) { + dc->dc_func = _dispatch_call_block_and_release; + } else { + dc->dc_func = _dispatch_Block_invoke(handler); + } + dc->dc_ctxt = _dispatch_Block_copy(handler); +#endif /* __BLOCKS__ */ + } else { + dc->dc_func = handler; + dc->dc_ctxt = ds->do_ctxt; + } + _dispatch_trace_continuation_push((dispatch_queue_t)ds, dc); + } else { + dc->dc_func = NULL; } - dr->ds_handler_func = context ? _dispatch_Block_invoke(context) : NULL; - dr->ds_handler_ctxt = context; - ds->ds_handler_is_block = true; -} - -void -dispatch_source_set_event_handler(dispatch_source_t ds, - dispatch_block_t handler) -{ - handler = _dispatch_Block_copy(handler); - _dispatch_barrier_trysync_f((dispatch_queue_t)ds, handler, - _dispatch_source_set_event_handler2); + dc->dc_data = (void*)kind; + return dc; } -#endif /* __BLOCKS__ */ -static void -_dispatch_source_set_event_handler_f(void *context) +static inline void +_dispatch_source_handler_replace(dispatch_source_refs_t dr, long kind, + dispatch_continuation_t dc_new) { - dispatch_source_t ds = (dispatch_source_t)_dispatch_queue_get_current(); - dispatch_assert(dx_type(ds) == DISPATCH_SOURCE_KEVENT_TYPE); - dispatch_source_refs_t dr = ds->ds_refs; - + dispatch_continuation_t dc = dr->ds_handler[kind]; + if (dc) { #ifdef __BLOCKS__ - if (ds->ds_handler_is_block && dr->ds_handler_ctxt) { - Block_release(dr->ds_handler_ctxt); + if ((long)dc->do_vtable & DISPATCH_OBJ_BLOCK_RELEASE_BIT) { + Block_release(dc->dc_ctxt); + } +#endif /* __BLOCKS__ */ + if (dc->dc_voucher) { + _voucher_release(dc->dc_voucher); + dc->dc_voucher = NULL; + } + _dispatch_continuation_free(dc); } -#endif - dr->ds_handler_func = context; - dr->ds_handler_ctxt = ds->do_ctxt; - ds->ds_handler_is_block = false; + dr->ds_handler[kind] = dc_new; } -void -dispatch_source_set_event_handler_f(dispatch_source_t ds, - dispatch_function_t handler) +static inline void +_dispatch_source_handler_free(dispatch_source_refs_t dr, long kind) { - _dispatch_barrier_trysync_f((dispatch_queue_t)ds, handler, - _dispatch_source_set_event_handler_f); + _dispatch_source_handler_replace(dr, kind, NULL); } -#ifdef __BLOCKS__ -// 6618342 Contact the team that owns the Instrument DTrace probe before -// renaming this symbol static void -_dispatch_source_set_cancel_handler2(void *context) +_dispatch_source_set_handler(void *context) { dispatch_source_t ds = (dispatch_source_t)_dispatch_queue_get_current(); dispatch_assert(dx_type(ds) == DISPATCH_SOURCE_KEVENT_TYPE); - dispatch_source_refs_t dr = ds->ds_refs; - - if (ds->ds_cancel_is_block && dr->ds_cancel_handler) { - Block_release(dr->ds_cancel_handler); + dispatch_continuation_t dc = context; + long kind = (long)dc->dc_data; + dc->dc_data = 0; + if (!dc->dc_func) { + _dispatch_continuation_free(dc); + dc = NULL; + } else if ((long)dc->do_vtable & DISPATCH_OBJ_CTXT_FETCH_BIT) { + dc->dc_ctxt = ds->do_ctxt; + } + _dispatch_source_handler_replace(ds->ds_refs, kind, dc); + if (kind == DS_EVENT_HANDLER && dc && dc->dc_priority) { +#if HAVE_PTHREAD_WORKQUEUE_QOS + ds->dq_priority = dc->dc_priority & ~_PTHREAD_PRIORITY_FLAGS_MASK; + _dispatch_queue_set_override_priority((dispatch_queue_t)ds); +#endif } - dr->ds_cancel_handler = context; - ds->ds_cancel_is_block = true; } +#ifdef __BLOCKS__ void -dispatch_source_set_cancel_handler(dispatch_source_t ds, - dispatch_block_t handler) +dispatch_source_set_event_handler(dispatch_source_t ds, + dispatch_block_t handler) { - handler = _dispatch_Block_copy(handler); - _dispatch_barrier_trysync_f((dispatch_queue_t)ds, handler, - _dispatch_source_set_cancel_handler2); + dispatch_continuation_t dc; + dc = _dispatch_source_handler_alloc(ds, handler, DS_EVENT_HANDLER, true); + _dispatch_barrier_trysync_f((dispatch_queue_t)ds, dc, + _dispatch_source_set_handler); } #endif /* __BLOCKS__ */ -static void -_dispatch_source_set_cancel_handler_f(void *context) +void +dispatch_source_set_event_handler_f(dispatch_source_t ds, + dispatch_function_t handler) { - dispatch_source_t ds = (dispatch_source_t)_dispatch_queue_get_current(); - dispatch_assert(dx_type(ds) == DISPATCH_SOURCE_KEVENT_TYPE); - dispatch_source_refs_t dr = ds->ds_refs; - -#ifdef __BLOCKS__ - if (ds->ds_cancel_is_block && dr->ds_cancel_handler) { - Block_release(dr->ds_cancel_handler); - } -#endif - dr->ds_cancel_handler = context; - ds->ds_cancel_is_block = false; + dispatch_continuation_t dc; + dc = _dispatch_source_handler_alloc(ds, handler, DS_EVENT_HANDLER, false); + _dispatch_barrier_trysync_f((dispatch_queue_t)ds, dc, + _dispatch_source_set_handler); } void -dispatch_source_set_cancel_handler_f(dispatch_source_t ds, - dispatch_function_t handler) +_dispatch_source_set_event_handler_with_context_f(dispatch_source_t ds, + void *ctxt, dispatch_function_t handler) { - _dispatch_barrier_trysync_f((dispatch_queue_t)ds, handler, - _dispatch_source_set_cancel_handler_f); + dispatch_continuation_t dc; + dc = _dispatch_source_handler_alloc(ds, handler, DS_EVENT_HANDLER, false); + dc->do_vtable = (void *)((long)dc->do_vtable &~DISPATCH_OBJ_CTXT_FETCH_BIT); + dc->dc_other = dc->dc_ctxt; + dc->dc_ctxt = ctxt; + _dispatch_barrier_trysync_f((dispatch_queue_t)ds, dc, + _dispatch_source_set_handler); } #ifdef __BLOCKS__ -static void -_dispatch_source_set_registration_handler2(void *context) -{ - dispatch_source_t ds = (dispatch_source_t)_dispatch_queue_get_current(); - dispatch_assert(dx_type(ds) == DISPATCH_SOURCE_KEVENT_TYPE); - dispatch_source_refs_t dr = ds->ds_refs; - - if (ds->ds_registration_is_block && dr->ds_registration_handler) { - Block_release(dr->ds_registration_handler); - } - dr->ds_registration_handler = context; - ds->ds_registration_is_block = true; -} - void -dispatch_source_set_registration_handler(dispatch_source_t ds, - dispatch_block_t handler) +dispatch_source_set_cancel_handler(dispatch_source_t ds, + dispatch_block_t handler) { - handler = _dispatch_Block_copy(handler); - _dispatch_barrier_trysync_f((dispatch_queue_t)ds, handler, - _dispatch_source_set_registration_handler2); + dispatch_continuation_t dc; + dc = _dispatch_source_handler_alloc(ds, handler, DS_CANCEL_HANDLER, true); + _dispatch_barrier_trysync_f((dispatch_queue_t)ds, dc, + _dispatch_source_set_handler); } #endif /* __BLOCKS__ */ -static void -_dispatch_source_set_registration_handler_f(void *context) +void +dispatch_source_set_cancel_handler_f(dispatch_source_t ds, + dispatch_function_t handler) { - dispatch_source_t ds = (dispatch_source_t)_dispatch_queue_get_current(); - dispatch_assert(dx_type(ds) == DISPATCH_SOURCE_KEVENT_TYPE); - dispatch_source_refs_t dr = ds->ds_refs; + dispatch_continuation_t dc; + dc = _dispatch_source_handler_alloc(ds, handler, DS_CANCEL_HANDLER, false); + _dispatch_barrier_trysync_f((dispatch_queue_t)ds, dc, + _dispatch_source_set_handler); +} #ifdef __BLOCKS__ - if (ds->ds_registration_is_block && dr->ds_registration_handler) { - Block_release(dr->ds_registration_handler); - } -#endif - dr->ds_registration_handler = context; - ds->ds_registration_is_block = false; +void +dispatch_source_set_registration_handler(dispatch_source_t ds, + dispatch_block_t handler) +{ + dispatch_continuation_t dc; + dc = _dispatch_source_handler_alloc(ds, handler, DS_REGISTN_HANDLER, true); + _dispatch_barrier_trysync_f((dispatch_queue_t)ds, dc, + _dispatch_source_set_handler); } +#endif /* __BLOCKS__ */ void dispatch_source_set_registration_handler_f(dispatch_source_t ds, dispatch_function_t handler) { - _dispatch_barrier_trysync_f((dispatch_queue_t)ds, handler, - _dispatch_source_set_registration_handler_f); + dispatch_continuation_t dc; + dc = _dispatch_source_handler_alloc(ds, handler, DS_REGISTN_HANDLER, false); + _dispatch_barrier_trysync_f((dispatch_queue_t)ds, dc, + _dispatch_source_set_handler); } #pragma mark - @@ -399,68 +437,43 @@ static void _dispatch_source_registration_callout(dispatch_source_t ds) { dispatch_source_refs_t dr = ds->ds_refs; - + dispatch_continuation_t dc = dr->ds_handler[DS_REGISTN_HANDLER]; if ((ds->ds_atomic_flags & DSF_CANCELED) || (ds->do_xref_cnt == -1)) { // no registration callout if source is canceled rdar://problem/8955246 -#ifdef __BLOCKS__ - if (ds->ds_registration_is_block) { - Block_release(dr->ds_registration_handler); - } - } else if (ds->ds_registration_is_block) { - dispatch_block_t b = dr->ds_registration_handler; - _dispatch_client_callout_block(b); - Block_release(dr->ds_registration_handler); -#endif - } else { - dispatch_function_t f = dr->ds_registration_handler; - _dispatch_client_callout(ds->do_ctxt, f); + return _dispatch_source_handler_free(dr, DS_REGISTN_HANDLER); } - ds->ds_registration_is_block = false; - dr->ds_registration_handler = NULL; + pthread_priority_t old_dp = _dispatch_set_defaultpriority(ds->dq_priority); + if ((long)dc->do_vtable & DISPATCH_OBJ_CTXT_FETCH_BIT) { + dc->dc_ctxt = ds->do_ctxt; + } + _dispatch_continuation_pop(dc); + dr->ds_handler[DS_REGISTN_HANDLER] = NULL; + _dispatch_reset_defaultpriority(old_dp); } static void _dispatch_source_cancel_callout(dispatch_source_t ds) { dispatch_source_refs_t dr = ds->ds_refs; - + dispatch_continuation_t dc = dr->ds_handler[DS_CANCEL_HANDLER]; ds->ds_pending_data_mask = 0; ds->ds_pending_data = 0; ds->ds_data = 0; - -#ifdef __BLOCKS__ - if (ds->ds_handler_is_block) { - Block_release(dr->ds_handler_ctxt); - ds->ds_handler_is_block = false; - dr->ds_handler_func = NULL; - dr->ds_handler_ctxt = NULL; - } - if (ds->ds_registration_is_block) { - Block_release(dr->ds_registration_handler); - ds->ds_registration_is_block = false; - dr->ds_registration_handler = NULL; - } -#endif - - if (!dr->ds_cancel_handler) { + _dispatch_source_handler_free(dr, DS_EVENT_HANDLER); + _dispatch_source_handler_free(dr, DS_REGISTN_HANDLER); + if (!dc) { return; } - if (ds->ds_cancel_is_block) { -#ifdef __BLOCKS__ - dispatch_block_t b = dr->ds_cancel_handler; - if (ds->ds_atomic_flags & DSF_CANCELED) { - _dispatch_client_callout_block(b); - } - Block_release(dr->ds_cancel_handler); - ds->ds_cancel_is_block = false; -#endif - } else { - dispatch_function_t f = dr->ds_cancel_handler; - if (ds->ds_atomic_flags & DSF_CANCELED) { - _dispatch_client_callout(ds->do_ctxt, f); - } + if (!(ds->ds_atomic_flags & DSF_CANCELED)) { + return _dispatch_source_handler_free(dr, DS_CANCEL_HANDLER); } - dr->ds_cancel_handler = NULL; + pthread_priority_t old_dp = _dispatch_set_defaultpriority(ds->dq_priority); + if ((long)dc->do_vtable & DISPATCH_OBJ_CTXT_FETCH_BIT) { + dc->dc_ctxt = ds->do_ctxt; + } + _dispatch_continuation_pop(dc); + dr->ds_handler[DS_CANCEL_HANDLER] = NULL; + _dispatch_reset_defaultpriority(old_dp); } static void @@ -472,6 +485,7 @@ _dispatch_source_latch_and_call(dispatch_source_t ds) return; } dispatch_source_refs_t dr = ds->ds_refs; + dispatch_continuation_t dc = dr->ds_handler[DS_EVENT_HANDLER]; prev = dispatch_atomic_xchg2o(ds, ds_pending_data, 0, relaxed); if (ds->ds_is_level) { ds->ds_data = ~prev; @@ -480,9 +494,17 @@ _dispatch_source_latch_and_call(dispatch_source_t ds) } else { ds->ds_data = prev; } - if (dispatch_assume(prev) && dr->ds_handler_func) { - _dispatch_client_callout(dr->ds_handler_ctxt, dr->ds_handler_func); + if (!dispatch_assume(prev) || !dc) { + return; } + pthread_priority_t old_dp = _dispatch_set_defaultpriority(ds->dq_priority); + _dispatch_trace_continuation_pop(_dispatch_queue_get_current(), dc); + voucher_t voucher = dc->dc_voucher ? _voucher_retain(dc->dc_voucher) : NULL; + _dispatch_continuation_voucher_adopt(dc); // consumes voucher reference + _dispatch_client_callout(dc->dc_ctxt, dc->dc_func); + _dispatch_introspection_queue_item_complete(dc); + if (voucher) dc->dc_voucher = voucher; + _dispatch_reset_defaultpriority(old_dp); } static void @@ -568,7 +590,7 @@ _dispatch_source_invoke2(dispatch_object_t dou, } _dispatch_source_kevent_register(ds); ds->ds_is_installed = true; - if (dr->ds_registration_handler) { + if (dr->ds_handler[DS_REGISTN_HANDLER]) { return ds->do_targetq; } if (slowpath(ds->do_xref_cnt == -1)) { @@ -577,7 +599,7 @@ _dispatch_source_invoke2(dispatch_object_t dou, } else if (slowpath(DISPATCH_OBJECT_SUSPENDED(ds))) { // Source suspended by an item drained from the source queue. return NULL; - } else if (dr->ds_registration_handler) { + } else if (dr->ds_handler[DS_REGISTN_HANDLER]) { // The source has been registered and the registration handler needs // to be delivered on the target queue. if (dq != ds->do_targetq) { @@ -598,8 +620,9 @@ _dispatch_source_invoke2(dispatch_object_t dou, } _dispatch_source_kevent_unregister(ds); } - if (dr->ds_cancel_handler || ds->ds_handler_is_block || - ds->ds_registration_is_block) { + if (dr->ds_handler[DS_EVENT_HANDLER] || + dr->ds_handler[DS_CANCEL_HANDLER] || + dr->ds_handler[DS_REGISTN_HANDLER]) { if (dq != ds->do_targetq) { return ds->do_targetq; } @@ -645,18 +668,16 @@ _dispatch_source_probe(dispatch_source_t ds) if (!ds->ds_is_installed) { // The source needs to be installed on the manager queue. return true; - } else if (dr->ds_registration_handler) { + } else if (dr->ds_handler[DS_REGISTN_HANDLER]) { // The registration handler needs to be delivered to the target queue. return true; } else if ((ds->ds_atomic_flags & DSF_CANCELED) || (ds->do_xref_cnt == -1)){ // The source needs to be uninstalled from the manager queue, or the // cancellation handler needs to be delivered to the target queue. // Note: cancellation assumes installation. - if (ds->ds_dkev || dr->ds_cancel_handler -#ifdef __BLOCKS__ - || ds->ds_handler_is_block || ds->ds_registration_is_block -#endif - ) { + if (ds->ds_dkev || dr->ds_handler[DS_EVENT_HANDLER] || + dr->ds_handler[DS_CANCEL_HANDLER] || + dr->ds_handler[DS_REGISTN_HANDLER]) { return true; } } else if (ds->ds_pending_data) { @@ -666,7 +687,7 @@ _dispatch_source_probe(dispatch_source_t ds) // The source needs to be rearmed on the manager queue. return true; } - return (ds->dq_items_tail != NULL); + return _dispatch_queue_class_probe(ds); } static void @@ -957,18 +978,6 @@ _dispatch_kevent_drain(struct kevent64_s *ke) } else if (ke->data == ESRCH) { return _dispatch_kevent_proc_exit(ke); } -#if DISPATCH_USE_VM_PRESSURE - } else if (ke->filter == EVFILT_VM && ke->data == ENOTSUP) { - // Memory pressure kevent is not supported on all platforms - // - return; -#endif -#if DISPATCH_USE_MEMORYSTATUS - } else if (ke->filter == EVFILT_MEMORYSTATUS && - (ke->data == EINVAL || ke->data == ENOTSUP)) { - // Memory status kevent is not supported on all platforms - return; -#endif } return _dispatch_kevent_error(ke); } @@ -1060,7 +1069,7 @@ _dispatch_kevent_unguard(dispatch_kevent_t dk) #pragma mark - #pragma mark dispatch_source_timer -#if DISPATCH_USE_DTRACE && DISPATCH_USE_DTRACE_INTROSPECTION +#if DISPATCH_USE_DTRACE static dispatch_source_refs_t _dispatch_trace_next_timer[DISPATCH_TIMER_QOS_COUNT]; #define _dispatch_trace_next_timer_set(x, q) \ @@ -1175,7 +1184,7 @@ _dispatch_source_set_timer2(void *context) // Called on the source queue struct dispatch_set_timer_params *params = context; dispatch_suspend(params->ds); - dispatch_barrier_async_f(&_dispatch_mgr_q, params, + _dispatch_barrier_async_detached_f(&_dispatch_mgr_q, params, _dispatch_source_set_timer3); } @@ -1464,8 +1473,9 @@ _dispatch_timers_init(void) DISPATCH_KEVENT_TIMER_UDATA(tidx); } #endif // __LP64__ - _dispatch_timers_force_max_leeway = - getenv("LIBDISPATCH_TIMERS_FORCE_MAX_LEEWAY"); + if (slowpath(getenv("LIBDISPATCH_TIMERS_FORCE_MAX_LEEWAY"))) { + _dispatch_timers_force_max_leeway = true; + } } static inline void @@ -1517,8 +1527,9 @@ _dispatch_timers_update(dispatch_source_t ds) if (tidx != DISPATCH_TIMER_INDEX_DISARM) { (void)dispatch_atomic_or2o(ds, ds_atomic_flags, DSF_ARMED, relaxed); } - free(dk); _dispatch_object_debug(ds, "%s", __func__); + ds->ds_dkev = NULL; + free(dk); } else { _dispatch_timers_unregister(ds, dk); } @@ -1796,9 +1807,9 @@ dispatch_timer_aggregate_create(void) dispatch_timer_aggregate_t dta = _dispatch_alloc(DISPATCH_VTABLE(queue), sizeof(struct dispatch_timer_aggregate_s)); _dispatch_queue_init((dispatch_queue_t)dta); - dta->do_targetq = _dispatch_get_root_queue(DISPATCH_QUEUE_PRIORITY_HIGH, - true); - dta->dq_width = UINT32_MAX; + dta->do_targetq = _dispatch_get_root_queue( + _DISPATCH_QOS_CLASS_USER_INITIATED, true); + dta->dq_width = DISPATCH_QUEUE_WIDTH_MAX; //FIXME: aggregates need custom vtable //dta->dq_label = "timer-aggregate"; for (tidx = 0; tidx < DISPATCH_KEVENT_TIMER_COUNT; tidx++) { @@ -1869,7 +1880,7 @@ _dispatch_timer_aggregates_configure(void) } dtau = _dispatch_calloc(DISPATCH_TIMER_COUNT, sizeof(*dtau)); memcpy(dtau, dta->dta_timer, sizeof(dta->dta_timer)); - dispatch_barrier_async_f((dispatch_queue_t)dta, dtau, + _dispatch_barrier_async_detached_f((dispatch_queue_t)dta, dtau, _dispatch_timer_aggregate_update); } } @@ -2099,8 +2110,25 @@ _dispatch_kq_init(void *context DISPATCH_UNUSED) _dispatch_kq = kqueue(); #endif if (_dispatch_kq == -1) { - DISPATCH_CLIENT_CRASH("kqueue() create failed: " - "probably out of file descriptors"); + int err = errno; + switch (err) { + case EMFILE: + DISPATCH_CLIENT_CRASH("kqueue() failure: " + "process is out of file descriptors"); + break; + case ENFILE: + DISPATCH_CLIENT_CRASH("kqueue() failure: " + "system is out of file descriptors"); + break; + case ENOMEM: + DISPATCH_CLIENT_CRASH("kqueue() failure: " + "kernel is out of memory"); + break; + default: + (void)dispatch_assume_zero(err); + DISPATCH_CRASH("kqueue() failure"); + break; + } } else if (dispatch_assume(_dispatch_kq < FD_SETSIZE)) { // in case we fall back to select() FD_SET(_dispatch_kq, &_dispatch_rfds); @@ -2108,7 +2136,7 @@ _dispatch_kq_init(void *context DISPATCH_UNUSED) (void)dispatch_assume_zero(kevent64(_dispatch_kq, &kev, 1, NULL, 0, 0, NULL)); - _dispatch_queue_push(_dispatch_mgr_q.do_targetq, &_dispatch_mgr_q); + _dispatch_queue_push(_dispatch_mgr_q.do_targetq, &_dispatch_mgr_q, 0); } static int @@ -2239,6 +2267,7 @@ _dispatch_mgr_invoke(void) poll = _dispatch_mgr_select(poll); if (!poll) continue; } + poll = poll || _dispatch_queue_class_probe(&_dispatch_mgr_q); r = kevent64(_dispatch_kq, _dispatch_kevent_enable, _dispatch_kevent_enable ? 1 : 0, &kev, 1, 0, poll ? &timeout_immediately : NULL); @@ -2295,10 +2324,12 @@ _dispatch_memorystatus_handler(void *context DISPATCH_UNUSED) memorystatus = dispatch_source_get_data(_dispatch_memorystatus_source); if (memorystatus & DISPATCH_MEMORYSTATUS_PRESSURE_NORMAL) { _dispatch_continuation_cache_limit = DISPATCH_CONTINUATION_CACHE_LIMIT; + _voucher_activity_heap_pressure_normal(); return; } _dispatch_continuation_cache_limit = DISPATCH_CONTINUATION_CACHE_LIMIT_MEMORYSTATUS_PRESSURE_WARN; + _voucher_activity_heap_pressure_warn(); #endif malloc_zone_pressure_relief(0,0); } @@ -2309,7 +2340,7 @@ _dispatch_memorystatus_init(void) _dispatch_memorystatus_source = dispatch_source_create( DISPATCH_MEMORYSTATUS_SOURCE_TYPE, 0, DISPATCH_MEMORYSTATUS_SOURCE_MASK, - _dispatch_get_root_queue(0, true)); + _dispatch_get_root_queue(_DISPATCH_QOS_CLASS_DEFAULT, true)); dispatch_source_set_event_handler_f(_dispatch_memorystatus_source, _dispatch_memorystatus_handler); dispatch_resume(_dispatch_memorystatus_source); @@ -2352,11 +2383,15 @@ static inline void _dispatch_memorystatus_init(void) {} #ifndef MACH_RCV_LARGE_IDENTITY #define MACH_RCV_LARGE_IDENTITY 0x00000008 #endif +#ifndef MACH_RCV_VOUCHER +#define MACH_RCV_VOUCHER 0x00000800 +#endif #define DISPATCH_MACH_RCV_TRAILER MACH_RCV_TRAILER_CTX #define DISPATCH_MACH_RCV_OPTIONS ( \ MACH_RCV_MSG | MACH_RCV_LARGE | MACH_RCV_LARGE_IDENTITY | \ MACH_RCV_TRAILER_ELEMENTS(DISPATCH_MACH_RCV_TRAILER) | \ - MACH_RCV_TRAILER_TYPE(MACH_MSG_TRAILER_FORMAT_0)) + MACH_RCV_TRAILER_TYPE(MACH_MSG_TRAILER_FORMAT_0)) | \ + MACH_RCV_VOUCHER #define DISPATCH_MACH_KEVENT_ARMED(dk) ((dk)->dk_kevent.ext[0]) @@ -2373,18 +2408,21 @@ static kern_return_t _dispatch_mach_notify_update(dispatch_kevent_t dk, static void _dispatch_mach_notify_source_invoke(mach_msg_header_t *hdr); static void _dispatch_mach_reply_kevent_unregister(dispatch_mach_t dm, dispatch_mach_reply_refs_t dmr, bool disconnected); -static void _dispatch_mach_msg_recv(dispatch_mach_t dm, mach_msg_header_t *hdr, +static void _dispatch_mach_kevent_unregister(dispatch_mach_t dm); +static inline void _dispatch_mach_msg_set_options(dispatch_object_t dou, + mach_msg_option_t options); +static void _dispatch_mach_msg_recv(dispatch_mach_t dm, + dispatch_mach_reply_refs_t dmr, mach_msg_header_t *hdr, mach_msg_size_t siz); static void _dispatch_mach_merge_kevent(dispatch_mach_t dm, const struct kevent64_s *ke); -static void _dispatch_mach_kevent_unregister(dispatch_mach_t dm); +static inline mach_msg_option_t _dispatch_mach_checkin_options(void); static const size_t _dispatch_mach_recv_msg_size = DISPATCH_MACH_RECEIVE_MAX_INLINE_MESSAGE_SIZE; static const size_t dispatch_mach_trailer_size = sizeof(dispatch_mach_trailer_t); -static const size_t _dispatch_mach_recv_msg_buf_size = mach_vm_round_page( - _dispatch_mach_recv_msg_size + dispatch_mach_trailer_size); +static mach_msg_size_t _dispatch_mach_recv_msg_buf_size; static mach_port_t _dispatch_mach_portset, _dispatch_mach_recv_portset; static mach_port_t _dispatch_mach_notify_port; static struct kevent64_s _dispatch_mach_recv_kevent = { @@ -2405,7 +2443,9 @@ struct dispatch_source_type_s _dispatch_source_type_mach_recv_direct = { static void _dispatch_mach_recv_msg_buf_init(void) { - mach_vm_size_t vm_size = _dispatch_mach_recv_msg_buf_size; + mach_vm_size_t vm_size = mach_vm_round_page( + _dispatch_mach_recv_msg_size + dispatch_mach_trailer_size); + _dispatch_mach_recv_msg_buf_size = (mach_msg_size_t)vm_size; mach_vm_address_t vm_addr = vm_page_size; kern_return_t kr; @@ -2419,7 +2459,7 @@ _dispatch_mach_recv_msg_buf_init(void) vm_addr = vm_page_size; } _dispatch_mach_recv_kevent.ext[0] = (uintptr_t)vm_addr; - _dispatch_mach_recv_kevent.ext[1] = _dispatch_mach_recv_msg_buf_size; + _dispatch_mach_recv_kevent.ext[1] = vm_size; } static inline void* @@ -2457,8 +2497,11 @@ _dispatch_mach_recv_portset_init(void *context DISPATCH_UNUSED) _dispatch_mach_notify_source = dispatch_source_create( &_dispatch_source_type_mach_recv_direct, _dispatch_mach_notify_port, 0, &_dispatch_mgr_q); - _dispatch_mach_notify_source->ds_refs->ds_handler_func = - (void*)_dispatch_mach_notify_source_invoke; + static const struct dispatch_continuation_s dc = { + .dc_func = (void*)_dispatch_mach_notify_source_invoke, + }; + _dispatch_mach_notify_source->ds_refs->ds_handler[DS_EVENT_HANDLER] = + (dispatch_continuation_t)&dc; dispatch_assert(_dispatch_mach_notify_source); dispatch_resume(_dispatch_mach_notify_source); } @@ -2772,11 +2815,11 @@ _dispatch_source_merge_mach_msg(dispatch_source_t ds, dispatch_source_refs_t dr, _dispatch_mach_notify_source_invoke(hdr); return _dispatch_kevent_mach_msg_destroy(hdr); } + dispatch_mach_reply_refs_t dmr = NULL; if (dk->dk_kevent.fflags & DISPATCH_MACH_RECV_MESSAGE_DIRECT_ONCE) { - _dispatch_mach_reply_kevent_unregister((dispatch_mach_t)ds, - (dispatch_mach_reply_refs_t)dr, false); + dmr = (dispatch_mach_reply_refs_t)dr; } - return _dispatch_mach_msg_recv((dispatch_mach_t)ds, hdr, siz); + return _dispatch_mach_msg_recv((dispatch_mach_t)ds, dmr, hdr, siz); } DISPATCH_ALWAYS_INLINE @@ -2911,7 +2954,7 @@ _dispatch_mach_host_notify_update(void *context DISPATCH_UNUSED) { (void)_dispatch_get_mach_recv_portset(); _dispatch_debug("registering for calendar-change notification"); - kern_return_t kr = host_request_notification(mach_host_self(), + kern_return_t kr = host_request_notification(_dispatch_get_mach_host_port(), HOST_NOTIFY_CALENDAR_CHANGE, _dispatch_mach_notify_port); DISPATCH_VERIFY_MIG(kr); (void)dispatch_assume_zero(kr); @@ -2994,17 +3037,20 @@ _dispatch_mach_notify_send_possible(mach_port_t notify DISPATCH_UNUSED, #pragma mark dispatch_mach_t #define DISPATCH_MACH_NEVER_CONNECTED (UINT32_MAX/2) -#define DISPATCH_MACH_PSEUDO_RECEIVED 0x1 #define DISPATCH_MACH_REGISTER_FOR_REPLY 0x2 #define DISPATCH_MACH_OPTIONS_MASK 0xffff static mach_port_t _dispatch_mach_msg_get_remote_port(dispatch_object_t dou); static void _dispatch_mach_msg_disconnected(dispatch_mach_t dm, mach_port_t local_port, mach_port_t remote_port); +static dispatch_mach_msg_t _dispatch_mach_msg_create_reply_disconnected( + dispatch_object_t dou, dispatch_mach_reply_refs_t dmr); static bool _dispatch_mach_reconnect_invoke(dispatch_mach_t dm, dispatch_object_t dou); static inline mach_msg_header_t* _dispatch_mach_msg_get_msg( dispatch_mach_msg_t dmsg); +static void _dispatch_mach_push(dispatch_object_t dm, dispatch_object_t dou, + pthread_priority_t pp); static dispatch_mach_t _dispatch_mach_create(const char *label, dispatch_queue_t q, void *context, @@ -3028,7 +3074,7 @@ _dispatch_mach_create(const char *label, dispatch_queue_t q, void *context, dr->dm_handler_func = handler; dr->dm_handler_ctxt = context; dm->ds_refs = dr; - dm->ds_handler_is_block = handler_is_block; + dm->dm_handler_is_block = handler_is_block; dm->dm_refs = _dispatch_calloc(1ul, sizeof(struct dispatch_mach_send_refs_s)); @@ -3063,7 +3109,7 @@ _dispatch_mach_dispose(dispatch_mach_t dm) { _dispatch_object_debug(dm, "%s", __func__); dispatch_mach_refs_t dr = dm->ds_refs; - if (dm->ds_handler_is_block && dr->dm_handler_ctxt) { + if (dm->dm_handler_is_block && dr->dm_handler_ctxt) { Block_release(dr->dm_handler_ctxt); } free(dr); @@ -3093,6 +3139,8 @@ dispatch_mach_connect(dispatch_mach_t dm, mach_port_t receive, if (MACH_PORT_VALID(send)) { if (checkin) { dispatch_retain(checkin); + mach_msg_option_t options = _dispatch_mach_checkin_options(); + _dispatch_mach_msg_set_options(checkin, options); dr->dm_checkin_port = _dispatch_mach_msg_get_remote_port(checkin); } dr->dm_checkin = checkin; @@ -3112,21 +3160,23 @@ static void _dispatch_mach_reply_kevent_unregister(dispatch_mach_t dm, dispatch_mach_reply_refs_t dmr, bool disconnected) { - dispatch_kevent_t dk = dmr->dm_dkev; - mach_port_t local_port = (mach_port_t)dk->dk_kevent.ident; + dispatch_mach_msg_t dmsgr = NULL; + if (disconnected) { + dmsgr = _dispatch_mach_msg_create_reply_disconnected(NULL, dmr); + } + dispatch_kevent_t dk = dmr->dmr_dkev; TAILQ_REMOVE(&dk->dk_sources, (dispatch_source_refs_t)dmr, dr_list); _dispatch_kevent_unregister(dk, DISPATCH_MACH_RECV_MESSAGE_DIRECT_ONCE); - TAILQ_REMOVE(&dm->dm_refs->dm_replies, dmr, dm_list); + TAILQ_REMOVE(&dm->dm_refs->dm_replies, dmr, dmr_list); + if (dmr->dmr_voucher) _voucher_release(dmr->dmr_voucher); free(dmr); - if (disconnected) { - _dispatch_mach_msg_disconnected(dm, local_port, MACH_PORT_NULL); - } + if (dmsgr) _dispatch_mach_push(dm, dmsgr, dmsgr->dmsg_priority); } DISPATCH_NOINLINE static void _dispatch_mach_reply_kevent_register(dispatch_mach_t dm, mach_port_t reply, - void *ctxt) + dispatch_mach_msg_t dmsg) { dispatch_kevent_t dk; dispatch_mach_reply_refs_t dmr; @@ -3137,22 +3187,26 @@ _dispatch_mach_reply_kevent_register(dispatch_mach_t dm, mach_port_t reply, dk->dk_kevent.flags |= EV_ADD|EV_ENABLE; dk->dk_kevent.fflags = DISPATCH_MACH_RECV_MESSAGE_DIRECT_ONCE; dk->dk_kevent.udata = (uintptr_t)dk; - // make reply context visible to leaks rdar://11777199 - dk->dk_kevent.ext[1] = (uintptr_t)ctxt; TAILQ_INIT(&dk->dk_sources); dmr = _dispatch_calloc(1ul, sizeof(struct dispatch_mach_reply_refs_s)); dmr->dr_source_wref = _dispatch_ptr2wref(dm); - dmr->dm_dkev = dk; + dmr->dmr_dkev = dk; + if (dmsg->dmsg_voucher) { + dmr->dmr_voucher =_voucher_retain(dmsg->dmsg_voucher); + } + dmr->dmr_priority = dmsg->dmsg_priority; + // make reply context visible to leaks rdar://11777199 + dmr->dmr_ctxt = dmsg->do_ctxt; _dispatch_debug("machport[0x%08x]: registering for reply, ctxt %p", reply, - ctxt); + dmsg->do_ctxt); uint32_t flags; - bool do_resume = _dispatch_kevent_register(&dmr->dm_dkev, &flags); - TAILQ_INSERT_TAIL(&dmr->dm_dkev->dk_sources, (dispatch_source_refs_t)dmr, + bool do_resume = _dispatch_kevent_register(&dmr->dmr_dkev, &flags); + TAILQ_INSERT_TAIL(&dmr->dmr_dkev->dk_sources, (dispatch_source_refs_t)dmr, dr_list); - TAILQ_INSERT_TAIL(&dm->dm_refs->dm_replies, dmr, dm_list); - if (do_resume && _dispatch_kevent_resume(dmr->dm_dkev, flags, 0)) { + TAILQ_INSERT_TAIL(&dm->dm_refs->dm_replies, dmr, dmr_list); + if (do_resume && _dispatch_kevent_resume(dmr->dmr_dkev, flags, 0)) { _dispatch_mach_reply_kevent_unregister(dm, dmr, true); } } @@ -3198,9 +3252,10 @@ _dispatch_mach_kevent_register(dispatch_mach_t dm, mach_port_t send) } static inline void -_dispatch_mach_push(dispatch_object_t dm, dispatch_object_t dou) +_dispatch_mach_push(dispatch_object_t dm, dispatch_object_t dou, + pthread_priority_t pp) { - return _dispatch_queue_push(dm._dq, dou); + return _dispatch_queue_push(dm._dq, dou, pp); } static inline void @@ -3239,8 +3294,8 @@ _dispatch_mach_msg_get_reason(dispatch_object_t dou, mach_error_t *err_ptr) } static void -_dispatch_mach_msg_recv(dispatch_mach_t dm, mach_msg_header_t *hdr, - mach_msg_size_t siz) +_dispatch_mach_msg_recv(dispatch_mach_t dm, dispatch_mach_reply_refs_t dmr, + mach_msg_header_t *hdr, mach_msg_size_t siz) { _dispatch_debug_machport(hdr->msgh_remote_port); _dispatch_debug("machport[0x%08x]: received msg id 0x%x, reply on 0x%08x", @@ -3249,13 +3304,32 @@ _dispatch_mach_msg_recv(dispatch_mach_t dm, mach_msg_header_t *hdr, return _dispatch_kevent_mach_msg_destroy(hdr); } dispatch_mach_msg_t dmsg; + voucher_t voucher; + pthread_priority_t priority; + void *ctxt = NULL; + if (dmr) { + _voucher_mach_msg_clear(hdr, false); // deallocate reply message voucher + voucher = dmr->dmr_voucher; + dmr->dmr_voucher = NULL; // transfer reference + priority = dmr->dmr_priority; + ctxt = dmr->dmr_ctxt; + _dispatch_mach_reply_kevent_unregister(dm, dmr, false); + } else { + voucher = voucher_create_with_mach_msg(hdr); + priority = _voucher_get_priority(voucher); + } dispatch_mach_msg_destructor_t destructor; destructor = (hdr == _dispatch_get_mach_recv_msg_buf()) ? DISPATCH_MACH_MSG_DESTRUCTOR_DEFAULT : DISPATCH_MACH_MSG_DESTRUCTOR_FREE; dmsg = dispatch_mach_msg_create(hdr, siz, destructor, NULL); + dmsg->dmsg_voucher = voucher; + dmsg->dmsg_priority = priority; + dmsg->do_ctxt = ctxt; _dispatch_mach_msg_set_reason(dmsg, 0, DISPATCH_MACH_MESSAGE_RECEIVED); - return _dispatch_mach_push(dm, dmsg); + _dispatch_voucher_debug("mach-msg[%p] create", voucher, dmsg); + _dispatch_voucher_ktrace_dmsg_push(dmsg); + return _dispatch_mach_push(dm, dmsg, dmsg->dmsg_priority); } static inline mach_port_t @@ -3266,24 +3340,6 @@ _dispatch_mach_msg_get_remote_port(dispatch_object_t dou) return remote; } -static inline mach_port_t -_dispatch_mach_msg_get_reply_port(dispatch_mach_t dm, dispatch_object_t dou) -{ - mach_msg_header_t *hdr = _dispatch_mach_msg_get_msg(dou._dmsg); - mach_port_t reply = MACH_PORT_NULL; - mach_msg_option_t msg_opts = _dispatch_mach_msg_get_options(dou); - if (msg_opts & DISPATCH_MACH_PSEUDO_RECEIVED) { - reply = hdr->msgh_reserved; - hdr->msgh_reserved = 0; - } else if (MACH_MSGH_BITS_LOCAL(hdr->msgh_bits) == - MACH_MSG_TYPE_MAKE_SEND_ONCE && - MACH_PORT_VALID(hdr->msgh_local_port) && (!dm->ds_dkev || - dm->ds_dkev->dk_kevent.ident != hdr->msgh_local_port)) { - reply = hdr->msgh_local_port; - } - return reply; -} - static inline void _dispatch_mach_msg_disconnected(dispatch_mach_t dm, mach_port_t local_port, mach_port_t remote_port) @@ -3295,19 +3351,45 @@ _dispatch_mach_msg_disconnected(dispatch_mach_t dm, mach_port_t local_port, if (local_port) hdr->msgh_local_port = local_port; if (remote_port) hdr->msgh_remote_port = remote_port; _dispatch_mach_msg_set_reason(dmsg, 0, DISPATCH_MACH_DISCONNECTED); - return _dispatch_mach_push(dm, dmsg); + return _dispatch_mach_push(dm, dmsg, dmsg->dmsg_priority); +} + +static inline dispatch_mach_msg_t +_dispatch_mach_msg_create_reply_disconnected(dispatch_object_t dou, + dispatch_mach_reply_refs_t dmr) +{ + dispatch_mach_msg_t dmsg = dou._dmsg, dmsgr; + if (dmsg && !dmsg->dmsg_reply) return NULL; + mach_msg_header_t *hdr; + dmsgr = dispatch_mach_msg_create(NULL, sizeof(mach_msg_header_t), + DISPATCH_MACH_MSG_DESTRUCTOR_DEFAULT, &hdr); + if (dmsg) { + hdr->msgh_local_port = dmsg->dmsg_reply; + if (dmsg->dmsg_voucher) { + dmsgr->dmsg_voucher = _voucher_retain(dmsg->dmsg_voucher); + } + dmsgr->dmsg_priority = dmsg->dmsg_priority; + dmsgr->do_ctxt = dmsg->do_ctxt; + } else { + hdr->msgh_local_port = (mach_port_t)dmr->dmr_dkev->dk_kevent.ident; + dmsgr->dmsg_voucher = dmr->dmr_voucher; + dmr->dmr_voucher = NULL; // transfer reference + dmsgr->dmsg_priority = dmr->dmr_priority; + dmsgr->do_ctxt = dmr->dmr_ctxt; + } + _dispatch_mach_msg_set_reason(dmsgr, 0, DISPATCH_MACH_DISCONNECTED); + return dmsgr; } DISPATCH_NOINLINE static void _dispatch_mach_msg_not_sent(dispatch_mach_t dm, dispatch_object_t dou) { - mach_port_t reply = _dispatch_mach_msg_get_reply_port(dm, dou); - _dispatch_mach_msg_set_reason(dou, 0, DISPATCH_MACH_MESSAGE_NOT_SENT); - _dispatch_mach_push(dm, dou); - if (reply) { - _dispatch_mach_msg_disconnected(dm, reply, MACH_PORT_NULL); - } + dispatch_mach_msg_t dmsg = dou._dmsg, dmsgr; + dmsgr = _dispatch_mach_msg_create_reply_disconnected(dmsg, NULL); + _dispatch_mach_msg_set_reason(dmsg, 0, DISPATCH_MACH_MESSAGE_NOT_SENT); + _dispatch_mach_push(dm, dmsg, dmsg->dmsg_priority); + if (dmsgr) _dispatch_mach_push(dm, dmsgr, dmsgr->dmsg_priority); } DISPATCH_NOINLINE @@ -3315,7 +3397,10 @@ static dispatch_object_t _dispatch_mach_msg_send(dispatch_mach_t dm, dispatch_object_t dou) { dispatch_mach_send_refs_t dr = dm->dm_refs; - dispatch_mach_msg_t dmsg = dou._dmsg; + dispatch_mach_msg_t dmsg = dou._dmsg, dmsgr = NULL; + voucher_t voucher = dmsg->dmsg_voucher; + mach_voucher_t ipc_kvoucher = MACH_VOUCHER_NULL; + bool clear_voucher = false, kvoucher_move_send = false; dr->dm_needs_mgr = 0; if (slowpath(dr->dm_checkin) && dmsg != dr->dm_checkin) { // send initial checkin message @@ -3332,10 +3417,10 @@ _dispatch_mach_msg_send(dispatch_mach_t dm, dispatch_object_t dou) } mach_msg_header_t *msg = _dispatch_mach_msg_get_msg(dmsg); mach_msg_return_t kr = 0; - mach_port_t reply = _dispatch_mach_msg_get_reply_port(dm, dmsg); + mach_port_t reply = dmsg->dmsg_reply; mach_msg_option_t opts = 0, msg_opts = _dispatch_mach_msg_get_options(dmsg); if (!slowpath(msg_opts & DISPATCH_MACH_REGISTER_FOR_REPLY)) { - opts = MACH_SEND_MSG | (msg_opts & DISPATCH_MACH_OPTIONS_MASK); + opts = MACH_SEND_MSG | (msg_opts & ~DISPATCH_MACH_OPTIONS_MASK); if (MACH_MSGH_BITS_REMOTE(msg->msgh_bits) != MACH_MSG_TYPE_MOVE_SEND_ONCE) { if (dmsg != dr->dm_checkin) { @@ -3353,16 +3438,38 @@ _dispatch_mach_msg_send(dispatch_mach_t dm, dispatch_object_t dou) } } opts |= MACH_SEND_TIMEOUT; + if (dmsg->dmsg_priority != _voucher_get_priority(voucher)) { + ipc_kvoucher = _voucher_create_mach_voucher_with_priority( + voucher, dmsg->dmsg_priority); + } + _dispatch_voucher_debug("mach-msg[%p] msg_set", voucher, dmsg); + if (ipc_kvoucher) { + kvoucher_move_send = true; + clear_voucher = _voucher_mach_msg_set_mach_voucher(msg, + ipc_kvoucher, kvoucher_move_send); + } else { + clear_voucher = _voucher_mach_msg_set(msg, voucher); + } } + _voucher_activity_trace_msg(voucher, msg, send); _dispatch_debug_machport(msg->msgh_remote_port); if (reply) _dispatch_debug_machport(reply); kr = mach_msg(msg, opts, msg->msgh_size, 0, MACH_PORT_NULL, 0, MACH_PORT_NULL); + _dispatch_debug("machport[0x%08x]: sent msg id 0x%x, ctxt %p, " + "opts 0x%x, msg_opts 0x%x, kvoucher 0x%08x, reply on 0x%08x: " + "%s - 0x%x", msg->msgh_remote_port, msg->msgh_id, dmsg->do_ctxt, + opts, msg_opts, msg->msgh_voucher_port, reply, + mach_error_string(kr), kr); + if (clear_voucher) { + if (kr == MACH_SEND_INVALID_VOUCHER && msg->msgh_voucher_port) { + DISPATCH_CRASH("Voucher port corruption"); + } + mach_voucher_t kv; + kv = _voucher_mach_msg_clear(msg, kvoucher_move_send); + if (kvoucher_move_send) ipc_kvoucher = kv; + } } - _dispatch_debug("machport[0x%08x]: sent msg id 0x%x, ctxt %p, opts 0x%x, " - "msg_opts 0x%x, reply on 0x%08x: %s - 0x%x", msg->msgh_remote_port, - msg->msgh_id, dmsg->do_ctxt, opts, msg_opts, reply, - mach_error_string(kr), kr); if (kr == MACH_SEND_TIMED_OUT && (opts & MACH_SEND_TIMEOUT)) { if (opts & MACH_SEND_NOTIFY) { _dispatch_debug("machport[0x%08x]: send-possible notification " @@ -3372,42 +3479,50 @@ _dispatch_mach_msg_send(dispatch_mach_t dm, dispatch_object_t dou) // send kevent must be installed on the manager queue dr->dm_needs_mgr = 1; } - if (reply) { - _dispatch_mach_msg_set_options(dmsg, msg_opts | - DISPATCH_MACH_PSEUDO_RECEIVED); - msg->msgh_reserved = reply; // Remember the original reply port + if (ipc_kvoucher) { + _dispatch_kvoucher_debug("reuse on re-send", ipc_kvoucher); + voucher_t ipc_voucher; + ipc_voucher = _voucher_create_with_priority_and_mach_voucher( + voucher, dmsg->dmsg_priority, ipc_kvoucher); + _dispatch_voucher_debug("mach-msg[%p] replace voucher[%p]", + ipc_voucher, dmsg, voucher); + if (dmsg->dmsg_voucher) _voucher_release(dmsg->dmsg_voucher); + dmsg->dmsg_voucher = ipc_voucher; } goto out; + } else if (ipc_kvoucher && (kr || !kvoucher_move_send)) { + _voucher_dealloc_mach_voucher(ipc_kvoucher); } - if (fastpath(!kr) && reply) { + if (fastpath(!kr) && reply && + !(dm->ds_dkev && dm->ds_dkev->dk_kevent.ident == reply)) { if (_dispatch_queue_get_current() != &_dispatch_mgr_q) { // reply receive kevent must be installed on the manager queue dr->dm_needs_mgr = 1; _dispatch_mach_msg_set_options(dmsg, msg_opts | DISPATCH_MACH_REGISTER_FOR_REPLY); - if (msg_opts & DISPATCH_MACH_PSEUDO_RECEIVED) { - msg->msgh_reserved = reply; // Remember the original reply port - } goto out; } - _dispatch_mach_reply_kevent_register(dm, reply, dmsg->do_ctxt); + _dispatch_mach_reply_kevent_register(dm, reply, dmsg); } if (slowpath(dmsg == dr->dm_checkin) && dm->dm_dkev) { _dispatch_mach_kevent_unregister(dm); } - _dispatch_mach_msg_set_reason(dmsg, kr, 0); - _dispatch_mach_push(dm, dmsg); - dmsg = NULL; - if (slowpath(kr) && reply) { + if (slowpath(kr)) { // Send failed, so reply was never connected - _dispatch_mach_msg_disconnected(dm, reply, MACH_PORT_NULL); + dmsgr = _dispatch_mach_msg_create_reply_disconnected(dmsg, NULL); } + _dispatch_mach_msg_set_reason(dmsg, kr, 0); + _dispatch_mach_push(dm, dmsg, dmsg->dmsg_priority); + if (dmsgr) _dispatch_mach_push(dm, dmsgr, dmsgr->dmsg_priority); + dmsg = NULL; out: return (dispatch_object_t)dmsg; } -static void -_dispatch_mach_send_push(dispatch_mach_t dm, dispatch_object_t dou) +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_mach_send_push_wakeup(dispatch_mach_t dm, dispatch_object_t dou, + bool wakeup) { dispatch_mach_send_refs_t dr = dm->dm_refs; struct dispatch_object_s *prev, *dc = dou._do; @@ -3416,10 +3531,19 @@ _dispatch_mach_send_push(dispatch_mach_t dm, dispatch_object_t dou) prev = dispatch_atomic_xchg2o(dr, dm_tail, dc, release); if (fastpath(prev)) { prev->do_next = dc; - return; + } else { + dr->dm_head = dc; + } + if (wakeup || !prev) { + _dispatch_wakeup(dm); } - dr->dm_head = dc; - _dispatch_wakeup(dm); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_mach_send_push(dispatch_mach_t dm, dispatch_object_t dou) +{ + return _dispatch_mach_send_push_wakeup(dm, dou, false); } DISPATCH_NOINLINE @@ -3429,25 +3553,21 @@ _dispatch_mach_send_drain(dispatch_mach_t dm) dispatch_mach_send_refs_t dr = dm->dm_refs; struct dispatch_object_s *dc = NULL, *next_dc = NULL; while (dr->dm_tail) { - while (!(dc = fastpath(dr->dm_head))) { - dispatch_hardware_pause(); - } + _dispatch_wait_until(dc = fastpath(dr->dm_head)); do { next_dc = fastpath(dc->do_next); dr->dm_head = next_dc; if (!next_dc && !dispatch_atomic_cmpxchg2o(dr, dm_tail, dc, NULL, relaxed)) { - // Enqueue is TIGHTLY controlled, we won't wait long. - while (!(next_dc = fastpath(dc->do_next))) { - dispatch_hardware_pause(); - } + _dispatch_wait_until(next_dc = fastpath(dc->do_next)); dr->dm_head = next_dc; } if (!DISPATCH_OBJ_IS_VTABLE(dc)) { if ((long)dc->do_vtable & DISPATCH_OBJ_BARRIER_BIT) { // send barrier // leave send queue locked until barrier has completed - return _dispatch_mach_push(dm, dc); + return _dispatch_mach_push(dm, dc, + ((dispatch_continuation_t)dc)->dc_priority); } #if DISPATCH_MACH_SEND_SYNC if (slowpath((long)dc->do_vtable & DISPATCH_OBJ_SYNC_SLOW_BIT)){ @@ -3461,6 +3581,7 @@ _dispatch_mach_send_drain(dispatch_mach_t dm) } continue; } + _dispatch_voucher_ktrace_dmsg_pop((dispatch_mach_msg_t)dc); if (slowpath(dr->dm_disconnect_cnt) || slowpath(dm->ds_atomic_flags & DSF_CANCELED)) { _dispatch_mach_msg_not_sent(dm, dc); @@ -3477,9 +3598,7 @@ out: if (!next_dc && !dispatch_atomic_cmpxchg2o(dr, dm_tail, NULL, dc, relaxed)) { // wait for enqueue slow path to finish - while (!(next_dc = fastpath(dr->dm_head))) { - dispatch_hardware_pause(); - } + _dispatch_wait_until(next_dc = fastpath(dr->dm_head)); dc->do_next = next_dc; } dr->dm_head = dc; @@ -3510,6 +3629,24 @@ _dispatch_mach_merge_kevent(dispatch_mach_t dm, const struct kevent64_s *ke) _dispatch_mach_send(dm); } +static inline mach_msg_option_t +_dispatch_mach_checkin_options(void) +{ + mach_msg_option_t options = 0; +#if DISPATCH_USE_CHECKIN_NOIMPORTANCE + options = MACH_SEND_NOIMPORTANCE; // +#endif + return options; +} + + +static inline mach_msg_option_t +_dispatch_mach_send_options(void) +{ + mach_msg_option_t options = 0; + return options; +} + DISPATCH_NOINLINE void dispatch_mach_send(dispatch_mach_t dm, dispatch_mach_msg_t dmsg, @@ -3521,18 +3658,32 @@ dispatch_mach_send(dispatch_mach_t dm, dispatch_mach_msg_t dmsg, } dispatch_retain(dmsg); dispatch_assert_zero(options & DISPATCH_MACH_OPTIONS_MASK); + options |= _dispatch_mach_send_options(); _dispatch_mach_msg_set_options(dmsg, options & ~DISPATCH_MACH_OPTIONS_MASK); - if (slowpath(dr->dm_tail) || slowpath(dr->dm_disconnect_cnt) || + mach_msg_header_t *msg = _dispatch_mach_msg_get_msg(dmsg); + dmsg->dmsg_reply = (MACH_MSGH_BITS_LOCAL(msg->msgh_bits) == + MACH_MSG_TYPE_MAKE_SEND_ONCE && + MACH_PORT_VALID(msg->msgh_local_port) ? msg->msgh_local_port : + MACH_PORT_NULL); + bool is_reply = (MACH_MSGH_BITS_REMOTE(msg->msgh_bits) == + MACH_MSG_TYPE_MOVE_SEND_ONCE); + dmsg->dmsg_priority = _dispatch_priority_propagate(); + dmsg->dmsg_voucher = _voucher_copy(); + _dispatch_voucher_debug("mach-msg[%p] set", dmsg->dmsg_voucher, dmsg); + if ((!is_reply && slowpath(dr->dm_tail)) || + slowpath(dr->dm_disconnect_cnt) || slowpath(dm->ds_atomic_flags & DSF_CANCELED) || slowpath(!dispatch_atomic_cmpxchg2o(dr, dm_sending, 0, 1, acquire))) { + _dispatch_voucher_ktrace_dmsg_push(dmsg); return _dispatch_mach_send_push(dm, dmsg); } if (slowpath(dmsg = _dispatch_mach_msg_send(dm, dmsg)._dmsg)) { (void)dispatch_atomic_dec2o(dr, dm_sending, release); - return _dispatch_mach_send_push(dm, dmsg); + _dispatch_voucher_ktrace_dmsg_push(dmsg); + return _dispatch_mach_send_push_wakeup(dm, dmsg, true); } - if (slowpath(dr->dm_tail)) { + if (!is_reply && slowpath(dr->dm_tail)) { return _dispatch_mach_send_drain(dm); } (void)dispatch_atomic_dec2o(dr, dm_sending, release); @@ -3556,7 +3707,7 @@ _dispatch_mach_disconnect(dispatch_mach_t dm) } if (!TAILQ_EMPTY(&dm->dm_refs->dm_replies)) { dispatch_mach_reply_refs_t dmr, tmp; - TAILQ_FOREACH_SAFE(dmr, &dm->dm_refs->dm_replies, dm_list, tmp){ + TAILQ_FOREACH_SAFE(dmr, &dm->dm_refs->dm_replies, dmr_list, tmp){ _dispatch_mach_reply_kevent_unregister(dm, dmr, true); } } @@ -3610,6 +3761,8 @@ dispatch_mach_reconnect(dispatch_mach_t dm, mach_port_t send, (void)dispatch_atomic_inc2o(dr, dm_disconnect_cnt, relaxed); if (MACH_PORT_VALID(send) && checkin) { dispatch_retain(checkin); + mach_msg_option_t options = _dispatch_mach_checkin_options(); + _dispatch_mach_msg_set_options(checkin, options); dr->dm_checkin_port = _dispatch_mach_msg_get_remote_port(checkin); } else { checkin = NULL; @@ -3672,6 +3825,11 @@ _dispatch_mach_msg_invoke(dispatch_mach_msg_t dmsg) dmsg->do_next = DISPATCH_OBJECT_LISTLESS; _dispatch_thread_setspecific(dispatch_queue_key, dm->do_targetq); + _dispatch_voucher_ktrace_dmsg_pop(dmsg); + _dispatch_voucher_debug("mach-msg[%p] adopt", dmsg->dmsg_voucher, dmsg); + _dispatch_adopt_priority_and_replace_voucher(dmsg->dmsg_priority, + dmsg->dmsg_voucher, DISPATCH_PRIORITY_ENFORCE); + dmsg->dmsg_voucher = NULL; if (slowpath(!dm->dm_connect_handler_called)) { _dispatch_mach_connect_invoke(dm); } @@ -3717,6 +3875,8 @@ dispatch_mach_send_barrier_f(dispatch_mach_t dm, void *context, dc->dc_ctxt = dc; dc->dc_data = context; dc->dc_other = barrier; + _dispatch_continuation_voucher_set(dc, 0); + _dispatch_continuation_priority_set(dc, 0, 0); dispatch_mach_send_refs_t dr = dm->dm_refs; if (slowpath(dr->dm_tail) || slowpath(!dispatch_atomic_cmpxchg2o(dr, @@ -3724,7 +3884,7 @@ dispatch_mach_send_barrier_f(dispatch_mach_t dm, void *context, return _dispatch_mach_send_push(dm, dc); } // leave send queue locked until barrier has completed - return _dispatch_mach_push(dm, dc); + return _dispatch_mach_push(dm, dc, dc->dc_priority); } DISPATCH_NOINLINE @@ -3738,7 +3898,10 @@ dispatch_mach_receive_barrier_f(dispatch_mach_t dm, void *context, dc->dc_ctxt = dc; dc->dc_data = context; dc->dc_other = barrier; - return _dispatch_mach_push(dm, dc); + _dispatch_continuation_voucher_set(dc, 0); + _dispatch_continuation_priority_set(dc, 0, 0); + + return _dispatch_mach_push(dm, dc, dc->dc_priority); } DISPATCH_NOINLINE @@ -3824,6 +3987,10 @@ _dispatch_mach_invoke2(dispatch_object_t dou, // An item on the channel changed the target queue return dm->do_targetq; } + } else if (dr->dm_sending) { + // Sending and uninstallation below require the send lock, the channel + // will be woken up when the lock is dropped + return NULL; } else if (dr->dm_tail) { if (slowpath(dr->dm_needs_mgr) || (slowpath(dr->dm_disconnect_cnt) && (dm->dm_dkev || !TAILQ_EMPTY(&dm->dm_refs->dm_replies)))) { @@ -3878,9 +4045,13 @@ _dispatch_mach_probe(dispatch_mach_t dm) if (slowpath(!dm->ds_is_installed)) { // The channel needs to be installed on the manager queue. return true; - } else if (dm->dq_items_tail) { + } else if (_dispatch_queue_class_probe(dm)) { // The source has pending messages to deliver to the target queue. return true; + } else if (dr->dm_sending) { + // Sending and uninstallation below require the send lock, the channel + // will be woken up when the lock is dropped + return false; } else if (dr->dm_tail && (!(dm->dm_dkev && DISPATCH_MACH_KEVENT_ARMED(dm->dm_dkev)) || (dm->ds_atomic_flags & DSF_CANCELED) || dr->dm_disconnect_cnt)) { @@ -3913,16 +4084,17 @@ dispatch_mach_msg_create(mach_msg_header_t *msg, size_t size, } dispatch_mach_msg_t dmsg = _dispatch_alloc(DISPATCH_VTABLE(mach_msg), sizeof(struct dispatch_mach_msg_s) + - (destructor ? 0 : size - sizeof(dmsg->msg))); + (destructor ? 0 : size - sizeof(dmsg->dmsg_msg))); if (destructor) { - dmsg->msg = msg; + dmsg->dmsg_msg = msg; } else if (msg) { - memcpy(dmsg->buf, msg, size); + memcpy(dmsg->dmsg_buf, msg, size); } dmsg->do_next = DISPATCH_OBJECT_LISTLESS; - dmsg->do_targetq = _dispatch_get_root_queue(0, false); - dmsg->destructor = destructor; - dmsg->size = size; + dmsg->do_targetq = _dispatch_get_root_queue(_DISPATCH_QOS_CLASS_DEFAULT, + false); + dmsg->dmsg_destructor = destructor; + dmsg->dmsg_size = size; if (msg_ptr) { *msg_ptr = _dispatch_mach_msg_get_msg(dmsg); } @@ -3932,15 +4104,19 @@ dispatch_mach_msg_create(mach_msg_header_t *msg, size_t size, void _dispatch_mach_msg_dispose(dispatch_mach_msg_t dmsg) { - switch (dmsg->destructor) { + if (dmsg->dmsg_voucher) { + _voucher_release(dmsg->dmsg_voucher); + dmsg->dmsg_voucher = NULL; + } + switch (dmsg->dmsg_destructor) { case DISPATCH_MACH_MSG_DESTRUCTOR_DEFAULT: break; case DISPATCH_MACH_MSG_DESTRUCTOR_FREE: - free(dmsg->msg); + free(dmsg->dmsg_msg); break; case DISPATCH_MACH_MSG_DESTRUCTOR_VM_DEALLOCATE: { - mach_vm_size_t vm_size = dmsg->size; - mach_vm_address_t vm_addr = (uintptr_t)dmsg->msg; + mach_vm_size_t vm_size = dmsg->dmsg_size; + mach_vm_address_t vm_addr = (uintptr_t)dmsg->dmsg_msg; (void)dispatch_assume_zero(mach_vm_deallocate(mach_task_self(), vm_addr, vm_size)); break; @@ -3950,14 +4126,15 @@ _dispatch_mach_msg_dispose(dispatch_mach_msg_t dmsg) static inline mach_msg_header_t* _dispatch_mach_msg_get_msg(dispatch_mach_msg_t dmsg) { - return dmsg->destructor ? dmsg->msg : (mach_msg_header_t*)dmsg->buf; + return dmsg->dmsg_destructor ? dmsg->dmsg_msg : + (mach_msg_header_t*)dmsg->dmsg_buf; } mach_msg_header_t* dispatch_mach_msg_get_msg(dispatch_mach_msg_t dmsg, size_t *size_ptr) { if (size_ptr) { - *size_ptr = dmsg->size; + *size_ptr = dmsg->dmsg_size; } return _dispatch_mach_msg_get_msg(dmsg); } @@ -3971,7 +4148,7 @@ _dispatch_mach_msg_debug(dispatch_mach_msg_t dmsg, char* buf, size_t bufsiz) offset += dsnprintf(&buf[offset], bufsiz - offset, "xrefcnt = 0x%x, " "refcnt = 0x%x, ", dmsg->do_xref_cnt + 1, dmsg->do_ref_cnt + 1); offset += dsnprintf(&buf[offset], bufsiz - offset, "opts/err = 0x%x, " - "msgh[%p] = { ", dmsg->do_suspend_cnt, dmsg->buf); + "msgh[%p] = { ", dmsg->do_suspend_cnt, dmsg->dmsg_buf); mach_msg_header_t *hdr = _dispatch_mach_msg_get_msg(dmsg); if (hdr->msgh_id) { offset += dsnprintf(&buf[offset], bufsiz - offset, "id 0x%x, ", @@ -4016,7 +4193,7 @@ dispatch_mig_server(dispatch_source_t ds, size_t maxmsgsz, { mach_msg_options_t options = MACH_RCV_MSG | MACH_RCV_TIMEOUT | MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_CTX) - | MACH_RCV_TRAILER_TYPE(MACH_MSG_TRAILER_FORMAT_0); + | MACH_RCV_TRAILER_TYPE(MACH_MSG_TRAILER_FORMAT_0) | MACH_RCV_VOUCHER; mach_msg_options_t tmp_options; mig_reply_error_t *bufTemp, *bufRequest, *bufReply; mach_msg_return_t kr = 0; @@ -4126,7 +4303,7 @@ dispatch_mig_server(dispatch_source_t ds, size_t maxmsgsz, (void)dispatch_assume_zero(r); } #endif - + _voucher_replace(voucher_create_with_mach_msg(&bufRequest->Head)); demux_success = callback(&bufRequest->Head, &bufReply->Head); if (!demux_success) { @@ -4264,7 +4441,8 @@ _dispatch_mach_debug(dispatch_mach_t dm, char* buf, size_t bufsiz) { size_t offset = 0; offset += dsnprintf(&buf[offset], bufsiz - offset, "%s[%p] = { ", - dm->dq_label ? dm->dq_label : dx_kind(dm), dm); + dm->dq_label && !dm->dm_cancel_handler_called ? dm->dq_label : + dx_kind(dm), dm); offset += _dispatch_object_debug_attr(dm, &buf[offset], bufsiz - offset); offset += _dispatch_mach_debug_attr(dm, &buf[offset], bufsiz - offset); offset += dsnprintf(&buf[offset], bufsiz - offset, "}"); @@ -4483,8 +4661,8 @@ dispatch_debug_machport(mach_port_t name, const char* str) MACH_PORT_RIGHT_DEAD_NAME, &nd)); } if (type & (MACH_PORT_TYPE_RECEIVE|MACH_PORT_TYPE_SEND)) { - (void)dispatch_assume_zero(mach_port_dnrequest_info(mach_task_self(), - name, &dnrsiz, &dnreqs)); + kr = mach_port_dnrequest_info(mach_task_self(), name, &dnrsiz, &dnreqs); + if (kr != KERN_INVALID_RIGHT) (void)dispatch_assume_zero(kr); } if (type & MACH_PORT_TYPE_RECEIVE) { mach_port_status_t status = { .mps_pset = 0, }; diff --git a/src/source_internal.h b/src/source_internal.h index 1a023cf..12ccdda 100644 --- a/src/source_internal.h +++ b/src/source_internal.h @@ -123,15 +123,18 @@ struct dispatch_timer_source_s { unsigned long missed; }; +enum { + DS_EVENT_HANDLER = 0, + DS_CANCEL_HANDLER, + DS_REGISTN_HANDLER, +}; + // Source state which may contain references to the source object // Separately allocated so that 'leaks' can see sources typedef struct dispatch_source_refs_s { TAILQ_ENTRY(dispatch_source_refs_s) dr_list; uintptr_t dr_source_wref; // "weak" backref to dispatch_source_t - dispatch_function_t ds_handler_func; - void *ds_handler_ctxt; - void *ds_cancel_handler; - void *ds_registration_handler; + dispatch_continuation_t ds_handler[3]; } *dispatch_source_refs_t; typedef struct dispatch_timer_source_refs_s { @@ -176,9 +179,9 @@ _dispatch_source_timer_idx(dispatch_source_refs_t dr) ds_is_installed:1, \ ds_needs_rearm:1, \ ds_is_timer:1, \ - ds_cancel_is_block:1, \ - ds_handler_is_block:1, \ - ds_registration_is_block:1, \ + ds_vmpressure_override:1, \ + ds_memorystatus_override:1, \ + dm_handler_is_block:1, \ dm_connect_handler_called:1, \ dm_cancel_handler_called:1; \ unsigned long ds_pending_data_mask; @@ -206,8 +209,11 @@ typedef struct dispatch_mach_refs_s *dispatch_mach_refs_t; struct dispatch_mach_reply_refs_s { TAILQ_ENTRY(dispatch_mach_reply_refs_s) dr_list; uintptr_t dr_source_wref; // "weak" backref to dispatch_mach_t - dispatch_kevent_t dm_dkev; - TAILQ_ENTRY(dispatch_mach_reply_refs_s) dm_list; + dispatch_kevent_t dmr_dkev; + void *dmr_ctxt; + pthread_priority_t dmr_priority; + voucher_t dmr_voucher; + TAILQ_ENTRY(dispatch_mach_reply_refs_s) dmr_list; }; typedef struct dispatch_mach_reply_refs_s *dispatch_mach_reply_refs_t; @@ -237,11 +243,14 @@ struct dispatch_mach_s { DISPATCH_CLASS_DECL(mach_msg); struct dispatch_mach_msg_s { DISPATCH_STRUCT_HEADER(mach_msg); - dispatch_mach_msg_destructor_t destructor; - size_t size; + mach_port_t dmsg_reply; + pthread_priority_t dmsg_priority; + voucher_t dmsg_voucher; + dispatch_mach_msg_destructor_t dmsg_destructor; + size_t dmsg_size; union { - mach_msg_header_t *msg; - char buf[0]; + mach_msg_header_t *dmsg_msg; + char dmsg_buf[0]; }; }; @@ -257,6 +266,8 @@ void _dispatch_source_invoke(dispatch_source_t ds); unsigned long _dispatch_source_probe(dispatch_source_t ds); size_t _dispatch_source_debug(dispatch_source_t ds, char* buf, size_t bufsiz); void _dispatch_source_set_interval(dispatch_source_t ds, uint64_t interval); +void _dispatch_source_set_event_handler_with_context_f(dispatch_source_t ds, + void *ctxt, dispatch_function_t handler); void _dispatch_mach_dispose(dispatch_mach_t dm); void _dispatch_mach_invoke(dispatch_mach_t dm); diff --git a/src/time.c b/src/time.c index a1a8924..35b0e52 100644 --- a/src/time.c +++ b/src/time.c @@ -44,7 +44,9 @@ _dispatch_get_nanoseconds(void) #if !(defined(__i386__) || defined(__x86_64__) || !HAVE_MACH_ABSOLUTE_TIME) \ || TARGET_OS_WIN32 -DISPATCH_CACHELINE_ALIGN _dispatch_host_time_data_s _dispatch_host_time_data; +DISPATCH_CACHELINE_ALIGN _dispatch_host_time_data_s _dispatch_host_time_data = { + .ratio_1_to_1 = true, +}; void _dispatch_get_host_time_init(void *context DISPATCH_UNUSED) diff --git a/src/trace.h b/src/trace.h index d6b31fe..df27ca8 100644 --- a/src/trace.h +++ b/src/trace.h @@ -27,16 +27,17 @@ #ifndef __DISPATCH_TRACE__ #define __DISPATCH_TRACE__ -#if DISPATCH_USE_DTRACE && !__OBJC2__ +#if !__OBJC2__ +#if DISPATCH_USE_DTRACE || DISPATCH_USE_DTRACE_INTROSPECTION typedef struct dispatch_trace_timer_params_s { int64_t deadline, interval, leeway; } *dispatch_trace_timer_params_t; #include "provider.h" +#endif // DISPATCH_USE_DTRACE || DISPATCH_USE_DTRACE_INTROSPECTION #if DISPATCH_USE_DTRACE_INTROSPECTION - #define _dispatch_trace_callout(_c, _f, _dcc) do { \ if (slowpath(DISPATCH_CALLOUT_ENTRY_ENABLED()) || \ slowpath(DISPATCH_CALLOUT_RETURN_ENABLED())) { \ @@ -51,7 +52,12 @@ typedef struct dispatch_trace_timer_params_s { _dcc; \ } \ } while (0) +#elif DISPATCH_INTROSPECTION +#define _dispatch_trace_callout(_c, _f, _dcc) \ + do { (void)(_c); (void)(_f); _dcc; } while (0) +#endif // DISPATCH_USE_DTRACE_INTROSPECTION || DISPATCH_INTROSPECTION +#if DISPATCH_USE_DTRACE_INTROSPECTION || DISPATCH_INTROSPECTION DISPATCH_ALWAYS_INLINE static inline void _dispatch_trace_client_callout(void *ctxt, dispatch_function_t f) @@ -73,40 +79,33 @@ _dispatch_trace_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t)) _dispatch_introspection_callout_return(ctxt, func); } -#ifdef __BLOCKS__ -DISPATCH_ALWAYS_INLINE -static inline void -_dispatch_trace_client_callout_block(dispatch_block_t b) -{ - dispatch_function_t func = _dispatch_Block_invoke(b); - _dispatch_introspection_callout_entry(b, func); - _dispatch_trace_callout(b, func, _dispatch_client_callout(b, func)); - _dispatch_introspection_callout_return(b, func); -} -#endif - #define _dispatch_client_callout _dispatch_trace_client_callout #define _dispatch_client_callout2 _dispatch_trace_client_callout2 -#define _dispatch_client_callout_block _dispatch_trace_client_callout_block +#endif // DISPATCH_USE_DTRACE_INTROSPECTION || DISPATCH_INTROSPECTION +#if DISPATCH_USE_DTRACE_INTROSPECTION #define _dispatch_trace_continuation(_q, _o, _t) do { \ dispatch_queue_t _dq = (_q); \ const char *_label = _dq && _dq->dq_label ? _dq->dq_label : ""; \ struct dispatch_object_s *_do = (_o); \ + dispatch_continuation_t _dc; \ char *_kind; \ dispatch_function_t _func; \ void *_ctxt; \ if (DISPATCH_OBJ_IS_VTABLE(_do)) { \ - _ctxt = _do->do_ctxt; \ _kind = (char*)dx_kind(_do); \ if ((dx_type(_do) & _DISPATCH_META_TYPE_MASK) == \ _DISPATCH_SOURCE_TYPE && (_dq) != &_dispatch_mgr_q) { \ - _func = ((dispatch_source_t)_do)->ds_refs->ds_handler_func; \ + dispatch_source_t _ds = (dispatch_source_t)_do; \ + _dc = _ds->ds_refs->ds_handler[DS_EVENT_HANDLER]; \ + _func = _dc->dc_func; \ + _ctxt = _dc->dc_ctxt; \ } else { \ _func = (dispatch_function_t)_dispatch_queue_invoke; \ + _ctxt = _do->do_ctxt; \ } \ } else { \ - struct dispatch_continuation_s *_dc = (void*)(_do); \ + _dc = (void*)_do; \ _ctxt = _dc->dc_ctxt; \ if ((long)_dc->do_vtable & DISPATCH_OBJ_SYNC_SLOW_BIT) { \ _kind = "semaphore"; \ @@ -121,11 +120,18 @@ _dispatch_trace_client_callout_block(dispatch_block_t b) } \ _t(_dq, _label, _do, _kind, _func, _ctxt); \ } while (0) - +#elif DISPATCH_INTROSPECTION +#define _dispatch_trace_continuation(_q, _o, _t) \ + do { (void)(_q); (void)(_o); } while(0) +#define DISPATCH_QUEUE_PUSH_ENABLED() 0 +#define DISPATCH_QUEUE_POP_ENABLED() 0 +#endif // DISPATCH_USE_DTRACE_INTROSPECTION || DISPATCH_INTROSPECTION + +#if DISPATCH_USE_DTRACE_INTROSPECTION || DISPATCH_INTROSPECTION DISPATCH_ALWAYS_INLINE static inline void _dispatch_trace_queue_push_list(dispatch_queue_t dq, dispatch_object_t _head, - dispatch_object_t _tail, unsigned int n) + dispatch_object_t _tail, pthread_priority_t pp, unsigned int n) { if (slowpath(DISPATCH_QUEUE_PUSH_ENABLED())) { struct dispatch_object_s *dou = _head._do; @@ -134,39 +140,50 @@ _dispatch_trace_queue_push_list(dispatch_queue_t dq, dispatch_object_t _head, } while (dou != _tail._do && (dou = dou->do_next)); } _dispatch_introspection_queue_push_list(dq, _head, _tail); - _dispatch_queue_push_list(dq, _head, _tail, n); + _dispatch_queue_push_list(dq, _head, _tail, pp, n); } DISPATCH_ALWAYS_INLINE static inline void -_dispatch_trace_queue_push(dispatch_queue_t dq, dispatch_object_t _tail) +_dispatch_trace_queue_push(dispatch_queue_t dq, dispatch_object_t _tail, pthread_priority_t pp) { if (slowpath(DISPATCH_QUEUE_PUSH_ENABLED())) { struct dispatch_object_s *dou = _tail._do; _dispatch_trace_continuation(dq, dou, DISPATCH_QUEUE_PUSH); } _dispatch_introspection_queue_push(dq, _tail); - _dispatch_queue_push(dq, _tail); + _dispatch_queue_push(dq, _tail, pp); } DISPATCH_ALWAYS_INLINE static inline void _dispatch_trace_queue_push_wakeup(dispatch_queue_t dq, dispatch_object_t _tail, - bool wakeup) + pthread_priority_t pp, bool wakeup) +{ + if (slowpath(DISPATCH_QUEUE_PUSH_ENABLED())) { + struct dispatch_object_s *dou = _tail._do; + _dispatch_trace_continuation(dq, dou, DISPATCH_QUEUE_PUSH); + } + _dispatch_introspection_queue_push(dq, _tail); + _dispatch_queue_push_wakeup(dq, _tail, pp, wakeup); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_trace_continuation_push(dispatch_queue_t dq, dispatch_object_t _tail) { if (slowpath(DISPATCH_QUEUE_PUSH_ENABLED())) { struct dispatch_object_s *dou = _tail._do; _dispatch_trace_continuation(dq, dou, DISPATCH_QUEUE_PUSH); } _dispatch_introspection_queue_push(dq, _tail); - _dispatch_queue_push_wakeup(dq, _tail, wakeup); } DISPATCH_ALWAYS_INLINE static inline void -_dispatch_queue_push_notrace(dispatch_queue_t dq, dispatch_object_t dou) +_dispatch_queue_push_notrace(dispatch_queue_t dq, dispatch_object_t dou, pthread_priority_t pp) { - _dispatch_queue_push(dq, dou); + _dispatch_queue_push(dq, dou, pp); } #define _dispatch_queue_push_list _dispatch_trace_queue_push_list @@ -175,24 +192,30 @@ _dispatch_queue_push_notrace(dispatch_queue_t dq, dispatch_object_t dou) DISPATCH_ALWAYS_INLINE static inline void -_dispatch_trace_continuation_pop(dispatch_queue_t dq, - dispatch_object_t dou) +_dispatch_trace_continuation_pop(dispatch_queue_t dq, dispatch_object_t dou) { if (slowpath(DISPATCH_QUEUE_POP_ENABLED())) { _dispatch_trace_continuation(dq, dou._do, DISPATCH_QUEUE_POP); } _dispatch_introspection_queue_pop(dq, dou); } +#else +#define _dispatch_queue_push_notrace _dispatch_queue_push +#define _dispatch_trace_continuation_push(dq, dou) \ + do { (void)(dq); (void)(dou); } while(0) +#define _dispatch_trace_continuation_pop(dq, dou) \ + do { (void)(dq); (void)(dou); } while(0) +#endif // DISPATCH_USE_DTRACE_INTROSPECTION || DISPATCH_INTROSPECTION -#endif // DISPATCH_USE_DTRACE_INTROSPECTION - +#if DISPATCH_USE_DTRACE static inline dispatch_function_t _dispatch_trace_timer_function(dispatch_source_t ds, dispatch_source_refs_t dr) { - dispatch_function_t func = dr->ds_handler_func; + dispatch_continuation_t dc = dr->ds_handler[DS_EVENT_HANDLER]; + dispatch_function_t func = dc ? dc->dc_func : NULL; if (func == _dispatch_after_timer_callback && - !(ds->ds_atomic_flags & DSF_CANCELED)) { - dispatch_continuation_t dc = ds->do_ctxt; + !(ds->ds_atomic_flags & DSF_CANCELED)) { + dc = ds->do_ctxt; func = dc->dc_func != _dispatch_call_block_and_release ? dc->dc_func : dc->dc_ctxt ? _dispatch_Block_invoke(dc->dc_ctxt) : NULL; } @@ -296,14 +319,8 @@ _dispatch_trace_timer_fire(dispatch_source_refs_t dr, unsigned long data, #define _dispatch_trace_timer_fire(dr, data, missed) \ do { (void)(dr); (void)(data); (void)(missed); } while(0) -#endif // DISPATCH_USE_DTRACE && !__OBJC2__ - -#if !DISPATCH_USE_DTRACE_INTROSPECTION - -#define _dispatch_queue_push_notrace _dispatch_queue_push -#define _dispatch_trace_continuation_pop(dq, dou) \ - do { (void)(dq); (void)(dou); } while(0) +#endif // DISPATCH_USE_DTRACE -#endif // !DISPATCH_USE_DTRACE_INTROSPECTION +#endif // !__OBJC2__ #endif // __DISPATCH_TRACE__ diff --git a/src/voucher.c b/src/voucher.c new file mode 100644 index 0000000..e886faf --- /dev/null +++ b/src/voucher.c @@ -0,0 +1,2774 @@ +/* + * Copyright (c) 2013 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +#include "internal.h" + +#if VOUCHER_USE_MACH_VOUCHER + +#include + +// +#ifndef VM_MEMORY_GENEALOGY +#define VM_MEMORY_GENEALOGY 78 +#endif + +#ifndef VOUCHER_ATM_COLLECT_THRESHOLD +#define VOUCHER_ATM_COLLECT_THRESHOLD 1 +#endif +#define VATM_COLLECT_THRESHOLD_VALUE(t) (((t) - 1) * 2) +static volatile long _voucher_atm_collect_level; +static long _voucher_atm_collect_threshold = + VATM_COLLECT_THRESHOLD_VALUE(VOUCHER_ATM_COLLECT_THRESHOLD); +static unsigned long _voucher_atm_subid_bits; + +typedef struct _voucher_atm_s *_voucher_atm_t; + +static void _voucher_activity_atfork_child(void); +static inline mach_voucher_t _voucher_get_atm_mach_voucher(voucher_t voucher); +static inline mach_voucher_t _voucher_activity_get_atm_mach_voucher( + _voucher_activity_t act); +static inline _voucher_activity_t _voucher_activity_get(voucher_t voucher); +static _voucher_activity_t _voucher_activity_copy_from_mach_voucher( + mach_voucher_t kv, voucher_activity_id_t va_id); +static inline _voucher_activity_t _voucher_activity_retain( + _voucher_activity_t act); +static inline void _voucher_activity_release(_voucher_activity_t act); + +#pragma mark - +#pragma mark voucher_t + +#if USE_OBJC +OS_OBJECT_OBJC_CLASS_DECL(voucher); +#define VOUCHER_CLASS OS_OBJECT_OBJC_CLASS(voucher) +#else +const _os_object_class_s _voucher_class = { + ._os_obj_xref_dispose = (void(*)(_os_object_t))_voucher_xref_dispose, + ._os_obj_dispose = (void(*)(_os_object_t))_voucher_dispose, +}; +#define VOUCHER_CLASS &_voucher_class +#endif // USE_OBJC + +static const voucher_activity_trace_id_t _voucher_activity_trace_id_release = + (voucher_activity_trace_id_t)voucher_activity_tracepoint_type_release << + _voucher_activity_trace_id_type_shift; +static const unsigned int _voucher_max_activities = 16; + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_recipes_init(mach_voucher_attr_recipe_data_t *recipes, + mach_voucher_attr_content_size_t bits_size) +{ + static const mach_voucher_attr_recipe_data_t base_recipe = { + .key = MACH_VOUCHER_ATTR_KEY_ALL, + .command = MACH_VOUCHER_ATTR_COPY, + }; + _voucher_recipes_base(recipes) = base_recipe; + static const mach_voucher_attr_recipe_data_t atm_recipe = { + .key = MACH_VOUCHER_ATTR_KEY_ATM, + .command = MACH_VOUCHER_ATTR_COPY, + }; + _voucher_recipes_atm(recipes) = atm_recipe; + static const mach_voucher_attr_recipe_data_t bits_recipe = { + .key = MACH_VOUCHER_ATTR_KEY_USER_DATA, + .command = MACH_VOUCHER_ATTR_USER_DATA_STORE, + }; + _voucher_recipes_bits(recipes) = bits_recipe; + if (!bits_size) return; + _voucher_recipes_bits(recipes).content_size = bits_size; + *_voucher_recipes_magic(recipes) = _voucher_magic_v1; +} + +static inline voucher_t +_voucher_alloc(unsigned int activities, pthread_priority_t priority, + mach_voucher_attr_recipe_size_t extra) +{ + if (activities > _voucher_max_activities) { + activities = _voucher_max_activities; + } + voucher_t voucher; + size_t voucher_size, recipes_size; + mach_voucher_attr_content_size_t bits_size; + recipes_size = (priority||activities||extra) ? _voucher_recipes_size() : 0; + bits_size = recipes_size ? _voucher_bits_size(activities) : 0; + voucher_size = sizeof(voucher_s) + recipes_size + bits_size + extra; + voucher = (voucher_t)_os_object_alloc_realized(VOUCHER_CLASS, voucher_size); +#if VOUCHER_ENABLE_RECIPE_OBJECTS + voucher->v_recipe_extra_size = extra; + voucher->v_recipe_extra_offset = voucher_size - extra; +#else + dispatch_assert(!extra); +#endif + voucher->v_has_priority = priority ? 1 : 0; + voucher->v_activities = activities; + if (!recipes_size) return voucher; + _voucher_recipes_init(voucher->v_recipes, bits_size); + *_voucher_priority(voucher) = (_voucher_priority_t)priority; + _dispatch_voucher_debug("alloc", voucher); + return voucher; +} + +#if VOUCHER_ENABLE_RECIPE_OBJECTS +voucher_t +voucher_create(voucher_recipe_t recipe) +{ + // TODO: capture current activities or current kvoucher ? + mach_voucher_attr_recipe_size_t extra = recipe ? recipe->vr_size : 0; + voucher_t voucher = _voucher_alloc(0, 0, extra); + if (extra) { + memcpy(_voucher_extra_recipes(voucher), recipe->vr_data, extra); + } + return voucher; +} +#endif + +voucher_t +voucher_adopt(voucher_t voucher) +{ + return _voucher_adopt(voucher); +} + +voucher_t +voucher_copy(void) +{ + return _voucher_copy(); +} + +voucher_t +voucher_copy_without_importance(void) +{ + return _voucher_copy_without_importance(); +} + +void +_voucher_thread_cleanup(void *voucher) +{ + _voucher_swap(voucher, NULL); +} + +DISPATCH_CACHELINE_ALIGN +static TAILQ_HEAD(, voucher_s) _vouchers[VL_HASH_SIZE]; +#define _vouchers(kv) (&_vouchers[VL_HASH((kv))]) +static os_lock_handoff_s _vouchers_lock = OS_LOCK_HANDOFF_INIT; +#define _vouchers_lock_lock() os_lock_lock(&_vouchers_lock) +#define _vouchers_lock_unlock() os_lock_unlock(&_vouchers_lock) + +static voucher_t +_voucher_find_and_retain(mach_voucher_t kv) +{ + voucher_t v; + if (!kv) return NULL; + _vouchers_lock_lock(); + TAILQ_FOREACH(v, _vouchers(kv), v_list) { + if (v->v_ipc_kvoucher == kv) { + int xref_cnt = dispatch_atomic_inc2o(v, os_obj_xref_cnt, relaxed); + _dispatch_voucher_debug("retain -> %d", v, xref_cnt + 1); + if (slowpath(xref_cnt < 0)) { + _dispatch_voucher_debug("overrelease", v); + DISPATCH_CRASH("Voucher overrelease"); + } + if (xref_cnt == 0) { + // resurrection: raced with _voucher_remove + (void)dispatch_atomic_inc2o(v, os_obj_ref_cnt, relaxed); + } + break; + } + } + _vouchers_lock_unlock(); + return v; +} + +static void +_voucher_insert(voucher_t v) +{ + mach_voucher_t kv = v->v_ipc_kvoucher; + if (!kv) return; + _vouchers_lock_lock(); + if (slowpath(_TAILQ_IS_ENQUEUED(v, v_list))) { + _dispatch_voucher_debug("corruption", v); + DISPATCH_CRASH("Voucher corruption"); + } + TAILQ_INSERT_TAIL(_vouchers(kv), v, v_list); + _vouchers_lock_unlock(); +} + +static void +_voucher_remove(voucher_t v) +{ + mach_voucher_t kv = v->v_ipc_kvoucher; + if (!_TAILQ_IS_ENQUEUED(v, v_list)) return; + _vouchers_lock_lock(); + if (slowpath(!kv)) { + _dispatch_voucher_debug("corruption", v); + DISPATCH_CRASH("Voucher corruption"); + } + // check for resurrection race with _voucher_find_and_retain + if (dispatch_atomic_load2o(v, os_obj_xref_cnt, seq_cst) < 0 && + _TAILQ_IS_ENQUEUED(v, v_list)) { + TAILQ_REMOVE(_vouchers(kv), v, v_list); + _TAILQ_MARK_NOT_ENQUEUED(v, v_list); + v->v_list.tqe_next = (void*)~0ull; + } + _vouchers_lock_unlock(); +} + +void +_voucher_dealloc_mach_voucher(mach_voucher_t kv) +{ + _dispatch_kvoucher_debug("dealloc", kv); + _dispatch_voucher_debug_machport(kv); + kern_return_t kr = mach_voucher_deallocate(kv); + DISPATCH_VERIFY_MIG(kr); + (void)dispatch_assume_zero(kr); +} + +static inline kern_return_t +_voucher_create_mach_voucher(const mach_voucher_attr_recipe_data_t *recipes, + size_t recipes_size, mach_voucher_t *kvp) +{ + kern_return_t kr; + mach_port_t mhp = _dispatch_get_mach_host_port(); + mach_voucher_t kv = MACH_VOUCHER_NULL; + mach_voucher_attr_raw_recipe_array_t kvr; + mach_voucher_attr_recipe_size_t kvr_size; + kvr = (mach_voucher_attr_raw_recipe_array_t)recipes; + kvr_size = (mach_voucher_attr_recipe_size_t)recipes_size; + kr = host_create_mach_voucher(mhp, kvr, kvr_size, &kv); + DISPATCH_VERIFY_MIG(kr); + if (!kr) { + _dispatch_kvoucher_debug("create", kv); + _dispatch_voucher_debug_machport(kv); + } + *kvp = kv; + return kr; +} + +#if __has_include() && !defined(VOUCHER_USE_ATTR_BANK) +#include +#define VOUCHER_USE_ATTR_BANK 1 +mach_voucher_t _voucher_default_task_mach_voucher; +#endif + +void +_voucher_task_mach_voucher_init(void* ctxt DISPATCH_UNUSED) +{ +#if VOUCHER_USE_ATTR_BANK + kern_return_t kr; + mach_voucher_t kv; + static const mach_voucher_attr_recipe_data_t task_create_recipe = { + .key = MACH_VOUCHER_ATTR_KEY_BANK, + .command = MACH_VOUCHER_ATTR_BANK_CREATE, + }; + kr = _voucher_create_mach_voucher(&task_create_recipe, + sizeof(task_create_recipe), &kv); + if (dispatch_assume_zero(kr)) { + DISPATCH_CLIENT_CRASH("Could not create task mach voucher"); + } + _voucher_default_task_mach_voucher = kv; + _voucher_task_mach_voucher = kv; +#endif +} + +void +voucher_replace_default_voucher(void) +{ +#if VOUCHER_USE_ATTR_BANK + (void)_voucher_get_task_mach_voucher(); // initalize task mach voucher + mach_voucher_t kv, tkv = MACH_VOUCHER_NULL; + voucher_t v = _voucher_get(); + if (v && v->v_kvoucher) { + kern_return_t kr; + kv = v->v_ipc_kvoucher ? v->v_ipc_kvoucher : v->v_kvoucher; + const mach_voucher_attr_recipe_data_t task_copy_recipe = { + .key = MACH_VOUCHER_ATTR_KEY_BANK, + .command = MACH_VOUCHER_ATTR_COPY, + .previous_voucher = kv, + }; + kr = _voucher_create_mach_voucher(&task_copy_recipe, + sizeof(task_copy_recipe), &tkv); + if (dispatch_assume_zero(kr)) { + tkv = MACH_VOUCHER_NULL; + } + } + if (!tkv) tkv = _voucher_default_task_mach_voucher; + kv = dispatch_atomic_xchg(&_voucher_task_mach_voucher, tkv, relaxed); + if (kv && kv != _voucher_default_task_mach_voucher) { + _voucher_dealloc_mach_voucher(kv); + } + _dispatch_voucher_debug("kvoucher[0x%08x] replace default voucher", v, tkv); +#endif +} + +static inline mach_voucher_t +_voucher_get_atm_mach_voucher(voucher_t voucher) +{ + _voucher_activity_t act = _voucher_activity_get(voucher); + return _voucher_activity_get_atm_mach_voucher(act); +} + +mach_voucher_t +_voucher_get_mach_voucher(voucher_t voucher) +{ + if (!voucher) return MACH_VOUCHER_NULL; + if (voucher->v_ipc_kvoucher) return voucher->v_ipc_kvoucher; + mach_voucher_t kvb = voucher->v_kvoucher; + if (!kvb) kvb = _voucher_get_task_mach_voucher(); + if (!voucher->v_has_priority && !voucher->v_activities && + !_voucher_extra_size(voucher)) { + return kvb; + } + kern_return_t kr; + mach_voucher_t kv, kvo; + _voucher_base_recipe(voucher).previous_voucher = kvb; + _voucher_atm_recipe(voucher).previous_voucher = + _voucher_get_atm_mach_voucher(voucher); + kr = _voucher_create_mach_voucher(voucher->v_recipes, + _voucher_recipes_size() + _voucher_extra_size(voucher) + + _voucher_bits_recipe(voucher).content_size, &kv); + if (dispatch_assume_zero(kr) || !kv){ + return MACH_VOUCHER_NULL; + } + if (!dispatch_atomic_cmpxchgv2o(voucher, v_ipc_kvoucher, MACH_VOUCHER_NULL, + kv, &kvo, relaxed)) { + _voucher_dealloc_mach_voucher(kv); + kv = kvo; + } else { + if (kv == voucher->v_kvoucher) { + // if v_kvoucher == v_ipc_kvoucher we keep only one reference + _voucher_dealloc_mach_voucher(kv); + } + _voucher_insert(voucher); + _dispatch_voucher_debug("kvoucher[0x%08x] create", voucher, kv); + } + return kv; +} + +mach_voucher_t +_voucher_create_mach_voucher_with_priority(voucher_t voucher, + pthread_priority_t priority) +{ + if (priority == _voucher_get_priority(voucher)) { + return MACH_VOUCHER_NULL; // caller will use _voucher_get_mach_voucher + } + kern_return_t kr; + mach_voucher_t kv, kvb = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL; + if (!kvb) kvb = _voucher_get_task_mach_voucher(); + mach_voucher_attr_recipe_data_t *recipes; + size_t recipes_size = _voucher_recipes_size(); + if (voucher && (voucher->v_has_priority || voucher->v_activities || + _voucher_extra_size(voucher))) { + recipes_size += _voucher_bits_recipe(voucher).content_size + + _voucher_extra_size(voucher); + recipes = alloca(recipes_size); + memcpy(recipes, voucher->v_recipes, recipes_size); + _voucher_recipes_atm(recipes).previous_voucher = + _voucher_get_atm_mach_voucher(voucher); + } else { + mach_voucher_attr_content_size_t bits_size = _voucher_bits_size(0); + recipes_size += bits_size; + recipes = alloca(recipes_size); + _voucher_recipes_init(recipes, bits_size); + } + _voucher_recipes_base(recipes).previous_voucher = kvb; + *_voucher_recipes_priority(recipes) = (_voucher_priority_t)priority; + kr = _voucher_create_mach_voucher(recipes, recipes_size, &kv); + if (dispatch_assume_zero(kr) || !kv){ + return MACH_VOUCHER_NULL; + } + _dispatch_kvoucher_debug("create with priority from voucher[%p]", kv, + voucher); + return kv; +} + +static voucher_t +_voucher_create_with_mach_voucher(mach_voucher_t kv) +{ + if (!kv) return NULL; + kern_return_t kr; + mach_voucher_t rkv; + mach_voucher_attr_recipe_t vr; + size_t vr_size; + mach_voucher_attr_recipe_size_t kvr_size = 0; + const mach_voucher_attr_recipe_data_t redeem_recipe[] = { + [0] = { + .key = MACH_VOUCHER_ATTR_KEY_ALL, + .command = MACH_VOUCHER_ATTR_COPY, + .previous_voucher = kv, + }, +#if VOUCHER_USE_ATTR_BANK + [1] = { + .key = MACH_VOUCHER_ATTR_KEY_BANK, + .command = MACH_VOUCHER_ATTR_REDEEM, + }, +#endif + }; + kr = _voucher_create_mach_voucher(redeem_recipe, sizeof(redeem_recipe), + &rkv); + if (!dispatch_assume_zero(kr)) { + _voucher_dealloc_mach_voucher(kv); + } else { + _dispatch_voucher_debug_machport(kv); + rkv = kv; + } + voucher_t v = _voucher_find_and_retain(rkv); + if (v) { + _dispatch_voucher_debug("kvoucher[0x%08x] find with 0x%08x", v, rkv,kv); + _voucher_dealloc_mach_voucher(rkv); + return v; + } + vr_size = sizeof(*vr) + _voucher_bits_size(_voucher_max_activities); + vr = alloca(vr_size); + if (rkv) { + kvr_size = (mach_voucher_attr_recipe_size_t)vr_size; + kr = mach_voucher_extract_attr_recipe(rkv, + MACH_VOUCHER_ATTR_KEY_USER_DATA, (void*)vr, &kvr_size); + DISPATCH_VERIFY_MIG(kr); + if (dispatch_assume_zero(kr)) kvr_size = 0; + } + mach_voucher_attr_content_size_t content_size = vr->content_size; + uint8_t *content = vr->content; + bool valid = false, has_priority = false; + unsigned int activities = 0; + if (kvr_size >= sizeof(*vr) + sizeof(_voucher_magic_t)) { + valid = (*(_voucher_magic_t*)content == _voucher_magic_v1); + content += sizeof(_voucher_magic_t); + content_size -= sizeof(_voucher_magic_t); + } + if (valid) { + has_priority = (content_size >= sizeof(_voucher_priority_t)); + activities = has_priority ? (content_size - sizeof(_voucher_priority_t)) + / sizeof(voucher_activity_id_t) : 0; + } + pthread_priority_t priority = 0; + if (has_priority) { + priority = (pthread_priority_t)*(_voucher_priority_t*)content; + content += sizeof(_voucher_priority_t); + content_size -= sizeof(_voucher_priority_t); + } + voucher_activity_id_t va_id = 0, va_base_id = 0; + _voucher_activity_t act = NULL; + if (activities) { + va_id = *(voucher_activity_id_t*)content; + act = _voucher_activity_copy_from_mach_voucher(rkv, va_id); + if (!act && _voucher_activity_default) { + activities++; + // default to _voucher_activity_default base activity + va_base_id = _voucher_activity_default->va_id; + } else if (act && act->va_id != va_id) { + activities++; + va_base_id = act->va_id; + } + } + v = _voucher_alloc(activities, priority, 0); + v->v_activity = act; + voucher_activity_id_t *activity_ids = _voucher_activity_ids(v); + if (activities && va_base_id) { + *activity_ids++ = va_base_id; + activities--; + } + if (activities) { + memcpy(activity_ids, content, content_size); + } + v->v_ipc_kvoucher = v->v_kvoucher = rkv; + _voucher_insert(v); + _dispatch_voucher_debug("kvoucher[0x%08x] create with 0x%08x", v, rkv, kv); + return v; +} + +voucher_t +_voucher_create_with_priority_and_mach_voucher(voucher_t ov, + pthread_priority_t priority, mach_voucher_t kv) +{ + if (priority == _voucher_get_priority(ov)) { + if (kv) _voucher_dealloc_mach_voucher(kv); + return ov ? _voucher_retain(ov) : NULL; + } + voucher_t v = _voucher_find_and_retain(kv); + if (v) { + _dispatch_voucher_debug("kvoucher[0x%08x] find", v, kv); + _voucher_dealloc_mach_voucher(kv); + return v; + } + unsigned int activities = ov ? ov->v_activities : 0; + mach_voucher_attr_recipe_size_t extra = ov ? _voucher_extra_size(ov) : 0; + v = _voucher_alloc(activities, priority, extra); + if (extra) { + memcpy(_voucher_extra_recipes(v), _voucher_extra_recipes(ov), extra); + } + if (activities) { + if (ov->v_activity) { + v->v_activity = _voucher_activity_retain(ov->v_activity); + } + memcpy(_voucher_activity_ids(v), _voucher_activity_ids(ov), + activities * sizeof(voucher_activity_id_t)); + } + if (kv) { + v->v_ipc_kvoucher = v->v_kvoucher = kv; + _voucher_insert(v); + _dispatch_voucher_debug("kvoucher[0x%08x] create with priority from " + "voucher[%p]", v, kv, ov); + _dispatch_voucher_debug_machport(kv); + } else if (ov && ov->v_kvoucher) { + voucher_t kvb = ov->v_kvbase ? ov->v_kvbase : ov; + v->v_kvbase = _voucher_retain(kvb); + v->v_kvoucher = kvb->v_kvoucher; + } + return v; +} + +voucher_t +_voucher_create_without_importance(voucher_t ov) +{ + // Nothing to do unless the old voucher has a kernel voucher. If it + // doesn't, it can't have any importance, now or in the future. + if (!ov) return NULL; + // TODO: 17487167: track presence of importance attribute + if (!ov->v_kvoucher) return _voucher_retain(ov); + kern_return_t kr; + mach_voucher_t kv, okv; + // Copy kernel voucher, removing importance. + okv = ov->v_ipc_kvoucher ? ov->v_ipc_kvoucher : ov->v_kvoucher; + const mach_voucher_attr_recipe_data_t importance_remove_recipe[] = { + [0] = { + .key = MACH_VOUCHER_ATTR_KEY_ALL, + .command = MACH_VOUCHER_ATTR_COPY, + .previous_voucher = okv, + }, + [1] = { + .key = MACH_VOUCHER_ATTR_KEY_IMPORTANCE, + .command = MACH_VOUCHER_ATTR_REMOVE, + }, + }; + kr = _voucher_create_mach_voucher(importance_remove_recipe, + sizeof(importance_remove_recipe), &kv); + if (dispatch_assume_zero(kr) || !kv){ + if (ov->v_ipc_kvoucher) return NULL; + kv = MACH_VOUCHER_NULL; + } + if (kv == okv) { + _voucher_dealloc_mach_voucher(kv); + return _voucher_retain(ov); + } + voucher_t v = _voucher_find_and_retain(kv); + if (v && ov->v_ipc_kvoucher) { + _dispatch_voucher_debug("kvoucher[0x%08x] find without importance " + "from voucher[%p]", v, kv, ov); + _voucher_dealloc_mach_voucher(kv); + return v; + } + voucher_t kvbase = v; + // Copy userspace contents + unsigned int activities = ov->v_activities; + pthread_priority_t priority = _voucher_get_priority(ov); + mach_voucher_attr_recipe_size_t extra = _voucher_extra_size(ov); + v = _voucher_alloc(activities, priority, extra); + if (extra) { + memcpy(_voucher_extra_recipes(v), _voucher_extra_recipes(ov), extra); + } + if (activities) { + if (ov->v_activity) { + v->v_activity = _voucher_activity_retain(ov->v_activity); + } + memcpy(_voucher_activity_ids(v), _voucher_activity_ids(ov), + activities * sizeof(voucher_activity_id_t)); + } + v->v_kvoucher = kv; + if (ov->v_ipc_kvoucher) { + v->v_ipc_kvoucher = kv; + _voucher_insert(v); + } else if (kvbase) { + v->v_kvbase = kvbase; + _voucher_dealloc_mach_voucher(kv); // borrow base reference + } + if (!kvbase) { + _dispatch_voucher_debug("kvoucher[0x%08x] create without importance " + "from voucher[%p]", v, kv, ov); + } + return v; +} + +voucher_t +voucher_create_with_mach_msg(mach_msg_header_t *msg) +{ + voucher_t v = _voucher_create_with_mach_voucher(_voucher_mach_msg_get(msg)); + _voucher_activity_trace_msg(v, msg, receive); + return v; +} + +#ifndef MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL +#define MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL 2 +#endif + +void +voucher_decrement_importance_count4CF(voucher_t v) +{ + if (!v || !v->v_kvoucher) return; + // TODO: 17487167: track presence of importance attribute + kern_return_t kr; + mach_voucher_t kv = v->v_ipc_kvoucher ? v->v_ipc_kvoucher : v->v_kvoucher; + uint32_t dec = 1; + mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&dec; + mach_voucher_attr_content_size_t kvc_in_size = sizeof(dec); + mach_voucher_attr_content_t kvc_out = NULL; + mach_voucher_attr_content_size_t kvc_out_size = 0; +#if DISPATCH_DEBUG + uint32_t count = UINT32_MAX; + kvc_out = (mach_voucher_attr_content_t)&count; + kvc_out_size = sizeof(count); +#endif + kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_IMPORTANCE, + MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL, kvc_in, kvc_in_size, + kvc_out, &kvc_out_size); + DISPATCH_VERIFY_MIG(kr); +#if DISPATCH_DEBUG + _dispatch_voucher_debug("kvoucher[0x%08x] decrement importance count to %u:" + " %s - 0x%x", v, kv, count, mach_error_string(kr), kr); +#endif + if (kr != KERN_INVALID_ARGUMENT && + dispatch_assume_zero(kr) == KERN_FAILURE) { + // TODO: 17487167: skip KERN_INVALID_ARGUMENT check + DISPATCH_CLIENT_CRASH("Voucher importance count underflow"); + } +} + +#if VOUCHER_ENABLE_GET_MACH_VOUCHER +mach_voucher_t +voucher_get_mach_voucher(voucher_t voucher) +{ + return _voucher_get_mach_voucher(voucher); +} +#endif + +void +_voucher_xref_dispose(voucher_t voucher) +{ + _dispatch_voucher_debug("xref_dispose", voucher); + _voucher_remove(voucher); + return _os_object_release_internal_inline((_os_object_t)voucher); +} + +void +_voucher_dispose(voucher_t voucher) +{ + _dispatch_voucher_debug("dispose", voucher); + if (slowpath(_TAILQ_IS_ENQUEUED(voucher, v_list))) { + _dispatch_voucher_debug("corruption", voucher); + DISPATCH_CRASH("Voucher corruption"); + } + voucher->v_list.tqe_next = DISPATCH_OBJECT_LISTLESS; + if (voucher->v_ipc_kvoucher) { + if (voucher->v_ipc_kvoucher != voucher->v_kvoucher) { + _voucher_dealloc_mach_voucher(voucher->v_ipc_kvoucher); + } + voucher->v_ipc_kvoucher = MACH_VOUCHER_NULL; + } + if (voucher->v_kvoucher) { + if (!voucher->v_kvbase) { + _voucher_dealloc_mach_voucher(voucher->v_kvoucher); + } + voucher->v_kvoucher = MACH_VOUCHER_NULL; + } + if (voucher->v_kvbase) { + _voucher_release(voucher->v_kvbase); + voucher->v_kvbase = NULL; + } + if (voucher->v_activity) { + _voucher_activity_release(voucher->v_activity); + voucher->v_activity = NULL; + } + voucher->v_has_priority= 0; + voucher->v_activities = 0; +#if VOUCHER_ENABLE_RECIPE_OBJECTS + voucher->v_recipe_extra_size = 0; + voucher->v_recipe_extra_offset = 0; +#endif + return _os_object_dealloc((_os_object_t)voucher); +} + +void +_voucher_atfork_child(void) +{ + _voucher_activity_atfork_child(); + _dispatch_thread_setspecific(dispatch_voucher_key, NULL); + _voucher_task_mach_voucher_pred = 0; + _voucher_task_mach_voucher = MACH_VOUCHER_NULL; + + // TODO: voucher/activity inheritance on fork ? +} + +#pragma mark - +#pragma mark _voucher_init + +boolean_t +voucher_mach_msg_set(mach_msg_header_t *msg) +{ + voucher_t v = _voucher_get(); + bool clear_voucher = _voucher_mach_msg_set(msg, v); + if (clear_voucher) _voucher_activity_trace_msg(v, msg, send); + return clear_voucher; +} + +void +voucher_mach_msg_clear(mach_msg_header_t *msg) +{ + (void)_voucher_mach_msg_clear(msg, false); +} + +voucher_mach_msg_state_t +voucher_mach_msg_adopt(mach_msg_header_t *msg) +{ + mach_voucher_t kv = _voucher_mach_msg_get(msg); + if (!kv) return VOUCHER_MACH_MSG_STATE_UNCHANGED; + voucher_t v = _voucher_create_with_mach_voucher(kv); + _voucher_activity_trace_msg(v, msg, receive); + return (voucher_mach_msg_state_t)_voucher_adopt(v); +} + +void +voucher_mach_msg_revert(voucher_mach_msg_state_t state) +{ + if (state == VOUCHER_MACH_MSG_STATE_UNCHANGED) return; + _voucher_replace((voucher_t)state); +} + +#if DISPATCH_USE_LIBKERNEL_VOUCHER_INIT +#include <_libkernel_init.h> + +static const struct _libkernel_voucher_functions _voucher_libkernel_functions = +{ + .version = 1, + .voucher_mach_msg_set = voucher_mach_msg_set, + .voucher_mach_msg_clear = voucher_mach_msg_clear, + .voucher_mach_msg_adopt = voucher_mach_msg_adopt, + .voucher_mach_msg_revert = voucher_mach_msg_revert, +}; + +static void +_voucher_libkernel_init(void) +{ + kern_return_t kr = __libkernel_voucher_init(&_voucher_libkernel_functions); + dispatch_assert(!kr); +} +#else +#define _voucher_libkernel_init() +#endif + +void +_voucher_init(void) +{ + _voucher_libkernel_init(); + char *e, *end; + unsigned int i; + for (i = 0; i < VL_HASH_SIZE; i++) { + TAILQ_INIT(&_vouchers[i]); + } + voucher_activity_mode_t mode; + mode = DISPATCH_DEBUG ? voucher_activity_mode_debug + : voucher_activity_mode_release; + e = getenv("OS_ACTIVITY_MODE"); + if (e) { + if (strcmp(e, "release") == 0) { + mode = voucher_activity_mode_release; + } else if (strcmp(e, "debug") == 0) { + mode = voucher_activity_mode_debug; + } else if (strcmp(e, "stream") == 0) { + mode = voucher_activity_mode_stream; + } else if (strcmp(e, "disable") == 0) { + mode = voucher_activity_mode_disable; + } + } + _voucher_activity_mode = mode; + if (_voucher_activity_disabled()) return; + + e = getenv("LIBDISPATCH_ACTIVITY_ATM_SUBID_BITS"); + if (e) { + unsigned long v = strtoul(e, &end, 0); + if (v && !*end) { + _voucher_atm_subid_bits = v; + } + } + e = getenv("LIBDISPATCH_ACTIVITY_ATM_COLLECT_THRESHOLD"); + if (e) { + unsigned long v = strtoul(e, &end, 0); + if (v && v < LONG_MAX/2 && !*end) { + _voucher_atm_collect_threshold = + VATM_COLLECT_THRESHOLD_VALUE((long)v); + } + } + // default task activity + bool default_task_activity = DISPATCH_DEBUG; + e = getenv("LIBDISPATCH_DEFAULT_TASK_ACTIVITY"); + if (e) default_task_activity = atoi(e); + if (default_task_activity) { + (void)voucher_activity_start(_voucher_activity_trace_id_release, 0); + } +} + +#pragma mark - +#pragma mark _voucher_activity_lock_s + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_activity_lock_init(_voucher_activity_lock_s *lock) { + static const os_lock_handoff_s _os_lock_handoff_init = OS_LOCK_HANDOFF_INIT; + *lock = _os_lock_handoff_init; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_activity_lock_lock(_voucher_activity_lock_s *lock) { + return os_lock_lock(lock); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_activity_lock_unlock(_voucher_activity_lock_s *lock) { + return os_lock_unlock(lock); +} + +#pragma mark - +#pragma mark _voucher_activity_heap + +#if __has_extension(c_static_assert) +_Static_assert(sizeof(struct _voucher_activity_tracepoint_s) == 64, + "Tracepoint too large"); +_Static_assert(sizeof(struct _voucher_activity_buffer_header_s) <= + sizeof(struct _voucher_activity_tracepoint_s), + "Buffer header too large"); +_Static_assert(offsetof(struct _voucher_activity_s, va_flags2) == + sizeof(struct _voucher_activity_tracepoint_s), + "Extended activity object misaligned"); +#if __LP64__ +_Static_assert(sizeof(struct _voucher_activity_s) == + 3 * sizeof(struct _voucher_activity_tracepoint_s), + "Activity object too large"); +_Static_assert(offsetof(struct _voucher_activity_s, va_flags3) == + 2 * sizeof(struct _voucher_activity_tracepoint_s), + "Extended activity object misaligned"); +_Static_assert(offsetof(struct _voucher_atm_s, vatm_activities_lock) % 64 == 0, + "Bad ATM padding"); +_Static_assert(sizeof(struct _voucher_atm_s) <= 128, + "ATM too large"); +#else +_Static_assert(sizeof(struct _voucher_activity_s) == + 2 * sizeof(struct _voucher_activity_tracepoint_s), + "Activity object too large"); +_Static_assert(sizeof(struct _voucher_atm_s) <= 64, + "ATM too large"); +#endif +_Static_assert(sizeof(_voucher_activity_buffer_t) == + sizeof(struct {char x[_voucher_activity_buffer_size];}), + "Buffer too large"); +_Static_assert(sizeof(struct _voucher_activity_metadata_s) <= + sizeof(struct _voucher_activity_metadata_opaque_s), + "Metadata too large"); +_Static_assert(sizeof(_voucher_activity_bitmap_t) % 64 == 0, + "Bad metadata bitmap size"); +_Static_assert(offsetof(struct _voucher_activity_metadata_s, + vam_atm_mbox_bitmap) % 64 == 0, + "Bad metadata padding"); +_Static_assert(offsetof(struct _voucher_activity_metadata_s, + vam_base_atm_subid) % 64 == 0, + "Bad metadata padding"); +_Static_assert(offsetof(struct _voucher_activity_metadata_s, vam_base_atm_lock) + % 32 == 0, + "Bad metadata padding"); +_Static_assert(offsetof(struct _voucher_activity_metadata_s, vam_atms) % 64 ==0, + "Bad metadata padding"); +_Static_assert(sizeof(_voucher_activity_bitmap_t) * 8 * + sizeof(atm_mailbox_offset_t) <= + sizeof(((_voucher_activity_metadata_t)NULL)->vam_kernel_metadata), + "Bad kernel metadata bitmap"); +_Static_assert(sizeof(atm_mailbox_offset_t) == 2 * sizeof(atm_subaid32_t), + "Bad kernel ATM mailbox sizes"); +#endif + +static const size_t _voucher_atm_mailboxes = + sizeof(((_voucher_activity_metadata_t)NULL)->vam_kernel_metadata) / + sizeof(atm_mailbox_offset_t); + +#define va_buffers_lock(va) (&(va)->va_buffers_lock) +#define vatm_activities_lock(vatm) (&(vatm)->vatm_activities_lock) +#define vatm_activities(vatm) (&(vatm)->vatm_activities) +#define vatm_used_activities(vatm) (&(vatm)->vatm_used_activities) +#define vam_base_atm_lock() (&_voucher_activity_heap->vam_base_atm_lock) +#define vam_nested_atm_lock() (&_voucher_activity_heap->vam_nested_atm_lock) +#define vam_atms_lock() (&_voucher_activity_heap->vam_atms_lock) +#define vam_activities_lock() (&_voucher_activity_heap->vam_activities_lock) +#define vam_atms(hash) (&_voucher_activity_heap->vam_atms[hash]) +#define vam_activities(hash) (&_voucher_activity_heap->vam_activities[hash]) +#define vam_buffer_bitmap() (_voucher_activity_heap->vam_buffer_bitmap) +#define vam_atm_mbox_bitmap() (_voucher_activity_heap->vam_atm_mbox_bitmap) +#define vam_pressure_locked_bitmap() \ + (_voucher_activity_heap->vam_pressure_locked_bitmap) +#define vam_buffer(i) ((void*)((char*)_voucher_activity_heap + \ + (i) * _voucher_activity_buffer_size)) + +static _voucher_activity_t _voucher_activity_create_with_atm( + _voucher_atm_t vatm, voucher_activity_id_t va_id, + voucher_activity_trace_id_t trace_id, uint64_t location, + _voucher_activity_buffer_header_t buffer); +static _voucher_atm_t _voucher_atm_create(mach_voucher_t kv, atm_aid_t atm_id); +static voucher_activity_id_t _voucher_atm_nested_atm_id_make(void); + +DISPATCH_ALWAYS_INLINE +static inline uint32_t +_voucher_default_activity_buffer_limit() +{ + switch (_voucher_activity_mode) { + case voucher_activity_mode_debug: + case voucher_activity_mode_stream: + // High-profile modes: Default activity can use 1/32nd of the heap + // (twice as much as non-default activities) + return MAX(_voucher_activity_buffers_per_heap / 32, 3) - 1; + } +#if TARGET_OS_EMBEDDED + // Low-profile modes: Default activity can use a total of 3 buffers. + return 2; +#else + // Low-profile modes: Default activity can use a total of 8 buffers. + return 7; +#endif +} + +DISPATCH_ALWAYS_INLINE +static inline uint32_t +_voucher_activity_buffer_limit() +{ + switch (_voucher_activity_mode) { + case voucher_activity_mode_debug: + case voucher_activity_mode_stream: + // High-profile modes: 64 activities, each of which can use 1/64th + // of the entire heap. + return MAX(_voucher_activity_buffers_per_heap / 64, 2) - 1; + } +#if TARGET_OS_EMBEDDED + // Low-profile modes: Each activity can use a total of 2 buffers. + return 1; +#else + // Low-profile modes: Each activity can use a total of 4 buffers. + return 3; +#endif +} + +// The two functions above return the number of *additional* buffers activities +// may allocate, hence the gymnastics with - 1. + +DISPATCH_ALWAYS_INLINE +static inline uint32_t +_voucher_heap_buffer_limit() +{ + switch (_voucher_activity_mode) { + case voucher_activity_mode_debug: + case voucher_activity_mode_stream: + // High-profile modes: Use it all. + return _voucher_activity_buffers_per_heap; + } +#if TARGET_OS_EMBEDDED + // Low-profile modes: 3 activities, each of which can use 2 buffers; + // plus the default activity, which can use 3; plus 3 buffers of overhead. + return 12; +#else + // Low-profile modes: 13 activities, each of which can use 4 buffers; + // plus the default activity, which can use 8; plus 3 buffers of overhead. + return 64; +#endif +} + +#define NO_BITS_WERE_UNSET (UINT_MAX) + +DISPATCH_ALWAYS_INLINE +static inline size_t +_voucher_activity_bitmap_set_first_unset_bit_upto( + _voucher_activity_bitmap_t volatile bitmap, + unsigned int max_index) +{ + dispatch_assert(max_index != 0); + unsigned int index = NO_BITS_WERE_UNSET, max_map, max_bit, i; + max_map = max_index / _voucher_activity_bits_per_bitmap_base_t; + max_map = MIN(max_map, _voucher_activity_bitmaps_per_heap - 1); + max_bit = max_index % _voucher_activity_bits_per_bitmap_base_t; + for (i = 0; i < max_map; i++) { + index = dispatch_atomic_set_first_bit(&bitmap[i], UINT_MAX); + if (fastpath(index < NO_BITS_WERE_UNSET)) { + return index + i * _voucher_activity_bits_per_bitmap_base_t; + } + } + index = dispatch_atomic_set_first_bit(&bitmap[i], max_bit); + if (fastpath(index < NO_BITS_WERE_UNSET)) { + return index + i * _voucher_activity_bits_per_bitmap_base_t; + } + return index; +} + +DISPATCH_ALWAYS_INLINE +static inline size_t +_voucher_activity_bitmap_set_first_unset_bit( + _voucher_activity_bitmap_t volatile bitmap) +{ + return _voucher_activity_bitmap_set_first_unset_bit_upto(bitmap, UINT_MAX); +} + + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_activity_bitmap_clear_bit( + _voucher_activity_bitmap_t volatile bitmap, size_t index) +{ + size_t i = index / _voucher_activity_bits_per_bitmap_base_t; + _voucher_activity_bitmap_base_t mask = ((typeof(mask))1) << + (index % _voucher_activity_bits_per_bitmap_base_t); + if (slowpath((bitmap[i] & mask) == 0)) { + DISPATCH_CRASH("Corruption: failed to clear bit exclusively"); + } + (void)dispatch_atomic_and(&bitmap[i], ~mask, release); +} + +_voucher_activity_metadata_t _voucher_activity_heap; +static dispatch_once_t _voucher_activity_heap_pred; + +static void +_voucher_activity_heap_init(void *ctxt DISPATCH_UNUSED) +{ + if (_voucher_activity_disabled()) return; + kern_return_t kr; + mach_vm_size_t vm_size = _voucher_activity_buffer_size * + _voucher_activity_buffers_per_heap; + mach_vm_address_t vm_addr = vm_page_size; + while (slowpath(kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size, + 0, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_MEMORY_GENEALOGY), + MEMORY_OBJECT_NULL, 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, + VM_INHERIT_NONE))) { + if (kr != KERN_NO_SPACE) { + (void)dispatch_assume_zero(kr); + _voucher_activity_mode = voucher_activity_mode_disable; + return; + } + _dispatch_temporary_resource_shortage(); + vm_addr = vm_page_size; + } + _voucher_activity_metadata_t heap; + task_trace_memory_info_data_t trace_memory_info = { + .user_memory_address = vm_addr, + .buffer_size = vm_size, + .mailbox_array_size = sizeof(heap->vam_kernel_metadata), + }; + kr = task_set_info(mach_task_self(), TASK_TRACE_MEMORY_INFO, + (task_info_t)&trace_memory_info, TASK_TRACE_MEMORY_INFO_COUNT); + DISPATCH_VERIFY_MIG(kr); + if (kr) { + if (kr != KERN_NOT_SUPPORTED) (void)dispatch_assume_zero(kr); + kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size); + (void)dispatch_assume_zero(kr); + _voucher_activity_mode = voucher_activity_mode_disable; + return; + } + heap = (void*)vm_addr; + heap->vam_self_metadata.vasm_baseaddr = (void*)vm_addr; + heap->vam_buffer_bitmap[0] = 0xf; // first four buffers are reserved + uint32_t i; + for (i = 0; i < _voucher_activity_hash_size; i++) { + TAILQ_INIT(&heap->vam_activities[i]); + TAILQ_INIT(&heap->vam_atms[i]); + } + uint32_t subid_max = VATM_SUBID_MAX; + if (_voucher_atm_subid_bits && + _voucher_atm_subid_bits < VATM_SUBID_MAXBITS) { + subid_max = MIN(VATM_SUBID_BITS2MAX(_voucher_atm_subid_bits), + VATM_SUBID_MAX); + } + heap->vam_base_atm_subid_max = subid_max; + _voucher_activity_lock_init(&heap->vam_base_atm_lock); + _voucher_activity_lock_init(&heap->vam_nested_atm_lock); + _voucher_activity_lock_init(&heap->vam_atms_lock); + _voucher_activity_lock_init(&heap->vam_activities_lock); + _voucher_activity_heap = heap; + + _voucher_atm_t vatm = _voucher_atm_create(0, 0); + dispatch_assert(vatm->vatm_kvoucher); + heap->vam_default_activity_atm = vatm; + _voucher_activity_buffer_header_t buffer = vam_buffer(3); // reserved index + // consumes vatm reference: + _voucher_activity_t va = _voucher_activity_create_with_atm(vatm, + VATM_ACTID(vatm, _voucher_default_activity_subid), 0, 0, buffer); + dispatch_assert(va); + va->va_buffer_limit = _voucher_default_activity_buffer_limit(); + _voucher_activity_default = va; + heap->vam_base_atm = _voucher_atm_create(0, 0); + heap->vam_nested_atm_id = _voucher_atm_nested_atm_id_make(); +} + +static void +_voucher_activity_atfork_child(void) +{ + _voucher_activity_heap_pred = 0; + _voucher_activity_heap = NULL; // activity heap is VM_INHERIT_NONE + _voucher_activity_default = NULL; +} + +void* +voucher_activity_get_metadata_buffer(size_t *length) +{ + dispatch_once_f(&_voucher_activity_heap_pred, NULL, + _voucher_activity_heap_init); + if (_voucher_activity_disabled()) { + *length = 0; + return NULL; + } + *length = sizeof(_voucher_activity_heap->vam_client_metadata); + return _voucher_activity_heap->vam_client_metadata; +} + +DISPATCH_ALWAYS_INLINE +static inline _voucher_activity_buffer_header_t +_voucher_activity_heap_buffer_alloc(void) +{ + _voucher_activity_buffer_header_t buffer = NULL; + size_t index; + index = _voucher_activity_bitmap_set_first_unset_bit_upto( + vam_buffer_bitmap(), _voucher_heap_buffer_limit() - 1); + if (index < NO_BITS_WERE_UNSET) { + buffer = vam_buffer(index); + } +#if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG + _dispatch_debug("activity heap alloc %zd (%p)", index, buffer); +#endif + return buffer; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_activity_heap_buffer_free(_voucher_activity_buffer_header_t buffer) +{ + buffer->vabh_flags = _voucher_activity_trace_flag_buffer_empty; + size_t index = (size_t)((char*)buffer - (char*)_voucher_activity_heap) / + _voucher_activity_buffer_size; +#if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG + _dispatch_debug("activity heap free %zd (%p)", index, buffer); +#endif + _voucher_activity_bitmap_clear_bit(vam_buffer_bitmap(), index); +} + +#define _voucher_activity_heap_can_madvise() \ + (PAGE_SIZE == _voucher_activity_buffer_size) // + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_activity_heap_madvise(size_t bitmap_num, unsigned int start, + unsigned int len) +{ + size_t base = bitmap_num * _voucher_activity_bits_per_bitmap_base_t; +#if DISPATCH_DEBUG +#if DISPATCH_VOUCHER_ACTIVITY_DEBUG + _dispatch_debug("activity heap madvise %zd (%p) -> %zd (%p)", base + start, + vam_buffer(base + start), base + start + len, + vam_buffer(base + start + len)); +#endif + dispatch_assert(!(len * _voucher_activity_buffer_size % vm_page_size)); + const uint64_t pattern = 0xFACEFACEFACEFACE; + _voucher_activity_buffer_header_t buffer = vam_buffer(base + start); + for (unsigned int i = 0; i < len; i++, buffer++) { + memset_pattern8((char*)buffer + sizeof(buffer->vabh_flags), &pattern, + _voucher_activity_buffer_size - sizeof(buffer->vabh_flags)); + } +#endif + (void)dispatch_assume_zero(madvise(vam_buffer(base + start), + len * _voucher_activity_buffer_size, MADV_FREE)); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_activity_heap_madvise_contiguous(size_t bitmap_num, + _voucher_activity_bitmap_base_t bits) +{ + // TODO: x86 has fast ctz; arm has fast clz; haswell has fast ctz + dispatch_assert(_voucher_activity_heap_can_madvise()); + if (bits == 0) { + return; + } else if (~bits == 0) { + _voucher_activity_heap_madvise(bitmap_num, 0, + _voucher_activity_bits_per_bitmap_base_t); + } else while (bits != 0) { + unsigned int start = (typeof(start))__builtin_ctzl(bits), len; + typeof(bits) inverse = ~bits >> start; + if (inverse) { + len = (typeof(len))__builtin_ctzl(inverse); + } else { + len = _voucher_activity_bits_per_bitmap_base_t - start; + } + typeof(bits) mask = ((((typeof(bits))1) << len) - 1) << start; + bits &= ~mask; + _voucher_activity_heap_madvise(bitmap_num, start, len); + } +} + +void +_voucher_activity_heap_pressure_warn(void) +{ + if (!_voucher_activity_heap_can_madvise() || !_voucher_activity_heap) { + return; + } + volatile _voucher_activity_bitmap_base_t *bitmap, *pressure_locked_bitmap; + bitmap = vam_buffer_bitmap(); + pressure_locked_bitmap = vam_pressure_locked_bitmap(); + + // number of bitmaps needed to map the current buffer limit = + // ceil(buffer limit / bits per bitmap) + size_t nbuffers = _voucher_heap_buffer_limit(); + size_t nbitmaps_quot = nbuffers / _voucher_activity_bits_per_bitmap_base_t; + size_t nbitmaps_rem = nbuffers % _voucher_activity_bits_per_bitmap_base_t; + size_t nbitmaps = nbitmaps_quot + ((nbitmaps_rem == 0) ? 0 : 1); + + for (size_t i = 0; i < nbitmaps; i++) { + _voucher_activity_bitmap_base_t got_bits; + got_bits = dispatch_atomic_or_orig(&bitmap[i], ~((typeof(bitmap[i]))0), + relaxed); + got_bits = ~got_bits; // Now 1 means 'acquired this one, madvise it' + _voucher_activity_heap_madvise_contiguous(i, got_bits); + pressure_locked_bitmap[i] |= got_bits; + } +} + +void +_voucher_activity_heap_pressure_normal(void) +{ + if (!_voucher_activity_heap_can_madvise() || !_voucher_activity_heap) { + return; + } + volatile _voucher_activity_bitmap_base_t *bitmap, *pressure_locked_bitmap; + bitmap = vam_buffer_bitmap(); + pressure_locked_bitmap = vam_pressure_locked_bitmap(); + for (size_t i = 0; i < _voucher_activity_bitmaps_per_heap; i++) { + _voucher_activity_bitmap_base_t free_bits = pressure_locked_bitmap[i]; + pressure_locked_bitmap[i] = 0; + if (free_bits != 0) { + (void)dispatch_atomic_and(&bitmap[i], ~free_bits, release); + } + } +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_activity_buffer_init(_voucher_activity_t act, + _voucher_activity_buffer_header_t buffer, bool reuse) +{ + if (!reuse) { + buffer->vabh_flags = _voucher_activity_trace_flag_buffer_header; + buffer->vabh_activity_id = act->va_id; + } + buffer->vabh_timestamp = _voucher_activity_timestamp(); + buffer->vabh_next_tracepoint_idx = 1; + buffer->vabh_sequence_no = dispatch_atomic_inc2o(act, va_max_sequence_no, + relaxed); +} + +static _voucher_activity_buffer_header_t +_voucher_activity_buffer_alloc_slow(_voucher_activity_t act, + _voucher_activity_buffer_header_t current) +{ + _voucher_activity_buffer_header_t buffer; + _voucher_activity_lock_lock(va_buffers_lock(act)); // TODO: revisit locking + buffer = act->va_current_buffer; + if (buffer != current) goto out; + buffer = TAILQ_FIRST(&act->va_buffers); + if (buffer) { + _voucher_activity_buffer_init(act, buffer, true); + if (buffer != TAILQ_LAST(&act->va_buffers, + _voucher_activity_buffer_list_s)) { + TAILQ_REMOVE(&act->va_buffers, buffer, vabh_list); + TAILQ_INSERT_TAIL(&act->va_buffers, buffer, vabh_list); + } + } + if (!dispatch_atomic_cmpxchgv2o(act, va_current_buffer, current, buffer, + ¤t, release)) { + if (buffer) { + TAILQ_REMOVE(&act->va_buffers, buffer, vabh_list); + _voucher_activity_heap_buffer_free(buffer); + } + buffer = current; + } +out: + _voucher_activity_lock_unlock(va_buffers_lock(act)); + _dispatch_voucher_activity_debug("buffer reuse %p", act, buffer); + return buffer; +} + +static _voucher_activity_buffer_header_t +_voucher_activity_buffer_alloc(_voucher_activity_t act, + _voucher_activity_buffer_header_t current) +{ + _voucher_activity_buffer_header_t buffer = NULL; + if (act->va_max_sequence_no < act->va_buffer_limit) { + buffer = _voucher_activity_heap_buffer_alloc(); + } + if (!buffer) return _voucher_activity_buffer_alloc_slow(act, current); + _voucher_activity_buffer_init(act, buffer, false); + if (dispatch_atomic_cmpxchgv2o(act, va_current_buffer, current, buffer, + ¤t, release)) { + _voucher_activity_lock_lock(va_buffers_lock(act)); + TAILQ_INSERT_TAIL(&act->va_buffers, buffer, vabh_list); + _voucher_activity_lock_unlock(va_buffers_lock(act)); + } else { + _voucher_activity_heap_buffer_free(buffer); + buffer = current; + } + _dispatch_voucher_activity_debug("buffer alloc %p", act, buffer); + return buffer; +} + +#pragma mark - +#pragma mark _voucher_activity_t + +#define _voucher_activity_ordered_insert(_act, head, field) do { \ + typeof(_act) _vai; \ + TAILQ_FOREACH(_vai, (head), field) { \ + if (_act->va_id < _vai->va_id) break; \ + } \ + if (_vai) { \ + TAILQ_INSERT_BEFORE(_vai, _act, field); \ + } else { \ + TAILQ_INSERT_TAIL((head), _act, field); \ + } } while (0); + +static void _voucher_activity_dispose(_voucher_activity_t act); +static _voucher_activity_t _voucher_atm_activity_mark_used( + _voucher_activity_t act); +static void _voucher_atm_activity_mark_unused(_voucher_activity_t act); +static _voucher_atm_t _voucher_atm_copy(atm_aid_t atm_id); +static inline void _voucher_atm_release(_voucher_atm_t vatm); +static void _voucher_atm_activity_insert(_voucher_atm_t vatm, + _voucher_activity_t act); +static void _voucher_atm_activity_remove(_voucher_activity_t act); +static atm_aid_t _voucher_mach_voucher_get_atm_id(mach_voucher_t kv); + +DISPATCH_ALWAYS_INLINE +static inline bool +_voucher_activity_copy(_voucher_activity_t act) +{ + int use_cnt = dispatch_atomic_inc2o(act, va_use_count, relaxed); + _dispatch_voucher_activity_debug("retain -> %d", act, use_cnt + 1); + if (slowpath(use_cnt < 0)) { + _dispatch_voucher_activity_debug("overrelease", act); + DISPATCH_CRASH("Activity overrelease"); + } + return (use_cnt == 0); +} + +DISPATCH_ALWAYS_INLINE +static inline _voucher_activity_t +_voucher_activity_retain(_voucher_activity_t act) +{ + if (_voucher_activity_copy(act)) { + _dispatch_voucher_activity_debug("invalid resurrection", act); + DISPATCH_CRASH("Invalid activity resurrection"); + } + return act; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_activity_release(_voucher_activity_t act) +{ + int use_cnt = dispatch_atomic_dec2o(act, va_use_count, relaxed); + _dispatch_voucher_activity_debug("release -> %d", act, use_cnt + 1); + if (fastpath(use_cnt >= 0)) { + return; + } + if (slowpath(use_cnt < -1)) { + _dispatch_voucher_activity_debug("overrelease", act); + DISPATCH_CRASH("Activity overrelease"); + } + return _voucher_atm_activity_mark_unused(act); +} + +DISPATCH_ALWAYS_INLINE +static inline _voucher_activity_t +_voucher_activity_atm_retain(_voucher_activity_t act) +{ + int refcnt = dispatch_atomic_inc2o(act, va_refcnt, relaxed); + _dispatch_voucher_activity_debug("atm retain -> %d", act, refcnt + 1); + if (slowpath(refcnt <= 0)) { + _dispatch_voucher_activity_debug("atm resurrection", act); + DISPATCH_CRASH("Activity ATM resurrection"); + } + return act; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_activity_atm_release(_voucher_activity_t act) +{ + int refcnt = dispatch_atomic_dec2o(act, va_refcnt, relaxed); + _dispatch_voucher_activity_debug("atm release -> %d", act, refcnt + 1); + if (fastpath(refcnt >= 0)) { + return; + } + if (slowpath(refcnt < -1)) { + _dispatch_voucher_activity_debug("atm overrelease", act); + DISPATCH_CRASH("Activity ATM overrelease"); + } + return _voucher_activity_dispose(act); +} + +static inline _voucher_activity_t +_voucher_activity_get(voucher_t v) +{ + _voucher_activity_t act; + act = v && v->v_activity ? v->v_activity : _voucher_activity_default; + return act; +} + +static _voucher_activity_t +_voucher_activity_find(voucher_activity_id_t va_id, uint32_t hash) +{ + // assumes vam_activities_lock held + _voucher_activity_t act; + TAILQ_FOREACH(act, vam_activities(hash), va_list){ + if (act->va_id == va_id) break; + } + return act; +} + +static _voucher_activity_t +_voucher_activity_copy_from_id(voucher_activity_id_t va_id) +{ + bool resurrect = false; + uint32_t hash = VACTID_HASH(va_id); + _voucher_activity_lock_lock(vam_activities_lock()); + _voucher_activity_t act = _voucher_activity_find(va_id, hash); + if (act) { + resurrect = _voucher_activity_copy(act); + _dispatch_voucher_activity_debug("copy from id 0x%llx", act, va_id); + } + _voucher_activity_lock_unlock(vam_activities_lock()); + if (resurrect) return _voucher_atm_activity_mark_used(act); + return act; +} + +static _voucher_activity_t +_voucher_activity_try_insert(_voucher_activity_t act_new) +{ + bool resurrect = false; + voucher_activity_id_t va_id = act_new->va_id; + uint32_t hash = VACTID_HASH(va_id); + _voucher_activity_lock_lock(vam_activities_lock()); + _voucher_activity_t act = _voucher_activity_find(va_id, hash); + if (act) { + resurrect = _voucher_activity_copy(act); + _dispatch_voucher_activity_debug("try insert: failed (%p)", act,act_new); + } else { + if (slowpath(_TAILQ_IS_ENQUEUED(act_new, va_list))) { + _dispatch_voucher_activity_debug("corruption", act_new); + DISPATCH_CRASH("Activity corruption"); + } + TAILQ_INSERT_TAIL(vam_activities(hash), act_new, va_list); + _dispatch_voucher_activity_debug("try insert: succeeded", act_new); + } + _voucher_activity_lock_unlock(vam_activities_lock()); + if (resurrect) return _voucher_atm_activity_mark_used(act); + return act; +} + +static bool +_voucher_activity_try_remove(_voucher_activity_t act) +{ + bool r; + voucher_activity_id_t va_id = act->va_id; + uint32_t hash = VACTID_HASH(va_id); + _voucher_activity_lock_lock(vam_activities_lock()); + if (slowpath(!va_id)) { + _dispatch_voucher_activity_debug("corruption", act); + DISPATCH_CRASH("Activity corruption"); + } + if ((r = (dispatch_atomic_load2o(act, va_use_count, seq_cst) < 0 && + _TAILQ_IS_ENQUEUED(act, va_list)))) { + TAILQ_REMOVE(vam_activities(hash), act, va_list); + _TAILQ_MARK_NOT_ENQUEUED(act, va_list); + act->va_list.tqe_next = (void*)~0ull; + } + _dispatch_voucher_activity_debug("try remove: %s", act, r ? "succeeded" : + "failed"); + _voucher_activity_lock_unlock(vam_activities_lock()); + return r; +} + +static _voucher_activity_t +_voucher_activity_create_with_atm(_voucher_atm_t vatm, + voucher_activity_id_t va_id, voucher_activity_trace_id_t trace_id, + uint64_t location, _voucher_activity_buffer_header_t buffer) +{ + if (!buffer) buffer = _voucher_activity_heap_buffer_alloc(); + if (!buffer) { + _dispatch_voucher_atm_debug("no buffer", vatm); + _voucher_atm_release(vatm); // consume vatm reference + return NULL; + } + if (!trace_id) trace_id = _voucher_activity_trace_id_release; + _voucher_activity_tracepoint_t vat = (_voucher_activity_tracepoint_t)buffer; + _voucher_activity_tracepoint_init_with_id(vat, trace_id, ~1ull); + _voucher_activity_t act = (_voucher_activity_t)buffer; + act->va_flags = _voucher_activity_trace_flag_buffer_header | + _voucher_activity_trace_flag_activity | + _voucher_activity_trace_flag_start | + _voucher_activity_trace_flag_wide_first; + act->vabh_next_tracepoint_idx = sizeof(*act)/sizeof(*vat); + act->va_max_sequence_no = 0; + act->va_id = va_id ? va_id : VATM_ACTID(vatm, 0); + act->va_use_count = 0; + act->va_buffer_limit = _voucher_activity_buffer_limit(); + TAILQ_INIT(&act->va_buffers); + act->va_flags2 = _voucher_activity_trace_flag_activity | + _voucher_activity_trace_flag_wide_second; +#if __LP64__ + act->va_flags3 = act->va_flags2; +#endif + act->va_refcnt = 0; + act->va_location = location; + act->va_current_buffer = buffer; + act->va_atm = vatm; // transfer vatm reference + _voucher_activity_lock_init(va_buffers_lock(act)); + _TAILQ_MARK_NOT_ENQUEUED(act, va_list); + _TAILQ_MARK_NOT_ENQUEUED(act, va_atm_list); + _TAILQ_MARK_NOT_ENQUEUED(act, va_atm_used_list); + _voucher_activity_t actx = _voucher_activity_try_insert(act); + if (actx) { + _voucher_activity_dispose(act); + act = actx; + } else { + _voucher_atm_activity_insert(vatm, act); + } + _dispatch_voucher_activity_debug("create", act); + return act; +} + +static void +_voucher_activity_dispose(_voucher_activity_t act) +{ + _dispatch_voucher_activity_debug("dispose", act); + _voucher_atm_release(act->va_atm); + if (slowpath(_TAILQ_IS_ENQUEUED(act, va_list))) { + _dispatch_voucher_activity_debug("corruption", act); + DISPATCH_CRASH("Activity corruption"); + } + act->va_list.tqe_next = DISPATCH_OBJECT_LISTLESS; + dispatch_assert(!_TAILQ_IS_ENQUEUED(act, va_atm_list)); + dispatch_assert(!_TAILQ_IS_ENQUEUED(act, va_atm_used_list)); + _voucher_activity_buffer_header_t buffer, tmp; + TAILQ_FOREACH_SAFE(buffer, &act->va_buffers, vabh_list, tmp) { + _dispatch_voucher_activity_debug("buffer free %p", act, buffer); + TAILQ_REMOVE(&act->va_buffers, buffer, vabh_list); + _voucher_activity_heap_buffer_free(buffer); + } + buffer = (_voucher_activity_buffer_header_t)act; + _voucher_activity_heap_buffer_free(buffer); +} + +static void +_voucher_activity_collect(_voucher_activity_t act) +{ + _dispatch_voucher_activity_debug("collect", act); + if (_voucher_activity_try_remove(act)) { + _voucher_atm_activity_remove(act); + } +} + +static _voucher_activity_t +_voucher_activity_copy_from_mach_voucher(mach_voucher_t kv, + voucher_activity_id_t va_id) +{ + dispatch_once_f(&_voucher_activity_heap_pred, NULL, + _voucher_activity_heap_init); + if (_voucher_activity_disabled()) return NULL; + _voucher_activity_t act = NULL; + if (dispatch_assume(va_id)) { + if ((act = _voucher_activity_copy_from_id(va_id))) return act; + } + atm_aid_t atm_id = _voucher_mach_voucher_get_atm_id(kv); + if (!dispatch_assume(atm_id)) return NULL; + _voucher_activity_buffer_header_t buffer; + buffer = _voucher_activity_heap_buffer_alloc(); + if (!buffer) return NULL; + _dispatch_kvoucher_debug("atm copy/create from <%lld>", kv, atm_id); + _voucher_atm_t vatm = _voucher_atm_copy(atm_id); + if (!vatm) vatm = _voucher_atm_create(kv, atm_id); + if (!vatm) { + _voucher_activity_heap_buffer_free(buffer); + return NULL; + } + if (VACTID_BASEID(va_id) != VATMID2ACTID(atm_id)) va_id = 0; + // consumes vatm reference: + act = _voucher_activity_create_with_atm(vatm, va_id, 0, 0, buffer); + _dispatch_voucher_activity_debug("copy from kvoucher[0x%08x]", act, kv); + return act; +} + +#pragma mark - +#pragma mark _voucher_atm_mailbox + +DISPATCH_ALWAYS_INLINE +static inline atm_mailbox_offset_t +_voucher_atm_mailbox_alloc(void) +{ + atm_mailbox_offset_t mailbox_offset = MAILBOX_OFFSET_UNSET; + size_t index; + index = _voucher_activity_bitmap_set_first_unset_bit(vam_atm_mbox_bitmap()); + if (index < NO_BITS_WERE_UNSET) { + mailbox_offset = index * sizeof(atm_mailbox_offset_t); +#if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG + _dispatch_debug("mailbox alloc %zd (%lld)", index, mailbox_offset); +#endif + } + return mailbox_offset; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_atm_mailbox_free(atm_mailbox_offset_t mailbox_offset) +{ + if (mailbox_offset == MAILBOX_OFFSET_UNSET) return; + size_t index = (size_t)mailbox_offset / sizeof(atm_mailbox_offset_t); +#if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG + _dispatch_debug("mailbox free %zd (%lld)", index, mailbox_offset); +#endif + _voucher_activity_bitmap_clear_bit(vam_atm_mbox_bitmap(), index); +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_voucher_atm_mailbox_set(atm_mailbox_offset_t mailbox_offset, + atm_subaid32_t subaid, bool max_present) +{ + if (mailbox_offset == MAILBOX_OFFSET_UNSET) return false; + char *mailbox_base = (char*)_voucher_activity_heap->vam_kernel_metadata; + atm_subaid32_t *mailbox = (atm_subaid32_t*)(mailbox_base + mailbox_offset); + if (max_present) mailbox++; // second atm_subaid32_t in atm_mailbox_offset_t + if (*mailbox == subaid) return false; + *mailbox = subaid; + return true; +} + +#pragma mark - +#pragma mark _voucher_atm_t + +static bool _voucher_atm_try_remove(_voucher_atm_t vatm); +static void _voucher_atm_dispose(_voucher_atm_t vatm, bool unregister); +static inline void _voucher_atm_collect_if_needed(bool updated); + +DISPATCH_ALWAYS_INLINE +static inline _voucher_atm_t +_voucher_atm_retain(_voucher_atm_t vatm) +{ + // assumes vam_atms_lock or vam_base_atm_lock held + int refcnt = dispatch_atomic_inc2o(vatm, vatm_refcnt, relaxed); + _dispatch_voucher_atm_debug("retain -> %d", vatm, refcnt + 1); + if (slowpath(refcnt < 0)) { + _dispatch_voucher_atm_debug("overrelease", vatm); + DISPATCH_CRASH("ATM overrelease"); + } + return vatm; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_atm_release(_voucher_atm_t vatm) +{ + int refcnt = dispatch_atomic_dec2o(vatm, vatm_refcnt, relaxed); + _dispatch_voucher_atm_debug("release -> %d", vatm, refcnt + 1); + if (fastpath(refcnt >= 0)) { + return; + } + if (slowpath(refcnt < -1)) { + _dispatch_voucher_atm_debug("overrelease", vatm); + DISPATCH_CRASH("ATM overrelease"); + } + if (_voucher_atm_try_remove(vatm)) { + _voucher_atm_dispose(vatm, true); + } +} + +static _voucher_atm_t +_voucher_atm_find(atm_aid_t atm_id, uint32_t hash) +{ + // assumes vam_atms_lock held + _voucher_atm_t vatm; + TAILQ_FOREACH(vatm, vam_atms(hash), vatm_list){ + if (vatm->vatm_id == atm_id) break; + } + return vatm; +} + +static _voucher_atm_t +_voucher_atm_copy(atm_aid_t atm_id) +{ + uint32_t hash = VATMID_HASH(atm_id); + _voucher_activity_lock_lock(vam_atms_lock()); + _voucher_atm_t vatm = _voucher_atm_find(atm_id, hash); + if (vatm) { + _voucher_atm_retain(vatm); + _dispatch_voucher_atm_debug("copy", vatm); + } + _voucher_activity_lock_unlock(vam_atms_lock()); + return vatm; +} + +static _voucher_atm_t +_voucher_atm_try_insert(_voucher_atm_t vatm_new) +{ + atm_aid_t atm_id = vatm_new->vatm_id; + uint32_t hash = VATMID_HASH(atm_id); + _voucher_activity_lock_lock(vam_atms_lock()); + _voucher_atm_t vatm = _voucher_atm_find(atm_id, hash); + if (vatm) { + _voucher_atm_retain(vatm); + _dispatch_voucher_atm_debug("try insert: failed (%p)", vatm, vatm_new); + } else { + if (slowpath(_TAILQ_IS_ENQUEUED(vatm_new, vatm_list))) { + _dispatch_voucher_atm_debug("corruption", vatm_new); + DISPATCH_CRASH("ATM corruption"); + } + TAILQ_INSERT_TAIL(vam_atms(hash), vatm_new, vatm_list); + _dispatch_voucher_atm_debug("try insert: succeeded", vatm_new); + } + _voucher_activity_lock_unlock(vam_atms_lock()); + return vatm; +} + +static bool +_voucher_atm_try_remove(_voucher_atm_t vatm) +{ + bool r; + atm_aid_t atm_id = vatm->vatm_id; + uint32_t hash = VATMID_HASH(atm_id); + _voucher_activity_lock_lock(vam_atms_lock()); + if (slowpath(!atm_id)) { + _dispatch_voucher_atm_debug("corruption", vatm); + DISPATCH_CRASH("ATM corruption"); + } + if ((r = (dispatch_atomic_load2o(vatm, vatm_refcnt, seq_cst) < 0 && + _TAILQ_IS_ENQUEUED(vatm, vatm_list)))) { + TAILQ_REMOVE(vam_atms(hash), vatm, vatm_list); + _TAILQ_MARK_NOT_ENQUEUED(vatm, vatm_list); + vatm->vatm_list.tqe_next = (void*)~0ull; + } + _dispatch_voucher_atm_debug("try remove: %s", vatm, r ? "succeeded" : + "failed"); + _voucher_activity_lock_unlock(vam_atms_lock()); + return r; +} + +static bool +_voucher_atm_update_mailbox(_voucher_atm_t vatm) +{ + // Update kernel mailbox with largest allocated subaid for this atm_id + // assumes atm_activities_lock held + _voucher_activity_t act = TAILQ_LAST(vatm_activities(vatm), + _voucher_atm_activities_s); + atm_subaid32_t subaid = act ? VACTID_SUBID(act->va_id) : 0; + bool r = _voucher_atm_mailbox_set(vatm->vatm_mailbox_offset, subaid, true); + if (r) { + _dispatch_voucher_atm_debug("update max-present subaid 0x%x", vatm, + subaid); + } + return r; +} + +static bool +_voucher_atm_update_used_mailbox(_voucher_atm_t vatm) +{ + // Update kernel mailbox with smallest in-use subaid for this atm_id + // assumes atm_activities_lock held + _voucher_activity_t act = TAILQ_FIRST(vatm_used_activities(vatm)); + atm_subaid32_t subaid = act ? VACTID_SUBID(act->va_id) : ATM_SUBAID32_MAX; + bool r = _voucher_atm_mailbox_set(vatm->vatm_mailbox_offset, subaid, false); + if (r) { + _dispatch_voucher_atm_debug("update min-used subaid 0x%x", vatm, + subaid); + } + return r; +} + +static void +_voucher_atm_activity_insert(_voucher_atm_t vatm, _voucher_activity_t act) +{ + _voucher_activity_lock_lock(vatm_activities_lock(vatm)); + if (!_TAILQ_IS_ENQUEUED(act, va_atm_list)) { + _voucher_activity_ordered_insert(act, vatm_activities(vatm), + va_atm_list); + _voucher_atm_update_mailbox(vatm); + } + if (!_TAILQ_IS_ENQUEUED(act, va_atm_used_list)) { + _voucher_activity_ordered_insert(act, vatm_used_activities(vatm), + va_atm_used_list); + _voucher_atm_update_used_mailbox(vatm); + } + _dispatch_voucher_activity_debug("atm insert", act); + _voucher_activity_lock_unlock(vatm_activities_lock(vatm)); +} + +static void +_voucher_atm_activity_remove(_voucher_activity_t act) +{ + _voucher_atm_t vatm = act->va_atm; + _voucher_activity_lock_lock(vatm_activities_lock(vatm)); + _dispatch_voucher_activity_debug("atm remove", act); + if (_TAILQ_IS_ENQUEUED(act, va_atm_used_list)) { + TAILQ_REMOVE(vatm_activities(vatm), act, va_atm_used_list); + _TAILQ_MARK_NOT_ENQUEUED(act, va_atm_used_list); + _voucher_atm_update_used_mailbox(vatm); + } + if (_TAILQ_IS_ENQUEUED(act, va_atm_list)) { + TAILQ_REMOVE(vatm_activities(vatm), act, va_atm_list); + _TAILQ_MARK_NOT_ENQUEUED(act, va_atm_list); + _voucher_atm_update_mailbox(vatm); + // Balance initial creation refcnt. Caller must hold additional + // reference to ensure this does not release vatm before the unlock, + // see _voucher_atm_activity_collect + _voucher_activity_atm_release(act); + } + _voucher_activity_lock_unlock(vatm_activities_lock(vatm)); +} + +static _voucher_activity_t +_voucher_atm_activity_mark_used(_voucher_activity_t act) +{ + _voucher_atm_t vatm = act->va_atm; + _voucher_activity_lock_lock(vatm_activities_lock(vatm)); + if (!_TAILQ_IS_ENQUEUED(act, va_atm_used_list)) { + _voucher_activity_ordered_insert(act, vatm_used_activities(vatm), + va_atm_used_list); + _voucher_atm_update_used_mailbox(vatm); + _dispatch_voucher_activity_debug("mark used", act); + } + _voucher_activity_lock_unlock(vatm_activities_lock(vatm)); + return act; +} + +static void +_voucher_atm_activity_mark_unused(_voucher_activity_t act) +{ + bool atm_collect = false, updated = false; + _voucher_atm_t vatm = act->va_atm; + _voucher_activity_lock_lock(vatm_activities_lock(vatm)); + if (_TAILQ_IS_ENQUEUED(act, va_atm_used_list)) { + _dispatch_voucher_activity_debug("mark unused", act); + TAILQ_REMOVE(&vatm->vatm_used_activities, act, va_atm_used_list); + _TAILQ_MARK_NOT_ENQUEUED(act, va_atm_used_list); + atm_collect = true; + _voucher_atm_retain(vatm); + updated = _voucher_atm_update_used_mailbox(vatm); + } + _voucher_activity_lock_unlock(vatm_activities_lock(vatm)); + if (atm_collect) { + _voucher_atm_release(vatm); + _voucher_atm_collect_if_needed(updated); + } +} + +static void +_voucher_atm_activity_collect(_voucher_atm_t vatm, atm_subaid32_t min_subaid) +{ + _dispatch_voucher_atm_debug("collect min subaid 0x%x", vatm, min_subaid); + voucher_activity_id_t min_va_id = VATM_ACTID(vatm, min_subaid); + _voucher_activity_t act; + do { + _voucher_activity_lock_lock(vatm_activities_lock(vatm)); + TAILQ_FOREACH(act, vatm_activities(vatm), va_atm_list) { + if (act->va_id >= min_va_id) { + act = NULL; + break; + } + if (!_TAILQ_IS_ENQUEUED(act, va_atm_used_list)) { + _voucher_activity_atm_retain(act); + break; + } + } + _voucher_activity_lock_unlock(vatm_activities_lock(vatm)); + if (act) { + _voucher_activity_collect(act); + _voucher_activity_atm_release(act); + } + } while (act); +} + +DISPATCH_NOINLINE +static void +_voucher_atm_collect(void) +{ + _voucher_atm_t vatms[_voucher_atm_mailboxes], vatm; + atm_aid_t aids[_voucher_atm_mailboxes]; + mach_atm_subaid_t subaids[_voucher_atm_mailboxes]; + uint32_t i, a = 0, s; + + _voucher_activity_lock_lock(vam_atms_lock()); + for (i = 0; i < _voucher_activity_hash_size; i++) { + TAILQ_FOREACH(vatm, vam_atms(i), vatm_list){ + if (vatm == _voucher_activity_heap->vam_default_activity_atm || + vatm->vatm_mailbox_offset == MAILBOX_OFFSET_UNSET) continue; + _dispatch_voucher_atm_debug("find min subaid", vatm); + vatms[a] = _voucher_atm_retain(vatm); + aids[a] = vatm->vatm_id; + if (++a == _voucher_atm_mailboxes) goto out; + } + } +out: + _voucher_activity_lock_unlock(vam_atms_lock()); + if (!a) return; + kern_return_t kr; + mach_voucher_t kv = vatms[0]->vatm_kvoucher; + mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&aids; + mach_voucher_attr_content_size_t kvc_in_size = sizeof(atm_aid_t) * a; + mach_voucher_attr_content_t kvc_out = (mach_voucher_attr_content_t)&subaids; + mach_voucher_attr_content_size_t kvc_out_size = sizeof(mach_atm_subaid_t)*a; + kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_ATM, + ATM_FIND_MIN_SUB_AID, kvc_in, kvc_in_size, kvc_out, &kvc_out_size); + DISPATCH_VERIFY_MIG(kr); + (void)dispatch_assume_zero(kr); + s = kvc_out_size / sizeof(mach_atm_subaid_t); +#if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG + _dispatch_debug("found min subaids (%u out of %u)", s, a); +#endif + for (i = 0; i < a; i++) { + if (i < s) _voucher_atm_activity_collect(vatms[i], + (atm_subaid32_t)subaids[i]); + _voucher_atm_release(vatms[i]); + } +} + +static inline void +_voucher_atm_collect_if_needed(bool updated) +{ + long level; + if (updated) { + level = dispatch_atomic_add(&_voucher_atm_collect_level, 2ul, relaxed); + } else { + level = _voucher_atm_collect_level; + if (!level) return; + } + if (level & 1 || level <= _voucher_atm_collect_threshold) return; + if (!dispatch_atomic_cmpxchg(&_voucher_atm_collect_level, level, level + 1, + acquire)) return; +#if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG + _dispatch_debug("atm collect: reached level %ld", level/2); +#endif + if (slowpath(level < 0)) { + DISPATCH_CRASH("ATM collection level corruption"); + } + _voucher_atm_collect(); + dispatch_atomic_sub(&_voucher_atm_collect_level, level + 1, release); +} + +DISPATCH_NOINLINE +static void +_voucher_atm_fault(mach_voucher_attr_command_t kvc_cmd) +{ + _voucher_activity_t act = _voucher_activity_get(_voucher_get()); + mach_voucher_t kv = _voucher_activity_get_atm_mach_voucher(act); + if (!kv) return; + + kern_return_t kr; + mach_atm_subaid_t subaid = VACTID_SUBID(act->va_id); + mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&subaid; + mach_voucher_attr_content_size_t kvc_in_size = sizeof(mach_atm_subaid_t); + mach_voucher_attr_content_t kvc_out = (mach_voucher_attr_content_t)&subaid; + mach_voucher_attr_content_size_t kvc_out_size = sizeof(mach_atm_subaid_t); + kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_ATM, + kvc_cmd, kvc_in, kvc_in_size, kvc_out, &kvc_out_size); + DISPATCH_VERIFY_MIG(kr); + (void)dispatch_assume_zero(kr); +} + +static atm_aid_t +_voucher_mach_voucher_get_atm_id(mach_voucher_t kv) +{ + kern_return_t kr; + atm_aid_t atm_id = 0; + mach_voucher_attr_content_t kvc = (mach_voucher_attr_content_t)&atm_id; + mach_voucher_attr_content_size_t kvc_size = sizeof(atm_id); + kr = mach_voucher_extract_attr_content(kv, MACH_VOUCHER_ATTR_KEY_ATM, kvc, + &kvc_size); + DISPATCH_VERIFY_MIG(kr); + (void)dispatch_assume_zero(kr); + return atm_id; +} + +static mach_voucher_t +_voucher_atm_mach_voucher_create(atm_aid_t *atm_id_ptr) +{ + kern_return_t kr; + mach_voucher_t kv; + static const mach_voucher_attr_recipe_data_t atm_create_recipe = { + .key = MACH_VOUCHER_ATTR_KEY_ATM, + .command = MACH_VOUCHER_ATTR_ATM_CREATE, + }; + kr = _voucher_create_mach_voucher(&atm_create_recipe, + sizeof(atm_create_recipe), &kv); + if (dispatch_assume_zero(kr)) { + DISPATCH_CLIENT_CRASH("Could not create ATM mach voucher"); + } + atm_aid_t atm_id = _voucher_mach_voucher_get_atm_id(kv); + if (!dispatch_assume(atm_id)) { + DISPATCH_CLIENT_CRASH("Could not extract ATM ID"); + } + _dispatch_kvoucher_debug("atm create <%lld>", kv, atm_id); + *atm_id_ptr = atm_id; + return kv; +} + +static void +_voucher_atm_mailbox_mach_voucher_register(_voucher_atm_t vatm, + mach_voucher_t kv) +{ + _dispatch_voucher_atm_debug("mailbox register %lld with kvoucher[0x%08x]", + vatm, vatm->vatm_mailbox_offset, kv); + kern_return_t kr; + mach_voucher_t akv; + atm_mailbox_offset_t offset = vatm->vatm_mailbox_offset; + mach_voucher_attr_recipe_t vr; + size_t vr_size; + static const mach_voucher_attr_recipe_data_t atm_register_recipe = { + .key = MACH_VOUCHER_ATTR_KEY_ATM, + .command = MACH_VOUCHER_ATTR_ATM_REGISTER, + .content_size = sizeof(offset), + }; + vr_size = sizeof(atm_register_recipe) + atm_register_recipe.content_size; + vr = alloca(vr_size); + *vr = atm_register_recipe; + vr->previous_voucher = kv; + memcpy(&vr->content, &offset, sizeof(offset)); + kr = _voucher_create_mach_voucher(vr, vr_size, &akv); + if (dispatch_assume_zero(kr)) { + DISPATCH_CLIENT_CRASH("Could not register ATM ID"); + } + if (!vatm->vatm_kvoucher) { + vatm->vatm_kvoucher = akv; + } else { +#if !RDAR_17510224 + if (akv != vatm->vatm_kvoucher) { + DISPATCH_CRASH("Unexpected mach voucher returned by ATM ID " + "registration"); + } + _voucher_dealloc_mach_voucher(akv); +#else + DISPATCH_CRASH("Registered invalid ATM object"); +#endif + } + _dispatch_voucher_atm_debug("mailbox registered %lld", vatm, + vatm->vatm_mailbox_offset); +} + +static void +_voucher_atm_mailbox_register(_voucher_atm_t vatm) +{ + mach_voucher_t kv = vatm->vatm_kvoucher; + if (!kv) return; +#if !RDAR_17510224 + _voucher_atm_mailbox_mach_voucher_register(vatm, kv); +#else // RDAR_17510224 + _dispatch_voucher_atm_debug("mailbox register %lld", vatm, + vatm->vatm_mailbox_offset); + kern_return_t kr; + atm_mailbox_offset_t offset = vatm->vatm_mailbox_offset; + mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&offset; + mach_voucher_attr_content_size_t kvc_in_size = sizeof(offset); + mach_voucher_attr_content_t kvc_out = NULL; + mach_voucher_attr_content_size_t kvc_out_size = 0; + kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_ATM, + ATM_ACTION_REGISTER, kvc_in, kvc_in_size, kvc_out, + &kvc_out_size); + DISPATCH_VERIFY_MIG(kr); + if (dispatch_assume_zero(kr)) { + DISPATCH_CLIENT_CRASH("Could not register ATM ID"); + } + _dispatch_voucher_atm_debug("mailbox registered %lld", vatm, + vatm->vatm_mailbox_offset); +#endif // RDAR_17510224 +} + +static bool +_voucher_atm_mailbox_unregister(_voucher_atm_t vatm) +{ + if (vatm->vatm_mailbox_offset == MAILBOX_OFFSET_UNSET) return false; + _dispatch_voucher_atm_debug("mailbox unregister %lld", vatm, + vatm->vatm_mailbox_offset); + mach_voucher_t kv = vatm->vatm_kvoucher; + dispatch_assert(kv); + kern_return_t kr; + atm_mailbox_offset_t offset = vatm->vatm_mailbox_offset; + mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&offset; + mach_voucher_attr_content_size_t kvc_in_size = sizeof(offset); + mach_voucher_attr_content_t kvc_out = NULL; + mach_voucher_attr_content_size_t kvc_out_size = 0; + kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_ATM, + ATM_ACTION_UNREGISTER, kvc_in, kvc_in_size, kvc_out, &kvc_out_size); + DISPATCH_VERIFY_MIG(kr); + if (kr && kr != KERN_INVALID_VALUE) { + (void)dispatch_assume_zero(kr); + DISPATCH_CLIENT_CRASH("Could not unregister ATM ID"); + } + _dispatch_voucher_atm_debug("mailbox unregistered %lld", vatm, + vatm->vatm_mailbox_offset); + return true; +} + +static _voucher_atm_t +_voucher_atm_create(mach_voucher_t kv, atm_aid_t atm_id) +{ + atm_mailbox_offset_t mailbox_offset = _voucher_atm_mailbox_alloc(); + if (kv && mailbox_offset == MAILBOX_OFFSET_UNSET) return NULL; + _voucher_atm_t vatm = _dispatch_calloc(1ul, sizeof(struct _voucher_atm_s)); + if (!kv) { + kv = _voucher_atm_mach_voucher_create(&atm_id); + if (mailbox_offset == MAILBOX_OFFSET_UNSET) { + _voucher_dealloc_mach_voucher(kv); + } else { + vatm->vatm_kvoucher = kv; + } + kv = MACH_VOUCHER_NULL; + } + vatm->vatm_id = atm_id; + vatm->vatm_mailbox_offset = mailbox_offset; + _voucher_activity_lock_init(vatm_activities_lock(vatm)); + TAILQ_INIT(&vatm->vatm_activities); + TAILQ_INIT(&vatm->vatm_used_activities); + _voucher_atm_mailbox_set(mailbox_offset, 0, true); + _voucher_atm_mailbox_set(mailbox_offset, ATM_SUBAID32_MAX, false); + _voucher_atm_t vatmx = _voucher_atm_try_insert(vatm); + if (vatmx) { + _voucher_atm_dispose(vatm, false); + vatm = vatmx; + } else if (kv) { + _voucher_atm_mailbox_mach_voucher_register(vatm, kv); + } else { + _voucher_atm_mailbox_register(vatm); + } + _dispatch_voucher_atm_debug("create with kvoucher[0x%08x]", vatm, kv); + return vatm; +} + +static void +_voucher_atm_dispose(_voucher_atm_t vatm, bool unregister) +{ + _dispatch_voucher_atm_debug("dispose", vatm); + dispatch_assert(TAILQ_EMPTY(&vatm->vatm_activities)); + dispatch_assert(TAILQ_EMPTY(&vatm->vatm_used_activities)); + if (slowpath(_TAILQ_IS_ENQUEUED(vatm, vatm_list))) { + _dispatch_voucher_atm_debug("corruption", vatm); + DISPATCH_CRASH("ATM corruption"); + } + vatm->vatm_list.tqe_next = DISPATCH_OBJECT_LISTLESS; + bool free_mailbox = (vatm->vatm_mailbox_offset != MAILBOX_OFFSET_UNSET); + if (vatm->vatm_kvoucher) { + if (unregister) free_mailbox = _voucher_atm_mailbox_unregister(vatm); + _voucher_dealloc_mach_voucher(vatm->vatm_kvoucher); + vatm->vatm_kvoucher = MACH_VOUCHER_NULL; + } + if (free_mailbox) { + _voucher_atm_mailbox_free(vatm->vatm_mailbox_offset); + vatm->vatm_mailbox_offset = MAILBOX_OFFSET_UNSET; + } + free(vatm); +} + +static inline mach_voucher_t +_voucher_activity_get_atm_mach_voucher(_voucher_activity_t act) +{ + mach_voucher_t kv; + kv = act && act->va_atm ? act->va_atm->vatm_kvoucher : MACH_VOUCHER_NULL; + return kv; +} + +DISPATCH_NOINLINE +static _voucher_atm_t +_voucher_atm_base_copy_and_activity_id_make(voucher_activity_id_t *va_id_ptr) +{ + _voucher_atm_subid_t subid; + _voucher_atm_t vatm, vatm_old = NULL, vatm_new = NULL; + if (_voucher_activity_heap->vam_base_atm_subid_max == 1) { + vatm = _voucher_atm_create(0, 0); + subid = 1; + goto out; + } + _voucher_activity_lock_lock(vam_base_atm_lock()); + vatm = _voucher_activity_heap->vam_base_atm; +retry: + _voucher_atm_retain(vatm); + subid = _voucher_activity_heap->vam_base_atm_subid; + if (subid++ >= _voucher_activity_heap->vam_base_atm_subid_max) { + _voucher_activity_lock_unlock(vam_base_atm_lock()); + if (!vatm_new) vatm_new = _voucher_atm_create(0, 0); + _voucher_activity_lock_lock(vam_base_atm_lock()); + _voucher_atm_release(vatm); + vatm_old = vatm; + vatm = _voucher_activity_heap->vam_base_atm; + if (vatm != vatm_old) { + vatm_old = NULL; + goto retry; + } + _voucher_activity_heap->vam_base_atm = vatm = vatm_new; + _voucher_activity_heap->vam_base_atm_subid = subid = 1; + vatm_new = NULL; + _voucher_atm_retain(vatm); + _dispatch_voucher_atm_debug("base replace", vatm); + } else { + _voucher_activity_heap->vam_base_atm_subid = subid; + _dispatch_voucher_atm_debug("base copy", vatm); + } + _voucher_activity_lock_unlock(vam_base_atm_lock()); + if (vatm_old) _voucher_atm_release(vatm_old); + if (vatm_new) _voucher_atm_release(vatm_new); +out: + *va_id_ptr = VATM_ACTID(vatm, subid); + return vatm; +} + +static voucher_activity_id_t +_voucher_atm_nested_atm_id_make(void) +{ + atm_aid_t atm_id; + mach_voucher_t kv = _voucher_atm_mach_voucher_create(&atm_id); + _voucher_dealloc_mach_voucher(kv); // just need the unique ID + return VATMID2ACTID(atm_id); +} + +static voucher_activity_id_t +_voucher_atm_nested_activity_id_make(void) +{ + voucher_activity_id_t va_id, va_id_old, va_id_new; + _voucher_atm_subid_t subid; + _voucher_activity_lock_lock(vam_nested_atm_lock()); + va_id = _voucher_activity_heap->vam_nested_atm_id; +retry: + subid = _voucher_activity_heap->vam_nested_atm_subid; + if (subid++ >= VATM_SUBID_MAX) { + _voucher_activity_lock_unlock(vam_nested_atm_lock()); + va_id_new = _voucher_atm_nested_atm_id_make(); + va_id_old = va_id; + _voucher_activity_lock_lock(vam_nested_atm_lock()); + va_id = _voucher_activity_heap->vam_nested_atm_id; + if (va_id != va_id_old) goto retry; + _voucher_activity_heap->vam_nested_atm_id = va_id = va_id_new; + subid = 1; + } + _voucher_activity_heap->vam_nested_atm_subid = subid; + _voucher_activity_lock_unlock(vam_nested_atm_lock()); + return va_id + subid; +} + +#pragma mark - +#pragma mark voucher_activity_id_t + +voucher_activity_id_t +voucher_activity_start_with_location(voucher_activity_trace_id_t trace_id, + uint64_t location, voucher_activity_flag_t flags) +{ + dispatch_once_f(&_voucher_activity_heap_pred, NULL, + _voucher_activity_heap_init); + if (!_voucher_activity_trace_id_enabled(trace_id)) return 0; + voucher_activity_id_t va_id = 0, va_base_id = 0; + _voucher_atm_t vatm = NULL; + _voucher_activity_t act = NULL; + _voucher_activity_tracepoint_t vat = NULL; + unsigned int activities = 1, oactivities = 0; + voucher_t ov = _voucher_get(); + if (!(flags & voucher_activity_flag_force) && ov && ov->v_activities) { + oactivities = ov->v_activities; + activities += oactivities; + if (activities > _voucher_max_activities) { + va_id = _voucher_atm_nested_activity_id_make(); + goto out; + } + } + if (activities == 1) { + vatm = _voucher_atm_base_copy_and_activity_id_make(&va_id); + if (vatm->vatm_kvoucher) { + // consumes vatm reference: + act = _voucher_activity_create_with_atm(vatm, va_id, trace_id, + location, NULL); + vat = (_voucher_activity_tracepoint_t)act; + } else { + _voucher_atm_release(vatm); + } + if (!act) { + activities++; + // default to _voucher_activity_default base activity + va_base_id = _voucher_activity_default->va_id; + } + } + pthread_priority_t priority = _voucher_get_priority(ov); + mach_voucher_attr_recipe_size_t extra = ov ? _voucher_extra_size(ov) : 0; + voucher_t v = _voucher_alloc(activities, priority, extra); + if (extra) { + memcpy(_voucher_extra_recipes(v), _voucher_extra_recipes(ov), extra); + } + if (ov && ov->v_kvoucher) { + voucher_t kvb = ov->v_kvbase ? ov->v_kvbase : ov; + v->v_kvbase = _voucher_retain(kvb); + v->v_kvoucher = kvb->v_kvoucher; + } + voucher_activity_id_t *activity_ids = _voucher_activity_ids(v); + if (oactivities) { + memcpy(activity_ids, _voucher_activity_ids(ov), + oactivities * sizeof(voucher_activity_id_t)); + } + if (!va_id) { + va_id = _voucher_atm_nested_activity_id_make(); + if (ov && ov->v_activity) { + act = _voucher_activity_retain(ov->v_activity); + } + } + if (va_base_id) activity_ids[0] = va_base_id; + activity_ids[activities-1] = va_id; + v->v_activity = act; + _voucher_swap(ov, v); + if (vat) return va_id; // new _voucher_activity_s contains trace info +out: + vat = _voucher_activity_trace_with_id(trace_id); + if (vat) { + vat->vat_flags |= _voucher_activity_trace_flag_activity | + _voucher_activity_trace_flag_start; + vat->vat_data[0] = va_id; + } + return va_id; +} + +voucher_activity_id_t +voucher_activity_start(voucher_activity_trace_id_t trace_id, + voucher_activity_flag_t flags) +{ + return voucher_activity_start_with_location(trace_id, 0, flags); +} + +void +voucher_activity_end(voucher_activity_id_t va_id) +{ + if (!va_id) return; + _voucher_activity_tracepoint_t vat; + vat = _voucher_activity_trace_with_id(_voucher_activity_trace_id_release); + if (vat) { + vat->vat_flags |= _voucher_activity_trace_flag_activity | + _voucher_activity_trace_flag_end; + vat->vat_data[0] = va_id; + } + voucher_t v = _voucher_get(); + if (!v) return; + unsigned int activities = v->v_activities, act_idx = activities; + voucher_activity_id_t *activity_ids = _voucher_activity_ids(v); + while (act_idx) { + if (activity_ids[act_idx-1] == va_id) break; + act_idx--; + } + if (!act_idx) return; // activity_id not found + pthread_priority_t priority = _voucher_get_priority(v); + mach_voucher_attr_recipe_size_t extra = _voucher_extra_size(v); + voucher_t nv = NULL; + if (act_idx > 1 || activities == 1) --activities; + if (priority || activities || extra || v->v_kvoucher) { + nv = _voucher_alloc(activities, priority, extra); + if (extra) { + memcpy(_voucher_extra_recipes(nv), _voucher_extra_recipes(v),extra); + } + } + if (v->v_kvoucher) { + voucher_t kvb = v->v_kvbase ? v->v_kvbase : v; + nv->v_kvbase = _voucher_retain(kvb); + nv->v_kvoucher = kvb->v_kvoucher; + } + bool atm_collect = !activities; + if (activities) { + voucher_activity_id_t *new_activity_ids = _voucher_activity_ids(nv); + if (act_idx == 1 && _voucher_activity_default) { + atm_collect = true; + // default to _voucher_activity_default base activity + new_activity_ids[0] = _voucher_activity_default->va_id; + memcpy(&new_activity_ids[1], &activity_ids[1], + (activities - 1) * sizeof(voucher_activity_id_t)); + } else { + if (v->v_activity) { + nv->v_activity = _voucher_activity_retain(v->v_activity); + } + memcpy(new_activity_ids, activity_ids, + --act_idx * sizeof(voucher_activity_id_t)); + if (act_idx < activities) { + memcpy(&new_activity_ids[act_idx], &activity_ids[act_idx+1], + (activities - act_idx) * sizeof(voucher_activity_id_t)); + } + } + } + _voucher_swap(v, nv); +} + +unsigned int +voucher_get_activities(voucher_activity_id_t *entries, unsigned int *count) +{ + voucher_t v = _voucher_get(); + if (!v || !count) return 0; + unsigned int activities = v->v_activities; + if (*count < activities) activities = *count; + *count = v->v_activities; + voucher_activity_id_t *activity_ids = _voucher_activity_ids(v); + if (activities && entries) { + memcpy(entries, activity_ids, activities * + sizeof(voucher_activity_id_t)); + } + return activities; +} + +uint8_t +voucher_activity_get_namespace(void) +{ + voucher_t v = _voucher_get(); + if (!v || !v->v_activity) return 0; + return v->v_activity->va_namespace; +} + +DISPATCH_NOINLINE +_voucher_activity_tracepoint_t +_voucher_activity_tracepoint_get_slow(unsigned int slots) +{ + _voucher_activity_t act; + _voucher_activity_buffer_header_t vab; + _voucher_activity_tracepoint_t vat = NULL; + voucher_t v = _voucher_get(); + if (v && v->v_activity) { + act = v->v_activity; + } else { + dispatch_once_f(&_voucher_activity_heap_pred, NULL, + _voucher_activity_heap_init); + if (_voucher_activity_disabled()) return NULL; + act = _voucher_activity_default; + } + vab = act->va_current_buffer; + if (vab && vab->vabh_next_tracepoint_idx <= + _voucher_activity_tracepoints_per_buffer) { + goto retry; // another slowpath raced us + } + do { + vab = _voucher_activity_buffer_alloc(act, vab); + if (!vab) break; +retry: + vat = _voucher_activity_buffer_tracepoint_get(vab, slots); + } while (!vat); + return vat; +} + +static inline void +_voucher_activity_trace_fault(voucher_activity_trace_id_t trace_id) +{ + if (!slowpath(_voucher_activity_trace_id_is_subtype(trace_id, error))) { + return; + } + mach_voucher_attr_command_t atm_cmd = ATM_ACTION_COLLECT; + if (_voucher_activity_trace_id_is_subtype(trace_id, fault)) { + atm_cmd = ATM_ACTION_LOGFAIL; + } + return _voucher_atm_fault(atm_cmd); +} + +uint64_t +voucher_activity_trace(voucher_activity_trace_id_t trace_id, uint64_t location, + void *buffer, size_t length) +{ + if (!_voucher_activity_trace_id_enabled(trace_id)) return 0; + _voucher_activity_tracepoint_t vat; + const unsigned int slots = length <= sizeof(vat->vat_data) ? 1 : 2; + vat = _voucher_activity_tracepoint_get(slots); + if (!vat) vat = _voucher_activity_tracepoint_get_slow(slots); + if (!vat) return 0; + uint64_t timestamp = _voucher_activity_tracepoint_init_with_id(vat, + trace_id, location); + void *tbuf = vat->vat_data; + size_t tlen = sizeof(vat->vat_data); + if (length < tlen) { + memcpy(tbuf, buffer, length); + } else { + memcpy(tbuf, buffer, tlen); + } + if (length > tlen) { + vat->vat_flags |= _voucher_activity_trace_flag_wide_first; + buffer += tlen; + length -= tlen; + (++vat)->vat_flags = _voucher_activity_trace_flag_tracepoint | + _voucher_activity_trace_flag_wide_second; + vat->vat_type = 0; vat->vat_namespace = 0; + tbuf = (void*)vat + offsetof(typeof(*vat), vat_code); + tlen = sizeof(*vat) - offsetof(typeof(*vat), vat_code); + if (length < tlen) { + memcpy(tbuf, buffer, length); + } else { + memcpy(tbuf, buffer, tlen); + } + } + _voucher_activity_trace_fault(trace_id); + return timestamp; +} + +uint64_t +voucher_activity_trace_args(voucher_activity_trace_id_t trace_id, + uint64_t location, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, + uintptr_t arg4) +{ + if (!_voucher_activity_trace_id_enabled(trace_id)) return 0; + _voucher_activity_tracepoint_t vat; + vat = _voucher_activity_tracepoint_get(1); + if (!vat) vat = _voucher_activity_tracepoint_get_slow(1); + if (!vat) return 0; + uint64_t timestamp = _voucher_activity_tracepoint_init_with_id(vat, + trace_id, location); + vat->vat_flags |= _voucher_activity_trace_flag_tracepoint_args; + vat->vat_data[0] = arg1; + vat->vat_data[1] = arg2; + vat->vat_data[2] = arg3; + vat->vat_data[3] = arg4; + _voucher_activity_trace_fault(trace_id); + return timestamp; +} + +#pragma mark - +#pragma mark _voucher_debug + +size_t +_voucher_debug(voucher_t v, char* buf, size_t bufsiz) +{ + size_t offset = 0; + #define bufprintf(...) \ + offset += dsnprintf(&buf[offset], bufsiz - offset, ##__VA_ARGS__) + bufprintf("voucher[%p] = { xrefcnt = 0x%x, refcnt = 0x%x, ", v, + v->os_obj_xref_cnt + 1, v->os_obj_ref_cnt + 1); + + if (v->v_kvbase) { + bufprintf("base voucher %p, ", v->v_kvbase); + } + if (v->v_kvoucher) { + bufprintf("kvoucher%s 0x%x, ", v->v_kvoucher == v->v_ipc_kvoucher ? + " & ipc kvoucher" : "", v->v_kvoucher); + } + if (v->v_ipc_kvoucher && v->v_ipc_kvoucher != v->v_kvoucher) { + bufprintf("ipc kvoucher 0x%x, ", v->v_ipc_kvoucher); + } + if (v->v_has_priority) { + bufprintf("QOS 0x%x, ", *_voucher_priority(v)); + } + if (v->v_activities) { + voucher_activity_id_t *activity_ids = _voucher_activity_ids(v); + bufprintf("activity IDs = { "); + unsigned int i; + for (i = 0; i < v->v_activities; i++) { + bufprintf("0x%llx, ", *activity_ids++); + } + bufprintf("}, "); + } + if (v->v_activity) { + _voucher_activity_t va = v->v_activity; + _voucher_atm_t vatm = va->va_atm; + bufprintf("activity[%p] = { ID 0x%llx, use %d, atm[%p] = { " + "AID 0x%llx, ref %d, kvoucher 0x%x } }, ", va, va->va_id, + va->va_use_count + 1, va->va_atm, vatm->vatm_id, + vatm->vatm_refcnt + 1, vatm->vatm_kvoucher); + } + bufprintf("}"); + return offset; +} + +#else // VOUCHER_USE_MACH_VOUCHER + +#pragma mark - +#pragma mark Simulator / vouchers disabled + +#if VOUCHER_ENABLE_RECIPE_OBJECTS +voucher_t +voucher_create(voucher_recipe_t recipe) +{ + (void)recipe; + return NULL; +} +#endif + +voucher_t +voucher_adopt(voucher_t voucher) +{ + return voucher; +} + +voucher_t +voucher_copy(void) +{ + return NULL; +} + +voucher_t +voucher_copy_without_importance(void) +{ + return NULL; +} + +void +voucher_replace_default_voucher(void) +{ +} + +void +voucher_decrement_importance_count4CF(voucher_t v) +{ + (void)v; +} + +void +_voucher_thread_cleanup(void *voucher) +{ + (void)voucher; +} + +void +_voucher_dealloc_mach_voucher(mach_voucher_t kv) +{ + (void)kv; +} + +mach_voucher_t +_voucher_create_mach_voucher_with_priority(voucher_t voucher, + pthread_priority_t priority) +{ + (void)voucher; (void)priority; + return MACH_VOUCHER_NULL; +} + +voucher_t +_voucher_create_with_priority_and_mach_voucher(voucher_t voucher, + pthread_priority_t priority, mach_voucher_t kv) +{ + (void)voucher; (void)priority; (void)kv; + return NULL; +} + +voucher_t +voucher_create_with_mach_msg(mach_msg_header_t *msg) +{ + (void)msg; + return NULL; +} + +#if VOUCHER_ENABLE_GET_MACH_VOUCHER +mach_voucher_t +voucher_get_mach_voucher(voucher_t voucher) +{ + (void)voucher; + return 0; +} +#endif + +void +_voucher_xref_dispose(voucher_t voucher) +{ + (void)voucher; +} + +void +_voucher_dispose(voucher_t voucher) +{ + (void)voucher; +} + +void +_voucher_atfork_child(void) +{ +} + +void +_voucher_init(void) +{ +} + +void* +voucher_activity_get_metadata_buffer(size_t *length) +{ + *length = 0; + return NULL; +} + +void +_voucher_activity_heap_pressure_normal(void) +{ +} + +void +_voucher_activity_heap_pressure_warn(void) +{ +} + +voucher_activity_id_t +voucher_activity_start_with_location(voucher_activity_trace_id_t trace_id, + uint64_t location, voucher_activity_flag_t flags) +{ + (void)trace_id; (void)location; (void)flags; + return 0; +} + +voucher_activity_id_t +voucher_activity_start(voucher_activity_trace_id_t trace_id, + voucher_activity_flag_t flags) +{ + (void)trace_id; (void)flags; + return 0; +} + +void +voucher_activity_end(voucher_activity_id_t activity_id) +{ + (void)activity_id; +} + +unsigned int +voucher_get_activities(voucher_activity_id_t *entries, unsigned int *count) +{ + (void)entries; (void)count; + return 0; +} + +uint8_t +voucher_activity_get_namespace(void) +{ + return 0; +} + +uint64_t +voucher_activity_trace(voucher_activity_trace_id_t trace_id, uint64_t location, + void *buffer, size_t length) +{ + (void)trace_id; (void)location; (void)buffer; (void)length; + return 0; +} + +uint64_t +voucher_activity_trace_args(voucher_activity_trace_id_t trace_id, + uint64_t location, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, + uintptr_t arg4) +{ + (void)trace_id; (void)location; + (void)arg1; (void)arg2; (void)arg3; (void)arg4; + return 0; +} + +size_t +_voucher_debug(voucher_t v, char* buf, size_t bufsiz) +{ + (void)v; (void)buf; (void)bufsiz; + return 0; +} + +#endif // VOUCHER_USE_MACH_VOUCHER diff --git a/src/voucher_internal.h b/src/voucher_internal.h new file mode 100644 index 0000000..6fa1538 --- /dev/null +++ b/src/voucher_internal.h @@ -0,0 +1,929 @@ +/* + * Copyright (c) 2013 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +/* + * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch + * which are subject to change in future releases of Mac OS X. Any applications + * relying on these interfaces WILL break. + */ + +#ifndef __DISPATCH_VOUCHER_INTERNAL__ +#define __DISPATCH_VOUCHER_INTERNAL__ + +#ifndef __DISPATCH_INDIRECT__ +#error "Please #include instead of this file directly." +#include // for HeaderDoc +#endif + +#pragma mark - +#pragma mark voucher_recipe_t (disabled) + +#if VOUCHER_ENABLE_RECIPE_OBJECTS +/*! + * @group Voucher Creation SPI + * SPI intended for clients that need to create vouchers. + */ + +#if OS_OBJECT_USE_OBJC +OS_OBJECT_DECL(voucher_recipe); +#else +typedef struct voucher_recipe_s *voucher_recipe_t; +#endif + +/*! + * @function voucher_create + * + * @abstract + * Creates a new voucher object from a recipe. + * + * @discussion + * Error handling TBD + * + * @result + * The newly created voucher object. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_EXPORT OS_OBJECT_RETURNS_RETAINED OS_WARN_RESULT OS_NOTHROW +voucher_t +voucher_create(voucher_recipe_t recipe); +#endif // VOUCHER_ENABLE_RECIPE_OBJECTS + +#if VOUCHER_ENABLE_GET_MACH_VOUCHER +/*! + * @function voucher_get_mach_voucher + * + * @abstract + * Returns the mach voucher port underlying the specified voucher object. + * + * @discussion + * The caller must either maintain a reference on the voucher object while the + * returned mach voucher port is in use to ensure it stays valid for the + * duration, or it must retain the mach voucher port with mach_port_mod_refs(). + * + * @param voucher + * The voucher object to query. + * + * @result + * A mach voucher port. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) +OS_VOUCHER_EXPORT OS_WARN_RESULT OS_NOTHROW +mach_voucher_t +voucher_get_mach_voucher(voucher_t voucher); +#endif // VOUCHER_ENABLE_GET_MACH_VOUCHER + +#pragma mark - +#pragma mark voucher_t + +#if TARGET_IPHONE_SIMULATOR && \ + IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101000 +#undef VOUCHER_USE_MACH_VOUCHER +#define VOUCHER_USE_MACH_VOUCHER 0 +#endif +#ifndef VOUCHER_USE_MACH_VOUCHER +#if __has_include() +#define VOUCHER_USE_MACH_VOUCHER 1 +#endif +#endif + +#if VOUCHER_USE_MACH_VOUCHER +#undef DISPATCH_USE_IMPORTANCE_ASSERTION +#define DISPATCH_USE_IMPORTANCE_ASSERTION 0 +#else +#undef MACH_RCV_VOUCHER +#define MACH_RCV_VOUCHER 0 +#endif // VOUCHER_USE_MACH_VOUCHER + +void _voucher_init(void); +void _voucher_atfork_child(void); +void _voucher_activity_heap_pressure_warn(void); +void _voucher_activity_heap_pressure_normal(void); +void _voucher_xref_dispose(voucher_t voucher); +void _voucher_dispose(voucher_t voucher); +size_t _voucher_debug(voucher_t v, char* buf, size_t bufsiz); +void _voucher_thread_cleanup(void *voucher); +mach_voucher_t _voucher_get_mach_voucher(voucher_t voucher); +voucher_t _voucher_create_without_importance(voucher_t voucher); +mach_voucher_t _voucher_create_mach_voucher_with_priority(voucher_t voucher, + pthread_priority_t priority); +voucher_t _voucher_create_with_priority_and_mach_voucher(voucher_t voucher, + pthread_priority_t priority, mach_voucher_t kv); +void _voucher_dealloc_mach_voucher(mach_voucher_t kv); + +#if OS_OBJECT_USE_OBJC +_OS_OBJECT_DECL_SUBCLASS_INTERFACE(voucher, object) +#if VOUCHER_ENABLE_RECIPE_OBJECTS +_OS_OBJECT_DECL_SUBCLASS_INTERFACE(voucher_recipe, object) +#endif +#endif + +#define _TAILQ_IS_ENQUEUED(elm, field) \ + ((elm)->field.tqe_prev != NULL) +#define _TAILQ_MARK_NOT_ENQUEUED(elm, field) \ + do { (elm)->field.tqe_prev = NULL; } while (0) + +#define VOUCHER_NO_MACH_VOUCHER MACH_PORT_DEAD + +#if VOUCHER_USE_MACH_VOUCHER + +#if DISPATCH_DEBUG +#define DISPATCH_VOUCHER_DEBUG 1 +#define DISPATCH_VOUCHER_ACTIVITY_DEBUG 1 +#endif + +typedef struct voucher_s { + _OS_OBJECT_HEADER( + void *os_obj_isa, + os_obj_ref_cnt, + os_obj_xref_cnt); + TAILQ_ENTRY(voucher_s) v_list; + mach_voucher_t v_kvoucher, v_ipc_kvoucher; // if equal, only one reference + voucher_t v_kvbase; // if non-NULL, v_kvoucher is a borrowed reference + _voucher_activity_t v_activity; +#if VOUCHER_ENABLE_RECIPE_OBJECTS + size_t v_recipe_extra_offset; + mach_voucher_attr_recipe_size_t v_recipe_extra_size; +#endif + unsigned int v_has_priority:1; + unsigned int v_activities; + mach_voucher_attr_recipe_data_t v_recipes[]; +} voucher_s; + +#if VOUCHER_ENABLE_RECIPE_OBJECTS +typedef struct voucher_recipe_s { + _OS_OBJECT_HEADER( + const _os_object_class_s *os_obj_isa, + os_obj_ref_cnt, + os_obj_xref_cnt); + size_t vr_allocation_size; + mach_voucher_attr_recipe_size_t volatile vr_size; + mach_voucher_attr_recipe_t vr_data; +} voucher_recipe_s; +#endif + +#define _voucher_recipes_base(r) (r[0]) +#define _voucher_recipes_atm(r) (r[1]) +#define _voucher_recipes_bits(r) (r[2]) +#define _voucher_base_recipe(v) (_voucher_recipes_base((v)->v_recipes)) +#define _voucher_atm_recipe(v) (_voucher_recipes_atm((v)->v_recipes)) +#define _voucher_bits_recipe(v) (_voucher_recipes_bits((v)->v_recipes)) +#define _voucher_recipes_size() (3 * sizeof(mach_voucher_attr_recipe_data_t)) + +#if TARGET_OS_EMBEDDED +#define VL_HASH_SIZE 64u // must be a power of two +#else +#define VL_HASH_SIZE 256u // must be a power of two +#endif +#define VL_HASH(kv) (MACH_PORT_INDEX(kv) & (VL_HASH_SIZE - 1)) + +typedef uint32_t _voucher_magic_t; +const _voucher_magic_t _voucher_magic_v1 = 0x0190cefa; // little-endian FACE9001 +#define _voucher_recipes_magic(r) ((_voucher_magic_t*) \ + (_voucher_recipes_bits(r).content)) +#define _voucher_magic(v) _voucher_recipes_magic((v)->v_recipes) +typedef uint32_t _voucher_priority_t; +#define _voucher_recipes_priority(r) ((_voucher_priority_t*) \ + (_voucher_recipes_bits(r).content + sizeof(_voucher_magic_t))) +#define _voucher_priority(v) _voucher_recipes_priority((v)->v_recipes) +#define _voucher_activity_ids(v) ((voucher_activity_id_t*) \ + (_voucher_bits_recipe(v).content + sizeof(_voucher_magic_t) + \ + sizeof(_voucher_priority_t))) +#define _voucher_bits_size(activities) \ + (sizeof(_voucher_magic_t) + sizeof(_voucher_priority_t) + \ + (activities) * sizeof(voucher_activity_id_t)) + +#if VOUCHER_ENABLE_RECIPE_OBJECTS +#define _voucher_extra_size(v) ((v)->v_recipe_extra_size) +#define _voucher_extra_recipes(v) ((char*)(v) + (v)->v_recipe_extra_offset) +#else +#define _voucher_extra_size(v) 0 +#define _voucher_extra_recipes(v) NULL +#endif + +#if DISPATCH_DEBUG && DISPATCH_VOUCHER_DEBUG +#define _dispatch_voucher_debug(msg, v, ...) \ + _dispatch_debug("voucher[%p]: " msg, v, ##__VA_ARGS__) +#define _dispatch_kvoucher_debug(msg, kv, ...) \ + _dispatch_debug("kvoucher[0x%08x]: " msg, kv, ##__VA_ARGS__) +#define _dispatch_voucher_debug_machport(name) \ + dispatch_debug_machport((name), __func__) +#else +#define _dispatch_voucher_debug(msg, v, ...) +#define _dispatch_kvoucher_debug(msg, kv, ...) +#define _dispatch_voucher_debug_machport(name) ((void)(name)) +#endif + +#if !(USE_OBJC && __OBJC2__) + +DISPATCH_ALWAYS_INLINE +static inline voucher_t +_voucher_retain(voucher_t voucher) +{ +#if !DISPATCH_VOUCHER_OBJC_DEBUG + int xref_cnt = dispatch_atomic_inc2o(voucher, os_obj_xref_cnt, relaxed); + _dispatch_voucher_debug("retain -> %d", voucher, xref_cnt + 1); + if (slowpath(xref_cnt <= 0)) { + _dispatch_voucher_debug("resurrection", voucher); + DISPATCH_CRASH("Voucher resurrection"); + } +#else + os_retain(voucher); + _dispatch_voucher_debug("retain -> %d", voucher, + voucher->os_obj_xref_cnt + 1); +#endif // DISPATCH_DEBUG + return voucher; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_release(voucher_t voucher) +{ +#if !DISPATCH_VOUCHER_OBJC_DEBUG + int xref_cnt = dispatch_atomic_dec2o(voucher, os_obj_xref_cnt, relaxed); + _dispatch_voucher_debug("release -> %d", voucher, xref_cnt + 1); + if (fastpath(xref_cnt >= 0)) { + return; + } + if (slowpath(xref_cnt < -1)) { + _dispatch_voucher_debug("overrelease", voucher); + DISPATCH_CRASH("Voucher overrelease"); + } + return _os_object_xref_dispose((_os_object_t)voucher); +#else + _dispatch_voucher_debug("release -> %d", voucher, voucher->os_obj_xref_cnt); + return os_release(voucher); +#endif // DISPATCH_DEBUG +} + +DISPATCH_ALWAYS_INLINE +static inline voucher_t +_voucher_get(void) +{ + return _dispatch_thread_getspecific(dispatch_voucher_key); +} + +DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT +static inline voucher_t +_voucher_copy(void) +{ + voucher_t voucher = _voucher_get(); + if (voucher) _voucher_retain(voucher); + return voucher; +} + +DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT +static inline voucher_t +_voucher_copy_without_importance(void) +{ + voucher_t voucher = _voucher_get(); + if (voucher) voucher = _voucher_create_without_importance(voucher); + return voucher; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_mach_voucher_set(mach_voucher_t kv) +{ + if (kv == VOUCHER_NO_MACH_VOUCHER) return; + _dispatch_set_priority_and_mach_voucher(0, kv); +} + +DISPATCH_ALWAYS_INLINE +static inline mach_voucher_t +_voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher) +{ + if (ov == voucher) return VOUCHER_NO_MACH_VOUCHER; + _dispatch_voucher_debug("swap from voucher[%p]", voucher, ov); + _dispatch_thread_setspecific(dispatch_voucher_key, voucher); + mach_voucher_t kv = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL; + mach_voucher_t okv = ov ? ov->v_kvoucher : MACH_VOUCHER_NULL; + return (kv != okv) ? kv : VOUCHER_NO_MACH_VOUCHER; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_swap(voucher_t ov, voucher_t voucher) +{ + _voucher_mach_voucher_set(_voucher_swap_and_get_mach_voucher(ov, voucher)); + if (ov) _voucher_release(ov); +} + +DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT +static inline voucher_t +_voucher_adopt(voucher_t voucher) +{ + voucher_t ov = _voucher_get(); + _voucher_mach_voucher_set(_voucher_swap_and_get_mach_voucher(ov, voucher)); + return ov; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_replace(voucher_t voucher) +{ + voucher_t ov = _voucher_get(); + _voucher_swap(ov, voucher); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_clear(void) +{ + _voucher_replace(NULL); +} + +DISPATCH_ALWAYS_INLINE +static inline pthread_priority_t +_voucher_get_priority(voucher_t voucher) +{ + return voucher && voucher->v_has_priority ? + (pthread_priority_t)*_voucher_priority(voucher) : 0; +} + +void _voucher_task_mach_voucher_init(void* ctxt); +extern dispatch_once_t _voucher_task_mach_voucher_pred; +extern mach_voucher_t _voucher_task_mach_voucher; + +DISPATCH_ALWAYS_INLINE +static inline mach_voucher_t +_voucher_get_task_mach_voucher(void) +{ + dispatch_once_f(&_voucher_task_mach_voucher_pred, NULL, + _voucher_task_mach_voucher_init); + return _voucher_task_mach_voucher; +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv, + bool move_send) +{ + if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false; + if (!kv) return false; + msg->msgh_voucher_port = kv; + msg->msgh_bits |= MACH_MSGH_BITS_SET_PORTS(0, 0, move_send ? + MACH_MSG_TYPE_MOVE_SEND : MACH_MSG_TYPE_COPY_SEND); + _dispatch_kvoucher_debug("msg[%p] set %s", kv, msg, move_send ? + "move-send" : "copy-send"); + _dispatch_voucher_debug_machport(kv); + return true; +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher) +{ + if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false; + mach_voucher_t kv; + if (voucher) { + kv = _voucher_get_mach_voucher(voucher); + } else { + kv = _voucher_get_task_mach_voucher(); + } + return _voucher_mach_msg_set_mach_voucher(msg, kv, false); +} + +DISPATCH_ALWAYS_INLINE +static inline mach_voucher_t +_voucher_mach_msg_get(mach_msg_header_t *msg) +{ + if (!MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return MACH_VOUCHER_NULL; + mach_voucher_t kv = msg->msgh_voucher_port; + msg->msgh_voucher_port = MACH_VOUCHER_NULL; + msg->msgh_bits &= (mach_msg_bits_t)~MACH_MSGH_BITS_VOUCHER_MASK; + return kv; +} + +DISPATCH_ALWAYS_INLINE +static inline mach_voucher_t +_voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send) +{ + mach_msg_bits_t kvbits = MACH_MSGH_BITS_VOUCHER(msg->msgh_bits); + mach_voucher_t kv = msg->msgh_voucher_port, kvm = MACH_VOUCHER_NULL; + if ((kvbits == MACH_MSG_TYPE_COPY_SEND || + kvbits == MACH_MSG_TYPE_MOVE_SEND) && kv) { + _dispatch_kvoucher_debug("msg[%p] clear %s", kv, msg, move_send ? + "move-send" : "copy-send"); + _dispatch_voucher_debug_machport(kv); + if (kvbits == MACH_MSG_TYPE_MOVE_SEND) { + // return/drop received or pseudo-received + // voucher reference (e.g. due to send failure). + if (move_send) { + kvm = kv; + } else { + _voucher_dealloc_mach_voucher(kv); + } + } + msg->msgh_voucher_port = MACH_VOUCHER_NULL; + msg->msgh_bits &= (mach_msg_bits_t)~MACH_MSGH_BITS_VOUCHER_MASK; + } + return kvm; +} + +#pragma mark - +#pragma mark dispatch_continuation_t + voucher_t + +#if DISPATCH_USE_KDEBUG_TRACE +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_voucher_ktrace(int code, natural_t voucher, void *container) +{ + if (!voucher) return; + __kdebug_trace(APPSDBG_CODE(DBG_MACH_CHUD, (0xfac >> 2)) | DBG_FUNC_NONE, + code, (int)voucher, (int)(uintptr_t)container, +#ifdef __LP64__ + (int)((uintptr_t)container >> 32) +#else + 0 +#endif + ); +} +#define _dispatch_voucher_ktrace_dc_push(dc) \ + _dispatch_voucher_ktrace(0x1, (dc)->dc_voucher ? \ + (dc)->dc_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dc)) +#define _dispatch_voucher_ktrace_dc_pop(dc) \ + _dispatch_voucher_ktrace(0x2, (dc)->dc_voucher ? \ + (dc)->dc_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dc)) +#define _dispatch_voucher_ktrace_dmsg_push(dmsg) \ + _dispatch_voucher_ktrace(0x3, (dmsg)->dmsg_voucher ? \ + (dmsg)->dmsg_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dmsg)) +#define _dispatch_voucher_ktrace_dmsg_pop(dmsg) \ + _dispatch_voucher_ktrace(0x4, (dmsg)->dmsg_voucher ? \ + (dmsg)->dmsg_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dmsg)) +#else +#define _dispatch_voucher_ktrace_dc_push(dc) +#define _dispatch_voucher_ktrace_dc_pop(dc) +#define _dispatch_voucher_ktrace_dmsg_push(dmsg) +#define _dispatch_voucher_ktrace_dmsg_pop(dmsg) +#endif // DISPATCH_USE_KDEBUG_TRACE + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_continuation_voucher_set(dispatch_continuation_t dc, + dispatch_block_flags_t flags) +{ + unsigned long bits = (unsigned long)dc->do_vtable; + voucher_t v = NULL; + + if (flags & DISPATCH_BLOCK_HAS_VOUCHER) { + bits |= DISPATCH_OBJ_HAS_VOUCHER_BIT; + } else if (!(flags & DISPATCH_BLOCK_NO_VOUCHER)) { + v = _voucher_copy(); + } + dc->do_vtable = (void*)bits; + dc->dc_voucher = v; + _dispatch_voucher_debug("continuation[%p] set", dc->dc_voucher, dc); + _dispatch_voucher_ktrace_dc_push(dc); +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_continuation_voucher_adopt(dispatch_continuation_t dc) +{ + unsigned long bits = (unsigned long)dc->do_vtable; + voucher_t v = DISPATCH_NO_VOUCHER; + if (!(bits & DISPATCH_OBJ_HAS_VOUCHER_BIT)) { + _dispatch_voucher_ktrace_dc_pop(dc); + _dispatch_voucher_debug("continuation[%p] adopt", dc->dc_voucher, dc); + v = dc->dc_voucher; + dc->dc_voucher = NULL; + } + _dispatch_adopt_priority_and_replace_voucher(dc->dc_priority, v, 0); +} + +#pragma mark - +#pragma mark _voucher_activity_heap + +typedef uint32_t _voucher_atm_subid_t; +static const size_t _voucher_activity_hash_bits = 6; +static const size_t _voucher_activity_hash_size = + 1 << _voucher_activity_hash_bits; +#define VACTID_HASH(x) ((((uint32_t)((x) >> 32) + (uint32_t)(x)) * \ + 2654435761u) >> (32-_voucher_activity_hash_bits)) +#define VATMID_HASH(x) \ + (((uint32_t)(x) * 2654435761u) >> (32-_voucher_activity_hash_bits)) +#define VATMID2ACTID(x) ((uint64_t)(x) << 32) +#define VACTID_BASEID(x) ((uint64_t)(x) & (((uint64_t)UINT32_MAX) << 32)) +#define VACTID_SUBID(x) ((uint32_t)(x)) +#define VATM_ACTID(vatm, subid) (VATMID2ACTID((vatm)->vatm_id) + (subid)) +#define VATM_SUBID_BITS2MAX(bits) ((1u << (bits)) - 1) +#define VATM_SUBID_MAXBITS (32) +#define VATM_SUBID_MAX (ATM_SUBAID32_MAX) +#define MAILBOX_OFFSET_UNSET UINT64_MAX + +static const size_t _voucher_activity_buffers_per_heap = 512; +typedef unsigned long _voucher_activity_bitmap_base_t; +static const size_t _voucher_activity_bits_per_bitmap_base_t = + 8 * sizeof(_voucher_activity_bitmap_base_t); +static const size_t _voucher_activity_bitmaps_per_heap = + _voucher_activity_buffers_per_heap / + _voucher_activity_bits_per_bitmap_base_t; +typedef _voucher_activity_bitmap_base_t + _voucher_activity_bitmap_t[_voucher_activity_bitmaps_per_heap]; + +typedef struct _voucher_activity_metadata_s { + _voucher_activity_buffer_t vam_kernel_metadata; + _voucher_activity_buffer_t vam_client_metadata; + struct _voucher_activity_self_metadata_s vam_self_metadata; +#if __LP64__ + uintptr_t vam_pad0[7]; +#else + uintptr_t vam_pad0[15]; +#endif + // cacheline + _voucher_activity_bitmap_t volatile vam_atm_mbox_bitmap; + _voucher_activity_bitmap_t volatile vam_buffer_bitmap; + _voucher_activity_bitmap_t volatile vam_pressure_locked_bitmap; + // cacheline + _voucher_atm_subid_t vam_base_atm_subid; + _voucher_atm_subid_t vam_base_atm_subid_max; + _voucher_atm_subid_t vam_nested_atm_subid; + _voucher_atm_t vam_default_activity_atm; + _voucher_atm_t volatile vam_base_atm; + voucher_activity_id_t volatile vam_nested_atm_id; +#if __LP64__ + uintptr_t vam_pad2[3]; +#else + uintptr_t vam_pad2[1]; +#endif + _voucher_activity_lock_s vam_base_atm_lock; + _voucher_activity_lock_s vam_nested_atm_lock; + _voucher_activity_lock_s vam_atms_lock; + _voucher_activity_lock_s vam_activities_lock; + // cacheline + TAILQ_HEAD(, _voucher_atm_s) vam_atms[_voucher_activity_hash_size]; + TAILQ_HEAD(, _voucher_activity_s) + vam_activities[_voucher_activity_hash_size]; +} *_voucher_activity_metadata_t; + +#pragma mark - +#pragma mark _voucher_activity_t + +_voucher_activity_tracepoint_t _voucher_activity_tracepoint_get_slow( + unsigned int slots); +extern _voucher_activity_t _voucher_activity_default; +extern voucher_activity_mode_t _voucher_activity_mode; + +#if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG +#define _dispatch_voucher_activity_debug(msg, act, ...) \ + _dispatch_debug("activity[%p] <0x%x>: atm[%p] <%lld>: " msg, (act), \ + (act) ? VACTID_SUBID((act)->va_id) : 0, (act) ? (act)->va_atm : NULL, \ + (act) && (act)->va_atm ? (act)->va_atm->vatm_id : 0, ##__VA_ARGS__) +#define _dispatch_voucher_atm_debug(msg, atm, ...) \ + _dispatch_debug("atm[%p] <%lld> kvoucher[0x%08x]: " msg, (atm), \ + (atm) ? (atm)->vatm_id : 0, (atm) ? (atm)->vatm_kvoucher : 0, \ + ##__VA_ARGS__) +#else +#define _dispatch_voucher_activity_debug(msg, act, ...) +#define _dispatch_voucher_atm_debug(msg, atm, ...) +#endif + +DISPATCH_ALWAYS_INLINE +static inline uint64_t +_voucher_activity_timestamp(void) +{ +#if TARGET_IPHONE_SIMULATOR && \ + IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101000 + return mach_absolute_time(); +#else + return mach_approximate_time(); +#endif +} + +DISPATCH_ALWAYS_INLINE +static inline uint64_t +_voucher_activity_thread_id(void) +{ + uint64_t thread_id; + pthread_threadid_np(NULL, &thread_id); // TODO: 15923074: use TSD thread_id + return thread_id; +} + +DISPATCH_ALWAYS_INLINE +static inline _voucher_activity_tracepoint_t +_voucher_activity_buffer_tracepoint_get(_voucher_activity_buffer_header_t vab, + unsigned int slots) +{ + uint32_t idx = dispatch_atomic_add2o(vab, vabh_next_tracepoint_idx, + slots, relaxed); + if (idx <= _voucher_activity_tracepoints_per_buffer) { + return (_voucher_activity_tracepoint_t)vab + (idx - slots); + } + return NULL; +} + +DISPATCH_ALWAYS_INLINE +static inline _voucher_activity_tracepoint_t +_voucher_activity_tracepoint_get_from_activity(_voucher_activity_t va, + unsigned int slots) +{ + _voucher_activity_buffer_header_t vab = va ? va->va_current_buffer : NULL; + return vab ? _voucher_activity_buffer_tracepoint_get(vab, slots) : NULL; +} + +DISPATCH_ALWAYS_INLINE +static inline _voucher_activity_tracepoint_t +_voucher_activity_tracepoint_get(unsigned int slots) +{ + _voucher_activity_t va; + voucher_t v = _voucher_get(); + va = v && v->v_activity ? v->v_activity : _voucher_activity_default; + return _voucher_activity_tracepoint_get_from_activity(va, slots); +} + +DISPATCH_ALWAYS_INLINE +static inline uint64_t +_voucher_activity_tracepoint_init(_voucher_activity_tracepoint_t vat, + uint8_t type, uint8_t code_namespace, uint32_t code, uint64_t location) +{ + if (!location) location = (uint64_t)__builtin_return_address(0); + uint64_t timestamp = _voucher_activity_timestamp(); + vat->vat_flags = _voucher_activity_trace_flag_tracepoint, + vat->vat_type = type, + vat->vat_namespace = code_namespace, + vat->vat_code = code, + vat->vat_timestamp = timestamp, + vat->vat_thread = _voucher_activity_thread_id(), + vat->vat_location = location; + return timestamp; +} + +DISPATCH_ALWAYS_INLINE +static inline uint64_t +_voucher_activity_tracepoint_init_with_id(_voucher_activity_tracepoint_t vat, + voucher_activity_trace_id_t trace_id, uint64_t location) +{ + uint8_t type = (uint8_t)(trace_id >> _voucher_activity_trace_id_type_shift); + uint8_t cns = (uint8_t)(trace_id >> + _voucher_activity_trace_id_code_namespace_shift); + uint32_t code = (uint32_t)trace_id; + return _voucher_activity_tracepoint_init(vat, type, cns, code, location); +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_voucher_activity_trace_id_is_subtype(voucher_activity_trace_id_t trace_id, + uint8_t type) +{ + voucher_activity_trace_id_t type_id = voucher_activity_trace_id(type, 0, 0); + return (trace_id & type_id) == type_id; +} +#define _voucher_activity_trace_id_is_subtype(trace_id, name) \ + _voucher_activity_trace_id_is_subtype(trace_id, \ + voucher_activity_tracepoint_type_ ## name) + +DISPATCH_ALWAYS_INLINE +static inline bool +_voucher_activity_trace_id_enabled(voucher_activity_trace_id_t trace_id) +{ + switch (_voucher_activity_mode) { + case voucher_activity_mode_release: + return _voucher_activity_trace_id_is_subtype(trace_id, release); + case voucher_activity_mode_stream: + case voucher_activity_mode_debug: + return _voucher_activity_trace_id_is_subtype(trace_id, debug) || + _voucher_activity_trace_id_is_subtype(trace_id, release); + } + return false; +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_voucher_activity_trace_type_enabled(uint8_t type) +{ + voucher_activity_trace_id_t type_id = voucher_activity_trace_id(type, 0, 0); + return _voucher_activity_trace_id_enabled(type_id); +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_voucher_activity_disabled(void) +{ + return slowpath(_voucher_activity_mode == voucher_activity_mode_disable); +} + +DISPATCH_ALWAYS_INLINE +static inline _voucher_activity_tracepoint_t +_voucher_activity_trace_args_inline(uint8_t type, uint8_t code_namespace, + uint32_t code, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, + uintptr_t arg4) +{ + if (!_voucher_activity_trace_type_enabled(type)) return NULL; + _voucher_activity_tracepoint_t vat; + vat = _voucher_activity_tracepoint_get(1); + if (!vat) return NULL; + _voucher_activity_tracepoint_init(vat, type, code_namespace, code, 0); + vat->vat_flags |= _voucher_activity_trace_flag_tracepoint_args; + vat->vat_data[0] = arg1; + vat->vat_data[1] = arg2; + vat->vat_data[2] = arg3; + vat->vat_data[3] = arg4; + return vat; +} + +DISPATCH_ALWAYS_INLINE +static inline _voucher_activity_tracepoint_t +_voucher_activity_trace_with_id_inline(voucher_activity_trace_id_t trace_id) +{ + _voucher_activity_tracepoint_t vat = _voucher_activity_tracepoint_get(1); + if (!vat) return NULL; + _voucher_activity_tracepoint_init_with_id(vat, trace_id, 0); + return vat; +} + +DISPATCH_ALWAYS_INLINE +static inline _voucher_activity_tracepoint_t +_voucher_activity_trace_with_id(voucher_activity_trace_id_t trace_id) +{ + _voucher_activity_tracepoint_t vat = _voucher_activity_tracepoint_get(1); + if (!vat) vat = _voucher_activity_tracepoint_get_slow(1); + if (!vat) return NULL; + _voucher_activity_tracepoint_init_with_id(vat, trace_id, 0); + return vat; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_activity_trace_msg(voucher_t v, mach_msg_header_t *msg, uint32_t code) +{ + if (!v || !v->v_activity) return; // Don't use default activity for IPC + const uint8_t type = voucher_activity_tracepoint_type_release; + const uint8_t code_namespace = _voucher_activity_tracepoint_namespace_ipc; + if (!_voucher_activity_trace_type_enabled(type)) return; + _voucher_activity_tracepoint_t vat; + vat = _voucher_activity_tracepoint_get_from_activity(v->v_activity, 1); + if (!vat) return; // TODO: slowpath ? + _voucher_activity_tracepoint_init(vat, type, code_namespace, code, 0); + vat->vat_flags |= _voucher_activity_trace_flag_libdispatch; +#if __has_extension(c_static_assert) + _Static_assert(sizeof(mach_msg_header_t) <= sizeof(vat->vat_data), + "mach_msg_header_t too large"); +#endif + memcpy(vat->vat_data, msg, sizeof(mach_msg_header_t)); +} +#define _voucher_activity_trace_msg(v, msg, type) \ + _voucher_activity_trace_msg(v, msg, \ + _voucher_activity_tracepoint_namespace_ipc_ ## type) + +#endif // !(USE_OBJC && __OBJC2__) + +#else // VOUCHER_USE_MACH_VOUCHER + +#pragma mark - +#pragma mark Simulator / vouchers disabled + +#define _dispatch_voucher_debug(msg, v, ...) +#define _dispatch_kvoucher_debug(msg, kv, ...) + +DISPATCH_ALWAYS_INLINE +static inline voucher_t +_voucher_retain(voucher_t voucher) +{ + return voucher; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_release(voucher_t voucher) +{ + (void)voucher; +} + +DISPATCH_ALWAYS_INLINE +static inline voucher_t +_voucher_get(void) +{ + return NULL; +} + +DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT +static inline voucher_t +_voucher_copy(void) +{ + return NULL; +} + +DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT +static inline voucher_t +_voucher_copy_without_importance(void) +{ + return NULL; +} + +DISPATCH_ALWAYS_INLINE +static inline mach_voucher_t +_voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher) +{ + (void)ov; (void)voucher; + return MACH_VOUCHER_NULL; +} + +DISPATCH_ALWAYS_INLINE +static inline voucher_t +_voucher_adopt(voucher_t voucher) +{ + return voucher; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_replace(voucher_t voucher) +{ + (void)voucher; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_clear(void) +{ +} + +DISPATCH_ALWAYS_INLINE +static inline pthread_priority_t +_voucher_get_priority(voucher_t voucher) +{ + (void)voucher; + return 0; +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv, + bool move_send) +{ + (void)msg; (void)kv; (void)move_send; + return false; + +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher) +{ + (void)msg; (void)voucher; + return false; +} + +DISPATCH_ALWAYS_INLINE +static inline mach_voucher_t +_voucher_mach_msg_get(mach_msg_header_t *msg) +{ + (void)msg; + return 0; +} + +DISPATCH_ALWAYS_INLINE +static inline mach_voucher_t +_voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send) +{ + (void)msg; (void)move_send; + return MACH_VOUCHER_NULL; +} + +#define _dispatch_voucher_ktrace_dmsg_push(dmsg) +#define _dispatch_voucher_ktrace_dmsg_pop(dmsg) + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_continuation_voucher_set(dispatch_continuation_t dc, + dispatch_block_flags_t flags) +{ + (void)dc; (void)flags; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_dispatch_continuation_voucher_adopt(dispatch_continuation_t dc) +{ + (void)dc; +} + +#define _voucher_activity_trace_msg(v, msg, type) + +DISPATCH_ALWAYS_INLINE +static inline bool +_voucher_activity_disabled(void) +{ + return true; +} + +#endif // VOUCHER_USE_MACH_VOUCHER + +#endif /* __DISPATCH_VOUCHER_INTERNAL__ */ diff --git a/xcodeconfig/libdispatch-introspection.xcconfig b/xcodeconfig/libdispatch-introspection.xcconfig index d0f431d..1644ea9 100644 --- a/xcodeconfig/libdispatch-introspection.xcconfig +++ b/xcodeconfig/libdispatch-introspection.xcconfig @@ -19,8 +19,8 @@ // BUILD_VARIANTS = normal -INSTALL_PATH = /usr/lib/system/introspection -INSTALL_PATH[sdk=iphonesimulator*] = $(SDKROOT)/usr/lib/system/introspection +INSTALL_PATH_ACTUAL = /usr/lib/system/introspection + GCC_PREPROCESSOR_DEFINITIONS = $(GCC_PREPROCESSOR_DEFINITIONS) DISPATCH_INTROSPECTION=1 CONFIGURATION_BUILD_DIR = $(BUILD_DIR)/introspection OTHER_LDFLAGS = $(OTHER_LDFLAGS) -Wl,-interposable_list,$(SRCROOT)/xcodeconfig/libdispatch.interposable diff --git a/xcodeconfig/libdispatch-resolver_iphoneos.order b/xcodeconfig/libdispatch-resolver_iphoneos.order new file mode 100644 index 0000000..eea9845 --- /dev/null +++ b/xcodeconfig/libdispatch-resolver_iphoneos.order @@ -0,0 +1,20 @@ +# +# Copyright (c) 2013 Apple Inc. All rights reserved. +# +# @APPLE_APACHE_LICENSE_HEADER_START@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# @APPLE_APACHE_LICENSE_HEADER_END@ +# + diff --git a/xcodeconfig/libdispatch.aliases b/xcodeconfig/libdispatch.aliases index 5877e50..c29b163 100644 --- a/xcodeconfig/libdispatch.aliases +++ b/xcodeconfig/libdispatch.aliases @@ -1,5 +1,5 @@ # -# Copyright (c) 2012-2013 Apple Inc. All rights reserved. +# Copyright (c) 2013-2014 Apple Inc. All rights reserved. # # @APPLE_APACHE_LICENSE_HEADER_START@ # @@ -18,19 +18,6 @@ # @APPLE_APACHE_LICENSE_HEADER_END@ # -_OBJC_CLASS_$_OS_dispatch_semaphore __dispatch_semaphore_vtable -_OBJC_CLASS_$_OS_dispatch_group __dispatch_group_vtable -_OBJC_CLASS_$_OS_dispatch_queue __dispatch_queue_vtable -_OBJC_CLASS_$_OS_dispatch_queue_root __dispatch_queue_root_vtable -_OBJC_CLASS_$_OS_dispatch_queue_runloop __dispatch_queue_runloop_vtable -_OBJC_CLASS_$_OS_dispatch_queue_mgr __dispatch_queue_mgr_vtable -_OBJC_CLASS_$_OS_dispatch_queue_specific_queue __dispatch_queue_specific_queue_vtable -_OBJC_CLASS_$_OS_dispatch_queue_attr __dispatch_queue_attr_vtable -_OBJC_CLASS_$_OS_dispatch_source __dispatch_source_vtable -_OBJC_CLASS_$_OS_dispatch_mach __dispatch_mach_vtable -_OBJC_CLASS_$_OS_dispatch_mach_msg __dispatch_mach_msg_vtable -_OBJC_CLASS_$_OS_dispatch_io __dispatch_io_vtable -_OBJC_CLASS_$_OS_dispatch_operation __dispatch_operation_vtable -_OBJC_CLASS_$_OS_dispatch_disk __dispatch_disk_vtable - __dispatch_data_destructor_vm_deallocate __dispatch_data_destructor_munmap +__dispatch_source_type_memorystatus __dispatch_source_type_memorypressure +__dispatch_queue_attrs __dispatch_queue_attr_concurrent diff --git a/xcodeconfig/libdispatch.order b/xcodeconfig/libdispatch.order index 8870ea9..8bb4550 100644 --- a/xcodeconfig/libdispatch.order +++ b/xcodeconfig/libdispatch.order @@ -50,8 +50,11 @@ _OBJC_CLASS_$_OS_dispatch_operation __OS_dispatch_operation_vtable _OBJC_CLASS_$_OS_dispatch_disk __OS_dispatch_disk_vtable -# non-dispatch_object_t classes +# os_object_t classes _OBJC_CLASS_$_OS_object +_OBJC_CLASS_$_OS_voucher +#_OBJC_CLASS_$_OS_voucher_recipe +# non-os_object_t classes _OBJC_CLASS_$_OS_dispatch_data _OBJC_CLASS_$_OS_dispatch_data_empty # metaclasses @@ -71,5 +74,7 @@ _OBJC_METACLASS_$_OS_dispatch_io _OBJC_METACLASS_$_OS_dispatch_operation _OBJC_METACLASS_$_OS_dispatch_disk _OBJC_METACLASS_$_OS_object +_OBJC_METACLASS_$_OS_voucher +_OBJC_METACLASS_$_OS_voucher_recipe _OBJC_METACLASS_$_OS_dispatch_data _OBJC_METACLASS_$_OS_dispatch_data_empty diff --git a/xcodeconfig/libdispatch.xcconfig b/xcodeconfig/libdispatch.xcconfig index 4904b9d..1d22933 100644 --- a/xcodeconfig/libdispatch.xcconfig +++ b/xcodeconfig/libdispatch.xcconfig @@ -19,25 +19,25 @@ // #include "/Makefiles/CoreOS/Xcode/BSD.xcconfig" -SUPPORTED_PLATFORMS = macosx iphoneos iphonesimulator +#include "/AppleInternal/XcodeConfig/SimulatorSupport.xcconfig" + +// Set INSTALL_PATH[sdk=macosx*] when SimulatorSupport.xcconfig is unavailable +INSTALL_PATH[sdk=macosx*] = $(INSTALL_PATH_ACTUAL) + +SUPPORTED_PLATFORMS = macosx iphoneos iphonesimulator iphoneosnano iphonesimulatornano ARCHS[sdk=iphonesimulator*] = $(NATIVE_ARCH_32_BIT) // Override BSD.xcconfig ARCHS PRODUCT_NAME = libdispatch EXECUTABLE_PREFIX = -LD_DYLIB_INSTALL_NAME = /usr/lib/system/$(EXECUTABLE_NAME) -INSTALL_PATH = /usr/lib/system -INSTALL_PATH[sdk=iphonesimulator*] = $(SDKROOT)/usr/lib/system -PUBLIC_HEADERS_FOLDER_PATH = /usr/include/dispatch -PUBLIC_HEADERS_FOLDER_PATH[sdk=iphonesimulator*] = $(SDKROOT)/usr/include/dispatch -PRIVATE_HEADERS_FOLDER_PATH = /usr/local/include/dispatch -PRIVATE_HEADERS_FOLDER_PATH[sdk=iphonesimulator*] = $(SDKROOT)/usr/local/include/dispatch -OS_PUBLIC_HEADERS_FOLDER_PATH = /usr/include/os -OS_PUBLIC_HEADERS_FOLDER_PATH[sdk=iphonesimulator*] = $(SDKROOT)/usr/include/os -OS_PRIVATE_HEADERS_FOLDER_PATH = /usr/local/include/os -OS_PRIVATE_HEADERS_FOLDER_PATH[sdk=iphonesimulator*] = $(SDKROOT)/usr/local/include/os -HEADER_SEARCH_PATHS = $(PROJECT_DIR) +INSTALL_PATH_ACTUAL = /usr/lib/system +PUBLIC_HEADERS_FOLDER_PATH = $(INSTALL_PATH_PREFIX)/usr/include/dispatch +PRIVATE_HEADERS_FOLDER_PATH = $(INSTALL_PATH_PREFIX)/usr/local/include/dispatch +OS_PUBLIC_HEADERS_FOLDER_PATH = $(INSTALL_PATH_PREFIX)/usr/include/os +OS_PRIVATE_HEADERS_FOLDER_PATH = $(INSTALL_PATH_PREFIX)/usr/local/include/os +HEADER_SEARCH_PATHS = $(PROJECT_DIR) $(PROJECT_DIR)/private $(PROJECT_DIR)/os LIBRARY_SEARCH_PATHS = $(SDKROOT)/usr/lib/system INSTALLHDRS_SCRIPT_PHASE = YES ALWAYS_SEARCH_USER_PATHS = NO +USE_HEADERMAP = NO BUILD_VARIANTS = normal debug profile ONLY_ACTIVE_ARCH = NO CLANG_LINK_OBJC_RUNTIME = NO @@ -71,11 +71,13 @@ OTHER_CFLAGS_profile = $(OTHER_CFLAGS_normal) -DDISPATCH_PROFILE=1 OTHER_CFLAGS_debug = -fstack-protector -fno-inline -O0 -DDISPATCH_DEBUG=1 GENERATE_PROFILING_CODE = NO DYLIB_CURRENT_VERSION = $(CURRENT_PROJECT_VERSION) -UMBRELLA_LDFLAGS = -umbrella System -nodefaultlibs -ldyld -lcompiler_rt -lsystem_kernel -lsystem_platform -lsystem_pthread -lsystem_malloc -lsystem_c -lsystem_blocks -lunwind -Wl,-upward-lsystem_asl +UMBRELLA_LDFLAGS = -umbrella System -nodefaultlibs -ldyld -lcompiler_rt -lsystem_kernel -lsystem_platform -lsystem_pthread -lsystem_malloc -lsystem_c -lsystem_blocks -lunwind UMBRELLA_LDFLAGS[sdk=iphonesimulator*] = -umbrella System -nodefaultlibs -ldyld_sim -lcompiler_rt_sim -lsystem_sim_c -lsystem_sim_blocks -lunwind_sim -Wl,-upward-lSystem -OBJC_LDFLAGS = -Wl,-upward-lobjc -Wl,-order_file,$(SRCROOT)/xcodeconfig/libdispatch.order -Wl,-alias_list,$(SRCROOT)/xcodeconfig/libdispatch.aliases -Wl,-unexported_symbols_list,$(SRCROOT)/xcodeconfig/libdispatch.unexport +OBJC_LDFLAGS = -Wl,-upward-lobjc -Wl,-order_file,$(SRCROOT)/xcodeconfig/libdispatch.order -Wl,-alias_list,$(SRCROOT)/xcodeconfig/libdispatch_objc.aliases -Wl,-unexported_symbols_list,$(SRCROOT)/xcodeconfig/libdispatch.unexport OBJC_LDFLAGS[sdk=macosx*] = $(OBJC_LDFLAGS) -Wl,-upward-lauto OBJC_LDFLAGS[arch=i386][sdk=macosx*] = OBJC_EXCLUDED_SOURCE_FILE_NAMES_i386_macosx = object.m data.m +ALIASES_LDFLAGS = -Wl,-alias_list,$(SRCROOT)/xcodeconfig/libdispatch.aliases PLATFORM_LDFLAGS[sdk=macosx*] = -Wl,-alias_list,$(SRCROOT)/xcodeconfig/libdispatch_macosx.aliases -OTHER_LDFLAGS = $(OTHER_LDFLAGS) $(UMBRELLA_LDFLAGS) $(CR_LDFLAGS) $(OBJC_LDFLAGS) $(PLATFORM_LDFLAGS) +OTHER_LDFLAGS = $(OTHER_LDFLAGS) $(UMBRELLA_LDFLAGS) $(CR_LDFLAGS) $(OBJC_LDFLAGS) $(ALIASES_LDFLAGS) $(PLATFORM_LDFLAGS) +OTHER_MIGFLAGS = -novouchers diff --git a/xcodeconfig/libdispatch_iphoneos.order b/xcodeconfig/libdispatch_iphoneos.order new file mode 100644 index 0000000..eea9845 --- /dev/null +++ b/xcodeconfig/libdispatch_iphoneos.order @@ -0,0 +1,20 @@ +# +# Copyright (c) 2013 Apple Inc. All rights reserved. +# +# @APPLE_APACHE_LICENSE_HEADER_START@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# @APPLE_APACHE_LICENSE_HEADER_END@ +# + diff --git a/xcodeconfig/libdispatch_macosx.aliases b/xcodeconfig/libdispatch_macosx.aliases index a7f61c5..66b24a3 100644 --- a/xcodeconfig/libdispatch_macosx.aliases +++ b/xcodeconfig/libdispatch_macosx.aliases @@ -17,5 +17,3 @@ # # @APPLE_APACHE_LICENSE_HEADER_END@ # - -__dispatch_source_type_memorystatus __dispatch_source_type_memorypressure diff --git a/xcodeconfig/libdispatch_objc.aliases b/xcodeconfig/libdispatch_objc.aliases new file mode 100644 index 0000000..ad104a1 --- /dev/null +++ b/xcodeconfig/libdispatch_objc.aliases @@ -0,0 +1,34 @@ +# +# Copyright (c) 2012-2013 Apple Inc. All rights reserved. +# +# @APPLE_APACHE_LICENSE_HEADER_START@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# @APPLE_APACHE_LICENSE_HEADER_END@ +# + +_OBJC_CLASS_$_OS_dispatch_semaphore __dispatch_semaphore_vtable +_OBJC_CLASS_$_OS_dispatch_group __dispatch_group_vtable +_OBJC_CLASS_$_OS_dispatch_queue __dispatch_queue_vtable +_OBJC_CLASS_$_OS_dispatch_queue_root __dispatch_queue_root_vtable +_OBJC_CLASS_$_OS_dispatch_queue_runloop __dispatch_queue_runloop_vtable +_OBJC_CLASS_$_OS_dispatch_queue_mgr __dispatch_queue_mgr_vtable +_OBJC_CLASS_$_OS_dispatch_queue_specific_queue __dispatch_queue_specific_queue_vtable +_OBJC_CLASS_$_OS_dispatch_queue_attr __dispatch_queue_attr_vtable +_OBJC_CLASS_$_OS_dispatch_source __dispatch_source_vtable +_OBJC_CLASS_$_OS_dispatch_mach __dispatch_mach_vtable +_OBJC_CLASS_$_OS_dispatch_mach_msg __dispatch_mach_msg_vtable +_OBJC_CLASS_$_OS_dispatch_io __dispatch_io_vtable +_OBJC_CLASS_$_OS_dispatch_operation __dispatch_operation_vtable +_OBJC_CLASS_$_OS_dispatch_disk __dispatch_disk_vtable diff --git a/xcodescripts/install-headers.sh b/xcodescripts/install-headers.sh index cb5e804..1610b81 100755 --- a/xcodescripts/install-headers.sh +++ b/xcodescripts/install-headers.sh @@ -27,3 +27,5 @@ mkdir -p "${DSTROOT}${OS_PUBLIC_HEADERS_FOLDER_PATH}" || true mkdir -p "${DSTROOT}${OS_PRIVATE_HEADERS_FOLDER_PATH}" || true cp -X "${SCRIPT_INPUT_FILE_1}" "${DSTROOT}${OS_PUBLIC_HEADERS_FOLDER_PATH}" cp -X "${SCRIPT_INPUT_FILE_2}" "${DSTROOT}${OS_PRIVATE_HEADERS_FOLDER_PATH}" +cp -X "${SCRIPT_INPUT_FILE_3}" "${DSTROOT}${OS_PRIVATE_HEADERS_FOLDER_PATH}" +cp -X "${SCRIPT_INPUT_FILE_4}" "${DSTROOT}${OS_PRIVATE_HEADERS_FOLDER_PATH}" diff --git a/xcodescripts/mig-headers.sh b/xcodescripts/mig-headers.sh index 3669ec2..f81eb85 100755 --- a/xcodescripts/mig-headers.sh +++ b/xcodescripts/mig-headers.sh @@ -23,7 +23,7 @@ export MIGCC="$(xcrun -find cc)" export MIGCOM="$(xcrun -find migcom)" export PATH="${PLATFORM_DEVELOPER_BIN_DIR}:${DEVELOPER_BIN_DIR}:${PATH}" for a in ${ARCHS}; do - xcrun mig -arch $a -header "${SCRIPT_OUTPUT_FILE_0}" \ + xcrun mig ${OTHER_MIGFLAGS} -arch $a -header "${SCRIPT_OUTPUT_FILE_0}" \ -sheader "${SCRIPT_OUTPUT_FILE_1}" -user /dev/null \ -server /dev/null "${SCRIPT_INPUT_FILE_0}" done -- 2.45.2