src/voucher_internal.h

   1 /*
   2  * Copyright (c) 2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_VOUCHER_INTERNAL__
  28 #define __DISPATCH_VOUCHER_INTERNAL__
  29
  30 #ifndef __DISPATCH_INDIRECT__
  31 #error "Please #include <dispatch/dispatch.h> instead of this file directly."
  32 #include <dispatch/base.h> // for HeaderDoc
  33 #endif
  34
  35 #pragma mark -
  36 #pragma mark voucher_recipe_t (disabled)
  37
  38 #if VOUCHER_ENABLE_RECIPE_OBJECTS
  39 /*!
  40  * @group Voucher Creation SPI
  41  * SPI intended for clients that need to create vouchers.
  42  */
  43 OS_OBJECT_DECL_CLASS(voucher_recipe);
  44
  45 /*!
  46  * @function voucher_create
  47  *
  48  * @abstract
  49  * Creates a new voucher object from a recipe.
  50  *
  51  * @discussion
  52  * Error handling TBD
  53  *
  54  * @result
  55  * The newly created voucher object.
  56  */
  57 __OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0)
  58 OS_EXPORT OS_OBJECT_RETURNS_RETAINED OS_WARN_RESULT OS_NOTHROW
  59 voucher_t
  60 voucher_create(voucher_recipe_t recipe);
  61 #endif // VOUCHER_ENABLE_RECIPE_OBJECTS
  62
  63 #if VOUCHER_ENABLE_GET_MACH_VOUCHER
  64 /*!
  65  * @function voucher_get_mach_voucher
  66  *
  67  * @abstract
  68  * Returns the mach voucher port underlying the specified voucher object.
  69  *
  70  * @discussion
  71  * The caller must either maintain a reference on the voucher object while the
  72  * returned mach voucher port is in use to ensure it stays valid for the
  73  * duration, or it must retain the mach voucher port with mach_port_mod_refs().
  74  *
  75  * @param voucher
  76  * The voucher object to query.
  77  *
  78  * @result
  79  * A mach voucher port.
  80  */
  81 __OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0)
  82 OS_VOUCHER_EXPORT OS_WARN_RESULT OS_NOTHROW
  83 mach_voucher_t
  84 voucher_get_mach_voucher(voucher_t voucher);
  85 #endif // VOUCHER_ENABLE_GET_MACH_VOUCHER
  86
  87 #pragma mark -
  88 #pragma mark voucher_t
  89
  90 void _voucher_init(void);
  91 void _voucher_atfork_child(void);
  92 void _voucher_activity_debug_channel_init(void);
  93 void _voucher_activity_swap(firehose_activity_id_t old_id,
  94                 firehose_activity_id_t new_id);
  95 void _voucher_xref_dispose(voucher_t voucher);
  96 void _voucher_dispose(voucher_t voucher);
  97 size_t _voucher_debug(voucher_t v, char* buf, size_t bufsiz);
  98 void _voucher_thread_cleanup(void *voucher);
  99 mach_voucher_t _voucher_get_mach_voucher(voucher_t voucher);
 100 voucher_t _voucher_create_without_importance(voucher_t voucher);
 101 voucher_t _voucher_create_accounting_voucher(voucher_t voucher);
 102 mach_voucher_t _voucher_create_mach_voucher_with_priority(voucher_t voucher,
 103                 pthread_priority_t priority);
 104 voucher_t _voucher_create_with_priority_and_mach_voucher(voucher_t voucher,
 105                 pthread_priority_t priority, mach_voucher_t kv);
 106 void _voucher_dealloc_mach_voucher(mach_voucher_t kv);
 107
 108 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 109 _OS_OBJECT_DECL_SUBCLASS_INTERFACE(voucher_recipe, object)
 110 #endif
 111
 112 voucher_t voucher_retain(voucher_t voucher);
 113 void voucher_release(voucher_t voucher);
 114
 115 #define VOUCHER_NO_MACH_VOUCHER MACH_PORT_DEAD
 116
 117 #if VOUCHER_USE_MACH_VOUCHER
 118
 119 #if DISPATCH_DEBUG
 120 #define DISPATCH_VOUCHER_DEBUG 1
 121 #define DISPATCH_VOUCHER_ACTIVITY_DEBUG 1
 122 #endif
 123
 124 #if VOUCHER_USE_MACH_VOUCHER_PRIORITY
 125 #include <voucher/ipc_pthread_priority_types.h>
 126 #endif
 127
 128 typedef uint32_t _voucher_magic_t;
 129 typedef uint32_t _voucher_priority_t;
 130
 131 #define VOUCHER_MAGIC_V3  ((_voucher_magic_t)0x0390cefa) // FACE9003
 132
 133 typedef struct _voucher_mach_udata_s {
 134         _voucher_magic_t vmu_magic;
 135         _voucher_priority_t vmu_priority;
 136         uint8_t _vmu_after_priority[0];
 137         firehose_activity_id_t vmu_activity;
 138         uint64_t vmu_activity_pid;
 139         firehose_activity_id_t vmu_parent_activity;
 140         uint8_t _vmu_after_activity[0];
 141 } _voucher_mach_udata_s;
 142
 143 OS_ENUM(voucher_fields, uint16_t,
 144         VOUCHER_FIELD_NONE              = 0,
 145         VOUCHER_FIELD_KVOUCHER  = 1u << 0,
 146         VOUCHER_FIELD_PRIORITY  = 1u << 1,
 147         VOUCHER_FIELD_ACTIVITY  = 1u << 2,
 148
 149 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 150         VOUCHER_FIELD_EXTRA             = 1u << 15,
 151 #else
 152         VOUCHER_FIELD_EXTRA             = 0,
 153 #endif
 154 );
 155
 156 typedef struct voucher_s {
 157         _OS_OBJECT_HEADER(
 158         struct voucher_vtable_s *os_obj_isa,
 159         os_obj_ref_cnt,
 160         os_obj_xref_cnt);
 161         TAILQ_ENTRY(voucher_s) v_list;
 162         mach_voucher_t v_kvoucher, v_ipc_kvoucher; // if equal, only one reference
 163         voucher_t v_kvbase; // if non-NULL, v_kvoucher is a borrowed reference
 164         firehose_activity_id_t v_activity;
 165         uint64_t v_activity_creator;
 166         firehose_activity_id_t v_parent_activity;
 167         _voucher_priority_t v_priority;
 168         unsigned int v_kv_has_importance:1;
 169 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 170         size_t v_recipe_extra_offset;
 171         mach_voucher_attr_recipe_size_t v_recipe_extra_size;
 172 #endif
 173 } voucher_s;
 174
 175 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 176 #define _voucher_extra_size(v) ((v)->v_recipe_extra_size)
 177 #define _voucher_extra_recipes(v) ((char*)(v) + (v)->v_recipe_extra_offset)
 178 #else
 179 #define _voucher_extra_size(v) 0
 180 #define _voucher_extra_recipes(v) NULL
 181 #endif
 182
 183 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 184 typedef struct voucher_recipe_s {
 185         _OS_OBJECT_HEADER(
 186         const _os_object_vtable_s *os_obj_isa,
 187         os_obj_ref_cnt,
 188         os_obj_xref_cnt);
 189         size_t vr_allocation_size;
 190         mach_voucher_attr_recipe_size_t volatile vr_size;
 191         mach_voucher_attr_recipe_t vr_data;
 192 } voucher_recipe_s;
 193 #endif
 194
 195 #if TARGET_OS_EMBEDDED
 196 #define VL_HASH_SIZE  64u // must be a power of two
 197 #else
 198 #define VL_HASH_SIZE 256u // must be a power of two
 199 #endif
 200 #define VL_HASH(kv) (MACH_PORT_INDEX(kv) & (VL_HASH_SIZE - 1))
 201
 202 #if DISPATCH_DEBUG && DISPATCH_VOUCHER_DEBUG
 203 #define _dispatch_voucher_debug(msg, v, ...) \
 204                 _dispatch_debug("voucher[%p]: " msg, v, ##__VA_ARGS__)
 205 #define _dispatch_kvoucher_debug(msg, kv, ...) \
 206                 _dispatch_debug("kvoucher[0x%08x]: " msg, kv, ##__VA_ARGS__)
 207 #if DISPATCH_MACHPORT_DEBUG
 208 #define _dispatch_voucher_debug_machport(name) \
 209                 dispatch_debug_machport((name), __func__)
 210 #else
 211 #define _dispatch_voucher_debug_machport(name) ((void)(name))
 212 #endif
 213 #else
 214 #define _dispatch_voucher_debug(msg, v, ...)
 215 #define _dispatch_kvoucher_debug(msg, kv, ...)
 216 #define _dispatch_voucher_debug_machport(name) ((void)(name))
 217 #endif
 218
 219 #if DISPATCH_PURE_C
 220
 221 DISPATCH_ALWAYS_INLINE
 222 static inline voucher_t
 223 _voucher_retain(voucher_t voucher)
 224 {
 225 #if !DISPATCH_VOUCHER_OBJC_DEBUG
 226         // not using _os_object_refcnt* because we don't need barriers:
 227         // vouchers are immutable and are in a hash table with a lock
 228         int xref_cnt = os_atomic_inc2o(voucher, os_obj_xref_cnt, relaxed);
 229         _dispatch_voucher_debug("retain  -> %d", voucher, xref_cnt + 1);
 230         if (unlikely(xref_cnt <= 0)) {
 231                 _OS_OBJECT_CLIENT_CRASH("Voucher resurrection");
 232         }
 233 #else
 234         os_retain(voucher);
 235         _dispatch_voucher_debug("retain  -> %d", voucher,
 236                         voucher->os_obj_xref_cnt + 1);
 237 #endif // DISPATCH_DEBUG
 238         return voucher;
 239 }
 240
 241 DISPATCH_ALWAYS_INLINE
 242 static inline void
 243 _voucher_release(voucher_t voucher)
 244 {
 245 #if !DISPATCH_VOUCHER_OBJC_DEBUG
 246         // not using _os_object_refcnt* because we don't need barriers:
 247         // vouchers are immutable and are in a hash table with a lock
 248         int xref_cnt = os_atomic_dec2o(voucher, os_obj_xref_cnt, relaxed);
 249         _dispatch_voucher_debug("release -> %d", voucher, xref_cnt + 1);
 250         if (likely(xref_cnt >= 0)) {
 251                 return;
 252         }
 253         if (unlikely(xref_cnt < -1)) {
 254                 _OS_OBJECT_CLIENT_CRASH("Voucher over-release");
 255         }
 256         return _os_object_xref_dispose((_os_object_t)voucher);
 257 #else
 258         _dispatch_voucher_debug("release -> %d", voucher, voucher->os_obj_xref_cnt);
 259         return os_release(voucher);
 260 #endif // DISPATCH_DEBUG
 261 }
 262
 263 DISPATCH_ALWAYS_INLINE
 264 static inline void
 265 _voucher_release_no_dispose(voucher_t voucher)
 266 {
 267 #if !DISPATCH_VOUCHER_OBJC_DEBUG
 268         // not using _os_object_refcnt* because we don't need barriers:
 269         // vouchers are immutable and are in a hash table with a lock
 270         int xref_cnt = os_atomic_dec2o(voucher, os_obj_xref_cnt, relaxed);
 271         _dispatch_voucher_debug("release -> %d", voucher, xref_cnt + 1);
 272         if (likely(xref_cnt >= 0)) {
 273                 return;
 274         }
 275         _OS_OBJECT_CLIENT_CRASH("Voucher over-release");
 276 #else
 277         _dispatch_voucher_debug("release -> %d", voucher, voucher->os_obj_xref_cnt);
 278         return os_release(voucher);
 279 #endif // DISPATCH_DEBUG
 280 }
 281
 282 DISPATCH_ALWAYS_INLINE
 283 static inline voucher_t
 284 _voucher_get(void)
 285 {
 286         return _dispatch_thread_getspecific(dispatch_voucher_key);
 287 }
 288
 289 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 290 static inline voucher_t
 291 _voucher_copy(void)
 292 {
 293         voucher_t voucher = _voucher_get();
 294         if (voucher) _voucher_retain(voucher);
 295         return voucher;
 296 }
 297
 298 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 299 static inline voucher_t
 300 _voucher_copy_without_importance(void)
 301 {
 302         voucher_t voucher = _voucher_get();
 303         if (voucher) voucher = _voucher_create_without_importance(voucher);
 304         return voucher;
 305 }
 306
 307 DISPATCH_ALWAYS_INLINE
 308 static inline void
 309 _voucher_mach_voucher_set(mach_voucher_t kv)
 310 {
 311         if (kv == VOUCHER_NO_MACH_VOUCHER) return;
 312         _dispatch_set_priority_and_mach_voucher_slow(0, kv);
 313 }
 314
 315 DISPATCH_ALWAYS_INLINE
 316 static inline mach_voucher_t
 317 _voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher)
 318 {
 319         if (ov == voucher) return VOUCHER_NO_MACH_VOUCHER;
 320         _dispatch_voucher_debug("swap from voucher[%p]", voucher, ov);
 321         _dispatch_thread_setspecific(dispatch_voucher_key, voucher);
 322         mach_voucher_t kv = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL;
 323         mach_voucher_t okv = ov ? ov->v_kvoucher : MACH_VOUCHER_NULL;
 324         firehose_activity_id_t aid = voucher ? voucher->v_activity : 0;
 325         firehose_activity_id_t oaid = ov ? ov->v_activity : 0;
 326         if (aid != oaid) _voucher_activity_swap(aid, oaid);
 327         return (kv != okv) ? kv : VOUCHER_NO_MACH_VOUCHER;
 328 }
 329
 330 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 331 static inline voucher_t
 332 _voucher_adopt(voucher_t voucher)
 333 {
 334         voucher_t ov = _voucher_get();
 335         _voucher_mach_voucher_set(_voucher_swap_and_get_mach_voucher(ov, voucher));
 336         return ov;
 337 }
 338
 339 DISPATCH_ALWAYS_INLINE
 340 static inline void
 341 _voucher_replace(voucher_t voucher)
 342 {
 343         voucher_t ov = _voucher_adopt(voucher);
 344         if (ov) _voucher_release(ov);
 345 }
 346
 347 DISPATCH_ALWAYS_INLINE
 348 static inline void
 349 _voucher_clear(void)
 350 {
 351         _voucher_replace(NULL);
 352 }
 353
 354 DISPATCH_ALWAYS_INLINE
 355 static inline pthread_priority_t
 356 _voucher_get_priority(voucher_t v)
 357 {
 358         return v ? (pthread_priority_t)v->v_priority : 0;
 359 }
 360
 361 DISPATCH_ALWAYS_INLINE
 362 static inline firehose_activity_id_t
 363 _voucher_get_activity_id(voucher_t v, uint64_t *creator_pid)
 364 {
 365         if (creator_pid) *creator_pid = v ? v->v_activity_creator : 0;
 366         return v ? v->v_activity : 0;
 367 }
 368
 369 void _voucher_task_mach_voucher_init(void* ctxt);
 370 extern dispatch_once_t _voucher_task_mach_voucher_pred;
 371 extern mach_voucher_t _voucher_task_mach_voucher;
 372 #if VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
 373 #define _voucher_default_task_mach_voucher MACH_VOUCHER_NULL
 374 #else
 375 extern mach_voucher_t _voucher_default_task_mach_voucher;
 376 #endif
 377 DISPATCH_ALWAYS_INLINE
 378 static inline mach_voucher_t
 379 _voucher_get_task_mach_voucher(void)
 380 {
 381         dispatch_once_f(&_voucher_task_mach_voucher_pred, NULL,
 382                         _voucher_task_mach_voucher_init);
 383         return _voucher_task_mach_voucher;
 384 }
 385
 386 DISPATCH_ALWAYS_INLINE
 387 static inline bool
 388 _voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv,
 389                 bool move_send)
 390 {
 391         if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false;
 392         if (!kv) return false;
 393         msg->msgh_voucher_port = kv;
 394         msg->msgh_bits |= MACH_MSGH_BITS_SET_PORTS(0, 0, move_send ?
 395                         MACH_MSG_TYPE_MOVE_SEND : MACH_MSG_TYPE_COPY_SEND);
 396         _dispatch_kvoucher_debug("msg[%p] set %s", kv, msg, move_send ?
 397                         "move-send" : "copy-send");
 398         _dispatch_voucher_debug_machport(kv);
 399         return true;
 400 }
 401
 402 DISPATCH_ALWAYS_INLINE
 403 static inline bool
 404 _voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher)
 405 {
 406         if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false;
 407         mach_voucher_t kv;
 408         if (voucher) {
 409                 kv = _voucher_get_mach_voucher(voucher);
 410         } else {
 411                 kv = _voucher_get_task_mach_voucher();
 412         }
 413         return _voucher_mach_msg_set_mach_voucher(msg, kv, false);
 414 }
 415
 416 DISPATCH_ALWAYS_INLINE
 417 static inline mach_voucher_t
 418 _voucher_mach_msg_get(mach_msg_header_t *msg, mach_msg_bits_t *msgh_bits)
 419 {
 420         if (!MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) {
 421                 *msgh_bits = 0;
 422                 return MACH_VOUCHER_NULL;
 423         }
 424         mach_voucher_t kv = msg->msgh_voucher_port;
 425         msg->msgh_voucher_port = MACH_VOUCHER_NULL;
 426         mach_msg_bits_t mask = MACH_MSGH_BITS_VOUCHER_MASK|MACH_MSGH_BITS_RAISEIMP;
 427         *msgh_bits = msg->msgh_bits & mask;
 428         msg->msgh_bits &= ~mask;
 429         return kv;
 430 }
 431
 432 DISPATCH_ALWAYS_INLINE
 433 static inline mach_voucher_t
 434 _voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send)
 435 {
 436         mach_msg_bits_t kvbits = MACH_MSGH_BITS_VOUCHER(msg->msgh_bits);
 437         mach_voucher_t kv = msg->msgh_voucher_port, kvm = MACH_VOUCHER_NULL;
 438         if ((kvbits == MACH_MSG_TYPE_COPY_SEND ||
 439                         kvbits == MACH_MSG_TYPE_MOVE_SEND) && kv) {
 440                 _dispatch_kvoucher_debug("msg[%p] clear %s", kv, msg, move_send ?
 441                                 "move-send" : "copy-send");
 442                 _dispatch_voucher_debug_machport(kv);
 443                 if (kvbits == MACH_MSG_TYPE_MOVE_SEND) {
 444                         // <rdar://problem/15694142> return/drop received or pseudo-received
 445                         // voucher reference (e.g. due to send failure).
 446                         if (move_send) {
 447                                 kvm = kv;
 448                         } else {
 449                                 _voucher_dealloc_mach_voucher(kv);
 450                         }
 451                 }
 452                 msg->msgh_voucher_port = MACH_VOUCHER_NULL;
 453                 msg->msgh_bits &= (mach_msg_bits_t)~MACH_MSGH_BITS_VOUCHER_MASK;
 454         }
 455         return kvm;
 456 }
 457
 458 #pragma mark -
 459 #pragma mark dispatch_continuation_t + voucher_t
 460
 461 #if DISPATCH_USE_VOUCHER_KDEBUG_TRACE
 462 #define DISPATCH_VOUCHER_CODE(code) DISPATCH_CODE(VOUCHER, code)
 463 #else
 464 #define DISPATCH_VOUCHER_CODE(code) 0
 465 #endif // DISPATCH_USE_VOUCHER_KDEBUG_TRACE
 466
 467 #define DISPATCH_TRACE_VOUCHER_DC_PUSH          DISPATCH_VOUCHER_CODE(0x1)
 468 #define DISPATCH_TRACE_VOUCHER_DC_POP           DISPATCH_VOUCHER_CODE(0x2)
 469 #define DISPATCH_TRACE_VOUCHER_DMSG_PUSH        DISPATCH_VOUCHER_CODE(0x3)
 470 #define DISPATCH_TRACE_VOUCHER_DMSG_POP         DISPATCH_VOUCHER_CODE(0x4)
 471 #define DISPATCH_TRACE_VOUCHER_ACTIVITY_ADOPT   DISPATCH_VOUCHER_CODE(0x5)
 472
 473 DISPATCH_ALWAYS_INLINE
 474 static inline void
 475 _dispatch_voucher_ktrace(uint32_t code, voucher_t v, const void *container)
 476 {
 477         if (v == DISPATCH_NO_VOUCHER) return;
 478         natural_t voucher = v ? v->v_kvoucher : MACH_VOUCHER_NULL;
 479         _dispatch_ktrace2(code, voucher, (uintptr_t)container);
 480 }
 481 #define _dispatch_voucher_ktrace(code, v, container) \
 482                 _dispatch_voucher_ktrace(DISPATCH_TRACE_VOUCHER_##code, v, container)
 483 #define _dispatch_voucher_ktrace_dc_push(dc) \
 484                 _dispatch_voucher_ktrace(DC_PUSH, (dc)->dc_voucher, (dc))
 485 #define _dispatch_voucher_ktrace_dc_pop(dc, v) \
 486                 _dispatch_voucher_ktrace(DC_POP, v, (dc))
 487 #define _dispatch_voucher_ktrace_dmsg_push(dmsg) \
 488                 _dispatch_voucher_ktrace(DMSG_PUSH, (dmsg)->dmsg_voucher, (dmsg))
 489 #define _dispatch_voucher_ktrace_dmsg_pop(dmsg) \
 490                 _dispatch_voucher_ktrace(DMSG_POP, (dmsg)->dmsg_voucher, (dmsg))
 491 #define _dispatch_voucher_ktrace_activity_adopt(aid) \
 492                 _dispatch_ktrace1(DISPATCH_TRACE_VOUCHER_ACTIVITY_ADOPT, aid);
 493
 494 DISPATCH_ALWAYS_INLINE
 495 static inline void
 496 _dispatch_continuation_voucher_set(dispatch_continuation_t dc,
 497                 dispatch_queue_class_t dqu, dispatch_block_flags_t flags)
 498 {
 499         voucher_t v = NULL;
 500
 501         // _dispatch_continuation_voucher_set is never called for blocks with
 502         // private data or with the DISPATCH_BLOCK_HAS_VOUCHER flag set.
 503         // only _dispatch_continuation_init_slow handles this bit.
 504         dispatch_assert(!(flags & DISPATCH_BLOCK_HAS_VOUCHER));
 505
 506         if (dqu._oq->oq_override_voucher != DISPATCH_NO_VOUCHER) {
 507                 // if the queue has an override voucher, we should not capture anything
 508                 //
 509                 // if the continuation is enqueued before the queue is activated, then
 510                 // this optimization fails and we do capture whatever is current
 511                 //
 512                 // _dispatch_continuation_voucher_adopt() would do the right thing
 513                 // but using DISPATCH_NO_VOUCHER here is more efficient.
 514                 v = DISPATCH_NO_VOUCHER;
 515         } else if (!(flags & DISPATCH_BLOCK_NO_VOUCHER)) {
 516                 v = _voucher_copy();
 517         }
 518         dc->dc_voucher = v;
 519         _dispatch_voucher_debug("continuation[%p] set", dc->dc_voucher, dc);
 520         _dispatch_voucher_ktrace_dc_push(dc);
 521 }
 522
 523 static inline dispatch_queue_t _dispatch_queue_get_current(void);
 524
 525 DISPATCH_ALWAYS_INLINE
 526 static inline void
 527 _dispatch_continuation_voucher_adopt(dispatch_continuation_t dc,
 528                 voucher_t ov, uintptr_t dc_flags)
 529 {
 530         voucher_t v = dc->dc_voucher;
 531         _dispatch_thread_set_self_t consume = (dc_flags & DISPATCH_OBJ_CONSUME_BIT);
 532         dispatch_assert(DISPATCH_OBJ_CONSUME_BIT == DISPATCH_VOUCHER_CONSUME);
 533
 534         if (consume) {
 535                 dc->dc_voucher = VOUCHER_INVALID;
 536         }
 537         if (likely(v != DISPATCH_NO_VOUCHER)) {
 538                 _dispatch_voucher_ktrace_dc_pop(dc, v);
 539                 _dispatch_voucher_debug("continuation[%p] adopt", v, dc);
 540
 541                 if (likely(!(dc_flags & DISPATCH_OBJ_ENFORCE_VOUCHER))) {
 542                         if (unlikely(ov != DISPATCH_NO_VOUCHER && v != ov)) {
 543                                 if (consume) _voucher_release(v);
 544                                 consume = 0;
 545                                 v = ov;
 546                         }
 547                 }
 548         } else {
 549                 consume = 0;
 550                 v = ov;
 551         }
 552         (void)_dispatch_adopt_priority_and_set_voucher(dc->dc_priority, v,
 553                         consume | DISPATCH_VOUCHER_REPLACE);
 554 }
 555
 556 #pragma mark -
 557 #pragma mark _voucher activity subsystem
 558
 559 extern dispatch_once_t _firehose_task_buffer_pred;
 560 extern union firehose_buffer_u *_firehose_task_buffer;
 561 extern uint64_t _voucher_unique_pid;
 562 extern dispatch_mach_t _voucher_activity_debug_channel;
 563 extern voucher_activity_hooks_t _voucher_libtrace_hooks;
 564
 565 #endif // DISPATCH_PURE_C
 566
 567 #else // VOUCHER_USE_MACH_VOUCHER
 568
 569 #pragma mark -
 570 #pragma mark Simulator / vouchers disabled
 571
 572 #define _dispatch_voucher_debug(msg, v, ...)
 573 #define _dispatch_kvoucher_debug(msg, kv, ...)
 574
 575 DISPATCH_ALWAYS_INLINE
 576 static inline voucher_t
 577 _voucher_retain(voucher_t voucher)
 578 {
 579         return voucher;
 580 }
 581
 582 DISPATCH_ALWAYS_INLINE
 583 static inline void
 584 _voucher_release(voucher_t voucher)
 585 {
 586         (void)voucher;
 587 }
 588
 589 DISPATCH_ALWAYS_INLINE
 590 static inline voucher_t
 591 _voucher_get(void)
 592 {
 593         return NULL;
 594 }
 595
 596 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 597 static inline voucher_t
 598 _voucher_copy(void)
 599 {
 600         return NULL;
 601 }
 602
 603 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 604 static inline voucher_t
 605 _voucher_copy_without_importance(void)
 606 {
 607         return NULL;
 608 }
 609
 610 DISPATCH_ALWAYS_INLINE
 611 static inline mach_voucher_t
 612 _voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher)
 613 {
 614         (void)ov; (void)voucher;
 615         return MACH_VOUCHER_NULL;
 616 }
 617
 618 DISPATCH_ALWAYS_INLINE
 619 static inline voucher_t
 620 _voucher_adopt(voucher_t voucher)
 621 {
 622         return voucher;
 623 }
 624
 625 DISPATCH_ALWAYS_INLINE
 626 static inline void
 627 _voucher_replace(voucher_t voucher)
 628 {
 629         (void)voucher;
 630 }
 631
 632 DISPATCH_ALWAYS_INLINE
 633 static inline void
 634 _voucher_clear(void)
 635 {
 636 }
 637
 638 DISPATCH_ALWAYS_INLINE
 639 static inline pthread_priority_t
 640 _voucher_get_priority(voucher_t voucher)
 641 {
 642         (void)voucher;
 643         return 0;
 644 }
 645
 646 DISPATCH_ALWAYS_INLINE
 647 static inline bool
 648 _voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv,
 649                 bool move_send)
 650 {
 651         (void)msg; (void)kv; (void)move_send;
 652         return false;
 653
 654 }
 655
 656 DISPATCH_ALWAYS_INLINE
 657 static inline bool
 658 _voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher)
 659 {
 660         (void)msg; (void)voucher;
 661         return false;
 662 }
 663
 664 DISPATCH_ALWAYS_INLINE
 665 static inline mach_voucher_t
 666 _voucher_mach_msg_get(mach_msg_header_t *msg, mach_msg_bits_t *msgh_bits)
 667 {
 668         (void)msg;(void)msgh_bits;
 669         return 0;
 670 }
 671
 672 DISPATCH_ALWAYS_INLINE
 673 static inline mach_voucher_t
 674 _voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send)
 675 {
 676         (void)msg; (void)move_send;
 677         return MACH_VOUCHER_NULL;
 678 }
 679
 680 #define _dispatch_voucher_ktrace_dc_push(dc)
 681 #define _dispatch_voucher_ktrace_dc_pop(dc, v)
 682 #define _dispatch_voucher_ktrace_dmsg_push(dmsg)
 683 #define _dispatch_voucher_ktrace_dmsg_pop(dmsg)
 684
 685 DISPATCH_ALWAYS_INLINE
 686 static inline void
 687 _dispatch_continuation_voucher_set(dispatch_continuation_t dc,
 688                 dispatch_queue_class_t dqu, dispatch_block_flags_t flags)
 689 {
 690         (void)dc; (void)dqu; (void)flags;
 691 }
 692
 693 DISPATCH_ALWAYS_INLINE
 694 static inline void
 695 _dispatch_continuation_voucher_adopt(dispatch_continuation_t dc, voucher_t ov,
 696                 uintptr_t dc_flags)
 697 {
 698         (void)dc; (void)ov; (void)dc_flags;
 699 }
 700
 701 #endif // VOUCHER_USE_MACH_VOUCHER
 702
 703 #endif /* __DISPATCH_VOUCHER_INTERNAL__ */