src/voucher_internal.h

   1 /*
   2  * Copyright (c) 2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_VOUCHER_INTERNAL__
  28 #define __DISPATCH_VOUCHER_INTERNAL__
  29
  30 #ifndef __DISPATCH_INDIRECT__
  31 #error "Please #include <dispatch/dispatch.h> instead of this file directly."
  32 #include <dispatch/base.h> // for HeaderDoc
  33 #endif
  34
  35 #pragma mark -
  36 #pragma mark voucher_recipe_t (disabled)
  37
  38 #if VOUCHER_ENABLE_RECIPE_OBJECTS
  39 /*!
  40  * @group Voucher Creation SPI
  41  * SPI intended for clients that need to create vouchers.
  42  */
  43 OS_OBJECT_DECL_CLASS(voucher_recipe);
  44
  45 /*!
  46  * @function voucher_create
  47  *
  48  * @abstract
  49  * Creates a new voucher object from a recipe.
  50  *
  51  * @discussion
  52  * Error handling TBD
  53  *
  54  * @result
  55  * The newly created voucher object.
  56  */
  57 __OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0)
  58 OS_EXPORT OS_OBJECT_RETURNS_RETAINED OS_WARN_RESULT OS_NOTHROW
  59 voucher_t
  60 voucher_create(voucher_recipe_t recipe);
  61 #endif // VOUCHER_ENABLE_RECIPE_OBJECTS
  62
  63 #if VOUCHER_ENABLE_GET_MACH_VOUCHER
  64 /*!
  65  * @function voucher_get_mach_voucher
  66  *
  67  * @abstract
  68  * Returns the mach voucher port underlying the specified voucher object.
  69  *
  70  * @discussion
  71  * The caller must either maintain a reference on the voucher object while the
  72  * returned mach voucher port is in use to ensure it stays valid for the
  73  * duration, or it must retain the mach voucher port with mach_port_mod_refs().
  74  *
  75  * @param voucher
  76  * The voucher object to query.
  77  *
  78  * @result
  79  * A mach voucher port.
  80  */
  81 __OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0)
  82 OS_VOUCHER_EXPORT OS_WARN_RESULT OS_NOTHROW
  83 mach_voucher_t
  84 voucher_get_mach_voucher(voucher_t voucher);
  85 #endif // VOUCHER_ENABLE_GET_MACH_VOUCHER
  86
  87 #pragma mark -
  88 #pragma mark voucher_t
  89
  90 void _voucher_init(void);
  91 void _voucher_atfork_child(void);
  92 void _voucher_activity_debug_channel_init(void);
  93 #if OS_VOUCHER_ACTIVITY_SPI && OS_VOUCHER_ACTIVITY_GENERATE_SWAPS
  94 void _voucher_activity_swap(firehose_activity_id_t old_id,
  95                 firehose_activity_id_t new_id);
  96 #endif
  97 void _voucher_xref_dispose(voucher_t voucher);
  98 void _voucher_dispose(voucher_t voucher);
  99 size_t _voucher_debug(voucher_t v, char* buf, size_t bufsiz);
 100 void _voucher_thread_cleanup(void *voucher);
 101 mach_voucher_t _voucher_get_mach_voucher(voucher_t voucher);
 102 voucher_t _voucher_create_without_importance(voucher_t voucher);
 103 voucher_t _voucher_create_accounting_voucher(voucher_t voucher);
 104 mach_voucher_t _voucher_create_mach_voucher_with_priority(voucher_t voucher,
 105                 pthread_priority_t priority);
 106 voucher_t _voucher_create_with_priority_and_mach_voucher(voucher_t voucher,
 107                 pthread_priority_t priority, mach_voucher_t kv);
 108 void _voucher_dealloc_mach_voucher(mach_voucher_t kv);
 109
 110 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 111 _OS_OBJECT_DECL_SUBCLASS_INTERFACE(voucher_recipe, object)
 112 #endif
 113
 114 voucher_t voucher_retain(voucher_t voucher);
 115 void voucher_release(voucher_t voucher);
 116
 117 #define VOUCHER_NO_MACH_VOUCHER MACH_PORT_DEAD
 118
 119 #if VOUCHER_USE_MACH_VOUCHER
 120
 121 #if DISPATCH_DEBUG
 122 #define DISPATCH_VOUCHER_DEBUG 1
 123 #define DISPATCH_VOUCHER_ACTIVITY_DEBUG 1
 124 #endif
 125
 126 #if VOUCHER_USE_MACH_VOUCHER_PRIORITY
 127 #include <voucher/ipc_pthread_priority_types.h>
 128 #endif
 129
 130 typedef uint32_t _voucher_magic_t;
 131 typedef uint32_t _voucher_priority_t;
 132
 133 #define VOUCHER_MAGIC_V3  ((_voucher_magic_t)0x0390cefa) // FACE9003
 134
 135 typedef struct _voucher_mach_udata_s {
 136         _voucher_magic_t vmu_magic;
 137         _voucher_priority_t vmu_priority;
 138         uint8_t _vmu_after_priority[0];
 139         firehose_activity_id_t vmu_activity;
 140         uint64_t vmu_activity_pid;
 141         firehose_activity_id_t vmu_parent_activity;
 142         uint8_t _vmu_after_activity[0];
 143 } _voucher_mach_udata_s;
 144
 145 OS_ENUM(voucher_fields, uint16_t,
 146         VOUCHER_FIELD_NONE              = 0,
 147         VOUCHER_FIELD_KVOUCHER  = 1u << 0,
 148         VOUCHER_FIELD_PRIORITY  = 1u << 1,
 149         VOUCHER_FIELD_ACTIVITY  = 1u << 2,
 150
 151 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 152         VOUCHER_FIELD_EXTRA             = 1u << 15,
 153 #else
 154         VOUCHER_FIELD_EXTRA             = 0,
 155 #endif
 156 );
 157
 158 typedef struct voucher_s {
 159         _OS_OBJECT_HEADER(
 160         struct voucher_vtable_s *os_obj_isa,
 161         os_obj_ref_cnt,
 162         os_obj_xref_cnt);
 163         TAILQ_ENTRY(voucher_s) v_list;
 164         mach_voucher_t v_kvoucher, v_ipc_kvoucher; // if equal, only one reference
 165         voucher_t v_kvbase; // if non-NULL, v_kvoucher is a borrowed reference
 166         firehose_activity_id_t v_activity;
 167         uint64_t v_activity_creator;
 168         firehose_activity_id_t v_parent_activity;
 169         _voucher_priority_t v_priority;
 170         unsigned int v_kv_has_importance:1;
 171 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 172         size_t v_recipe_extra_offset;
 173         mach_voucher_attr_recipe_size_t v_recipe_extra_size;
 174 #endif
 175 } voucher_s;
 176
 177 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 178 #define _voucher_extra_size(v) ((v)->v_recipe_extra_size)
 179 #define _voucher_extra_recipes(v) ((char*)(v) + (v)->v_recipe_extra_offset)
 180 #else
 181 #define _voucher_extra_size(v) 0
 182 #define _voucher_extra_recipes(v) NULL
 183 #endif
 184
 185 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 186 typedef struct voucher_recipe_s {
 187         _OS_OBJECT_HEADER(
 188         const _os_object_vtable_s *os_obj_isa,
 189         os_obj_ref_cnt,
 190         os_obj_xref_cnt);
 191         size_t vr_allocation_size;
 192         mach_voucher_attr_recipe_size_t volatile vr_size;
 193         mach_voucher_attr_recipe_t vr_data;
 194 } voucher_recipe_s;
 195 #endif
 196
 197 #if TARGET_OS_EMBEDDED
 198 #define VL_HASH_SIZE  64u // must be a power of two
 199 #else
 200 #define VL_HASH_SIZE 256u // must be a power of two
 201 #endif
 202 #define VL_HASH(kv) (MACH_PORT_INDEX(kv) & (VL_HASH_SIZE - 1))
 203
 204 #if DISPATCH_DEBUG && DISPATCH_VOUCHER_DEBUG
 205 #define _dispatch_voucher_debug(msg, v, ...) \
 206                 _dispatch_debug("voucher[%p]: " msg, v, ##__VA_ARGS__)
 207 #define _dispatch_kvoucher_debug(msg, kv, ...) \
 208                 _dispatch_debug("kvoucher[0x%08x]: " msg, kv, ##__VA_ARGS__)
 209 #if DISPATCH_MACHPORT_DEBUG
 210 #define _dispatch_voucher_debug_machport(name) \
 211                 dispatch_debug_machport((name), __func__)
 212 #else
 213 #define _dispatch_voucher_debug_machport(name) ((void)(name))
 214 #endif
 215 #else
 216 #define _dispatch_voucher_debug(msg, v, ...)
 217 #define _dispatch_kvoucher_debug(msg, kv, ...)
 218 #define _dispatch_voucher_debug_machport(name) ((void)(name))
 219 #endif
 220
 221 #if DISPATCH_PURE_C
 222
 223 DISPATCH_ALWAYS_INLINE
 224 static inline voucher_t
 225 _voucher_retain(voucher_t voucher)
 226 {
 227 #if !DISPATCH_VOUCHER_OBJC_DEBUG
 228         // not using _os_object_refcnt* because we don't need barriers:
 229         // vouchers are immutable and are in a hash table with a lock
 230         int xref_cnt = os_atomic_inc2o(voucher, os_obj_xref_cnt, relaxed);
 231         _dispatch_voucher_debug("retain  -> %d", voucher, xref_cnt + 1);
 232         if (unlikely(xref_cnt <= 0)) {
 233                 _OS_OBJECT_CLIENT_CRASH("Voucher resurrection");
 234         }
 235 #else
 236         os_retain(voucher);
 237         _dispatch_voucher_debug("retain  -> %d", voucher,
 238                         voucher->os_obj_xref_cnt + 1);
 239 #endif // DISPATCH_DEBUG
 240         return voucher;
 241 }
 242
 243 DISPATCH_ALWAYS_INLINE
 244 static inline void
 245 _voucher_release(voucher_t voucher)
 246 {
 247 #if !DISPATCH_VOUCHER_OBJC_DEBUG
 248         // not using _os_object_refcnt* because we don't need barriers:
 249         // vouchers are immutable and are in a hash table with a lock
 250         int xref_cnt = os_atomic_dec2o(voucher, os_obj_xref_cnt, relaxed);
 251         _dispatch_voucher_debug("release -> %d", voucher, xref_cnt + 1);
 252         if (likely(xref_cnt >= 0)) {
 253                 return;
 254         }
 255         if (unlikely(xref_cnt < -1)) {
 256                 _OS_OBJECT_CLIENT_CRASH("Voucher over-release");
 257         }
 258         return _os_object_xref_dispose((_os_object_t)voucher);
 259 #else
 260         _dispatch_voucher_debug("release -> %d", voucher, voucher->os_obj_xref_cnt);
 261         return os_release(voucher);
 262 #endif // DISPATCH_DEBUG
 263 }
 264
 265 DISPATCH_ALWAYS_INLINE
 266 static inline void
 267 _voucher_release_no_dispose(voucher_t voucher)
 268 {
 269 #if !DISPATCH_VOUCHER_OBJC_DEBUG
 270         // not using _os_object_refcnt* because we don't need barriers:
 271         // vouchers are immutable and are in a hash table with a lock
 272         int xref_cnt = os_atomic_dec2o(voucher, os_obj_xref_cnt, relaxed);
 273         _dispatch_voucher_debug("release -> %d", voucher, xref_cnt + 1);
 274         if (likely(xref_cnt >= 0)) {
 275                 return;
 276         }
 277         _OS_OBJECT_CLIENT_CRASH("Voucher over-release");
 278 #else
 279         _dispatch_voucher_debug("release -> %d", voucher, voucher->os_obj_xref_cnt);
 280         return os_release(voucher);
 281 #endif // DISPATCH_DEBUG
 282 }
 283
 284 DISPATCH_ALWAYS_INLINE
 285 static inline voucher_t
 286 _voucher_get(void)
 287 {
 288         return _dispatch_thread_getspecific(dispatch_voucher_key);
 289 }
 290
 291 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 292 static inline voucher_t
 293 _voucher_copy(void)
 294 {
 295         voucher_t voucher = _voucher_get();
 296         if (voucher) _voucher_retain(voucher);
 297         return voucher;
 298 }
 299
 300 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 301 static inline voucher_t
 302 _voucher_copy_without_importance(void)
 303 {
 304         voucher_t voucher = _voucher_get();
 305         if (voucher) voucher = _voucher_create_without_importance(voucher);
 306         return voucher;
 307 }
 308
 309 DISPATCH_ALWAYS_INLINE
 310 static inline void
 311 _voucher_mach_voucher_set(mach_voucher_t kv)
 312 {
 313         if (kv == VOUCHER_NO_MACH_VOUCHER) return;
 314         _dispatch_set_priority_and_mach_voucher_slow(0, kv);
 315 }
 316
 317 DISPATCH_ALWAYS_INLINE
 318 static inline mach_voucher_t
 319 _voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher)
 320 {
 321         if (ov == voucher) return VOUCHER_NO_MACH_VOUCHER;
 322         _dispatch_voucher_debug("swap from voucher[%p]", voucher, ov);
 323         _dispatch_thread_setspecific(dispatch_voucher_key, voucher);
 324         mach_voucher_t kv = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL;
 325         mach_voucher_t okv = ov ? ov->v_kvoucher : MACH_VOUCHER_NULL;
 326 #if OS_VOUCHER_ACTIVITY_GENERATE_SWAPS
 327         firehose_activity_id_t aid = voucher ? voucher->v_activity : 0;
 328         firehose_activity_id_t oaid = ov ? ov->v_activity : 0;
 329         if (aid != oaid) _voucher_activity_swap(aid, oaid);
 330 #endif
 331         return (kv != okv) ? kv : VOUCHER_NO_MACH_VOUCHER;
 332 }
 333
 334 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 335 static inline voucher_t
 336 _voucher_adopt(voucher_t voucher)
 337 {
 338         voucher_t ov = _voucher_get();
 339         _voucher_mach_voucher_set(_voucher_swap_and_get_mach_voucher(ov, voucher));
 340         return ov;
 341 }
 342
 343 DISPATCH_ALWAYS_INLINE
 344 static inline void
 345 _voucher_replace(voucher_t voucher)
 346 {
 347         voucher_t ov = _voucher_adopt(voucher);
 348         if (ov) _voucher_release(ov);
 349 }
 350
 351 DISPATCH_ALWAYS_INLINE
 352 static inline void
 353 _voucher_clear(void)
 354 {
 355         _voucher_replace(NULL);
 356 }
 357
 358 DISPATCH_ALWAYS_INLINE
 359 static inline pthread_priority_t
 360 _voucher_get_priority(voucher_t v)
 361 {
 362         return v ? (pthread_priority_t)v->v_priority : 0;
 363 }
 364
 365 DISPATCH_ALWAYS_INLINE
 366 static inline firehose_activity_id_t
 367 _voucher_get_activity_id(voucher_t v, uint64_t *creator_pid)
 368 {
 369         if (creator_pid) *creator_pid = v ? v->v_activity_creator : 0;
 370         return v ? v->v_activity : 0;
 371 }
 372
 373 void _voucher_task_mach_voucher_init(void* ctxt);
 374 extern dispatch_once_t _voucher_task_mach_voucher_pred;
 375 extern mach_voucher_t _voucher_task_mach_voucher;
 376 #if VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
 377 #define _voucher_default_task_mach_voucher MACH_VOUCHER_NULL
 378 #else
 379 extern mach_voucher_t _voucher_default_task_mach_voucher;
 380 #endif
 381 DISPATCH_ALWAYS_INLINE
 382 static inline mach_voucher_t
 383 _voucher_get_task_mach_voucher(void)
 384 {
 385         dispatch_once_f(&_voucher_task_mach_voucher_pred, NULL,
 386                         _voucher_task_mach_voucher_init);
 387         return _voucher_task_mach_voucher;
 388 }
 389
 390 DISPATCH_ALWAYS_INLINE
 391 static inline bool
 392 _voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv,
 393                 bool move_send)
 394 {
 395         if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false;
 396         if (!kv) return false;
 397         msg->msgh_voucher_port = kv;
 398         msg->msgh_bits |= MACH_MSGH_BITS_SET_PORTS(0, 0, move_send ?
 399                         MACH_MSG_TYPE_MOVE_SEND : MACH_MSG_TYPE_COPY_SEND);
 400         _dispatch_kvoucher_debug("msg[%p] set %s", kv, msg, move_send ?
 401                         "move-send" : "copy-send");
 402         _dispatch_voucher_debug_machport(kv);
 403         return true;
 404 }
 405
 406 DISPATCH_ALWAYS_INLINE
 407 static inline bool
 408 _voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher)
 409 {
 410         if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false;
 411         mach_voucher_t kv;
 412         if (voucher) {
 413                 kv = _voucher_get_mach_voucher(voucher);
 414         } else {
 415                 kv = _voucher_get_task_mach_voucher();
 416         }
 417         return _voucher_mach_msg_set_mach_voucher(msg, kv, false);
 418 }
 419
 420 DISPATCH_ALWAYS_INLINE
 421 static inline mach_voucher_t
 422 _voucher_mach_msg_get(mach_msg_header_t *msg, mach_msg_bits_t *msgh_bits)
 423 {
 424         if (!MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) {
 425                 *msgh_bits = 0;
 426                 return MACH_VOUCHER_NULL;
 427         }
 428         mach_voucher_t kv = msg->msgh_voucher_port;
 429         msg->msgh_voucher_port = MACH_VOUCHER_NULL;
 430         mach_msg_bits_t mask = MACH_MSGH_BITS_VOUCHER_MASK|MACH_MSGH_BITS_RAISEIMP;
 431         *msgh_bits = msg->msgh_bits & mask;
 432         msg->msgh_bits &= ~mask;
 433         return kv;
 434 }
 435
 436 DISPATCH_ALWAYS_INLINE
 437 static inline mach_voucher_t
 438 _voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send)
 439 {
 440         mach_msg_bits_t kvbits = MACH_MSGH_BITS_VOUCHER(msg->msgh_bits);
 441         mach_voucher_t kv = msg->msgh_voucher_port, kvm = MACH_VOUCHER_NULL;
 442         if ((kvbits == MACH_MSG_TYPE_COPY_SEND ||
 443                         kvbits == MACH_MSG_TYPE_MOVE_SEND) && kv) {
 444                 _dispatch_kvoucher_debug("msg[%p] clear %s", kv, msg, move_send ?
 445                                 "move-send" : "copy-send");
 446                 _dispatch_voucher_debug_machport(kv);
 447                 if (kvbits == MACH_MSG_TYPE_MOVE_SEND) {
 448                         // <rdar://problem/15694142> return/drop received or pseudo-received
 449                         // voucher reference (e.g. due to send failure).
 450                         if (move_send) {
 451                                 kvm = kv;
 452                         } else {
 453                                 _voucher_dealloc_mach_voucher(kv);
 454                         }
 455                 }
 456                 msg->msgh_voucher_port = MACH_VOUCHER_NULL;
 457                 msg->msgh_bits &= (mach_msg_bits_t)~MACH_MSGH_BITS_VOUCHER_MASK;
 458         }
 459         return kvm;
 460 }
 461
 462 #pragma mark -
 463 #pragma mark dispatch_continuation_t + voucher_t
 464
 465 #if DISPATCH_USE_VOUCHER_KDEBUG_TRACE
 466 #define DISPATCH_VOUCHER_CODE(code) DISPATCH_CODE(VOUCHER, code)
 467 #else
 468 #define DISPATCH_VOUCHER_CODE(code) 0
 469 #endif // DISPATCH_USE_VOUCHER_KDEBUG_TRACE
 470
 471 #define DISPATCH_TRACE_VOUCHER_DC_PUSH          DISPATCH_VOUCHER_CODE(0x1)
 472 #define DISPATCH_TRACE_VOUCHER_DC_POP           DISPATCH_VOUCHER_CODE(0x2)
 473 #define DISPATCH_TRACE_VOUCHER_DMSG_PUSH        DISPATCH_VOUCHER_CODE(0x3)
 474 #define DISPATCH_TRACE_VOUCHER_DMSG_POP         DISPATCH_VOUCHER_CODE(0x4)
 475 #define DISPATCH_TRACE_VOUCHER_ACTIVITY_ADOPT   DISPATCH_VOUCHER_CODE(0x5)
 476
 477 DISPATCH_ALWAYS_INLINE
 478 static inline void
 479 _dispatch_voucher_ktrace(uint32_t code, voucher_t v, const void *container)
 480 {
 481         if (v == DISPATCH_NO_VOUCHER) return;
 482         natural_t voucher = v ? v->v_kvoucher : MACH_VOUCHER_NULL;
 483         _dispatch_ktrace2(code, voucher, (uintptr_t)container);
 484 }
 485 #define _dispatch_voucher_ktrace(code, v, container) \
 486                 _dispatch_voucher_ktrace(DISPATCH_TRACE_VOUCHER_##code, v, container)
 487 #define _dispatch_voucher_ktrace_dc_push(dc) \
 488                 _dispatch_voucher_ktrace(DC_PUSH, (dc)->dc_voucher, (dc))
 489 #define _dispatch_voucher_ktrace_dc_pop(dc, v) \
 490                 _dispatch_voucher_ktrace(DC_POP, v, (dc))
 491 #define _dispatch_voucher_ktrace_dmsg_push(dmsg) \
 492                 _dispatch_voucher_ktrace(DMSG_PUSH, (dmsg)->dmsg_voucher, (dmsg))
 493 #define _dispatch_voucher_ktrace_dmsg_pop(dmsg) \
 494                 _dispatch_voucher_ktrace(DMSG_POP, (dmsg)->dmsg_voucher, (dmsg))
 495 #define _dispatch_voucher_ktrace_activity_adopt(aid) \
 496                 _dispatch_ktrace1(DISPATCH_TRACE_VOUCHER_ACTIVITY_ADOPT, aid);
 497
 498 DISPATCH_ALWAYS_INLINE
 499 static inline void
 500 _dispatch_continuation_voucher_set(dispatch_continuation_t dc,
 501                 dispatch_queue_class_t dqu, dispatch_block_flags_t flags)
 502 {
 503         voucher_t v = NULL;
 504
 505         // _dispatch_continuation_voucher_set is never called for blocks with
 506         // private data or with the DISPATCH_BLOCK_HAS_VOUCHER flag set.
 507         // only _dispatch_continuation_init_slow handles this bit.
 508         dispatch_assert(!(flags & DISPATCH_BLOCK_HAS_VOUCHER));
 509
 510         if (dqu._oq->oq_override_voucher != DISPATCH_NO_VOUCHER) {
 511                 // if the queue has an override voucher, we should not capture anything
 512                 //
 513                 // if the continuation is enqueued before the queue is activated, then
 514                 // this optimization fails and we do capture whatever is current
 515                 //
 516                 // _dispatch_continuation_voucher_adopt() would do the right thing
 517                 // but using DISPATCH_NO_VOUCHER here is more efficient.
 518                 v = DISPATCH_NO_VOUCHER;
 519         } else if (!(flags & DISPATCH_BLOCK_NO_VOUCHER)) {
 520                 v = _voucher_copy();
 521         }
 522         dc->dc_voucher = v;
 523         _dispatch_voucher_debug("continuation[%p] set", dc->dc_voucher, dc);
 524         _dispatch_voucher_ktrace_dc_push(dc);
 525 }
 526
 527 static inline dispatch_queue_t _dispatch_queue_get_current(void);
 528
 529 DISPATCH_ALWAYS_INLINE
 530 static inline void
 531 _dispatch_continuation_voucher_adopt(dispatch_continuation_t dc,
 532                 voucher_t ov, uintptr_t dc_flags)
 533 {
 534         voucher_t v = dc->dc_voucher;
 535         _dispatch_thread_set_self_t consume = (dc_flags & DISPATCH_OBJ_CONSUME_BIT);
 536         dispatch_assert(DISPATCH_OBJ_CONSUME_BIT == DISPATCH_VOUCHER_CONSUME);
 537
 538         if (consume) {
 539                 dc->dc_voucher = VOUCHER_INVALID;
 540         }
 541         if (likely(v != DISPATCH_NO_VOUCHER)) {
 542                 _dispatch_voucher_ktrace_dc_pop(dc, v);
 543                 _dispatch_voucher_debug("continuation[%p] adopt", v, dc);
 544
 545                 if (likely(!(dc_flags & DISPATCH_OBJ_ENFORCE_VOUCHER))) {
 546                         if (unlikely(ov != DISPATCH_NO_VOUCHER && v != ov)) {
 547                                 if (consume) _voucher_release(v);
 548                                 consume = 0;
 549                                 v = ov;
 550                         }
 551                 }
 552         } else {
 553                 consume = 0;
 554                 v = ov;
 555         }
 556         (void)_dispatch_adopt_priority_and_set_voucher(dc->dc_priority, v,
 557                         consume | DISPATCH_VOUCHER_REPLACE);
 558 }
 559
 560 #pragma mark -
 561 #pragma mark _voucher activity subsystem
 562
 563 extern dispatch_once_t _firehose_task_buffer_pred;
 564 extern union firehose_buffer_u *_firehose_task_buffer;
 565 extern uint64_t _voucher_unique_pid;
 566 extern dispatch_mach_t _voucher_activity_debug_channel;
 567 extern voucher_activity_hooks_t _voucher_libtrace_hooks;
 568
 569 #endif // DISPATCH_PURE_C
 570
 571 #else // VOUCHER_USE_MACH_VOUCHER
 572
 573 #pragma mark -
 574 #pragma mark Simulator / vouchers disabled
 575
 576 #define _dispatch_voucher_debug(msg, v, ...)
 577 #define _dispatch_kvoucher_debug(msg, kv, ...)
 578
 579 DISPATCH_ALWAYS_INLINE
 580 static inline voucher_t
 581 _voucher_retain(voucher_t voucher)
 582 {
 583         return voucher;
 584 }
 585
 586 DISPATCH_ALWAYS_INLINE
 587 static inline void
 588 _voucher_release(voucher_t voucher)
 589 {
 590         (void)voucher;
 591 }
 592
 593 DISPATCH_ALWAYS_INLINE
 594 static inline voucher_t
 595 _voucher_get(void)
 596 {
 597         return NULL;
 598 }
 599
 600 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 601 static inline voucher_t
 602 _voucher_copy(void)
 603 {
 604         return NULL;
 605 }
 606
 607 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 608 static inline voucher_t
 609 _voucher_copy_without_importance(void)
 610 {
 611         return NULL;
 612 }
 613
 614 DISPATCH_ALWAYS_INLINE
 615 static inline mach_voucher_t
 616 _voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher)
 617 {
 618         (void)ov; (void)voucher;
 619         return MACH_VOUCHER_NULL;
 620 }
 621
 622 DISPATCH_ALWAYS_INLINE
 623 static inline voucher_t
 624 _voucher_adopt(voucher_t voucher)
 625 {
 626         return voucher;
 627 }
 628
 629 DISPATCH_ALWAYS_INLINE
 630 static inline void
 631 _voucher_replace(voucher_t voucher)
 632 {
 633         (void)voucher;
 634 }
 635
 636 DISPATCH_ALWAYS_INLINE
 637 static inline void
 638 _voucher_clear(void)
 639 {
 640 }
 641
 642 DISPATCH_ALWAYS_INLINE
 643 static inline pthread_priority_t
 644 _voucher_get_priority(voucher_t voucher)
 645 {
 646         (void)voucher;
 647         return 0;
 648 }
 649
 650 DISPATCH_ALWAYS_INLINE
 651 static inline bool
 652 _voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv,
 653                 bool move_send)
 654 {
 655         (void)msg; (void)kv; (void)move_send;
 656         return false;
 657
 658 }
 659
 660 DISPATCH_ALWAYS_INLINE
 661 static inline bool
 662 _voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher)
 663 {
 664         (void)msg; (void)voucher;
 665         return false;
 666 }
 667
 668 DISPATCH_ALWAYS_INLINE
 669 static inline mach_voucher_t
 670 _voucher_mach_msg_get(mach_msg_header_t *msg, mach_msg_bits_t *msgh_bits)
 671 {
 672         (void)msg;(void)msgh_bits;
 673         return 0;
 674 }
 675
 676 DISPATCH_ALWAYS_INLINE
 677 static inline mach_voucher_t
 678 _voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send)
 679 {
 680         (void)msg; (void)move_send;
 681         return MACH_VOUCHER_NULL;
 682 }
 683
 684 #define _dispatch_voucher_ktrace_dc_push(dc)
 685 #define _dispatch_voucher_ktrace_dc_pop(dc, v)
 686 #define _dispatch_voucher_ktrace_dmsg_push(dmsg)
 687 #define _dispatch_voucher_ktrace_dmsg_pop(dmsg)
 688
 689 DISPATCH_ALWAYS_INLINE
 690 static inline void
 691 _dispatch_continuation_voucher_set(dispatch_continuation_t dc,
 692                 dispatch_queue_class_t dqu, dispatch_block_flags_t flags)
 693 {
 694         (void)dc; (void)dqu; (void)flags;
 695 }
 696
 697 DISPATCH_ALWAYS_INLINE
 698 static inline void
 699 _dispatch_continuation_voucher_adopt(dispatch_continuation_t dc, voucher_t ov,
 700                 uintptr_t dc_flags)
 701 {
 702         (void)dc; (void)ov; (void)dc_flags;
 703 }
 704
 705 #endif // VOUCHER_USE_MACH_VOUCHER
 706
 707 #endif /* __DISPATCH_VOUCHER_INTERNAL__ */