src/voucher_internal.h

   1 /*
   2  * Copyright (c) 2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_VOUCHER_INTERNAL__
  28 #define __DISPATCH_VOUCHER_INTERNAL__
  29
  30 #ifndef __DISPATCH_INDIRECT__
  31 #error "Please #include <dispatch/dispatch.h> instead of this file directly."
  32 #include <dispatch/base.h> // for HeaderDoc
  33 #endif
  34
  35 #pragma mark -
  36 #pragma mark voucher_recipe_t (disabled)
  37
  38 #if VOUCHER_ENABLE_RECIPE_OBJECTS
  39 /*!
  40  * @group Voucher Creation SPI
  41  * SPI intended for clients that need to create vouchers.
  42  */
  43 OS_OBJECT_DECL_CLASS(voucher_recipe);
  44
  45 /*!
  46  * @function voucher_create
  47  *
  48  * @abstract
  49  * Creates a new voucher object from a recipe.
  50  *
  51  * @discussion
  52  * Error handling TBD
  53  *
  54  * @result
  55  * The newly created voucher object.
  56  */
  57 API_AVAILABLE(macos(10.10), ios(8.0))
  58 OS_EXPORT OS_OBJECT_RETURNS_RETAINED OS_WARN_RESULT OS_NOTHROW
  59 voucher_t
  60 voucher_create(voucher_recipe_t recipe);
  61 #endif // VOUCHER_ENABLE_RECIPE_OBJECTS
  62
  63 #if VOUCHER_ENABLE_GET_MACH_VOUCHER
  64 /*!
  65  * @function voucher_get_mach_voucher
  66  *
  67  * @abstract
  68  * Returns the mach voucher port underlying the specified voucher object.
  69  *
  70  * @discussion
  71  * The caller must either maintain a reference on the voucher object while the
  72  * returned mach voucher port is in use to ensure it stays valid for the
  73  * duration, or it must retain the mach voucher port with mach_port_mod_refs().
  74  *
  75  * @param voucher
  76  * The voucher object to query.
  77  *
  78  * @result
  79  * A mach voucher port.
  80  */
  81 API_AVAILABLE(macos(10.10), ios(8.0))
  82 OS_VOUCHER_EXPORT OS_WARN_RESULT OS_NOTHROW
  83 mach_voucher_t
  84 voucher_get_mach_voucher(voucher_t voucher);
  85 #endif // VOUCHER_ENABLE_GET_MACH_VOUCHER
  86
  87 #pragma mark -
  88 #pragma mark voucher_t
  89
  90 void _voucher_init(void);
  91 void _voucher_atfork_child(void);
  92 void _voucher_activity_debug_channel_init(void);
  93 #if OS_VOUCHER_ACTIVITY_SPI && OS_VOUCHER_ACTIVITY_GENERATE_SWAPS
  94 void _voucher_activity_swap(firehose_activity_id_t old_id,
  95                 firehose_activity_id_t new_id);
  96 #endif
  97 void _voucher_xref_dispose(voucher_t voucher);
  98 void _voucher_dispose(voucher_t voucher);
  99 size_t _voucher_debug(voucher_t v, char* buf, size_t bufsiz);
 100 void _voucher_thread_cleanup(void *voucher);
 101 mach_voucher_t _voucher_get_mach_voucher(voucher_t voucher);
 102 voucher_t _voucher_create_without_importance(voucher_t voucher);
 103 voucher_t _voucher_create_accounting_voucher(voucher_t voucher);
 104 mach_voucher_t _voucher_create_mach_voucher_with_priority(voucher_t voucher,
 105                 pthread_priority_t priority);
 106 voucher_t _voucher_create_with_priority_and_mach_voucher(voucher_t voucher,
 107                 pthread_priority_t priority, mach_voucher_t kv);
 108 void _voucher_dealloc_mach_voucher(mach_voucher_t kv);
 109
 110 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 111 _OS_OBJECT_DECL_SUBCLASS_INTERFACE(voucher_recipe, object)
 112 #endif
 113
 114 voucher_t voucher_retain(voucher_t voucher);
 115 void voucher_release(voucher_t voucher);
 116
 117 #define VOUCHER_NO_MACH_VOUCHER MACH_PORT_DEAD
 118
 119 #if VOUCHER_USE_MACH_VOUCHER
 120
 121 #if DISPATCH_DEBUG
 122 #define DISPATCH_VOUCHER_DEBUG 1
 123 #define DISPATCH_VOUCHER_ACTIVITY_DEBUG 1
 124 #endif
 125
 126 #include <voucher/ipc_pthread_priority_types.h>
 127
 128 typedef uint32_t _voucher_magic_t;
 129 typedef uint32_t _voucher_priority_t;
 130
 131 #define VOUCHER_MAGIC_V3  ((_voucher_magic_t)0x0390cefa) // FACE9003
 132
 133 typedef struct _voucher_mach_udata_s {
 134         _voucher_magic_t vmu_magic;
 135         _voucher_priority_t vmu_priority;
 136         uint8_t _vmu_after_priority[0];
 137         firehose_activity_id_t vmu_activity;
 138         uint64_t vmu_activity_pid;
 139         firehose_activity_id_t vmu_parent_activity;
 140         uint8_t _vmu_after_activity[0];
 141 } _voucher_mach_udata_s;
 142
 143 OS_ENUM(voucher_fields, uint16_t,
 144         VOUCHER_FIELD_NONE              = 0,
 145         VOUCHER_FIELD_KVOUCHER  = 1u << 0,
 146         VOUCHER_FIELD_PRIORITY  = 1u << 1,
 147         VOUCHER_FIELD_ACTIVITY  = 1u << 2,
 148
 149 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 150         VOUCHER_FIELD_EXTRA             = 1u << 15,
 151 #else
 152         VOUCHER_FIELD_EXTRA             = 0,
 153 #endif
 154 );
 155
 156 typedef struct voucher_s {
 157         _OS_OBJECT_HEADER(
 158         struct voucher_vtable_s *os_obj_isa,
 159         os_obj_ref_cnt,
 160         os_obj_xref_cnt);
 161         struct voucher_hash_entry_s {
 162                 uintptr_t vhe_next;
 163                 uintptr_t vhe_prev_ptr;
 164         } v_list;
 165         mach_voucher_t v_kvoucher, v_ipc_kvoucher; // if equal, only one reference
 166         voucher_t v_kvbase; // if non-NULL, v_kvoucher is a borrowed reference
 167         firehose_activity_id_t v_activity;
 168         uint64_t v_activity_creator;
 169         firehose_activity_id_t v_parent_activity;
 170         _voucher_priority_t v_priority;
 171         unsigned int v_kv_has_importance:1;
 172 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 173         size_t v_recipe_extra_offset;
 174         mach_voucher_attr_recipe_size_t v_recipe_extra_size;
 175 #endif
 176 } voucher_s;
 177
 178 typedef struct voucher_hash_head_s {
 179         uintptr_t vhh_first;
 180         uintptr_t vhh_last_ptr;
 181 } voucher_hash_head_s;
 182
 183 DISPATCH_ALWAYS_INLINE
 184 static inline bool
 185 _voucher_hash_is_enqueued(const struct voucher_s *v)
 186 {
 187         return v->v_list.vhe_prev_ptr != 0;
 188 }
 189
 190 DISPATCH_ALWAYS_INLINE
 191 static inline void
 192 _voucher_hash_mark_not_enqueued(struct voucher_s *v)
 193 {
 194         v->v_list.vhe_prev_ptr = 0;
 195         v->v_list.vhe_next = (uintptr_t)DISPATCH_OBJECT_LISTLESS;
 196 }
 197
 198 DISPATCH_ALWAYS_INLINE
 199 static inline void
 200 _voucher_hash_set_next(uintptr_t *next, struct voucher_s *v)
 201 {
 202         *next = ~(uintptr_t)v;
 203 }
 204
 205 DISPATCH_ALWAYS_INLINE
 206 static inline voucher_t
 207 _voucher_hash_get_next(uintptr_t next)
 208 {
 209         return (voucher_t)~next;
 210 }
 211
 212 DISPATCH_ALWAYS_INLINE
 213 static inline void
 214 _voucher_hash_set_prev_ptr(uintptr_t *prev_ptr, uintptr_t *addr)
 215 {
 216         *prev_ptr = ~(uintptr_t)addr;
 217 }
 218
 219 DISPATCH_ALWAYS_INLINE
 220 static inline void
 221 _voucher_hash_store_to_prev_ptr(uintptr_t prev_ptr, struct voucher_s *v)
 222 {
 223         *(uintptr_t *)~prev_ptr = ~(uintptr_t)v;
 224 }
 225
 226 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 227 #define _voucher_extra_size(v) ((v)->v_recipe_extra_size)
 228 #define _voucher_extra_recipes(v) ((char*)(v) + (v)->v_recipe_extra_offset)
 229 #else
 230 #define _voucher_extra_size(v) 0
 231 #define _voucher_extra_recipes(v) NULL
 232 #endif
 233
 234 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 235 typedef struct voucher_recipe_s {
 236         _OS_OBJECT_HEADER(
 237         const _os_object_vtable_s *os_obj_isa,
 238         os_obj_ref_cnt,
 239         os_obj_xref_cnt);
 240         size_t vr_allocation_size;
 241         mach_voucher_attr_recipe_size_t volatile vr_size;
 242         mach_voucher_attr_recipe_t vr_data;
 243 } voucher_recipe_s;
 244 #endif
 245
 246 #if TARGET_OS_EMBEDDED
 247 #define VL_HASH_SIZE  64u // must be a power of two
 248 #else
 249 #define VL_HASH_SIZE 256u // must be a power of two
 250 #endif
 251 #define VL_HASH(kv) (MACH_PORT_INDEX(kv) & (VL_HASH_SIZE - 1))
 252
 253 #if DISPATCH_DEBUG && DISPATCH_VOUCHER_DEBUG
 254 #define _dispatch_voucher_debug(msg, v, ...) \
 255                 _dispatch_debug("voucher[%p]: " msg, v, ##__VA_ARGS__)
 256 #define _dispatch_kvoucher_debug(msg, kv, ...) \
 257                 _dispatch_debug("kvoucher[0x%08x]: " msg, kv, ##__VA_ARGS__)
 258 #define _dispatch_voucher_debug_machport(name) _dispatch_debug_machport(name)
 259 #else
 260 #define _dispatch_voucher_debug(msg, v, ...)
 261 #define _dispatch_kvoucher_debug(msg, kv, ...)
 262 #define _dispatch_voucher_debug_machport(name) ((void)(name))
 263 #endif
 264
 265 #if DISPATCH_USE_DTRACE
 266 #define _voucher_trace(how, ...)  ({ \
 267                 if (unlikely(VOUCHER_##how##_ENABLED())) { \
 268                         VOUCHER_##how(__VA_ARGS__); \
 269                 } \
 270         })
 271 #else
 272 #define _voucher_trace(how, ...) ((void)0)
 273 #endif
 274
 275 #ifndef DISPATCH_VOUCHER_OBJC_DEBUG
 276 #if DISPATCH_INTROSPECTION || DISPATCH_DEBUG
 277 #define DISPATCH_VOUCHER_OBJC_DEBUG 1
 278 #else
 279 #define DISPATCH_VOUCHER_OBJC_DEBUG 0
 280 #endif
 281 #endif // DISPATCH_VOUCHER_OBJC_DEBUG
 282
 283 DISPATCH_ALWAYS_INLINE
 284 static inline struct voucher_s *
 285 _voucher_retain_inline(struct voucher_s *voucher)
 286 {
 287         // not using _os_object_refcnt* because we don't need barriers:
 288         // vouchers are immutable and are in a hash table with a lock
 289         int xref_cnt = os_atomic_inc2o(voucher, os_obj_xref_cnt, relaxed);
 290         _voucher_trace(RETAIN, (voucher_t)voucher, xref_cnt + 1);
 291         _dispatch_voucher_debug("retain  -> %d", voucher, xref_cnt + 1);
 292         if (unlikely(xref_cnt <= 0)) {
 293                 _OS_OBJECT_CLIENT_CRASH("Voucher resurrection");
 294         }
 295         return voucher;
 296 }
 297
 298 DISPATCH_ALWAYS_INLINE
 299 static inline void
 300 _voucher_release_inline(struct voucher_s *voucher)
 301 {
 302         // not using _os_object_refcnt* because we don't need barriers:
 303         // vouchers are immutable and are in a hash table with a lock
 304         int xref_cnt = os_atomic_dec2o(voucher, os_obj_xref_cnt, relaxed);
 305         _voucher_trace(RELEASE, (voucher_t)voucher, xref_cnt + 1);
 306         _dispatch_voucher_debug("release -> %d", voucher, xref_cnt + 1);
 307         if (likely(xref_cnt >= 0)) {
 308                 return;
 309         }
 310         if (unlikely(xref_cnt < -1)) {
 311                 _OS_OBJECT_CLIENT_CRASH("Voucher over-release");
 312         }
 313         return _os_object_xref_dispose((_os_object_t)voucher);
 314 }
 315
 316 #if DISPATCH_PURE_C
 317
 318 DISPATCH_ALWAYS_INLINE
 319 static inline voucher_t
 320 _voucher_retain(voucher_t voucher)
 321 {
 322 #if DISPATCH_VOUCHER_OBJC_DEBUG
 323         os_retain(voucher);
 324 #else
 325         _voucher_retain_inline(voucher);
 326 #endif // DISPATCH_VOUCHER_OBJC_DEBUG
 327         return voucher;
 328 }
 329
 330 DISPATCH_ALWAYS_INLINE
 331 static inline void
 332 _voucher_release(voucher_t voucher)
 333 {
 334 #if DISPATCH_VOUCHER_OBJC_DEBUG
 335         os_release(voucher);
 336 #else
 337         _voucher_release_inline(voucher);
 338 #endif // DISPATCH_VOUCHER_OBJC_DEBUG
 339 }
 340
 341 DISPATCH_ALWAYS_INLINE
 342 static inline void
 343 _voucher_release_no_dispose(voucher_t voucher)
 344 {
 345 #if !DISPATCH_VOUCHER_OBJC_DEBUG
 346         // not using _os_object_refcnt* because we don't need barriers:
 347         // vouchers are immutable and are in a hash table with a lock
 348         int xref_cnt = os_atomic_dec2o(voucher, os_obj_xref_cnt, relaxed);
 349         _voucher_trace(RELEASE, voucher, xref_cnt + 1);
 350         _dispatch_voucher_debug("release -> %d", voucher, xref_cnt + 1);
 351         if (likely(xref_cnt >= 0)) {
 352                 return;
 353         }
 354         _OS_OBJECT_CLIENT_CRASH("Voucher over-release");
 355 #else
 356         return os_release(voucher);
 357 #endif // DISPATCH_DEBUG
 358 }
 359
 360 DISPATCH_ALWAYS_INLINE
 361 static inline voucher_t
 362 _voucher_get(void)
 363 {
 364         return _dispatch_thread_getspecific(dispatch_voucher_key);
 365 }
 366
 367 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 368 static inline voucher_t
 369 _voucher_copy(void)
 370 {
 371         voucher_t voucher = _voucher_get();
 372         if (voucher) _voucher_retain(voucher);
 373         return voucher;
 374 }
 375
 376 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 377 static inline voucher_t
 378 _voucher_copy_without_importance(void)
 379 {
 380         voucher_t voucher = _voucher_get();
 381         if (voucher) voucher = _voucher_create_without_importance(voucher);
 382         return voucher;
 383 }
 384
 385 DISPATCH_ALWAYS_INLINE
 386 static inline void
 387 _voucher_mach_voucher_set(mach_voucher_t kv)
 388 {
 389         if (kv == VOUCHER_NO_MACH_VOUCHER) return;
 390         _dispatch_set_priority_and_mach_voucher_slow(0, kv);
 391 }
 392
 393 DISPATCH_ALWAYS_INLINE
 394 static inline mach_voucher_t
 395 _voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher)
 396 {
 397         if (ov == voucher) return VOUCHER_NO_MACH_VOUCHER;
 398         if (ov) _voucher_trace(ORPHAN, ov);
 399         _dispatch_thread_setspecific(dispatch_voucher_key, voucher);
 400         if (voucher) _voucher_trace(ADOPT, voucher);
 401         _dispatch_voucher_debug("swap from voucher[%p]", voucher, ov);
 402         mach_voucher_t kv = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL;
 403         mach_voucher_t okv = ov ? ov->v_kvoucher : MACH_VOUCHER_NULL;
 404 #if OS_VOUCHER_ACTIVITY_GENERATE_SWAPS
 405         firehose_activity_id_t aid = voucher ? voucher->v_activity : 0;
 406         firehose_activity_id_t oaid = ov ? ov->v_activity : 0;
 407         if (aid != oaid) _voucher_activity_swap(aid, oaid);
 408 #endif
 409         return (kv != okv) ? kv : VOUCHER_NO_MACH_VOUCHER;
 410 }
 411
 412 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 413 static inline voucher_t
 414 _voucher_adopt(voucher_t voucher)
 415 {
 416         voucher_t ov = _voucher_get();
 417         _voucher_mach_voucher_set(_voucher_swap_and_get_mach_voucher(ov, voucher));
 418         return ov;
 419 }
 420
 421 DISPATCH_ALWAYS_INLINE
 422 static inline void
 423 _voucher_replace(voucher_t voucher)
 424 {
 425         voucher_t ov = _voucher_adopt(voucher);
 426         if (ov) _voucher_release(ov);
 427 }
 428
 429 DISPATCH_ALWAYS_INLINE
 430 static inline void
 431 _voucher_clear(void)
 432 {
 433         _voucher_replace(NULL);
 434 }
 435
 436 DISPATCH_ALWAYS_INLINE
 437 static inline pthread_priority_t
 438 _voucher_get_priority(voucher_t v)
 439 {
 440         return v ? (pthread_priority_t)v->v_priority : 0;
 441 }
 442
 443 DISPATCH_ALWAYS_INLINE
 444 static inline firehose_activity_id_t
 445 _voucher_get_activity_id(voucher_t v, uint64_t *creator_pid)
 446 {
 447         if (creator_pid) *creator_pid = v ? v->v_activity_creator : 0;
 448         return v ? v->v_activity : 0;
 449 }
 450
 451 void _voucher_task_mach_voucher_init(void* ctxt);
 452 extern dispatch_once_t _voucher_task_mach_voucher_pred;
 453 extern mach_voucher_t _voucher_task_mach_voucher;
 454 #if VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
 455 #define _voucher_default_task_mach_voucher MACH_VOUCHER_NULL
 456 #else
 457 extern mach_voucher_t _voucher_default_task_mach_voucher;
 458 #endif
 459 DISPATCH_ALWAYS_INLINE
 460 static inline mach_voucher_t
 461 _voucher_get_task_mach_voucher(void)
 462 {
 463         dispatch_once_f(&_voucher_task_mach_voucher_pred, NULL,
 464                         _voucher_task_mach_voucher_init);
 465         return _voucher_task_mach_voucher;
 466 }
 467
 468 DISPATCH_ALWAYS_INLINE
 469 static inline bool
 470 _voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv,
 471                 bool move_send)
 472 {
 473         if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false;
 474         if (!kv) return false;
 475         msg->msgh_voucher_port = kv;
 476         msg->msgh_bits |= MACH_MSGH_BITS_SET_PORTS(0, 0, move_send ?
 477                         MACH_MSG_TYPE_MOVE_SEND : MACH_MSG_TYPE_COPY_SEND);
 478         _dispatch_kvoucher_debug("msg[%p] set %s", kv, msg, move_send ?
 479                         "move-send" : "copy-send");
 480         _dispatch_voucher_debug_machport(kv);
 481         return true;
 482 }
 483
 484 DISPATCH_ALWAYS_INLINE
 485 static inline bool
 486 _voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher)
 487 {
 488         if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false;
 489         mach_voucher_t kv;
 490         if (voucher) {
 491                 kv = _voucher_get_mach_voucher(voucher);
 492         } else {
 493                 kv = _voucher_get_task_mach_voucher();
 494         }
 495         return _voucher_mach_msg_set_mach_voucher(msg, kv, false);
 496 }
 497
 498 DISPATCH_ALWAYS_INLINE
 499 static inline mach_voucher_t
 500 _voucher_mach_msg_get(mach_msg_header_t *msg, mach_msg_bits_t *msgh_bits)
 501 {
 502         if (!MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) {
 503                 *msgh_bits = 0;
 504                 return MACH_VOUCHER_NULL;
 505         }
 506         mach_voucher_t kv = msg->msgh_voucher_port;
 507         msg->msgh_voucher_port = MACH_VOUCHER_NULL;
 508         mach_msg_bits_t mask = MACH_MSGH_BITS_VOUCHER_MASK|MACH_MSGH_BITS_RAISEIMP;
 509         *msgh_bits = msg->msgh_bits & mask;
 510         msg->msgh_bits &= ~mask;
 511         return kv;
 512 }
 513
 514 DISPATCH_ALWAYS_INLINE
 515 static inline mach_voucher_t
 516 _voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send)
 517 {
 518         mach_msg_bits_t kvbits = MACH_MSGH_BITS_VOUCHER(msg->msgh_bits);
 519         mach_voucher_t kv = msg->msgh_voucher_port, kvm = MACH_VOUCHER_NULL;
 520         if ((kvbits == MACH_MSG_TYPE_COPY_SEND ||
 521                         kvbits == MACH_MSG_TYPE_MOVE_SEND) && kv) {
 522                 _dispatch_kvoucher_debug("msg[%p] clear %s", kv, msg, move_send ?
 523                                 "move-send" : "copy-send");
 524                 _dispatch_voucher_debug_machport(kv);
 525                 if (kvbits == MACH_MSG_TYPE_MOVE_SEND) {
 526                         // <rdar://problem/15694142> return/drop received or pseudo-received
 527                         // voucher reference (e.g. due to send failure).
 528                         if (move_send) {
 529                                 kvm = kv;
 530                         } else {
 531                                 _voucher_dealloc_mach_voucher(kv);
 532                         }
 533                 }
 534                 msg->msgh_voucher_port = MACH_VOUCHER_NULL;
 535                 msg->msgh_bits &= (mach_msg_bits_t)~MACH_MSGH_BITS_VOUCHER_MASK;
 536         }
 537         return kvm;
 538 }
 539
 540 #pragma mark -
 541 #pragma mark dispatch_continuation_t + voucher_t
 542
 543 #if DISPATCH_USE_VOUCHER_KDEBUG_TRACE
 544 #define DISPATCH_VOUCHER_CODE(code) DISPATCH_CODE(VOUCHER, code)
 545 #else
 546 #define DISPATCH_VOUCHER_CODE(code) 0
 547 #endif // DISPATCH_USE_VOUCHER_KDEBUG_TRACE
 548
 549 #define DISPATCH_TRACE_VOUCHER_DC_PUSH          DISPATCH_VOUCHER_CODE(0x1)
 550 #define DISPATCH_TRACE_VOUCHER_DC_POP           DISPATCH_VOUCHER_CODE(0x2)
 551 #define DISPATCH_TRACE_VOUCHER_DMSG_PUSH        DISPATCH_VOUCHER_CODE(0x3)
 552 #define DISPATCH_TRACE_VOUCHER_DMSG_POP         DISPATCH_VOUCHER_CODE(0x4)
 553 #define DISPATCH_TRACE_VOUCHER_ACTIVITY_ADOPT   DISPATCH_VOUCHER_CODE(0x5)
 554
 555 DISPATCH_ALWAYS_INLINE
 556 static inline void
 557 _dispatch_voucher_ktrace(uint32_t code, voucher_t v, const void *container)
 558 {
 559         if (v == DISPATCH_NO_VOUCHER) return;
 560         natural_t voucher = v ? v->v_kvoucher : MACH_VOUCHER_NULL;
 561         _dispatch_ktrace2(code, voucher, (uintptr_t)container);
 562 }
 563 #define _dispatch_voucher_ktrace(code, v, container) \
 564                 _dispatch_voucher_ktrace(DISPATCH_TRACE_VOUCHER_##code, v, container)
 565 #define _dispatch_voucher_ktrace_dc_push(dc) \
 566                 _dispatch_voucher_ktrace(DC_PUSH, (dc)->dc_voucher, (dc))
 567 #define _dispatch_voucher_ktrace_dc_pop(dc, v) \
 568                 _dispatch_voucher_ktrace(DC_POP, v, (dc))
 569 #define _dispatch_voucher_ktrace_dmsg_push(dmsg) \
 570                 _dispatch_voucher_ktrace(DMSG_PUSH, (dmsg)->dmsg_voucher, (dmsg))
 571 #define _dispatch_voucher_ktrace_dmsg_pop(dmsg) \
 572                 _dispatch_voucher_ktrace(DMSG_POP, (dmsg)->dmsg_voucher, (dmsg))
 573 #define _dispatch_voucher_ktrace_activity_adopt(aid) \
 574                 _dispatch_ktrace1(DISPATCH_TRACE_VOUCHER_ACTIVITY_ADOPT, aid);
 575
 576 DISPATCH_ALWAYS_INLINE
 577 static inline void
 578 _dispatch_continuation_voucher_set(dispatch_continuation_t dc,
 579                 dispatch_queue_class_t dqu, dispatch_block_flags_t flags)
 580 {
 581         voucher_t v = NULL;
 582
 583         (void)dqu;
 584         // _dispatch_continuation_voucher_set is never called for blocks with
 585         // private data or with the DISPATCH_BLOCK_HAS_VOUCHER flag set.
 586         // only _dispatch_continuation_init_slow handles this bit.
 587         dispatch_assert(!(flags & DISPATCH_BLOCK_HAS_VOUCHER));
 588
 589         if (!(flags & DISPATCH_BLOCK_NO_VOUCHER)) {
 590                 v = _voucher_copy();
 591         }
 592         dc->dc_voucher = v;
 593         _dispatch_voucher_debug("continuation[%p] set", dc->dc_voucher, dc);
 594         _dispatch_voucher_ktrace_dc_push(dc);
 595 }
 596
 597 static inline dispatch_queue_t _dispatch_queue_get_current(void);
 598
 599 DISPATCH_ALWAYS_INLINE
 600 static inline void
 601 _dispatch_continuation_voucher_adopt(dispatch_continuation_t dc,
 602                 voucher_t ov, uintptr_t dc_flags)
 603 {
 604         voucher_t v = dc->dc_voucher;
 605         dispatch_thread_set_self_t consume = (dc_flags & DISPATCH_OBJ_CONSUME_BIT);
 606         dispatch_assert(DISPATCH_OBJ_CONSUME_BIT == DISPATCH_VOUCHER_CONSUME);
 607
 608         if (consume) {
 609                 dc->dc_voucher = VOUCHER_INVALID;
 610         }
 611         if (likely(v != DISPATCH_NO_VOUCHER)) {
 612                 _dispatch_voucher_ktrace_dc_pop(dc, v);
 613                 _dispatch_voucher_debug("continuation[%p] adopt", v, dc);
 614
 615                 if (likely(!(dc_flags & DISPATCH_OBJ_ENFORCE_VOUCHER))) {
 616                         if (unlikely(ov != DISPATCH_NO_VOUCHER && v != ov)) {
 617                                 if (consume && v) _voucher_release(v);
 618                                 consume = 0;
 619                                 v = ov;
 620                         }
 621                 }
 622         } else {
 623                 consume = 0;
 624                 v = ov;
 625         }
 626         (void)_dispatch_adopt_priority_and_set_voucher(dc->dc_priority, v,
 627                         consume | DISPATCH_VOUCHER_REPLACE);
 628 }
 629
 630 #pragma mark -
 631 #pragma mark _voucher activity subsystem
 632
 633 extern dispatch_once_t _firehose_task_buffer_pred;
 634 extern union firehose_buffer_u *_firehose_task_buffer;
 635 extern uint64_t _voucher_unique_pid;
 636 extern dispatch_mach_t _voucher_activity_debug_channel;
 637 extern voucher_activity_hooks_t _voucher_libtrace_hooks;
 638
 639 #endif // DISPATCH_PURE_C
 640
 641 #else // VOUCHER_USE_MACH_VOUCHER
 642
 643 #pragma mark -
 644 #pragma mark Simulator / vouchers disabled
 645
 646 #define _dispatch_voucher_debug(msg, v, ...)
 647 #define _dispatch_kvoucher_debug(msg, kv, ...)
 648
 649 DISPATCH_ALWAYS_INLINE
 650 static inline voucher_t
 651 _voucher_retain(voucher_t voucher)
 652 {
 653         return voucher;
 654 }
 655
 656 DISPATCH_ALWAYS_INLINE
 657 static inline void
 658 _voucher_release(voucher_t voucher)
 659 {
 660         (void)voucher;
 661 }
 662
 663 DISPATCH_ALWAYS_INLINE
 664 static inline voucher_t
 665 _voucher_get(void)
 666 {
 667         return NULL;
 668 }
 669
 670 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 671 static inline voucher_t
 672 _voucher_copy(void)
 673 {
 674         return NULL;
 675 }
 676
 677 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 678 static inline voucher_t
 679 _voucher_copy_without_importance(void)
 680 {
 681         return NULL;
 682 }
 683
 684 DISPATCH_ALWAYS_INLINE
 685 static inline mach_voucher_t
 686 _voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher)
 687 {
 688         (void)ov; (void)voucher;
 689         return MACH_VOUCHER_NULL;
 690 }
 691
 692 DISPATCH_ALWAYS_INLINE
 693 static inline voucher_t
 694 _voucher_adopt(voucher_t voucher)
 695 {
 696         return voucher;
 697 }
 698
 699 DISPATCH_ALWAYS_INLINE
 700 static inline void
 701 _voucher_replace(voucher_t voucher)
 702 {
 703         (void)voucher;
 704 }
 705
 706 DISPATCH_ALWAYS_INLINE
 707 static inline void
 708 _voucher_clear(void)
 709 {
 710 }
 711
 712 DISPATCH_ALWAYS_INLINE
 713 static inline pthread_priority_t
 714 _voucher_get_priority(voucher_t voucher)
 715 {
 716         (void)voucher;
 717         return 0;
 718 }
 719
 720 DISPATCH_ALWAYS_INLINE
 721 static inline bool
 722 _voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv,
 723                 bool move_send)
 724 {
 725         (void)msg; (void)kv; (void)move_send;
 726         return false;
 727
 728 }
 729
 730 DISPATCH_ALWAYS_INLINE
 731 static inline bool
 732 _voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher)
 733 {
 734         (void)msg; (void)voucher;
 735         return false;
 736 }
 737
 738 DISPATCH_ALWAYS_INLINE
 739 static inline mach_voucher_t
 740 _voucher_mach_msg_get(mach_msg_header_t *msg, mach_msg_bits_t *msgh_bits)
 741 {
 742         (void)msg;(void)msgh_bits;
 743         return 0;
 744 }
 745
 746 DISPATCH_ALWAYS_INLINE
 747 static inline mach_voucher_t
 748 _voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send)
 749 {
 750         (void)msg; (void)move_send;
 751         return MACH_VOUCHER_NULL;
 752 }
 753
 754 #define _dispatch_voucher_ktrace_dc_push(dc)
 755 #define _dispatch_voucher_ktrace_dc_pop(dc, v)
 756 #define _dispatch_voucher_ktrace_dmsg_push(dmsg)
 757 #define _dispatch_voucher_ktrace_dmsg_pop(dmsg)
 758
 759 DISPATCH_ALWAYS_INLINE
 760 static inline void
 761 _dispatch_continuation_voucher_set(dispatch_continuation_t dc,
 762                 dispatch_queue_class_t dqu, dispatch_block_flags_t flags)
 763 {
 764         (void)dc; (void)dqu; (void)flags;
 765 }
 766
 767 DISPATCH_ALWAYS_INLINE
 768 static inline void
 769 _dispatch_continuation_voucher_adopt(dispatch_continuation_t dc, voucher_t ov,
 770                 uintptr_t dc_flags)
 771 {
 772         (void)dc; (void)ov; (void)dc_flags;
 773 }
 774
 775 #endif // VOUCHER_USE_MACH_VOUCHER
 776
 777 #endif /* __DISPATCH_VOUCHER_INTERNAL__ */