src/voucher_internal.h

   1 /*
   2  * Copyright (c) 2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_VOUCHER_INTERNAL__
  28 #define __DISPATCH_VOUCHER_INTERNAL__
  29
  30 #ifndef __DISPATCH_INDIRECT__
  31 #error "Please #include <dispatch/dispatch.h> instead of this file directly."
  32 #include <dispatch/base.h> // for HeaderDoc
  33 #endif
  34
  35 #pragma mark -
  36 #pragma mark voucher_recipe_t (disabled)
  37
  38 #if VOUCHER_ENABLE_RECIPE_OBJECTS
  39 /*!
  40  * @group Voucher Creation SPI
  41  * SPI intended for clients that need to create vouchers.
  42  */
  43
  44 #if OS_OBJECT_USE_OBJC
  45 OS_OBJECT_DECL(voucher_recipe);
  46 #else
  47 typedef struct voucher_recipe_s *voucher_recipe_t;
  48 #endif
  49
  50 /*!
  51  * @function voucher_create
  52  *
  53  * @abstract
  54  * Creates a new voucher object from a recipe.
  55  *
  56  * @discussion
  57  * Error handling TBD
  58  *
  59  * @result
  60  * The newly created voucher object.
  61  */
  62 __OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0)
  63 OS_EXPORT OS_OBJECT_RETURNS_RETAINED OS_WARN_RESULT OS_NOTHROW
  64 voucher_t
  65 voucher_create(voucher_recipe_t recipe);
  66 #endif // VOUCHER_ENABLE_RECIPE_OBJECTS
  67
  68 #if VOUCHER_ENABLE_GET_MACH_VOUCHER
  69 /*!
  70  * @function voucher_get_mach_voucher
  71  *
  72  * @abstract
  73  * Returns the mach voucher port underlying the specified voucher object.
  74  *
  75  * @discussion
  76  * The caller must either maintain a reference on the voucher object while the
  77  * returned mach voucher port is in use to ensure it stays valid for the
  78  * duration, or it must retain the mach voucher port with mach_port_mod_refs().
  79  *
  80  * @param voucher
  81  * The voucher object to query.
  82  *
  83  * @result
  84  * A mach voucher port.
  85  */
  86 __OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0)
  87 OS_VOUCHER_EXPORT OS_WARN_RESULT OS_NOTHROW
  88 mach_voucher_t
  89 voucher_get_mach_voucher(voucher_t voucher);
  90 #endif // VOUCHER_ENABLE_GET_MACH_VOUCHER
  91
  92 #pragma mark -
  93 #pragma mark voucher_t
  94
  95 #if TARGET_IPHONE_SIMULATOR && \
  96                 IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101100
  97 #undef VOUCHER_USE_MACH_VOUCHER
  98 #define VOUCHER_USE_MACH_VOUCHER 0
  99 #endif
 100 #ifndef VOUCHER_USE_MACH_VOUCHER
 101 #if __has_include(<mach/mach_voucher.h>)
 102 #define VOUCHER_USE_MACH_VOUCHER 1
 103 #endif
 104 #endif
 105
 106 #if VOUCHER_USE_MACH_VOUCHER
 107 #undef DISPATCH_USE_IMPORTANCE_ASSERTION
 108 #define DISPATCH_USE_IMPORTANCE_ASSERTION 0
 109 #else
 110 #undef MACH_RCV_VOUCHER
 111 #define MACH_RCV_VOUCHER 0
 112 #endif // VOUCHER_USE_MACH_VOUCHER
 113
 114 void _voucher_init(void);
 115 void _voucher_atfork_child(void);
 116 void _voucher_activity_heap_pressure_warn(void);
 117 void _voucher_activity_heap_pressure_normal(void);
 118 void _voucher_xref_dispose(voucher_t voucher);
 119 void _voucher_dispose(voucher_t voucher);
 120 size_t _voucher_debug(voucher_t v, char* buf, size_t bufsiz);
 121 void _voucher_thread_cleanup(void *voucher);
 122 mach_voucher_t _voucher_get_mach_voucher(voucher_t voucher);
 123 voucher_t _voucher_create_without_importance(voucher_t voucher);
 124 voucher_t _voucher_create_accounting_voucher(voucher_t voucher);
 125 mach_voucher_t _voucher_create_mach_voucher_with_priority(voucher_t voucher,
 126                 pthread_priority_t priority);
 127 voucher_t _voucher_create_with_priority_and_mach_voucher(voucher_t voucher,
 128                 pthread_priority_t priority, mach_voucher_t kv);
 129 void _voucher_dealloc_mach_voucher(mach_voucher_t kv);
 130
 131 #if OS_OBJECT_USE_OBJC
 132 _OS_OBJECT_DECL_SUBCLASS_INTERFACE(voucher, object)
 133 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 134 _OS_OBJECT_DECL_SUBCLASS_INTERFACE(voucher_recipe, object)
 135 #endif
 136 #endif
 137
 138 voucher_t voucher_retain(voucher_t voucher);
 139 void voucher_release(voucher_t voucher);
 140
 141 #define _TAILQ_IS_ENQUEUED(elm, field) \
 142                 ((elm)->field.tqe_prev != NULL)
 143 #define _TAILQ_MARK_NOT_ENQUEUED(elm, field) \
 144                 do { (elm)->field.tqe_prev = NULL; } while (0)
 145
 146 #define VOUCHER_NO_MACH_VOUCHER MACH_PORT_DEAD
 147
 148 #if VOUCHER_USE_MACH_VOUCHER
 149
 150 #if DISPATCH_DEBUG
 151 #define DISPATCH_VOUCHER_DEBUG 1
 152 #define DISPATCH_VOUCHER_ACTIVITY_DEBUG 1
 153 #endif
 154
 155 typedef struct voucher_s {
 156         _OS_OBJECT_HEADER(
 157         void *os_obj_isa,
 158         os_obj_ref_cnt,
 159         os_obj_xref_cnt);
 160         TAILQ_ENTRY(voucher_s) v_list;
 161         mach_voucher_t v_kvoucher, v_ipc_kvoucher; // if equal, only one reference
 162         voucher_t v_kvbase; // if non-NULL, v_kvoucher is a borrowed reference
 163         struct _voucher_atm_s *v_atm;
 164         struct _voucher_activity_s *v_activity;
 165 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 166         size_t v_recipe_extra_offset;
 167         mach_voucher_attr_recipe_size_t v_recipe_extra_size;
 168 #endif
 169         unsigned int v_has_priority:1;
 170         unsigned int v_activities;
 171         mach_voucher_attr_recipe_data_t v_recipes[];
 172 } voucher_s;
 173
 174 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 175 typedef struct voucher_recipe_s {
 176         _OS_OBJECT_HEADER(
 177         const _os_object_class_s *os_obj_isa,
 178         os_obj_ref_cnt,
 179         os_obj_xref_cnt);
 180         size_t vr_allocation_size;
 181         mach_voucher_attr_recipe_size_t volatile vr_size;
 182         mach_voucher_attr_recipe_t vr_data;
 183 } voucher_recipe_s;
 184 #endif
 185
 186 #define _voucher_recipes_base(r) (r[0])
 187 #define _voucher_recipes_atm(r) (r[1])
 188 #define _voucher_recipes_bits(r) (r[2])
 189 #define _voucher_base_recipe(v) (_voucher_recipes_base((v)->v_recipes))
 190 #define _voucher_atm_recipe(v) (_voucher_recipes_atm((v)->v_recipes))
 191 #define _voucher_bits_recipe(v) (_voucher_recipes_bits((v)->v_recipes))
 192 #define _voucher_recipes_size() (3 * sizeof(mach_voucher_attr_recipe_data_t))
 193
 194 #if TARGET_OS_EMBEDDED
 195 #define VL_HASH_SIZE  64u // must be a power of two
 196 #else
 197 #define VL_HASH_SIZE 256u // must be a power of two
 198 #endif
 199 #define VL_HASH(kv) (MACH_PORT_INDEX(kv) & (VL_HASH_SIZE - 1))
 200
 201 typedef uint32_t _voucher_magic_t;
 202 const _voucher_magic_t _voucher_magic_v1 = 0x0190cefa; // little-endian FACE9001
 203 #define _voucher_recipes_magic(r) ((_voucher_magic_t*) \
 204                 (_voucher_recipes_bits(r).content))
 205 #define _voucher_magic(v) _voucher_recipes_magic((v)->v_recipes)
 206 typedef uint32_t _voucher_priority_t;
 207 #define _voucher_recipes_priority(r) ((_voucher_priority_t*) \
 208                 (_voucher_recipes_bits(r).content + sizeof(_voucher_magic_t)))
 209 #define _voucher_priority(v) _voucher_recipes_priority((v)->v_recipes)
 210 #define _voucher_activity_ids(v) ((voucher_activity_id_t*) \
 211                 (_voucher_bits_recipe(v).content + sizeof(_voucher_magic_t) + \
 212                 sizeof(_voucher_priority_t)))
 213 #define _voucher_bits_size(activities) \
 214                 (sizeof(_voucher_magic_t) + sizeof(_voucher_priority_t) + \
 215                 (activities) * sizeof(voucher_activity_id_t))
 216
 217 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 218 #define _voucher_extra_size(v) ((v)->v_recipe_extra_size)
 219 #define _voucher_extra_recipes(v) ((char*)(v) + (v)->v_recipe_extra_offset)
 220 #else
 221 #define _voucher_extra_size(v) 0
 222 #define _voucher_extra_recipes(v) NULL
 223 #endif
 224
 225 #if DISPATCH_DEBUG && DISPATCH_VOUCHER_DEBUG
 226 #define _dispatch_voucher_debug(msg, v, ...) \
 227                 _dispatch_debug("voucher[%p]: " msg, v, ##__VA_ARGS__)
 228 #define _dispatch_kvoucher_debug(msg, kv, ...) \
 229                 _dispatch_debug("kvoucher[0x%08x]: " msg, kv, ##__VA_ARGS__)
 230 #define _dispatch_voucher_debug_machport(name) \
 231                 dispatch_debug_machport((name), __func__)
 232 #else
 233 #define _dispatch_voucher_debug(msg, v, ...)
 234 #define _dispatch_kvoucher_debug(msg, kv, ...)
 235 #define _dispatch_voucher_debug_machport(name) ((void)(name))
 236 #endif
 237
 238 #if !(USE_OBJC && __OBJC2__) && !defined(__cplusplus)
 239
 240 DISPATCH_ALWAYS_INLINE
 241 static inline voucher_t
 242 _voucher_retain(voucher_t voucher)
 243 {
 244 #if !DISPATCH_VOUCHER_OBJC_DEBUG
 245         // not using _os_object_refcnt* because we don't need barriers:
 246         // vouchers are immutable and are in a hash table with a lock
 247         int xref_cnt = dispatch_atomic_inc2o(voucher, os_obj_xref_cnt, relaxed);
 248         _dispatch_voucher_debug("retain  -> %d", voucher, xref_cnt + 1);
 249         if (slowpath(xref_cnt <= 0)) {
 250                 _dispatch_voucher_debug("resurrection", voucher);
 251                 DISPATCH_CRASH("Voucher resurrection");
 252         }
 253 #else
 254         os_retain(voucher);
 255         _dispatch_voucher_debug("retain  -> %d", voucher,
 256                         voucher->os_obj_xref_cnt + 1);
 257 #endif // DISPATCH_DEBUG
 258         return voucher;
 259 }
 260
 261 DISPATCH_ALWAYS_INLINE
 262 static inline void
 263 _voucher_release(voucher_t voucher)
 264 {
 265 #if !DISPATCH_VOUCHER_OBJC_DEBUG
 266         // not using _os_object_refcnt* because we don't need barriers:
 267         // vouchers are immutable and are in a hash table with a lock
 268         int xref_cnt = dispatch_atomic_dec2o(voucher, os_obj_xref_cnt, relaxed);
 269         _dispatch_voucher_debug("release -> %d", voucher, xref_cnt + 1);
 270         if (fastpath(xref_cnt >= 0)) {
 271                 return;
 272         }
 273         if (slowpath(xref_cnt < -1)) {
 274                 _dispatch_voucher_debug("overrelease", voucher);
 275                 DISPATCH_CRASH("Voucher overrelease");
 276         }
 277         return _os_object_xref_dispose((_os_object_t)voucher);
 278 #else
 279         _dispatch_voucher_debug("release -> %d", voucher, voucher->os_obj_xref_cnt);
 280         return os_release(voucher);
 281 #endif // DISPATCH_DEBUG
 282 }
 283
 284 DISPATCH_ALWAYS_INLINE
 285 static inline voucher_t
 286 _voucher_get(void)
 287 {
 288         return _dispatch_thread_getspecific(dispatch_voucher_key);
 289 }
 290
 291 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 292 static inline voucher_t
 293 _voucher_copy(void)
 294 {
 295         voucher_t voucher = _voucher_get();
 296         if (voucher) _voucher_retain(voucher);
 297         return voucher;
 298 }
 299
 300 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 301 static inline voucher_t
 302 _voucher_copy_without_importance(void)
 303 {
 304         voucher_t voucher = _voucher_get();
 305         if (voucher) voucher = _voucher_create_without_importance(voucher);
 306         return voucher;
 307 }
 308
 309 DISPATCH_ALWAYS_INLINE
 310 static inline void
 311 _voucher_mach_voucher_set(mach_voucher_t kv)
 312 {
 313         if (kv == VOUCHER_NO_MACH_VOUCHER) return;
 314         _dispatch_set_priority_and_mach_voucher(0, kv);
 315 }
 316
 317 DISPATCH_ALWAYS_INLINE
 318 static inline mach_voucher_t
 319 _voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher)
 320 {
 321         if (ov == voucher) return VOUCHER_NO_MACH_VOUCHER;
 322         _dispatch_voucher_debug("swap from voucher[%p]", voucher, ov);
 323         _dispatch_thread_setspecific(dispatch_voucher_key, voucher);
 324         mach_voucher_t kv = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL;
 325         mach_voucher_t okv = ov ? ov->v_kvoucher : MACH_VOUCHER_NULL;
 326         return (kv != okv) ? kv : VOUCHER_NO_MACH_VOUCHER;
 327 }
 328
 329 DISPATCH_ALWAYS_INLINE
 330 static inline void
 331 _voucher_swap(voucher_t ov, voucher_t voucher)
 332 {
 333         _voucher_mach_voucher_set(_voucher_swap_and_get_mach_voucher(ov, voucher));
 334         if (ov) _voucher_release(ov);
 335 }
 336
 337 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 338 static inline voucher_t
 339 _voucher_adopt(voucher_t voucher)
 340 {
 341         voucher_t ov = _voucher_get();
 342         _voucher_mach_voucher_set(_voucher_swap_and_get_mach_voucher(ov, voucher));
 343         return ov;
 344 }
 345
 346 DISPATCH_ALWAYS_INLINE
 347 static inline void
 348 _voucher_replace(voucher_t voucher)
 349 {
 350         voucher_t ov = _voucher_get();
 351         _voucher_swap(ov, voucher);
 352 }
 353
 354 DISPATCH_ALWAYS_INLINE
 355 static inline void
 356 _voucher_clear(void)
 357 {
 358         _voucher_replace(NULL);
 359 }
 360
 361 DISPATCH_ALWAYS_INLINE
 362 static inline pthread_priority_t
 363 _voucher_get_priority(voucher_t voucher)
 364 {
 365         return voucher && voucher->v_has_priority ?
 366                         (pthread_priority_t)*_voucher_priority(voucher) : 0;
 367 }
 368
 369 void _voucher_task_mach_voucher_init(void* ctxt);
 370 extern dispatch_once_t _voucher_task_mach_voucher_pred;
 371 extern mach_voucher_t _voucher_task_mach_voucher;
 372
 373 DISPATCH_ALWAYS_INLINE
 374 static inline mach_voucher_t
 375 _voucher_get_task_mach_voucher(void)
 376 {
 377         dispatch_once_f(&_voucher_task_mach_voucher_pred, NULL,
 378                         _voucher_task_mach_voucher_init);
 379         return _voucher_task_mach_voucher;
 380 }
 381
 382 DISPATCH_ALWAYS_INLINE
 383 static inline bool
 384 _voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv,
 385                 bool move_send)
 386 {
 387         if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false;
 388         if (!kv) return false;
 389         msg->msgh_voucher_port = kv;
 390         msg->msgh_bits |= MACH_MSGH_BITS_SET_PORTS(0, 0, move_send ?
 391                         MACH_MSG_TYPE_MOVE_SEND : MACH_MSG_TYPE_COPY_SEND);
 392         _dispatch_kvoucher_debug("msg[%p] set %s", kv, msg, move_send ?
 393                         "move-send" : "copy-send");
 394         _dispatch_voucher_debug_machport(kv);
 395         return true;
 396 }
 397
 398 DISPATCH_ALWAYS_INLINE
 399 static inline bool
 400 _voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher)
 401 {
 402         if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false;
 403         mach_voucher_t kv;
 404         if (voucher) {
 405                 kv = _voucher_get_mach_voucher(voucher);
 406         } else {
 407                 kv = _voucher_get_task_mach_voucher();
 408         }
 409         return _voucher_mach_msg_set_mach_voucher(msg, kv, false);
 410 }
 411
 412 DISPATCH_ALWAYS_INLINE
 413 static inline mach_voucher_t
 414 _voucher_mach_msg_get(mach_msg_header_t *msg)
 415 {
 416         if (!MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return MACH_VOUCHER_NULL;
 417         mach_voucher_t kv = msg->msgh_voucher_port;
 418         msg->msgh_voucher_port = MACH_VOUCHER_NULL;
 419         msg->msgh_bits &= (mach_msg_bits_t)~MACH_MSGH_BITS_VOUCHER_MASK;
 420         return kv;
 421 }
 422
 423 DISPATCH_ALWAYS_INLINE
 424 static inline mach_voucher_t
 425 _voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send)
 426 {
 427         mach_msg_bits_t kvbits = MACH_MSGH_BITS_VOUCHER(msg->msgh_bits);
 428         mach_voucher_t kv = msg->msgh_voucher_port, kvm = MACH_VOUCHER_NULL;
 429         if ((kvbits == MACH_MSG_TYPE_COPY_SEND ||
 430                         kvbits == MACH_MSG_TYPE_MOVE_SEND) && kv) {
 431                 _dispatch_kvoucher_debug("msg[%p] clear %s", kv, msg, move_send ?
 432                                 "move-send" : "copy-send");
 433                 _dispatch_voucher_debug_machport(kv);
 434                 if (kvbits == MACH_MSG_TYPE_MOVE_SEND) {
 435                         // <rdar://problem/15694142> return/drop received or pseudo-received
 436                         // voucher reference (e.g. due to send failure).
 437                         if (move_send) {
 438                                 kvm = kv;
 439                         } else {
 440                                 _voucher_dealloc_mach_voucher(kv);
 441                         }
 442                 }
 443                 msg->msgh_voucher_port = MACH_VOUCHER_NULL;
 444                 msg->msgh_bits &= (mach_msg_bits_t)~MACH_MSGH_BITS_VOUCHER_MASK;
 445         }
 446         return kvm;
 447 }
 448
 449 #pragma mark -
 450 #pragma mark dispatch_continuation_t + voucher_t
 451
 452 #if DISPATCH_USE_KDEBUG_TRACE
 453 DISPATCH_ALWAYS_INLINE
 454 static inline void
 455 _dispatch_voucher_ktrace(int code, natural_t voucher, void *container)
 456 {
 457         if (!voucher) return;
 458         __kdebug_trace(APPSDBG_CODE(DBG_MACH_CHUD, (0xfac >> 2)) | DBG_FUNC_NONE,
 459                         code, (int)voucher, (int)(uintptr_t)container,
 460 #ifdef __LP64__
 461                         (int)((uintptr_t)container >> 32)
 462 #else
 463                         0
 464 #endif
 465                         );
 466 }
 467 #define _dispatch_voucher_ktrace_dc_push(dc) \
 468                 _dispatch_voucher_ktrace(0x1, (dc)->dc_voucher ? \
 469                                 (dc)->dc_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dc))
 470 #define _dispatch_voucher_ktrace_dc_pop(dc) \
 471                 _dispatch_voucher_ktrace(0x2, (dc)->dc_voucher ? \
 472                                 (dc)->dc_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dc))
 473 #define _dispatch_voucher_ktrace_dmsg_push(dmsg) \
 474                 _dispatch_voucher_ktrace(0x3, (dmsg)->dmsg_voucher ? \
 475                                 (dmsg)->dmsg_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dmsg))
 476 #define _dispatch_voucher_ktrace_dmsg_pop(dmsg) \
 477                 _dispatch_voucher_ktrace(0x4, (dmsg)->dmsg_voucher ? \
 478                                 (dmsg)->dmsg_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dmsg))
 479 #else
 480 #define _dispatch_voucher_ktrace_dc_push(dc)
 481 #define _dispatch_voucher_ktrace_dc_pop(dc)
 482 #define _dispatch_voucher_ktrace_dmsg_push(dmsg)
 483 #define _dispatch_voucher_ktrace_dmsg_pop(dmsg)
 484 #endif // DISPATCH_USE_KDEBUG_TRACE
 485
 486 DISPATCH_ALWAYS_INLINE
 487 static inline void
 488 _dispatch_continuation_voucher_set(dispatch_continuation_t dc,
 489                 dispatch_block_flags_t flags)
 490 {
 491         unsigned long bits = (unsigned long)dc->do_vtable;
 492         voucher_t v = NULL;
 493
 494         if (flags & DISPATCH_BLOCK_HAS_VOUCHER) {
 495                 bits |= DISPATCH_OBJ_HAS_VOUCHER_BIT;
 496         } else if (!(flags & DISPATCH_BLOCK_NO_VOUCHER)) {
 497                 v = _voucher_copy();
 498         }
 499         dc->do_vtable = (void*)bits;
 500         dc->dc_voucher = v;
 501         _dispatch_voucher_debug("continuation[%p] set", dc->dc_voucher, dc);
 502         _dispatch_voucher_ktrace_dc_push(dc);
 503 }
 504
 505 DISPATCH_ALWAYS_INLINE
 506 static inline void
 507 _dispatch_continuation_voucher_adopt(dispatch_continuation_t dc)
 508 {
 509         unsigned long bits = (unsigned long)dc->do_vtable;
 510         voucher_t v = DISPATCH_NO_VOUCHER;
 511         if (!(bits & DISPATCH_OBJ_HAS_VOUCHER_BIT)) {
 512                 _dispatch_voucher_ktrace_dc_pop(dc);
 513                 _dispatch_voucher_debug("continuation[%p] adopt", dc->dc_voucher, dc);
 514                 v = dc->dc_voucher;
 515                 dc->dc_voucher = NULL;
 516         }
 517         _dispatch_adopt_priority_and_replace_voucher(dc->dc_priority, v, 0);
 518 }
 519
 520 #pragma mark -
 521 #pragma mark _voucher_activity_heap
 522
 523 typedef uint32_t _voucher_atm_subid_t;
 524 static const size_t _voucher_activity_hash_bits = 6;
 525 static const size_t _voucher_activity_hash_size =
 526                 1 << _voucher_activity_hash_bits;
 527 #define VACTID_HASH(x) \
 528                 (((uint32_t)(x) * 2654435761u) >> (32-_voucher_activity_hash_bits))
 529 #define VATMID_HASH(x) \
 530                 (((uint32_t)(x) * 2654435761u) >> (32-_voucher_activity_hash_bits))
 531 #define VATMID2ACTID(x, flags) \
 532                 (((voucher_activity_id_t)(x) & 0xffffffffffffff) | \
 533                 (((voucher_activity_id_t)(flags) & 0xfe) << 55))
 534
 535 typedef struct _voucher_activity_metadata_s {
 536         _voucher_activity_buffer_t vam_client_metadata;
 537         struct _voucher_activity_metadata_opaque_s *vasm_baseaddr;
 538         _voucher_activity_bitmap_t volatile vam_buffer_bitmap;
 539         _voucher_activity_bitmap_t volatile vam_pressure_locked_bitmap;
 540         _voucher_activity_lock_s vam_atms_lock;
 541         _voucher_activity_lock_s vam_activities_lock;
 542         TAILQ_HEAD(, _voucher_atm_s) vam_atms[_voucher_activity_hash_size];
 543         TAILQ_HEAD(, _voucher_activity_s)
 544                         vam_activities[_voucher_activity_hash_size];
 545 } *_voucher_activity_metadata_t;
 546
 547 #pragma mark -
 548 #pragma mark _voucher_atm_t
 549
 550 typedef struct _voucher_atm_s {
 551         int32_t volatile vatm_refcnt;
 552         mach_voucher_t vatm_kvoucher;
 553         atm_aid_t vatm_id;
 554         atm_guard_t vatm_generation;
 555         TAILQ_ENTRY(_voucher_atm_s) vatm_list;
 556 #if __LP64__
 557         uintptr_t vatm_pad[3];
 558         // cacheline
 559 #endif
 560 } *_voucher_atm_t;
 561
 562 extern _voucher_atm_t _voucher_task_atm;
 563
 564 #pragma mark -
 565 #pragma mark _voucher_activity_t
 566
 567 typedef struct _voucher_activity_s {
 568         voucher_activity_id_t va_id;
 569         voucher_activity_trace_id_t va_trace_id;
 570         uint64_t va_location;
 571         int32_t volatile va_refcnt;
 572         uint32_t volatile va_buffer_count;
 573         uint32_t va_buffer_limit;
 574         _voucher_activity_buffer_header_t volatile va_current_buffer;
 575         _voucher_atm_t va_atm;
 576 #if __LP64__
 577         uint64_t va_unused;
 578 #endif
 579         // cacheline
 580         _voucher_activity_lock_s va_buffers_lock;
 581         TAILQ_HEAD(_voucher_activity_buffer_list_s,
 582                         _voucher_activity_buffer_header_s) va_buffers;
 583         TAILQ_ENTRY(_voucher_activity_s) va_list;
 584         TAILQ_ENTRY(_voucher_activity_s) va_atm_list;
 585         TAILQ_ENTRY(_voucher_activity_s) va_atm_used_list;
 586         pthread_mutex_t va_mutex;
 587         pthread_cond_t va_cond;
 588 } *_voucher_activity_t;
 589
 590 _voucher_activity_tracepoint_t _voucher_activity_buffer_tracepoint_acquire_slow(
 591                 _voucher_activity_t *vap, _voucher_activity_buffer_header_t *vabp,
 592                 unsigned int slots, size_t strsize, uint16_t *stroffsetp);
 593 void _voucher_activity_firehose_push(_voucher_activity_t act,
 594                 _voucher_activity_buffer_header_t buffer);
 595 extern _voucher_activity_t _voucher_activity_default;
 596 extern voucher_activity_mode_t _voucher_activity_mode;
 597
 598 #if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG
 599 #define _dispatch_voucher_activity_debug(msg, act, ...) \
 600                 _dispatch_debug("activity[%p] <0x%llx>: atm[%p] <%lld>: " msg, (act), \
 601                 (act) ? (act)->va_id : 0, (act) ? (act)->va_atm : NULL, \
 602                 (act) && (act)->va_atm ? (act)->va_atm->vatm_id : 0, ##__VA_ARGS__)
 603 #define _dispatch_voucher_atm_debug(msg, atm, ...) \
 604                 _dispatch_debug("atm[%p] <%lld> kvoucher[0x%08x]: " msg, (atm), \
 605                 (atm) ? (atm)->vatm_id : 0, (atm) ? (atm)->vatm_kvoucher : 0, \
 606                 ##__VA_ARGS__)
 607 #else
 608 #define _dispatch_voucher_activity_debug(msg, act, ...)
 609 #define _dispatch_voucher_atm_debug(msg, atm, ...)
 610 #endif
 611
 612 DISPATCH_ALWAYS_INLINE
 613 static inline uint64_t
 614 _voucher_activity_timestamp(bool approx)
 615 {
 616 #if TARGET_IPHONE_SIMULATOR && \
 617                 IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101000
 618         (void)approx;
 619         return mach_absolute_time();
 620 #else
 621         return approx ? mach_approximate_time() : mach_absolute_time();
 622 #endif
 623 }
 624
 625 DISPATCH_ALWAYS_INLINE
 626 static inline uint64_t
 627 _voucher_activity_thread_id(void)
 628 {
 629         uint64_t thread_id;
 630         pthread_threadid_np(NULL, &thread_id); // TODO: 15923074: use TSD thread_id
 631         return thread_id;
 632 }
 633
 634 #define _voucher_activity_buffer_pos2length(pos) \
 635                 ({ _voucher_activity_buffer_position_u _pos = (pos); \
 636                 _pos.vabp_pos.vabp_next_tracepoint_idx * \
 637                 sizeof(struct _voucher_activity_tracepoint_s) + \
 638                 _pos.vabp_pos.vabp_string_offset; })
 639
 640 DISPATCH_ALWAYS_INLINE
 641 static inline _voucher_activity_tracepoint_t
 642 _voucher_activity_buffer_tracepoint_acquire(
 643                 _voucher_activity_buffer_header_t vab, unsigned int slots,
 644                 size_t strsize, uint16_t *stroffsetp)
 645 {
 646         if (!vab) return NULL;
 647         _voucher_activity_buffer_position_u pos_orig, pos;
 648         pos_orig.vabp_atomic_pos = vab->vabh_pos.vabp_atomic_pos;
 649         do {
 650                 pos.vabp_atomic_pos = pos_orig.vabp_atomic_pos;
 651                 pos.vabp_pos.vabp_next_tracepoint_idx += slots;
 652                 pos.vabp_pos.vabp_string_offset += strsize;
 653                 size_t len = _voucher_activity_buffer_pos2length(pos);
 654                 if (len > _voucher_activity_buffer_size || pos.vabp_pos.vabp_flags) {
 655                         return NULL;
 656                 }
 657                 if (len == _voucher_activity_buffer_size) {
 658                         pos.vabp_pos.vabp_flags |= _voucher_activity_buffer_full;
 659                 }
 660                 pos.vabp_pos.vabp_refcnt++;
 661         } while (!dispatch_atomic_cmpxchgvw2o(vab, vabh_pos.vabp_atomic_pos,
 662                         pos_orig.vabp_atomic_pos, pos.vabp_atomic_pos,
 663                         &pos_orig.vabp_atomic_pos, relaxed));
 664         if (stroffsetp) *stroffsetp = pos.vabp_pos.vabp_string_offset;
 665         return (_voucher_activity_tracepoint_t)vab +
 666                         pos_orig.vabp_pos.vabp_next_tracepoint_idx;
 667 }
 668
 669 DISPATCH_ALWAYS_INLINE
 670 static inline bool
 671 _voucher_activity_buffer_tracepoint_release(
 672                 _voucher_activity_buffer_header_t vab)
 673 {
 674         _voucher_activity_buffer_position_u pos_orig, pos;
 675         pos_orig.vabp_atomic_pos = vab->vabh_pos.vabp_atomic_pos;
 676         do {
 677                 pos.vabp_atomic_pos = pos_orig.vabp_atomic_pos;
 678                 pos.vabp_pos.vabp_refcnt--;
 679                 if (!pos.vabp_pos.vabp_refcnt &&
 680                                 (pos.vabp_pos.vabp_flags & _voucher_activity_buffer_full)) {
 681                         pos.vabp_pos.vabp_flags |= _voucher_activity_buffer_pushing;
 682                 }
 683         } while (!dispatch_atomic_cmpxchgvw2o(vab, vabh_pos.vabp_atomic_pos,
 684                         pos_orig.vabp_atomic_pos, pos.vabp_atomic_pos,
 685                         &pos_orig.vabp_atomic_pos, relaxed));
 686         return (pos.vabp_pos.vabp_flags & _voucher_activity_buffer_pushing);
 687 }
 688
 689 DISPATCH_ALWAYS_INLINE
 690 static inline bool
 691 _voucher_activity_buffer_mark_full(_voucher_activity_buffer_header_t vab)
 692 {
 693         _voucher_activity_buffer_position_u pos_orig, pos;
 694         pos_orig.vabp_atomic_pos = vab->vabh_pos.vabp_atomic_pos;
 695         do {
 696                 pos.vabp_atomic_pos = pos_orig.vabp_atomic_pos;
 697                 if (pos.vabp_pos.vabp_flags & _voucher_activity_buffer_full) {
 698                         return false;
 699                 }
 700                 pos.vabp_pos.vabp_flags |= _voucher_activity_buffer_full;
 701                 if (!pos.vabp_pos.vabp_refcnt) {
 702                         pos.vabp_pos.vabp_flags |= _voucher_activity_buffer_pushing;
 703                 }
 704         } while (!dispatch_atomic_cmpxchgvw2o(vab, vabh_pos.vabp_atomic_pos,
 705                         pos_orig.vabp_atomic_pos, pos.vabp_atomic_pos,
 706                         &pos_orig.vabp_atomic_pos, relaxed));
 707         return (pos.vabp_pos.vabp_flags & _voucher_activity_buffer_pushing);
 708 }
 709
 710 DISPATCH_ALWAYS_INLINE
 711 static inline bool
 712 _voucher_activity_buffer_is_full(_voucher_activity_buffer_header_t vab)
 713 {
 714         _voucher_activity_buffer_position_u pos;
 715         pos.vabp_atomic_pos = vab->vabh_pos.vabp_atomic_pos;
 716         return (pos.vabp_pos.vabp_flags);
 717 }
 718
 719 DISPATCH_ALWAYS_INLINE
 720 static inline _voucher_activity_buffer_header_t
 721 _voucher_activity_buffer_get_from_activity(_voucher_activity_t va)
 722 {
 723         return va ? va->va_current_buffer : NULL;
 724 }
 725
 726 DISPATCH_ALWAYS_INLINE
 727 static inline _voucher_activity_t
 728 _voucher_activity_get(void)
 729 {
 730         _voucher_activity_t va;
 731         voucher_t v = _voucher_get();
 732         va = v && v->v_activity ? v->v_activity : _voucher_activity_default;
 733         return va;
 734 }
 735
 736 DISPATCH_ALWAYS_INLINE
 737 static inline uint64_t
 738 _voucher_activity_tracepoint_init(_voucher_activity_tracepoint_t vat,
 739                 uint8_t type, uint8_t code_namespace, uint32_t code, uint64_t location,
 740                 bool approx)
 741 {
 742         if (!location) location = (uint64_t)__builtin_return_address(0);
 743         uint64_t timestamp = _voucher_activity_timestamp(approx);
 744         vat->vat_flags = _voucher_activity_trace_flag_tracepoint,
 745         vat->vat_type = type,
 746         vat->vat_namespace = code_namespace,
 747         vat->vat_code = code,
 748         vat->vat_timestamp = timestamp,
 749         vat->vat_thread = _voucher_activity_thread_id(),
 750         vat->vat_location = location;
 751         return timestamp;
 752 }
 753
 754 DISPATCH_ALWAYS_INLINE
 755 static inline uint64_t
 756 _voucher_activity_tracepoint_init_with_id(_voucher_activity_tracepoint_t vat,
 757                 voucher_activity_trace_id_t trace_id, uint64_t location, bool approx)
 758 {
 759         uint8_t type = (uint8_t)(trace_id >> _voucher_activity_trace_id_type_shift);
 760         uint8_t cns = (uint8_t)(trace_id >>
 761                         _voucher_activity_trace_id_code_namespace_shift);
 762         uint32_t code = (uint32_t)trace_id;
 763         return _voucher_activity_tracepoint_init(vat, type, cns, code, location,
 764                         approx);
 765 }
 766
 767 DISPATCH_ALWAYS_INLINE
 768 static inline bool
 769 _voucher_activity_trace_id_is_subtype(voucher_activity_trace_id_t trace_id,
 770                 uint8_t type)
 771 {
 772         voucher_activity_trace_id_t type_id = voucher_activity_trace_id(type, 0, 0);
 773         return (trace_id & type_id) == type_id;
 774 }
 775 #define _voucher_activity_trace_id_is_subtype(trace_id, name) \
 776         _voucher_activity_trace_id_is_subtype(trace_id, \
 777                         voucher_activity_tracepoint_type_ ## name)
 778
 779 DISPATCH_ALWAYS_INLINE
 780 static inline bool
 781 _voucher_activity_trace_id_enabled(voucher_activity_trace_id_t trace_id)
 782 {
 783         switch (_voucher_activity_mode) {
 784         case voucher_activity_mode_release:
 785                 return _voucher_activity_trace_id_is_subtype(trace_id, release);
 786         case voucher_activity_mode_stream:
 787         case voucher_activity_mode_debug:
 788                 return _voucher_activity_trace_id_is_subtype(trace_id, debug) ||
 789                                 _voucher_activity_trace_id_is_subtype(trace_id, release);
 790         }
 791         return false;
 792 }
 793
 794 DISPATCH_ALWAYS_INLINE
 795 static inline bool
 796 _voucher_activity_trace_type_enabled(uint8_t type)
 797 {
 798         voucher_activity_trace_id_t type_id = voucher_activity_trace_id(type, 0, 0);
 799         return _voucher_activity_trace_id_enabled(type_id);
 800 }
 801
 802 DISPATCH_ALWAYS_INLINE
 803 static inline bool
 804 _voucher_activity_disabled(void)
 805 {
 806         return slowpath(_voucher_activity_mode == voucher_activity_mode_disable);
 807 }
 808
 809 DISPATCH_ALWAYS_INLINE
 810 static inline void
 811 _voucher_activity_trace_args_inline(uint8_t type, uint8_t code_namespace,
 812                 uint32_t code, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3,
 813                 uintptr_t arg4)
 814 {
 815         if (!_voucher_activity_trace_type_enabled(type)) return;
 816         _voucher_activity_t act;
 817         _voucher_activity_buffer_header_t vab;
 818         _voucher_activity_tracepoint_t vat;
 819         act = _voucher_activity_get();
 820         vab = _voucher_activity_buffer_get_from_activity(act);
 821         vat = _voucher_activity_buffer_tracepoint_acquire(vab, 1, 0, NULL);
 822         if (!vat) return;
 823         _voucher_activity_tracepoint_init(vat, type, code_namespace, code, 0, true);
 824         vat->vat_flags |= _voucher_activity_trace_flag_tracepoint_args;
 825         vat->vat_data[0] = arg1;
 826         vat->vat_data[1] = arg2;
 827         vat->vat_data[2] = arg3;
 828         vat->vat_data[3] = arg4;
 829         if (_voucher_activity_buffer_tracepoint_release(vab)) {
 830                 _voucher_activity_firehose_push(act, vab);
 831         }
 832 }
 833
 834 DISPATCH_ALWAYS_INLINE
 835 static inline void
 836 _voucher_activity_trace_activity_event(voucher_activity_trace_id_t trace_id,
 837                 voucher_activity_id_t va_id, _voucher_activity_tracepoint_flag_t flags)
 838 {
 839         _voucher_activity_t act;
 840         _voucher_activity_buffer_header_t vab;
 841         _voucher_activity_tracepoint_t vat;
 842         act = _voucher_activity_get();
 843         vab = _voucher_activity_buffer_get_from_activity(act);
 844         vat = _voucher_activity_buffer_tracepoint_acquire(vab, 1, 0, NULL);
 845         if (!vat) return;
 846         _voucher_activity_tracepoint_init_with_id(vat, trace_id, 0, false);
 847         vat->vat_flags |= _voucher_activity_trace_flag_activity | flags;
 848         vat->vat_data[0] = va_id;
 849         if (_voucher_activity_buffer_tracepoint_release(vab)) {
 850                 _voucher_activity_firehose_push(act, vab);
 851         }
 852 }
 853 #define _voucher_activity_trace_activity_event(trace_id, va_id, type) \
 854                 _voucher_activity_trace_activity_event(trace_id, va_id, \
 855                                 _voucher_activity_trace_flag_ ## type)
 856
 857 DISPATCH_ALWAYS_INLINE
 858 static inline void
 859 _voucher_activity_trace_msg(voucher_t v, mach_msg_header_t *msg, uint32_t code)
 860 {
 861         if (!v || !v->v_activity) return; // Don't use default activity for IPC
 862         const uint8_t type = voucher_activity_tracepoint_type_debug;
 863         const uint8_t code_namespace = _voucher_activity_tracepoint_namespace_ipc;
 864         if (!_voucher_activity_trace_type_enabled(type)) return;
 865         _voucher_activity_buffer_header_t vab;
 866         _voucher_activity_tracepoint_t vat;
 867         vab = _voucher_activity_buffer_get_from_activity(v->v_activity);
 868         vat = _voucher_activity_buffer_tracepoint_acquire(vab, 1, 0, NULL);
 869         if (!vat) return; // TODO: slowpath ?
 870         _voucher_activity_tracepoint_init(vat, type, code_namespace, code, 0, true);
 871         vat->vat_flags |= _voucher_activity_trace_flag_libdispatch;
 872 #if __has_extension(c_static_assert)
 873         _Static_assert(sizeof(mach_msg_header_t) <= sizeof(vat->vat_data),
 874                         "mach_msg_header_t too large");
 875 #endif
 876         memcpy(vat->vat_data, msg, sizeof(mach_msg_header_t));
 877         if (_voucher_activity_buffer_tracepoint_release(vab)) {
 878                 _voucher_activity_firehose_push(v->v_activity, vab);
 879         }
 880 }
 881 #define _voucher_activity_trace_msg(v, msg, type) \
 882                 _voucher_activity_trace_msg(v, msg, \
 883                                 _voucher_activity_tracepoint_namespace_ipc_ ## type)
 884
 885 #endif // !(USE_OBJC && __OBJC2__) && !defined(__cplusplus)
 886
 887 #else // VOUCHER_USE_MACH_VOUCHER
 888
 889 #pragma mark -
 890 #pragma mark Simulator / vouchers disabled
 891
 892 #define _dispatch_voucher_debug(msg, v, ...)
 893 #define _dispatch_kvoucher_debug(msg, kv, ...)
 894
 895 DISPATCH_ALWAYS_INLINE
 896 static inline voucher_t
 897 _voucher_retain(voucher_t voucher)
 898 {
 899         return voucher;
 900 }
 901
 902 DISPATCH_ALWAYS_INLINE
 903 static inline void
 904 _voucher_release(voucher_t voucher)
 905 {
 906         (void)voucher;
 907 }
 908
 909 DISPATCH_ALWAYS_INLINE
 910 static inline voucher_t
 911 _voucher_get(void)
 912 {
 913         return NULL;
 914 }
 915
 916 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 917 static inline voucher_t
 918 _voucher_copy(void)
 919 {
 920         return NULL;
 921 }
 922
 923 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 924 static inline voucher_t
 925 _voucher_copy_without_importance(void)
 926 {
 927         return NULL;
 928 }
 929
 930 DISPATCH_ALWAYS_INLINE
 931 static inline mach_voucher_t
 932 _voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher)
 933 {
 934         (void)ov; (void)voucher;
 935         return MACH_VOUCHER_NULL;
 936 }
 937
 938 DISPATCH_ALWAYS_INLINE
 939 static inline voucher_t
 940 _voucher_adopt(voucher_t voucher)
 941 {
 942         return voucher;
 943 }
 944
 945 DISPATCH_ALWAYS_INLINE
 946 static inline void
 947 _voucher_replace(voucher_t voucher)
 948 {
 949         (void)voucher;
 950 }
 951
 952 DISPATCH_ALWAYS_INLINE
 953 static inline void
 954 _voucher_clear(void)
 955 {
 956 }
 957
 958 DISPATCH_ALWAYS_INLINE
 959 static inline pthread_priority_t
 960 _voucher_get_priority(voucher_t voucher)
 961 {
 962         (void)voucher;
 963         return 0;
 964 }
 965
 966 DISPATCH_ALWAYS_INLINE
 967 static inline bool
 968 _voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv,
 969                 bool move_send)
 970 {
 971         (void)msg; (void)kv; (void)move_send;
 972         return false;
 973
 974 }
 975
 976 DISPATCH_ALWAYS_INLINE
 977 static inline bool
 978 _voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher)
 979 {
 980         (void)msg; (void)voucher;
 981         return false;
 982 }
 983
 984 DISPATCH_ALWAYS_INLINE
 985 static inline mach_voucher_t
 986 _voucher_mach_msg_get(mach_msg_header_t *msg)
 987 {
 988         (void)msg;
 989         return 0;
 990 }
 991
 992 DISPATCH_ALWAYS_INLINE
 993 static inline mach_voucher_t
 994 _voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send)
 995 {
 996         (void)msg; (void)move_send;
 997         return MACH_VOUCHER_NULL;
 998 }
 999
1000 #define _dispatch_voucher_ktrace_dmsg_push(dmsg)
1001 #define _dispatch_voucher_ktrace_dmsg_pop(dmsg)
1002
1003 DISPATCH_ALWAYS_INLINE
1004 static inline void
1005 _dispatch_continuation_voucher_set(dispatch_continuation_t dc,
1006                 dispatch_block_flags_t flags)
1007 {
1008         (void)dc; (void)flags;
1009 }
1010
1011 DISPATCH_ALWAYS_INLINE
1012 static inline void
1013 _dispatch_continuation_voucher_adopt(dispatch_continuation_t dc)
1014 {
1015         (void)dc;
1016 }
1017
1018 #define _voucher_activity_trace_msg(v, msg, type)
1019
1020 DISPATCH_ALWAYS_INLINE
1021 static inline bool
1022 _voucher_activity_disabled(void)
1023 {
1024         return true;
1025 }
1026
1027 #endif // VOUCHER_USE_MACH_VOUCHER
1028
1029 #endif /* __DISPATCH_VOUCHER_INTERNAL__ */