src/voucher_internal.h

   1 /*
   2  * Copyright (c) 2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_VOUCHER_INTERNAL__
  28 #define __DISPATCH_VOUCHER_INTERNAL__
  29
  30 #ifndef __DISPATCH_INDIRECT__
  31 #error "Please #include <dispatch/dispatch.h> instead of this file directly."
  32 #include <dispatch/base.h> // for HeaderDoc
  33 #endif
  34
  35 #pragma mark -
  36 #pragma mark voucher_recipe_t (disabled)
  37
  38 #if VOUCHER_ENABLE_RECIPE_OBJECTS
  39 /*!
  40  * @group Voucher Creation SPI
  41  * SPI intended for clients that need to create vouchers.
  42  */
  43
  44 #if OS_OBJECT_USE_OBJC
  45 OS_OBJECT_DECL(voucher_recipe);
  46 #else
  47 typedef struct voucher_recipe_s *voucher_recipe_t;
  48 #endif
  49
  50 /*!
  51  * @function voucher_create
  52  *
  53  * @abstract
  54  * Creates a new voucher object from a recipe.
  55  *
  56  * @discussion
  57  * Error handling TBD
  58  *
  59  * @result
  60  * The newly created voucher object.
  61  */
  62 __OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0)
  63 OS_EXPORT OS_OBJECT_RETURNS_RETAINED OS_WARN_RESULT OS_NOTHROW
  64 voucher_t
  65 voucher_create(voucher_recipe_t recipe);
  66 #endif // VOUCHER_ENABLE_RECIPE_OBJECTS
  67
  68 #if VOUCHER_ENABLE_GET_MACH_VOUCHER
  69 /*!
  70  * @function voucher_get_mach_voucher
  71  *
  72  * @abstract
  73  * Returns the mach voucher port underlying the specified voucher object.
  74  *
  75  * @discussion
  76  * The caller must either maintain a reference on the voucher object while the
  77  * returned mach voucher port is in use to ensure it stays valid for the
  78  * duration, or it must retain the mach voucher port with mach_port_mod_refs().
  79  *
  80  * @param voucher
  81  * The voucher object to query.
  82  *
  83  * @result
  84  * A mach voucher port.
  85  */
  86 __OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0)
  87 OS_VOUCHER_EXPORT OS_WARN_RESULT OS_NOTHROW
  88 mach_voucher_t
  89 voucher_get_mach_voucher(voucher_t voucher);
  90 #endif // VOUCHER_ENABLE_GET_MACH_VOUCHER
  91
  92 #pragma mark -
  93 #pragma mark voucher_t
  94
  95 #if TARGET_IPHONE_SIMULATOR && \
  96                 IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101000
  97 #undef VOUCHER_USE_MACH_VOUCHER
  98 #define VOUCHER_USE_MACH_VOUCHER 0
  99 #endif
 100 #ifndef VOUCHER_USE_MACH_VOUCHER
 101 #if __has_include(<mach/mach_voucher.h>)
 102 #define VOUCHER_USE_MACH_VOUCHER 1
 103 #endif
 104 #endif
 105
 106 #if VOUCHER_USE_MACH_VOUCHER
 107 #undef DISPATCH_USE_IMPORTANCE_ASSERTION
 108 #define DISPATCH_USE_IMPORTANCE_ASSERTION 0
 109 #else
 110 #undef MACH_RCV_VOUCHER
 111 #define MACH_RCV_VOUCHER 0
 112 #endif // VOUCHER_USE_MACH_VOUCHER
 113
 114 void _voucher_init(void);
 115 void _voucher_atfork_child(void);
 116 void _voucher_activity_heap_pressure_warn(void);
 117 void _voucher_activity_heap_pressure_normal(void);
 118 void _voucher_xref_dispose(voucher_t voucher);
 119 void _voucher_dispose(voucher_t voucher);
 120 size_t _voucher_debug(voucher_t v, char* buf, size_t bufsiz);
 121 void _voucher_thread_cleanup(void *voucher);
 122 mach_voucher_t _voucher_get_mach_voucher(voucher_t voucher);
 123 voucher_t _voucher_create_without_importance(voucher_t voucher);
 124 mach_voucher_t _voucher_create_mach_voucher_with_priority(voucher_t voucher,
 125                 pthread_priority_t priority);
 126 voucher_t _voucher_create_with_priority_and_mach_voucher(voucher_t voucher,
 127                 pthread_priority_t priority, mach_voucher_t kv);
 128 void _voucher_dealloc_mach_voucher(mach_voucher_t kv);
 129
 130 #if OS_OBJECT_USE_OBJC
 131 _OS_OBJECT_DECL_SUBCLASS_INTERFACE(voucher, object)
 132 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 133 _OS_OBJECT_DECL_SUBCLASS_INTERFACE(voucher_recipe, object)
 134 #endif
 135 #endif
 136
 137 #define _TAILQ_IS_ENQUEUED(elm, field) \
 138                 ((elm)->field.tqe_prev != NULL)
 139 #define _TAILQ_MARK_NOT_ENQUEUED(elm, field) \
 140                 do { (elm)->field.tqe_prev = NULL; } while (0)
 141
 142 #define VOUCHER_NO_MACH_VOUCHER MACH_PORT_DEAD
 143
 144 #if VOUCHER_USE_MACH_VOUCHER
 145
 146 #if DISPATCH_DEBUG
 147 #define DISPATCH_VOUCHER_DEBUG 1
 148 #define DISPATCH_VOUCHER_ACTIVITY_DEBUG 1
 149 #endif
 150
 151 typedef struct voucher_s {
 152         _OS_OBJECT_HEADER(
 153         void *os_obj_isa,
 154         os_obj_ref_cnt,
 155         os_obj_xref_cnt);
 156         TAILQ_ENTRY(voucher_s) v_list;
 157         mach_voucher_t v_kvoucher, v_ipc_kvoucher; // if equal, only one reference
 158         voucher_t v_kvbase; // if non-NULL, v_kvoucher is a borrowed reference
 159         _voucher_activity_t v_activity;
 160 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 161         size_t v_recipe_extra_offset;
 162         mach_voucher_attr_recipe_size_t v_recipe_extra_size;
 163 #endif
 164         unsigned int v_has_priority:1;
 165         unsigned int v_activities;
 166         mach_voucher_attr_recipe_data_t v_recipes[];
 167 } voucher_s;
 168
 169 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 170 typedef struct voucher_recipe_s {
 171         _OS_OBJECT_HEADER(
 172         const _os_object_class_s *os_obj_isa,
 173         os_obj_ref_cnt,
 174         os_obj_xref_cnt);
 175         size_t vr_allocation_size;
 176         mach_voucher_attr_recipe_size_t volatile vr_size;
 177         mach_voucher_attr_recipe_t vr_data;
 178 } voucher_recipe_s;
 179 #endif
 180
 181 #define _voucher_recipes_base(r) (r[0])
 182 #define _voucher_recipes_atm(r) (r[1])
 183 #define _voucher_recipes_bits(r) (r[2])
 184 #define _voucher_base_recipe(v) (_voucher_recipes_base((v)->v_recipes))
 185 #define _voucher_atm_recipe(v) (_voucher_recipes_atm((v)->v_recipes))
 186 #define _voucher_bits_recipe(v) (_voucher_recipes_bits((v)->v_recipes))
 187 #define _voucher_recipes_size() (3 * sizeof(mach_voucher_attr_recipe_data_t))
 188
 189 #if TARGET_OS_EMBEDDED
 190 #define VL_HASH_SIZE  64u // must be a power of two
 191 #else
 192 #define VL_HASH_SIZE 256u // must be a power of two
 193 #endif
 194 #define VL_HASH(kv) (MACH_PORT_INDEX(kv) & (VL_HASH_SIZE - 1))
 195
 196 typedef uint32_t _voucher_magic_t;
 197 const _voucher_magic_t _voucher_magic_v1 = 0x0190cefa; // little-endian FACE9001
 198 #define _voucher_recipes_magic(r) ((_voucher_magic_t*) \
 199                 (_voucher_recipes_bits(r).content))
 200 #define _voucher_magic(v) _voucher_recipes_magic((v)->v_recipes)
 201 typedef uint32_t _voucher_priority_t;
 202 #define _voucher_recipes_priority(r) ((_voucher_priority_t*) \
 203                 (_voucher_recipes_bits(r).content + sizeof(_voucher_magic_t)))
 204 #define _voucher_priority(v) _voucher_recipes_priority((v)->v_recipes)
 205 #define _voucher_activity_ids(v) ((voucher_activity_id_t*) \
 206                 (_voucher_bits_recipe(v).content + sizeof(_voucher_magic_t) + \
 207                 sizeof(_voucher_priority_t)))
 208 #define _voucher_bits_size(activities) \
 209                 (sizeof(_voucher_magic_t) + sizeof(_voucher_priority_t) + \
 210                 (activities) * sizeof(voucher_activity_id_t))
 211
 212 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 213 #define _voucher_extra_size(v) ((v)->v_recipe_extra_size)
 214 #define _voucher_extra_recipes(v) ((char*)(v) + (v)->v_recipe_extra_offset)
 215 #else
 216 #define _voucher_extra_size(v) 0
 217 #define _voucher_extra_recipes(v) NULL
 218 #endif
 219
 220 #if DISPATCH_DEBUG && DISPATCH_VOUCHER_DEBUG
 221 #define _dispatch_voucher_debug(msg, v, ...) \
 222                 _dispatch_debug("voucher[%p]: " msg, v, ##__VA_ARGS__)
 223 #define _dispatch_kvoucher_debug(msg, kv, ...) \
 224                 _dispatch_debug("kvoucher[0x%08x]: " msg, kv, ##__VA_ARGS__)
 225 #define _dispatch_voucher_debug_machport(name) \
 226                 dispatch_debug_machport((name), __func__)
 227 #else
 228 #define _dispatch_voucher_debug(msg, v, ...)
 229 #define _dispatch_kvoucher_debug(msg, kv, ...)
 230 #define _dispatch_voucher_debug_machport(name) ((void)(name))
 231 #endif
 232
 233 #if !(USE_OBJC && __OBJC2__)
 234
 235 DISPATCH_ALWAYS_INLINE
 236 static inline voucher_t
 237 _voucher_retain(voucher_t voucher)
 238 {
 239 #if !DISPATCH_VOUCHER_OBJC_DEBUG
 240         int xref_cnt = dispatch_atomic_inc2o(voucher, os_obj_xref_cnt, relaxed);
 241         _dispatch_voucher_debug("retain  -> %d", voucher, xref_cnt + 1);
 242         if (slowpath(xref_cnt <= 0)) {
 243                 _dispatch_voucher_debug("resurrection", voucher);
 244                 DISPATCH_CRASH("Voucher resurrection");
 245         }
 246 #else
 247         os_retain(voucher);
 248         _dispatch_voucher_debug("retain  -> %d", voucher,
 249                         voucher->os_obj_xref_cnt + 1);
 250 #endif // DISPATCH_DEBUG
 251         return voucher;
 252 }
 253
 254 DISPATCH_ALWAYS_INLINE
 255 static inline void
 256 _voucher_release(voucher_t voucher)
 257 {
 258 #if !DISPATCH_VOUCHER_OBJC_DEBUG
 259         int xref_cnt = dispatch_atomic_dec2o(voucher, os_obj_xref_cnt, relaxed);
 260         _dispatch_voucher_debug("release -> %d", voucher, xref_cnt + 1);
 261         if (fastpath(xref_cnt >= 0)) {
 262                 return;
 263         }
 264         if (slowpath(xref_cnt < -1)) {
 265                 _dispatch_voucher_debug("overrelease", voucher);
 266                 DISPATCH_CRASH("Voucher overrelease");
 267         }
 268         return _os_object_xref_dispose((_os_object_t)voucher);
 269 #else
 270         _dispatch_voucher_debug("release -> %d", voucher, voucher->os_obj_xref_cnt);
 271         return os_release(voucher);
 272 #endif // DISPATCH_DEBUG
 273 }
 274
 275 DISPATCH_ALWAYS_INLINE
 276 static inline voucher_t
 277 _voucher_get(void)
 278 {
 279         return _dispatch_thread_getspecific(dispatch_voucher_key);
 280 }
 281
 282 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 283 static inline voucher_t
 284 _voucher_copy(void)
 285 {
 286         voucher_t voucher = _voucher_get();
 287         if (voucher) _voucher_retain(voucher);
 288         return voucher;
 289 }
 290
 291 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 292 static inline voucher_t
 293 _voucher_copy_without_importance(void)
 294 {
 295         voucher_t voucher = _voucher_get();
 296         if (voucher) voucher = _voucher_create_without_importance(voucher);
 297         return voucher;
 298 }
 299
 300 DISPATCH_ALWAYS_INLINE
 301 static inline void
 302 _voucher_mach_voucher_set(mach_voucher_t kv)
 303 {
 304         if (kv == VOUCHER_NO_MACH_VOUCHER) return;
 305         _dispatch_set_priority_and_mach_voucher(0, kv);
 306 }
 307
 308 DISPATCH_ALWAYS_INLINE
 309 static inline mach_voucher_t
 310 _voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher)
 311 {
 312         if (ov == voucher) return VOUCHER_NO_MACH_VOUCHER;
 313         _dispatch_voucher_debug("swap from voucher[%p]", voucher, ov);
 314         _dispatch_thread_setspecific(dispatch_voucher_key, voucher);
 315         mach_voucher_t kv = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL;
 316         mach_voucher_t okv = ov ? ov->v_kvoucher : MACH_VOUCHER_NULL;
 317         return (kv != okv) ? kv : VOUCHER_NO_MACH_VOUCHER;
 318 }
 319
 320 DISPATCH_ALWAYS_INLINE
 321 static inline void
 322 _voucher_swap(voucher_t ov, voucher_t voucher)
 323 {
 324         _voucher_mach_voucher_set(_voucher_swap_and_get_mach_voucher(ov, voucher));
 325         if (ov) _voucher_release(ov);
 326 }
 327
 328 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 329 static inline voucher_t
 330 _voucher_adopt(voucher_t voucher)
 331 {
 332         voucher_t ov = _voucher_get();
 333         _voucher_mach_voucher_set(_voucher_swap_and_get_mach_voucher(ov, voucher));
 334         return ov;
 335 }
 336
 337 DISPATCH_ALWAYS_INLINE
 338 static inline void
 339 _voucher_replace(voucher_t voucher)
 340 {
 341         voucher_t ov = _voucher_get();
 342         _voucher_swap(ov, voucher);
 343 }
 344
 345 DISPATCH_ALWAYS_INLINE
 346 static inline void
 347 _voucher_clear(void)
 348 {
 349         _voucher_replace(NULL);
 350 }
 351
 352 DISPATCH_ALWAYS_INLINE
 353 static inline pthread_priority_t
 354 _voucher_get_priority(voucher_t voucher)
 355 {
 356         return voucher && voucher->v_has_priority ?
 357                         (pthread_priority_t)*_voucher_priority(voucher) : 0;
 358 }
 359
 360 void _voucher_task_mach_voucher_init(void* ctxt);
 361 extern dispatch_once_t _voucher_task_mach_voucher_pred;
 362 extern mach_voucher_t _voucher_task_mach_voucher;
 363
 364 DISPATCH_ALWAYS_INLINE
 365 static inline mach_voucher_t
 366 _voucher_get_task_mach_voucher(void)
 367 {
 368         dispatch_once_f(&_voucher_task_mach_voucher_pred, NULL,
 369                         _voucher_task_mach_voucher_init);
 370         return _voucher_task_mach_voucher;
 371 }
 372
 373 DISPATCH_ALWAYS_INLINE
 374 static inline bool
 375 _voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv,
 376                 bool move_send)
 377 {
 378         if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false;
 379         if (!kv) return false;
 380         msg->msgh_voucher_port = kv;
 381         msg->msgh_bits |= MACH_MSGH_BITS_SET_PORTS(0, 0, move_send ?
 382                         MACH_MSG_TYPE_MOVE_SEND : MACH_MSG_TYPE_COPY_SEND);
 383         _dispatch_kvoucher_debug("msg[%p] set %s", kv, msg, move_send ?
 384                         "move-send" : "copy-send");
 385         _dispatch_voucher_debug_machport(kv);
 386         return true;
 387 }
 388
 389 DISPATCH_ALWAYS_INLINE
 390 static inline bool
 391 _voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher)
 392 {
 393         if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false;
 394         mach_voucher_t kv;
 395         if (voucher) {
 396                 kv = _voucher_get_mach_voucher(voucher);
 397         } else {
 398                 kv = _voucher_get_task_mach_voucher();
 399         }
 400         return _voucher_mach_msg_set_mach_voucher(msg, kv, false);
 401 }
 402
 403 DISPATCH_ALWAYS_INLINE
 404 static inline mach_voucher_t
 405 _voucher_mach_msg_get(mach_msg_header_t *msg)
 406 {
 407         if (!MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return MACH_VOUCHER_NULL;
 408         mach_voucher_t kv = msg->msgh_voucher_port;
 409         msg->msgh_voucher_port = MACH_VOUCHER_NULL;
 410         msg->msgh_bits &= (mach_msg_bits_t)~MACH_MSGH_BITS_VOUCHER_MASK;
 411         return kv;
 412 }
 413
 414 DISPATCH_ALWAYS_INLINE
 415 static inline mach_voucher_t
 416 _voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send)
 417 {
 418         mach_msg_bits_t kvbits = MACH_MSGH_BITS_VOUCHER(msg->msgh_bits);
 419         mach_voucher_t kv = msg->msgh_voucher_port, kvm = MACH_VOUCHER_NULL;
 420         if ((kvbits == MACH_MSG_TYPE_COPY_SEND ||
 421                         kvbits == MACH_MSG_TYPE_MOVE_SEND) && kv) {
 422                 _dispatch_kvoucher_debug("msg[%p] clear %s", kv, msg, move_send ?
 423                                 "move-send" : "copy-send");
 424                 _dispatch_voucher_debug_machport(kv);
 425                 if (kvbits == MACH_MSG_TYPE_MOVE_SEND) {
 426                         // <rdar://problem/15694142> return/drop received or pseudo-received
 427                         // voucher reference (e.g. due to send failure).
 428                         if (move_send) {
 429                                 kvm = kv;
 430                         } else {
 431                                 _voucher_dealloc_mach_voucher(kv);
 432                         }
 433                 }
 434                 msg->msgh_voucher_port = MACH_VOUCHER_NULL;
 435                 msg->msgh_bits &= (mach_msg_bits_t)~MACH_MSGH_BITS_VOUCHER_MASK;
 436         }
 437         return kvm;
 438 }
 439
 440 #pragma mark -
 441 #pragma mark dispatch_continuation_t + voucher_t
 442
 443 #if DISPATCH_USE_KDEBUG_TRACE
 444 DISPATCH_ALWAYS_INLINE
 445 static inline void
 446 _dispatch_voucher_ktrace(int code, natural_t voucher, void *container)
 447 {
 448         if (!voucher) return;
 449         __kdebug_trace(APPSDBG_CODE(DBG_MACH_CHUD, (0xfac >> 2)) | DBG_FUNC_NONE,
 450                         code, (int)voucher, (int)(uintptr_t)container,
 451 #ifdef __LP64__
 452                         (int)((uintptr_t)container >> 32)
 453 #else
 454                         0
 455 #endif
 456                         );
 457 }
 458 #define _dispatch_voucher_ktrace_dc_push(dc) \
 459                 _dispatch_voucher_ktrace(0x1, (dc)->dc_voucher ? \
 460                                 (dc)->dc_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dc))
 461 #define _dispatch_voucher_ktrace_dc_pop(dc) \
 462                 _dispatch_voucher_ktrace(0x2, (dc)->dc_voucher ? \
 463                                 (dc)->dc_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dc))
 464 #define _dispatch_voucher_ktrace_dmsg_push(dmsg) \
 465                 _dispatch_voucher_ktrace(0x3, (dmsg)->dmsg_voucher ? \
 466                                 (dmsg)->dmsg_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dmsg))
 467 #define _dispatch_voucher_ktrace_dmsg_pop(dmsg) \
 468                 _dispatch_voucher_ktrace(0x4, (dmsg)->dmsg_voucher ? \
 469                                 (dmsg)->dmsg_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dmsg))
 470 #else
 471 #define _dispatch_voucher_ktrace_dc_push(dc)
 472 #define _dispatch_voucher_ktrace_dc_pop(dc)
 473 #define _dispatch_voucher_ktrace_dmsg_push(dmsg)
 474 #define _dispatch_voucher_ktrace_dmsg_pop(dmsg)
 475 #endif // DISPATCH_USE_KDEBUG_TRACE
 476
 477 DISPATCH_ALWAYS_INLINE
 478 static inline void
 479 _dispatch_continuation_voucher_set(dispatch_continuation_t dc,
 480                 dispatch_block_flags_t flags)
 481 {
 482         unsigned long bits = (unsigned long)dc->do_vtable;
 483         voucher_t v = NULL;
 484
 485         if (flags & DISPATCH_BLOCK_HAS_VOUCHER) {
 486                 bits |= DISPATCH_OBJ_HAS_VOUCHER_BIT;
 487         } else if (!(flags & DISPATCH_BLOCK_NO_VOUCHER)) {
 488                 v = _voucher_copy();
 489         }
 490         dc->do_vtable = (void*)bits;
 491         dc->dc_voucher = v;
 492         _dispatch_voucher_debug("continuation[%p] set", dc->dc_voucher, dc);
 493         _dispatch_voucher_ktrace_dc_push(dc);
 494 }
 495
 496 DISPATCH_ALWAYS_INLINE
 497 static inline void
 498 _dispatch_continuation_voucher_adopt(dispatch_continuation_t dc)
 499 {
 500         unsigned long bits = (unsigned long)dc->do_vtable;
 501         voucher_t v = DISPATCH_NO_VOUCHER;
 502         if (!(bits & DISPATCH_OBJ_HAS_VOUCHER_BIT)) {
 503                 _dispatch_voucher_ktrace_dc_pop(dc);
 504                 _dispatch_voucher_debug("continuation[%p] adopt", dc->dc_voucher, dc);
 505                 v = dc->dc_voucher;
 506                 dc->dc_voucher = NULL;
 507         }
 508         _dispatch_adopt_priority_and_replace_voucher(dc->dc_priority, v, 0);
 509 }
 510
 511 #pragma mark -
 512 #pragma mark _voucher_activity_heap
 513
 514 typedef uint32_t _voucher_atm_subid_t;
 515 static const size_t _voucher_activity_hash_bits = 6;
 516 static const size_t _voucher_activity_hash_size =
 517                 1 << _voucher_activity_hash_bits;
 518 #define VACTID_HASH(x) ((((uint32_t)((x) >> 32) + (uint32_t)(x)) * \
 519                 2654435761u) >> (32-_voucher_activity_hash_bits))
 520 #define VATMID_HASH(x) \
 521                 (((uint32_t)(x) * 2654435761u) >> (32-_voucher_activity_hash_bits))
 522 #define VATMID2ACTID(x) ((uint64_t)(x) << 32)
 523 #define VACTID_BASEID(x) ((uint64_t)(x) & (((uint64_t)UINT32_MAX) << 32))
 524 #define VACTID_SUBID(x)  ((uint32_t)(x))
 525 #define VATM_ACTID(vatm, subid) (VATMID2ACTID((vatm)->vatm_id) + (subid))
 526 #define VATM_SUBID_BITS2MAX(bits) ((1u << (bits)) - 1)
 527 #define VATM_SUBID_MAXBITS (32)
 528 #define VATM_SUBID_MAX (ATM_SUBAID32_MAX)
 529 #define MAILBOX_OFFSET_UNSET UINT64_MAX
 530
 531 static const size_t _voucher_activity_buffers_per_heap = 512;
 532 typedef unsigned long _voucher_activity_bitmap_base_t;
 533 static const size_t _voucher_activity_bits_per_bitmap_base_t =
 534                 8 * sizeof(_voucher_activity_bitmap_base_t);
 535 static const size_t _voucher_activity_bitmaps_per_heap =
 536                 _voucher_activity_buffers_per_heap /
 537                 _voucher_activity_bits_per_bitmap_base_t;
 538 typedef _voucher_activity_bitmap_base_t
 539                 _voucher_activity_bitmap_t[_voucher_activity_bitmaps_per_heap];
 540
 541 typedef struct _voucher_activity_metadata_s {
 542         _voucher_activity_buffer_t vam_kernel_metadata;
 543         _voucher_activity_buffer_t vam_client_metadata;
 544         struct _voucher_activity_self_metadata_s vam_self_metadata;
 545 #if __LP64__
 546         uintptr_t vam_pad0[7];
 547 #else
 548         uintptr_t vam_pad0[15];
 549 #endif
 550         // cacheline
 551         _voucher_activity_bitmap_t volatile vam_atm_mbox_bitmap;
 552         _voucher_activity_bitmap_t volatile vam_buffer_bitmap;
 553         _voucher_activity_bitmap_t volatile vam_pressure_locked_bitmap;
 554         // cacheline
 555         _voucher_atm_subid_t vam_base_atm_subid;
 556         _voucher_atm_subid_t vam_base_atm_subid_max;
 557         _voucher_atm_subid_t vam_nested_atm_subid;
 558         _voucher_atm_t vam_default_activity_atm;
 559         _voucher_atm_t volatile vam_base_atm;
 560         voucher_activity_id_t volatile vam_nested_atm_id;
 561 #if __LP64__
 562         uintptr_t vam_pad2[3];
 563 #else
 564         uintptr_t vam_pad2[1];
 565 #endif
 566         _voucher_activity_lock_s vam_base_atm_lock;
 567         _voucher_activity_lock_s vam_nested_atm_lock;
 568         _voucher_activity_lock_s vam_atms_lock;
 569         _voucher_activity_lock_s vam_activities_lock;
 570         // cacheline
 571         TAILQ_HEAD(, _voucher_atm_s) vam_atms[_voucher_activity_hash_size];
 572         TAILQ_HEAD(, _voucher_activity_s)
 573                         vam_activities[_voucher_activity_hash_size];
 574 } *_voucher_activity_metadata_t;
 575
 576 #pragma mark -
 577 #pragma mark _voucher_activity_t
 578
 579 _voucher_activity_tracepoint_t _voucher_activity_tracepoint_get_slow(
 580                 unsigned int slots);
 581 extern _voucher_activity_t _voucher_activity_default;
 582 extern voucher_activity_mode_t _voucher_activity_mode;
 583
 584 #if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG
 585 #define _dispatch_voucher_activity_debug(msg, act, ...) \
 586                 _dispatch_debug("activity[%p] <0x%x>: atm[%p] <%lld>: " msg, (act), \
 587                 (act) ? VACTID_SUBID((act)->va_id) : 0, (act) ? (act)->va_atm : NULL, \
 588                 (act) && (act)->va_atm ? (act)->va_atm->vatm_id : 0, ##__VA_ARGS__)
 589 #define _dispatch_voucher_atm_debug(msg, atm, ...) \
 590                 _dispatch_debug("atm[%p] <%lld> kvoucher[0x%08x]: " msg, (atm), \
 591                 (atm) ? (atm)->vatm_id : 0, (atm) ? (atm)->vatm_kvoucher : 0, \
 592                 ##__VA_ARGS__)
 593 #else
 594 #define _dispatch_voucher_activity_debug(msg, act, ...)
 595 #define _dispatch_voucher_atm_debug(msg, atm, ...)
 596 #endif
 597
 598 DISPATCH_ALWAYS_INLINE
 599 static inline uint64_t
 600 _voucher_activity_timestamp(void)
 601 {
 602 #if TARGET_IPHONE_SIMULATOR && \
 603                 IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101000
 604         return mach_absolute_time();
 605 #else
 606         return mach_approximate_time();
 607 #endif
 608 }
 609
 610 DISPATCH_ALWAYS_INLINE
 611 static inline uint64_t
 612 _voucher_activity_thread_id(void)
 613 {
 614         uint64_t thread_id;
 615         pthread_threadid_np(NULL, &thread_id); // TODO: 15923074: use TSD thread_id
 616         return thread_id;
 617 }
 618
 619 DISPATCH_ALWAYS_INLINE
 620 static inline _voucher_activity_tracepoint_t
 621 _voucher_activity_buffer_tracepoint_get(_voucher_activity_buffer_header_t vab,
 622                 unsigned int slots)
 623 {
 624         uint32_t idx = dispatch_atomic_add2o(vab, vabh_next_tracepoint_idx,
 625                         slots, relaxed);
 626         if (idx <= _voucher_activity_tracepoints_per_buffer) {
 627                 return (_voucher_activity_tracepoint_t)vab + (idx - slots);
 628         }
 629         return NULL;
 630 }
 631
 632 DISPATCH_ALWAYS_INLINE
 633 static inline _voucher_activity_tracepoint_t
 634 _voucher_activity_tracepoint_get_from_activity(_voucher_activity_t va,
 635                 unsigned int slots)
 636 {
 637         _voucher_activity_buffer_header_t vab = va ? va->va_current_buffer : NULL;
 638         return vab ? _voucher_activity_buffer_tracepoint_get(vab, slots) : NULL;
 639 }
 640
 641 DISPATCH_ALWAYS_INLINE
 642 static inline _voucher_activity_tracepoint_t
 643 _voucher_activity_tracepoint_get(unsigned int slots)
 644 {
 645         _voucher_activity_t va;
 646         voucher_t v = _voucher_get();
 647         va = v && v->v_activity ? v->v_activity : _voucher_activity_default;
 648         return _voucher_activity_tracepoint_get_from_activity(va, slots);
 649 }
 650
 651 DISPATCH_ALWAYS_INLINE
 652 static inline uint64_t
 653 _voucher_activity_tracepoint_init(_voucher_activity_tracepoint_t vat,
 654                 uint8_t type, uint8_t code_namespace, uint32_t code, uint64_t location)
 655 {
 656         if (!location) location = (uint64_t)__builtin_return_address(0);
 657         uint64_t timestamp = _voucher_activity_timestamp();
 658         vat->vat_flags = _voucher_activity_trace_flag_tracepoint,
 659         vat->vat_type = type,
 660         vat->vat_namespace = code_namespace,
 661         vat->vat_code = code,
 662         vat->vat_timestamp = timestamp,
 663         vat->vat_thread = _voucher_activity_thread_id(),
 664         vat->vat_location = location;
 665         return timestamp;
 666 }
 667
 668 DISPATCH_ALWAYS_INLINE
 669 static inline uint64_t
 670 _voucher_activity_tracepoint_init_with_id(_voucher_activity_tracepoint_t vat,
 671                 voucher_activity_trace_id_t trace_id, uint64_t location)
 672 {
 673         uint8_t type = (uint8_t)(trace_id >> _voucher_activity_trace_id_type_shift);
 674         uint8_t cns = (uint8_t)(trace_id >>
 675                         _voucher_activity_trace_id_code_namespace_shift);
 676         uint32_t code = (uint32_t)trace_id;
 677         return _voucher_activity_tracepoint_init(vat, type, cns, code, location);
 678 }
 679
 680 DISPATCH_ALWAYS_INLINE
 681 static inline bool
 682 _voucher_activity_trace_id_is_subtype(voucher_activity_trace_id_t trace_id,
 683                 uint8_t type)
 684 {
 685         voucher_activity_trace_id_t type_id = voucher_activity_trace_id(type, 0, 0);
 686         return (trace_id & type_id) == type_id;
 687 }
 688 #define _voucher_activity_trace_id_is_subtype(trace_id, name) \
 689         _voucher_activity_trace_id_is_subtype(trace_id, \
 690                         voucher_activity_tracepoint_type_ ## name)
 691
 692 DISPATCH_ALWAYS_INLINE
 693 static inline bool
 694 _voucher_activity_trace_id_enabled(voucher_activity_trace_id_t trace_id)
 695 {
 696         switch (_voucher_activity_mode) {
 697         case voucher_activity_mode_release:
 698                 return _voucher_activity_trace_id_is_subtype(trace_id, release);
 699         case voucher_activity_mode_stream:
 700         case voucher_activity_mode_debug:
 701                 return _voucher_activity_trace_id_is_subtype(trace_id, debug) ||
 702                                 _voucher_activity_trace_id_is_subtype(trace_id, release);
 703         }
 704         return false;
 705 }
 706
 707 DISPATCH_ALWAYS_INLINE
 708 static inline bool
 709 _voucher_activity_trace_type_enabled(uint8_t type)
 710 {
 711         voucher_activity_trace_id_t type_id = voucher_activity_trace_id(type, 0, 0);
 712         return _voucher_activity_trace_id_enabled(type_id);
 713 }
 714
 715 DISPATCH_ALWAYS_INLINE
 716 static inline bool
 717 _voucher_activity_disabled(void)
 718 {
 719         return slowpath(_voucher_activity_mode == voucher_activity_mode_disable);
 720 }
 721
 722 DISPATCH_ALWAYS_INLINE
 723 static inline _voucher_activity_tracepoint_t
 724 _voucher_activity_trace_args_inline(uint8_t type, uint8_t code_namespace,
 725                 uint32_t code, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3,
 726                 uintptr_t arg4)
 727 {
 728         if (!_voucher_activity_trace_type_enabled(type)) return NULL;
 729         _voucher_activity_tracepoint_t vat;
 730         vat = _voucher_activity_tracepoint_get(1);
 731         if (!vat) return NULL;
 732         _voucher_activity_tracepoint_init(vat, type, code_namespace, code, 0);
 733         vat->vat_flags |= _voucher_activity_trace_flag_tracepoint_args;
 734         vat->vat_data[0] = arg1;
 735         vat->vat_data[1] = arg2;
 736         vat->vat_data[2] = arg3;
 737         vat->vat_data[3] = arg4;
 738         return vat;
 739 }
 740
 741 DISPATCH_ALWAYS_INLINE
 742 static inline _voucher_activity_tracepoint_t
 743 _voucher_activity_trace_with_id_inline(voucher_activity_trace_id_t trace_id)
 744 {
 745         _voucher_activity_tracepoint_t vat = _voucher_activity_tracepoint_get(1);
 746         if (!vat) return NULL;
 747         _voucher_activity_tracepoint_init_with_id(vat, trace_id, 0);
 748         return vat;
 749 }
 750
 751 DISPATCH_ALWAYS_INLINE
 752 static inline _voucher_activity_tracepoint_t
 753 _voucher_activity_trace_with_id(voucher_activity_trace_id_t trace_id)
 754 {
 755         _voucher_activity_tracepoint_t vat = _voucher_activity_tracepoint_get(1);
 756         if (!vat) vat = _voucher_activity_tracepoint_get_slow(1);
 757         if (!vat) return NULL;
 758         _voucher_activity_tracepoint_init_with_id(vat, trace_id, 0);
 759         return vat;
 760 }
 761
 762 DISPATCH_ALWAYS_INLINE
 763 static inline void
 764 _voucher_activity_trace_msg(voucher_t v, mach_msg_header_t *msg, uint32_t code)
 765 {
 766         if (!v || !v->v_activity) return; // Don't use default activity for IPC
 767         const uint8_t type = voucher_activity_tracepoint_type_release;
 768         const uint8_t code_namespace = _voucher_activity_tracepoint_namespace_ipc;
 769         if (!_voucher_activity_trace_type_enabled(type)) return;
 770         _voucher_activity_tracepoint_t vat;
 771         vat = _voucher_activity_tracepoint_get_from_activity(v->v_activity, 1);
 772         if (!vat) return; // TODO: slowpath ?
 773         _voucher_activity_tracepoint_init(vat, type, code_namespace, code, 0);
 774         vat->vat_flags |= _voucher_activity_trace_flag_libdispatch;
 775 #if __has_extension(c_static_assert)
 776         _Static_assert(sizeof(mach_msg_header_t) <= sizeof(vat->vat_data),
 777                         "mach_msg_header_t too large");
 778 #endif
 779         memcpy(vat->vat_data, msg, sizeof(mach_msg_header_t));
 780 }
 781 #define _voucher_activity_trace_msg(v, msg, type) \
 782                 _voucher_activity_trace_msg(v, msg, \
 783                                 _voucher_activity_tracepoint_namespace_ipc_ ## type)
 784
 785 #endif // !(USE_OBJC && __OBJC2__)
 786
 787 #else // VOUCHER_USE_MACH_VOUCHER
 788
 789 #pragma mark -
 790 #pragma mark Simulator / vouchers disabled
 791
 792 #define _dispatch_voucher_debug(msg, v, ...)
 793 #define _dispatch_kvoucher_debug(msg, kv, ...)
 794
 795 DISPATCH_ALWAYS_INLINE
 796 static inline voucher_t
 797 _voucher_retain(voucher_t voucher)
 798 {
 799         return voucher;
 800 }
 801
 802 DISPATCH_ALWAYS_INLINE
 803 static inline void
 804 _voucher_release(voucher_t voucher)
 805 {
 806         (void)voucher;
 807 }
 808
 809 DISPATCH_ALWAYS_INLINE
 810 static inline voucher_t
 811 _voucher_get(void)
 812 {
 813         return NULL;
 814 }
 815
 816 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 817 static inline voucher_t
 818 _voucher_copy(void)
 819 {
 820         return NULL;
 821 }
 822
 823 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
 824 static inline voucher_t
 825 _voucher_copy_without_importance(void)
 826 {
 827         return NULL;
 828 }
 829
 830 DISPATCH_ALWAYS_INLINE
 831 static inline mach_voucher_t
 832 _voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher)
 833 {
 834         (void)ov; (void)voucher;
 835         return MACH_VOUCHER_NULL;
 836 }
 837
 838 DISPATCH_ALWAYS_INLINE
 839 static inline voucher_t
 840 _voucher_adopt(voucher_t voucher)
 841 {
 842         return voucher;
 843 }
 844
 845 DISPATCH_ALWAYS_INLINE
 846 static inline void
 847 _voucher_replace(voucher_t voucher)
 848 {
 849         (void)voucher;
 850 }
 851
 852 DISPATCH_ALWAYS_INLINE
 853 static inline void
 854 _voucher_clear(void)
 855 {
 856 }
 857
 858 DISPATCH_ALWAYS_INLINE
 859 static inline pthread_priority_t
 860 _voucher_get_priority(voucher_t voucher)
 861 {
 862         (void)voucher;
 863         return 0;
 864 }
 865
 866 DISPATCH_ALWAYS_INLINE
 867 static inline bool
 868 _voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv,
 869                 bool move_send)
 870 {
 871         (void)msg; (void)kv; (void)move_send;
 872         return false;
 873
 874 }
 875
 876 DISPATCH_ALWAYS_INLINE
 877 static inline bool
 878 _voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher)
 879 {
 880         (void)msg; (void)voucher;
 881         return false;
 882 }
 883
 884 DISPATCH_ALWAYS_INLINE
 885 static inline mach_voucher_t
 886 _voucher_mach_msg_get(mach_msg_header_t *msg)
 887 {
 888         (void)msg;
 889         return 0;
 890 }
 891
 892 DISPATCH_ALWAYS_INLINE
 893 static inline mach_voucher_t
 894 _voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send)
 895 {
 896         (void)msg; (void)move_send;
 897         return MACH_VOUCHER_NULL;
 898 }
 899
 900 #define _dispatch_voucher_ktrace_dmsg_push(dmsg)
 901 #define _dispatch_voucher_ktrace_dmsg_pop(dmsg)
 902
 903 DISPATCH_ALWAYS_INLINE
 904 static inline void
 905 _dispatch_continuation_voucher_set(dispatch_continuation_t dc,
 906                 dispatch_block_flags_t flags)
 907 {
 908         (void)dc; (void)flags;
 909 }
 910
 911 DISPATCH_ALWAYS_INLINE
 912 static inline void
 913 _dispatch_continuation_voucher_adopt(dispatch_continuation_t dc)
 914 {
 915         (void)dc;
 916 }
 917
 918 #define _voucher_activity_trace_msg(v, msg, type)
 919
 920 DISPATCH_ALWAYS_INLINE
 921 static inline bool
 922 _voucher_activity_disabled(void)
 923 {
 924         return true;
 925 }
 926
 927 #endif // VOUCHER_USE_MACH_VOUCHER
 928
 929 #endif /* __DISPATCH_VOUCHER_INTERNAL__ */