src/voucher.c

   1 /*
   2  * Copyright (c) 2013-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 #include "internal.h"
  22
  23 #if !defined(VOUCHER_EXPORT_PERSONA_SPI)
  24 #if TARGET_OS_IPHONE
  25 #define VOUCHER_EXPORT_PERSONA_SPI 1
  26 #else
  27 #define VOUCHER_EXPORT_PERSONA_SPI 0
  28 #endif
  29 #endif
  30
  31 #ifndef PERSONA_ID_NONE
  32 #define PERSONA_ID_NONE ((uid_t)-1)
  33 #endif
  34
  35 #if !DISPATCH_VARIANT_DYLD_STUB
  36
  37 #if VOUCHER_USE_MACH_VOUCHER
  38 #if !HAVE_PTHREAD_WORKQUEUE_QOS
  39 #error Unsupported configuration, workqueue QoS support is required
  40 #endif
  41 #include <mach/mach_voucher.h>
  42 #include <sys/proc_info.h>
  43
  44 #define MACH_ACTIVITY_ID_RANGE_SIZE 16
  45 #define MACH_ACTIVITY_ID_MASK ((1ULL << FIREHOSE_ACTIVITY_ID_FLAGS_SHIFT) - 1)
  46 #define FIREHOSE_ACTIVITY_ID_MAKE(aid, flags) \
  47                 FIREHOSE_ACTIVITY_ID_MERGE_FLAGS((aid) & MACH_ACTIVITY_ID_MASK, flags)
  48
  49 static volatile uint64_t _voucher_aid_next;
  50
  51 #pragma mark -
  52 #pragma mark voucher_t
  53
  54 OS_OBJECT_CLASS_DECL(voucher, object);
  55 #if !USE_OBJC
  56 OS_OBJECT_VTABLE_INSTANCE(voucher,
  57                 (void (*)(_os_object_t))_voucher_xref_dispose,
  58                 (void (*)(_os_object_t))_voucher_dispose);
  59 #endif // USE_OBJC
  60 #define VOUCHER_CLASS OS_OBJECT_VTABLE(voucher)
  61
  62 static inline voucher_t
  63 _voucher_alloc(mach_voucher_attr_recipe_size_t extra)
  64 {
  65         voucher_t voucher;
  66         size_t voucher_size = sizeof(voucher_s) + extra;
  67         voucher = (voucher_t)_os_object_alloc_realized(VOUCHER_CLASS, voucher_size);
  68 #if VOUCHER_ENABLE_RECIPE_OBJECTS
  69         voucher->v_recipe_extra_size = extra;
  70         voucher->v_recipe_extra_offset = voucher_size - extra;
  71 #else
  72         dispatch_assert(!extra);
  73 #endif
  74         _dispatch_voucher_debug("alloc", voucher);
  75         return voucher;
  76 }
  77
  78 #if VOUCHER_ENABLE_RECIPE_OBJECTS
  79 voucher_t
  80 voucher_create(voucher_recipe_t recipe)
  81 {
  82         // TODO: capture current activities or current kvoucher ?
  83         mach_voucher_attr_recipe_size_t extra = recipe ? recipe->vr_size : 0;
  84         voucher_t voucher = _voucher_alloc(extra);
  85         if (extra) {
  86                 memcpy(_voucher_extra_recipes(voucher), recipe->vr_data, extra);
  87         }
  88         _voucher_trace(CREATE, voucher, MACH_PORT_NULL, 0);
  89         return voucher;
  90 }
  91 #endif
  92
  93 DISPATCH_ALWAYS_INLINE
  94 static inline voucher_t
  95 _voucher_clone(const voucher_t ov, voucher_fields_t ignore_fields)
  96 {
  97         mach_voucher_attr_recipe_size_t extra = 0;
  98         voucher_t v;
  99
 100         if (ov && !(ignore_fields & VOUCHER_FIELD_EXTRA)) {
 101                 extra = _voucher_extra_size(ov);
 102         }
 103         v = _voucher_alloc(extra);
 104         if (ov) {
 105                 voucher_fields_t fields = ~ignore_fields;
 106                 if ((fields & VOUCHER_FIELD_KVOUCHER) && ov->v_kvoucher) {
 107                         voucher_t kvb = ov->v_kvbase ? ov->v_kvbase : ov;
 108                         v->v_kvbase = _voucher_retain(kvb);
 109                         v->v_kvoucher = kvb->v_kvoucher;
 110                         v->v_kv_has_importance = kvb->v_kv_has_importance;
 111                 }
 112                 if (fields & VOUCHER_FIELD_PRIORITY) {
 113                         v->v_priority = ov->v_priority;
 114                 }
 115                 if (fields & VOUCHER_FIELD_ACTIVITY) {
 116                         v->v_activity = ov->v_activity;
 117                         v->v_activity_creator = ov->v_activity_creator;
 118                         v->v_parent_activity = ov->v_parent_activity;
 119                 }
 120                 if ((fields & VOUCHER_FIELD_EXTRA) && extra) {
 121                         memcpy(_voucher_extra_recipes(v), _voucher_extra_recipes(ov),extra);
 122                 }
 123         }
 124         return v;
 125 }
 126
 127 voucher_t
 128 voucher_adopt(voucher_t voucher)
 129 {
 130         if (voucher == VOUCHER_CURRENT) {
 131                 return _voucher_copy();
 132         }
 133         return _voucher_adopt(voucher);
 134 }
 135
 136 voucher_t
 137 voucher_copy(void)
 138 {
 139         return _voucher_copy();
 140 }
 141
 142 voucher_t
 143 voucher_copy_without_importance(void)
 144 {
 145         return _voucher_copy_without_importance();
 146 }
 147
 148 voucher_t
 149 voucher_retain(voucher_t voucher)
 150 {
 151         return _voucher_retain(voucher);
 152 }
 153
 154 void
 155 voucher_release(voucher_t voucher)
 156 {
 157         return _voucher_release(voucher);
 158 }
 159
 160 void
 161 _voucher_thread_cleanup(void *voucher)
 162 {
 163         // when a thread exits and has a voucher left, the kernel
 164         // will get rid of the voucher kernel object that is set on the thread,
 165         // we only need to release the voucher_t object.
 166         _voucher_release(voucher);
 167 }
 168
 169 #pragma mark -
 170 #pragma mark voucher_hash
 171
 172 DISPATCH_CACHELINE_ALIGN
 173 static voucher_hash_head_s _voucher_hash[VL_HASH_SIZE];
 174
 175 #define _voucher_hash_head(kv)   (&_voucher_hash[VL_HASH((kv))])
 176 static dispatch_unfair_lock_s _voucher_hash_lock;
 177 #define _voucher_hash_lock_lock() \
 178                 _dispatch_unfair_lock_lock(&_voucher_hash_lock)
 179 #define _voucher_hash_lock_unlock() \
 180                 _dispatch_unfair_lock_unlock(&_voucher_hash_lock)
 181
 182 DISPATCH_ALWAYS_INLINE
 183 static inline void
 184 _voucher_hash_head_init(voucher_hash_head_s *head)
 185 {
 186         _voucher_hash_set_next(&head->vhh_first, VOUCHER_NULL);
 187         _voucher_hash_set_prev_ptr(&head->vhh_last_ptr, &head->vhh_first);
 188 }
 189
 190 DISPATCH_ALWAYS_INLINE
 191 static inline void
 192 _voucher_hash_enqueue(mach_voucher_t kv, voucher_t v)
 193 {
 194         // same as TAILQ_INSERT_TAIL
 195         voucher_hash_head_s *head = _voucher_hash_head(kv);
 196         uintptr_t prev_ptr = head->vhh_last_ptr;
 197         _voucher_hash_set_next(&v->v_list.vhe_next, VOUCHER_NULL);
 198         v->v_list.vhe_prev_ptr = prev_ptr;
 199         _voucher_hash_store_to_prev_ptr(prev_ptr, v);
 200         _voucher_hash_set_prev_ptr(&head->vhh_last_ptr, &v->v_list.vhe_next);
 201 }
 202
 203 DISPATCH_ALWAYS_INLINE
 204 static inline void
 205 _voucher_hash_remove(mach_voucher_t kv, voucher_t v)
 206 {
 207         // same as TAILQ_REMOVE
 208         voucher_hash_head_s *head = _voucher_hash_head(kv);
 209         voucher_t next = _voucher_hash_get_next(v->v_list.vhe_next);
 210         uintptr_t prev_ptr = v->v_list.vhe_prev_ptr;
 211         if (next) {
 212                 next->v_list.vhe_prev_ptr = prev_ptr;
 213         } else {
 214                 head->vhh_last_ptr = prev_ptr;
 215         }
 216         _voucher_hash_store_to_prev_ptr(prev_ptr, next);
 217         _voucher_hash_mark_not_enqueued(v);
 218 }
 219
 220 static voucher_t
 221 _voucher_find_and_retain(mach_voucher_t kv)
 222 {
 223         if (!kv) return NULL;
 224         _voucher_hash_lock_lock();
 225         voucher_hash_head_s *head = _voucher_hash_head(kv);
 226         voucher_t v = _voucher_hash_get_next(head->vhh_first);
 227         while (v) {
 228                 if (v->v_ipc_kvoucher == kv) {
 229                         int xref_cnt = os_atomic_inc2o(v, os_obj_xref_cnt, relaxed);
 230                         _dispatch_voucher_debug("retain  -> %d", v, xref_cnt + 1);
 231                         if (unlikely(xref_cnt < 0)) {
 232                                 _dispatch_voucher_debug("over-release", v);
 233                                 _OS_OBJECT_CLIENT_CRASH("Voucher over-release");
 234                         }
 235                         if (xref_cnt == 0) {
 236                                 // resurrection: raced with _voucher_remove
 237                                 (void)os_atomic_inc2o(v, os_obj_ref_cnt, relaxed);
 238                         }
 239                         break;
 240                 }
 241                 v = _voucher_hash_get_next(v->v_list.vhe_next);
 242         }
 243         _voucher_hash_lock_unlock();
 244         return v;
 245 }
 246
 247 static void
 248 _voucher_insert(voucher_t v)
 249 {
 250         mach_voucher_t kv = v->v_ipc_kvoucher;
 251         if (!kv) return;
 252         _voucher_hash_lock_lock();
 253         if (unlikely(_voucher_hash_is_enqueued(v))) {
 254                 _dispatch_voucher_debug("corruption", v);
 255                 DISPATCH_CLIENT_CRASH(0, "Voucher corruption");
 256         }
 257         _voucher_hash_enqueue(kv, v);
 258         _voucher_hash_lock_unlock();
 259 }
 260
 261 static void
 262 _voucher_remove(voucher_t v)
 263 {
 264         mach_voucher_t kv = v->v_ipc_kvoucher;
 265         if (!_voucher_hash_is_enqueued(v)) return;
 266         _voucher_hash_lock_lock();
 267         if (unlikely(!kv)) {
 268                 _dispatch_voucher_debug("corruption", v);
 269                 DISPATCH_CLIENT_CRASH(0, "Voucher corruption");
 270         }
 271         // check for resurrection race with _voucher_find_and_retain
 272         if (os_atomic_load2o(v, os_obj_xref_cnt, ordered) < 0) {
 273                 if (_voucher_hash_is_enqueued(v)) _voucher_hash_remove(kv, v);
 274         }
 275         _voucher_hash_lock_unlock();
 276 }
 277
 278 #pragma mark -
 279 #pragma mark mach_voucher_t
 280
 281 void
 282 _voucher_dealloc_mach_voucher(mach_voucher_t kv)
 283 {
 284         _dispatch_kvoucher_debug("dealloc", kv);
 285         _dispatch_voucher_debug_machport(kv);
 286         kern_return_t kr = mach_voucher_deallocate(kv);
 287         DISPATCH_VERIFY_MIG(kr);
 288         (void)dispatch_assume_zero(kr);
 289 }
 290
 291 static inline kern_return_t
 292 _voucher_create_mach_voucher(const mach_voucher_attr_recipe_data_t *recipes,
 293                 size_t recipes_size, mach_voucher_t *kvp)
 294 {
 295         kern_return_t kr;
 296         mach_port_t mhp = _dispatch_get_mach_host_port();
 297         mach_voucher_t kv = MACH_VOUCHER_NULL;
 298         mach_voucher_attr_raw_recipe_array_t kvr;
 299         mach_voucher_attr_recipe_size_t kvr_size;
 300         kvr = (mach_voucher_attr_raw_recipe_array_t)recipes;
 301         kvr_size = (mach_voucher_attr_recipe_size_t)recipes_size;
 302         kr = host_create_mach_voucher(mhp, kvr, kvr_size, &kv);
 303         DISPATCH_VERIFY_MIG(kr);
 304         if (!kr) {
 305                 _dispatch_kvoucher_debug("create", kv);
 306                 _dispatch_voucher_debug_machport(kv);
 307         }
 308         *kvp = kv;
 309         return kr;
 310 }
 311
 312 void
 313 _voucher_task_mach_voucher_init(void* ctxt DISPATCH_UNUSED)
 314 {
 315         kern_return_t kr;
 316         mach_voucher_t kv = MACH_VOUCHER_NULL;
 317 #if !VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
 318         static const mach_voucher_attr_recipe_data_t task_create_recipe = {
 319                 .key = MACH_VOUCHER_ATTR_KEY_BANK,
 320                 .command = MACH_VOUCHER_ATTR_BANK_CREATE,
 321         };
 322         kr = _voucher_create_mach_voucher(&task_create_recipe,
 323                         sizeof(task_create_recipe), &kv);
 324         if (slowpath(kr)) {
 325                 DISPATCH_CLIENT_CRASH(kr, "Could not create task mach voucher");
 326         }
 327         _voucher_default_task_mach_voucher = kv;
 328 #endif
 329         _voucher_task_mach_voucher = kv;
 330 }
 331
 332 void
 333 voucher_replace_default_voucher(void)
 334 {
 335         (void)_voucher_get_task_mach_voucher(); // initalize task mach voucher
 336         mach_voucher_t kv, tkv = MACH_VOUCHER_NULL;
 337         voucher_t v = _voucher_get();
 338         if (v && v->v_kvoucher) {
 339                 kern_return_t kr;
 340                 kv = v->v_ipc_kvoucher ? v->v_ipc_kvoucher : v->v_kvoucher;
 341                 const mach_voucher_attr_recipe_data_t task_copy_recipe = {
 342                         .key = MACH_VOUCHER_ATTR_KEY_BANK,
 343                         .command = MACH_VOUCHER_ATTR_COPY,
 344                         .previous_voucher = kv,
 345                 };
 346                 kr = _voucher_create_mach_voucher(&task_copy_recipe,
 347                                 sizeof(task_copy_recipe), &tkv);
 348                 if (dispatch_assume_zero(kr)) {
 349                         tkv = MACH_VOUCHER_NULL;
 350                 }
 351         }
 352         if (!tkv) tkv = _voucher_default_task_mach_voucher;
 353         kv = os_atomic_xchg(&_voucher_task_mach_voucher, tkv, relaxed);
 354         if (kv && kv != _voucher_default_task_mach_voucher) {
 355                 _voucher_dealloc_mach_voucher(kv);
 356         }
 357         _dispatch_voucher_debug("kvoucher[0x%08x] replace default voucher", v, tkv);
 358 }
 359
 360 #define _voucher_mach_recipe_size(payload_size) \
 361         (sizeof(mach_voucher_attr_recipe_data_t) + (payload_size))
 362
 363 #define _voucher_mach_recipe_alloca(v) ((mach_voucher_attr_recipe_t)alloca(\
 364                 _voucher_mach_recipe_size(0) + \
 365                 _voucher_mach_recipe_size(sizeof(ipc_pthread_priority_value_t)) + \
 366                 _voucher_mach_recipe_size(sizeof(_voucher_mach_udata_s)) + \
 367                 _voucher_extra_size(v)))
 368
 369 DISPATCH_ALWAYS_INLINE
 370 static inline mach_voucher_attr_recipe_size_t
 371 _voucher_mach_recipe_init(mach_voucher_attr_recipe_t mvar_buf, voucher_s *v,
 372                 mach_voucher_t kvb, pthread_priority_t pp)
 373 {
 374         mach_voucher_attr_recipe_size_t extra = _voucher_extra_size(v);
 375         mach_voucher_attr_recipe_size_t size = 0;
 376
 377         // normalize to just the QoS class and 0 relative priority
 378         pp &= _PTHREAD_PRIORITY_QOS_CLASS_MASK;
 379         if (pp) pp |= _PTHREAD_PRIORITY_PRIORITY_MASK;
 380
 381         *mvar_buf++ = (mach_voucher_attr_recipe_data_t){
 382                 .key = MACH_VOUCHER_ATTR_KEY_ALL,
 383                 .command = MACH_VOUCHER_ATTR_COPY,
 384                 .previous_voucher = kvb,
 385         };
 386         size += _voucher_mach_recipe_size(0);
 387
 388         if (pp) {
 389                 ipc_pthread_priority_value_t value = (ipc_pthread_priority_value_t)pp;
 390                 *mvar_buf++ = (mach_voucher_attr_recipe_data_t){
 391                         .key = MACH_VOUCHER_ATTR_KEY_PTHPRIORITY,
 392                         .command = MACH_VOUCHER_ATTR_PTHPRIORITY_CREATE,
 393                         .content_size = sizeof(value),
 394                 };
 395                 mvar_buf = _dispatch_memappend(mvar_buf, &value);
 396                 size += _voucher_mach_recipe_size(sizeof(value));
 397         }
 398
 399         if ((v && v->v_activity) || pp) {
 400                 _voucher_mach_udata_s *udata_buf;
 401                 unsigned udata_size = 0;
 402
 403                 if (v && v->v_activity) {
 404                         udata_size = offsetof(_voucher_mach_udata_s, _vmu_after_activity);
 405                 } else {
 406                         udata_size = offsetof(_voucher_mach_udata_s, _vmu_after_priority);
 407                 }
 408                 *mvar_buf = (mach_voucher_attr_recipe_data_t){
 409                         .key = MACH_VOUCHER_ATTR_KEY_USER_DATA,
 410                         .command = MACH_VOUCHER_ATTR_USER_DATA_STORE,
 411                         .content_size = udata_size,
 412                 };
 413                 udata_buf = (_voucher_mach_udata_s *)(mvar_buf->content);
 414
 415                 if (v && v->v_activity) {
 416                         *udata_buf = (_voucher_mach_udata_s){
 417                                 .vmu_magic = VOUCHER_MAGIC_V3,
 418                                 .vmu_priority = (_voucher_priority_t)pp,
 419                                 .vmu_activity = v->v_activity,
 420                                 .vmu_activity_pid = v->v_activity_creator,
 421                                 .vmu_parent_activity = v->v_parent_activity,
 422                         };
 423                 } else {
 424                         *udata_buf = (_voucher_mach_udata_s){
 425                                 .vmu_magic = VOUCHER_MAGIC_V3,
 426                                 .vmu_priority = (_voucher_priority_t)pp,
 427                         };
 428                 }
 429
 430                 mvar_buf = (mach_voucher_attr_recipe_t)(mvar_buf->content + udata_size);
 431                 size += _voucher_mach_recipe_size(udata_size);
 432         }
 433
 434         if (extra) {
 435                 memcpy(mvar_buf, _voucher_extra_recipes(v), extra);
 436                 size += extra;
 437         }
 438         return size;
 439 }
 440
 441 mach_voucher_t
 442 _voucher_get_mach_voucher(voucher_t voucher)
 443 {
 444         if (!voucher) return MACH_VOUCHER_NULL;
 445         if (voucher->v_ipc_kvoucher) return voucher->v_ipc_kvoucher;
 446         mach_voucher_t kvb = voucher->v_kvoucher;
 447         if (!kvb) kvb = _voucher_get_task_mach_voucher();
 448         if (!voucher->v_activity && !voucher->v_priority &&
 449                         !_voucher_extra_size(voucher)) {
 450                 return kvb;
 451         }
 452
 453         mach_voucher_attr_recipe_t mvar = _voucher_mach_recipe_alloca(voucher);
 454         mach_voucher_attr_recipe_size_t size;
 455         mach_voucher_t kv, kvo;
 456         kern_return_t kr;
 457
 458         size = _voucher_mach_recipe_init(mvar, voucher, kvb, voucher->v_priority);
 459         kr = _voucher_create_mach_voucher(mvar, size, &kv);
 460         if (dispatch_assume_zero(kr) || !kv) {
 461                 return MACH_VOUCHER_NULL;
 462         }
 463         if (!os_atomic_cmpxchgv2o(voucher, v_ipc_kvoucher, MACH_VOUCHER_NULL,
 464                         kv, &kvo, relaxed)) {
 465                 _voucher_dealloc_mach_voucher(kv);
 466                 kv = kvo;
 467         } else {
 468                 if (kv == voucher->v_kvoucher) {
 469                         // if v_kvoucher == v_ipc_kvoucher we keep only one reference
 470                         _voucher_dealloc_mach_voucher(kv);
 471                 }
 472                 _voucher_insert(voucher);
 473                 _dispatch_voucher_debug("kvoucher[0x%08x] create", voucher, kv);
 474         }
 475         return kv;
 476 }
 477
 478 mach_voucher_t
 479 _voucher_create_mach_voucher_with_priority(voucher_t voucher,
 480                 pthread_priority_t priority)
 481 {
 482         if (priority == _voucher_get_priority(voucher)) {
 483                 return MACH_VOUCHER_NULL; // caller will use _voucher_get_mach_voucher
 484         }
 485         kern_return_t kr;
 486         mach_voucher_t kv, kvb = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL;
 487         if (!kvb) kvb = _voucher_get_task_mach_voucher();
 488
 489         mach_voucher_attr_recipe_t mvar = _voucher_mach_recipe_alloca(voucher);
 490         mach_voucher_attr_recipe_size_t size;
 491
 492         size = _voucher_mach_recipe_init(mvar, voucher, kvb, priority);
 493         kr = _voucher_create_mach_voucher(mvar, size, &kv);
 494         if (dispatch_assume_zero(kr) || !kv) {
 495                 return MACH_VOUCHER_NULL;
 496         }
 497         _dispatch_kvoucher_debug("create with priority from voucher[%p]", kv,
 498                         voucher);
 499         return kv;
 500 }
 501
 502 static voucher_t
 503 _voucher_create_with_mach_voucher(mach_voucher_t kv, mach_msg_bits_t msgh_bits)
 504 {
 505         if (!kv) return NULL;
 506         kern_return_t kr;
 507         mach_voucher_attr_recipe_t vr;
 508         size_t vr_size;
 509         mach_voucher_attr_recipe_size_t kvr_size = 0;
 510         mach_voucher_attr_content_size_t udata_sz = 0;
 511         _voucher_mach_udata_s *udata = NULL;
 512         voucher_t v = _voucher_find_and_retain(kv);
 513         if (v) {
 514                 _dispatch_voucher_debug("kvoucher[0x%08x] found", v, kv);
 515                 _voucher_dealloc_mach_voucher(kv);
 516                 return v;
 517         }
 518         vr_size = sizeof(*vr) + sizeof(_voucher_mach_udata_s);
 519         vr = alloca(vr_size);
 520         if (kv) {
 521                 kvr_size = (mach_voucher_attr_recipe_size_t)vr_size;
 522                 kr = mach_voucher_extract_attr_recipe(kv,
 523                                 MACH_VOUCHER_ATTR_KEY_USER_DATA, (void*)vr, &kvr_size);
 524                 DISPATCH_VERIFY_MIG(kr);
 525                 if (!dispatch_assume_zero(kr) && kvr_size >= sizeof(*vr)) {
 526                         udata_sz = vr->content_size;
 527                         udata = (_voucher_mach_udata_s*)vr->content;
 528                         dispatch_assume(udata_sz >= sizeof(_voucher_magic_t));
 529                 }
 530         }
 531         vr = NULL;
 532
 533         v = _voucher_alloc(0);
 534         v->v_ipc_kvoucher = v->v_kvoucher = kv;
 535         v->v_kv_has_importance = !!(msgh_bits & MACH_MSGH_BITS_RAISEIMP);
 536
 537         if (udata_sz >= offsetof(_voucher_mach_udata_s,_vmu_after_priority)){
 538                 if (udata->vmu_magic == VOUCHER_MAGIC_V3) {
 539                         v->v_priority = udata->vmu_priority;
 540                 }
 541         }
 542         bool remove_kv_userdata = false;
 543         if (udata_sz >= offsetof(_voucher_mach_udata_s, _vmu_after_activity)) {
 544 #if !RDAR_25050791
 545                 remove_kv_userdata = true;
 546 #endif
 547                 if (udata->vmu_magic == VOUCHER_MAGIC_V3 && udata->vmu_activity) {
 548                         v->v_activity = udata->vmu_activity;
 549                         v->v_activity_creator = udata->vmu_activity_pid;
 550                         v->v_parent_activity = udata->vmu_parent_activity;
 551                 }
 552         }
 553
 554         if (remove_kv_userdata) {
 555                 mach_voucher_t nkv = MACH_VOUCHER_NULL;
 556                 const mach_voucher_attr_recipe_data_t remove_userdata_recipe[] = {
 557                         [0] = {
 558                                 .key = MACH_VOUCHER_ATTR_KEY_ALL,
 559                                 .command = MACH_VOUCHER_ATTR_COPY,
 560                                 .previous_voucher = kv,
 561                         },
 562                         [1] = {
 563                                 .key = MACH_VOUCHER_ATTR_KEY_USER_DATA,
 564                                 .command = MACH_VOUCHER_ATTR_REMOVE,
 565                         },
 566                         [2] = {
 567                                 .key = MACH_VOUCHER_ATTR_KEY_PTHPRIORITY,
 568                                 .command = MACH_VOUCHER_ATTR_REMOVE,
 569                         },
 570                 };
 571                 mach_voucher_attr_recipe_size_t size = sizeof(remove_userdata_recipe);
 572                 kr = _voucher_create_mach_voucher(remove_userdata_recipe, size, &nkv);
 573                 if (!dispatch_assume_zero(kr)) {
 574                         _dispatch_voucher_debug("kvoucher[0x%08x] udata removal "
 575                                         "(created 0x%08x)", v, kv, nkv);
 576                         v->v_ipc_kvoucher = MACH_VOUCHER_NULL;
 577                         v->v_kvoucher = nkv;
 578                         v->v_kvbase = _voucher_find_and_retain(nkv);
 579                         if (v->v_kvbase) {
 580                                 _voucher_dealloc_mach_voucher(nkv); // borrow base reference
 581                         }
 582                         _voucher_dealloc_mach_voucher(kv);
 583                         kv = nkv;
 584                 } else {
 585                         _dispatch_voucher_debug_machport(kv);
 586                 }
 587         }
 588
 589         _voucher_trace(CREATE, v, v->v_kvoucher, v->v_activity);
 590         _voucher_insert(v);
 591         _dispatch_voucher_debug("kvoucher[0x%08x] create", v, kv);
 592         return v;
 593 }
 594
 595 voucher_t
 596 _voucher_create_with_priority_and_mach_voucher(voucher_t ov,
 597                 pthread_priority_t priority, mach_voucher_t kv)
 598 {
 599         if (priority == _voucher_get_priority(ov)) {
 600                 if (kv) _voucher_dealloc_mach_voucher(kv);
 601                 return ov ? _voucher_retain(ov) : NULL;
 602         }
 603         voucher_t v = _voucher_find_and_retain(kv);
 604         voucher_fields_t ignore_fields = VOUCHER_FIELD_PRIORITY;
 605
 606         if (v) {
 607                 _dispatch_voucher_debug("kvoucher[0x%08x] find", v, kv);
 608                 _voucher_dealloc_mach_voucher(kv);
 609                 return v;
 610         }
 611
 612         if (kv) ignore_fields |= VOUCHER_FIELD_KVOUCHER;
 613         v = _voucher_clone(ov, ignore_fields);
 614         if (priority) {
 615                 v->v_priority = (_voucher_priority_t)priority;
 616         }
 617         if (kv) {
 618                 v->v_ipc_kvoucher = v->v_kvoucher = kv;
 619                 _voucher_insert(v);
 620                 _dispatch_voucher_debug("kvoucher[0x%08x] create with priority from "
 621                                 "voucher[%p]", v, kv, ov);
 622                 _dispatch_voucher_debug_machport(kv);
 623         }
 624         _voucher_trace(CREATE, v, v->v_kvoucher, v->v_activity);
 625         return v;
 626 }
 627
 628 voucher_t
 629 _voucher_create_without_importance(voucher_t ov)
 630 {
 631         // Nothing to do unless the old voucher has a kernel voucher. If it
 632         // doesn't, it can't have any importance, now or in the future.
 633         if (!ov) return NULL;
 634         if (!ov->v_kvoucher || !ov->v_kv_has_importance) return _voucher_retain(ov);
 635         kern_return_t kr;
 636         mach_voucher_t kv, okv;
 637         // Copy kernel voucher, removing importance.
 638         okv = ov->v_ipc_kvoucher ? ov->v_ipc_kvoucher : ov->v_kvoucher;
 639         const mach_voucher_attr_recipe_data_t importance_remove_recipe[] = {
 640                 [0] = {
 641                         .key = MACH_VOUCHER_ATTR_KEY_ALL,
 642                         .command = MACH_VOUCHER_ATTR_COPY,
 643                         .previous_voucher = okv,
 644                 },
 645                 [1] = {
 646                         .key = MACH_VOUCHER_ATTR_KEY_IMPORTANCE,
 647                         .command = MACH_VOUCHER_ATTR_REMOVE,
 648                 },
 649         };
 650         kr = _voucher_create_mach_voucher(importance_remove_recipe,
 651                         sizeof(importance_remove_recipe), &kv);
 652         if (dispatch_assume_zero(kr) || !kv) {
 653                 if (ov->v_ipc_kvoucher) return NULL;
 654                 kv = MACH_VOUCHER_NULL;
 655         }
 656         if (kv == okv) {
 657                 _voucher_dealloc_mach_voucher(kv);
 658                 return _voucher_retain(ov);
 659         }
 660         voucher_t v = _voucher_find_and_retain(kv);
 661         if (v && ov->v_ipc_kvoucher) {
 662                 _dispatch_voucher_debug("kvoucher[0x%08x] find without importance "
 663                                 "from voucher[%p]", v, kv, ov);
 664                 _voucher_dealloc_mach_voucher(kv);
 665                 return v;
 666         }
 667         voucher_t kvbase = v;
 668         voucher_fields_t ignore_fields = VOUCHER_FIELD_KVOUCHER;
 669         v = _voucher_clone(ov, ignore_fields);
 670         v->v_kvoucher = kv;
 671         if (ov->v_ipc_kvoucher) {
 672                 v->v_ipc_kvoucher = kv;
 673                 _voucher_insert(v);
 674         } else if (kvbase) {
 675                 v->v_kvbase = kvbase;
 676                 _voucher_dealloc_mach_voucher(kv); // borrow base reference
 677         }
 678         if (!kvbase) {
 679                 _dispatch_voucher_debug("kvoucher[0x%08x] create without importance "
 680                                 "from voucher[%p]", v, kv, ov);
 681         }
 682         _voucher_trace(CREATE, v, v->v_kvoucher, v->v_activity);
 683         return v;
 684 }
 685
 686 voucher_t
 687 _voucher_create_accounting_voucher(voucher_t ov)
 688 {
 689         // Nothing to do unless the old voucher has a kernel voucher. If it does
 690         // doesn't, it can't have any accounting attributes.
 691         if (!ov || !ov->v_kvoucher) return NULL;
 692         kern_return_t kr = KERN_SUCCESS;
 693         mach_voucher_t okv, kv = MACH_VOUCHER_NULL;
 694         okv = ov->v_ipc_kvoucher ? ov->v_ipc_kvoucher : ov->v_kvoucher;
 695         const mach_voucher_attr_recipe_data_t accounting_copy_recipe = {
 696                 .key = MACH_VOUCHER_ATTR_KEY_BANK,
 697                 .command = MACH_VOUCHER_ATTR_COPY,
 698                 .previous_voucher = okv,
 699         };
 700         kr = _voucher_create_mach_voucher(&accounting_copy_recipe,
 701                         sizeof(accounting_copy_recipe), &kv);
 702         if (dispatch_assume_zero(kr) || !kv) {
 703                 return NULL;
 704         }
 705         voucher_t v = _voucher_find_and_retain(kv);
 706         if (v) {
 707                 _dispatch_voucher_debug("kvoucher[0x%08x] find accounting voucher "
 708                                 "from voucher[%p]", v, kv, ov);
 709                 _voucher_dealloc_mach_voucher(kv);
 710                 return v;
 711         }
 712         v = _voucher_alloc(0);
 713         v->v_ipc_kvoucher = v->v_kvoucher = kv;
 714         if (kv == okv) {
 715                 v->v_kvbase = _voucher_retain(ov);
 716                 _voucher_dealloc_mach_voucher(kv); // borrow base reference
 717         }
 718         _voucher_trace(CREATE, v, kv, v->v_activity);
 719         _voucher_insert(v);
 720         _dispatch_voucher_debug("kvoucher[0x%08x] create accounting voucher "
 721                         "from voucher[%p]", v, kv, ov);
 722         return v;
 723 }
 724
 725 voucher_t
 726 voucher_create_with_mach_msg(mach_msg_header_t *msg)
 727 {
 728         mach_msg_bits_t msgh_bits;
 729         mach_voucher_t kv = _voucher_mach_msg_get(msg, &msgh_bits);
 730         return _voucher_create_with_mach_voucher(kv, msgh_bits);
 731 }
 732
 733 void
 734 voucher_decrement_importance_count4CF(voucher_t v)
 735 {
 736         if (!v || !v->v_kvoucher || !v->v_kv_has_importance) return;
 737         kern_return_t kr;
 738         mach_voucher_t kv = v->v_ipc_kvoucher ? v->v_ipc_kvoucher : v->v_kvoucher;
 739         uint32_t dec = 1;
 740         mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&dec;
 741         mach_voucher_attr_content_size_t kvc_in_size = sizeof(dec);
 742         mach_voucher_attr_content_t kvc_out = NULL;
 743         mach_voucher_attr_content_size_t kvc_out_size = 0;
 744 #if DISPATCH_DEBUG
 745         uint32_t count = UINT32_MAX;
 746         kvc_out = (mach_voucher_attr_content_t)&count;
 747         kvc_out_size = sizeof(count);
 748 #endif
 749         kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_IMPORTANCE,
 750                         MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL, kvc_in, kvc_in_size,
 751                         kvc_out, &kvc_out_size);
 752         DISPATCH_VERIFY_MIG(kr);
 753         if (kr == KERN_INVALID_TASK) return; // non-denap receiver rdar://25643185
 754 #if DISPATCH_DEBUG
 755         _dispatch_voucher_debug("kvoucher[0x%08x] decrement importance count to %u:"
 756                         " %s - 0x%x", v, kv, count, mach_error_string(kr), kr);
 757 #endif
 758         if (slowpath(dispatch_assume_zero(kr) == KERN_FAILURE)) {
 759                 DISPATCH_CLIENT_CRASH(kr, "Voucher importance count underflow");
 760         }
 761 }
 762
 763 #if VOUCHER_ENABLE_GET_MACH_VOUCHER
 764 mach_voucher_t
 765 voucher_get_mach_voucher(voucher_t voucher)
 766 {
 767         return _voucher_get_mach_voucher(voucher);
 768 }
 769 #endif
 770
 771 void
 772 _voucher_xref_dispose(voucher_t voucher)
 773 {
 774         _dispatch_voucher_debug("xref_dispose", voucher);
 775         _voucher_remove(voucher);
 776         return _os_object_release_internal_n_inline((_os_object_t)voucher, 1);
 777 }
 778
 779 void
 780 _voucher_dispose(voucher_t voucher)
 781 {
 782         _voucher_trace(DISPOSE, voucher);
 783         _dispatch_voucher_debug("dispose", voucher);
 784         if (slowpath(_voucher_hash_is_enqueued(voucher))) {
 785                 _dispatch_voucher_debug("corruption", voucher);
 786                 DISPATCH_CLIENT_CRASH(0, "Voucher corruption");
 787         }
 788         _voucher_hash_mark_not_enqueued(voucher);
 789         if (voucher->v_ipc_kvoucher) {
 790                 if (voucher->v_ipc_kvoucher != voucher->v_kvoucher) {
 791                         _voucher_dealloc_mach_voucher(voucher->v_ipc_kvoucher);
 792                 }
 793                 voucher->v_ipc_kvoucher = MACH_VOUCHER_NULL;
 794         }
 795         if (voucher->v_kvoucher) {
 796                 if (!voucher->v_kvbase) {
 797                         _voucher_dealloc_mach_voucher(voucher->v_kvoucher);
 798                 }
 799                 voucher->v_kvoucher = MACH_VOUCHER_NULL;
 800         }
 801         if (voucher->v_kvbase) {
 802                 _voucher_release(voucher->v_kvbase);
 803                 voucher->v_kvbase = NULL;
 804         }
 805         voucher->v_activity = 0;
 806         voucher->v_activity_creator = 0;
 807         voucher->v_parent_activity = 0;
 808         voucher->v_priority = 0;
 809 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 810         voucher->v_recipe_extra_size = 0;
 811         voucher->v_recipe_extra_offset = 0;
 812 #endif
 813         return _os_object_dealloc((_os_object_t)voucher);
 814 }
 815
 816 static void
 817 _voucher_activity_debug_channel_barrier_nop(void *ctxt DISPATCH_UNUSED)
 818 {
 819 }
 820
 821 void
 822 _voucher_activity_debug_channel_init(void)
 823 {
 824         dispatch_mach_handler_function_t handler = NULL;
 825
 826         if (_voucher_libtrace_hooks) {
 827                 handler = _voucher_libtrace_hooks->vah_debug_channel_handler;
 828         }
 829         if (!handler) return;
 830
 831         dispatch_mach_t dm;
 832         mach_port_t dbgp;
 833         kern_return_t kr;
 834
 835         kr = task_get_debug_control_port(mach_task_self(), &dbgp);
 836         DISPATCH_VERIFY_MIG(kr);
 837         if (kr) {
 838                 DISPATCH_CLIENT_CRASH(kr, "Couldn't get debug control port");
 839         }
 840         if (dbgp) {
 841                 dm = dispatch_mach_create_f("com.apple.debug-channel",
 842                                 DISPATCH_TARGET_QUEUE_DEFAULT, NULL, handler);
 843                 dm->dm_recv_refs->du_can_be_wlh = false; // 29906118
 844                 dispatch_mach_connect(dm, dbgp, MACH_PORT_NULL, NULL);
 845                 // will force the DISPATCH_MACH_CONNECTED event
 846                 dispatch_mach_send_barrier_f(dm, NULL,
 847                                 _voucher_activity_debug_channel_barrier_nop);
 848                 _voucher_activity_debug_channel = dm;
 849         }
 850 }
 851
 852 void
 853 _voucher_atfork_child(void)
 854 {
 855         _dispatch_thread_setspecific(dispatch_voucher_key, NULL);
 856         _voucher_task_mach_voucher_pred = 0;
 857         _voucher_task_mach_voucher = MACH_VOUCHER_NULL;
 858 #if !VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
 859         _voucher_default_task_mach_voucher = MACH_PORT_NULL;
 860 #endif
 861         _voucher_aid_next = 0;
 862         _firehose_task_buffer_pred = 0;
 863         _firehose_task_buffer = NULL; // firehose buffer is VM_INHERIT_NONE
 864 }
 865
 866 #if VOUCHER_EXPORT_PERSONA_SPI
 867 #if VOUCHER_USE_PERSONA
 868 static kern_return_t
 869 _voucher_get_current_persona_token(struct persona_token *token)
 870 {
 871         kern_return_t kr = KERN_FAILURE;
 872         voucher_t v = _voucher_get();
 873
 874         if (v && v->v_kvoucher) {
 875                 mach_voucher_t kv = v->v_ipc_kvoucher ?: v->v_kvoucher;
 876                 mach_voucher_attr_content_t kvc_in = NULL;
 877                 mach_voucher_attr_content_size_t kvc_in_size = 0;
 878                 mach_voucher_attr_content_t kvc_out =
 879                         (mach_voucher_attr_content_t)token;
 880                 mach_voucher_attr_content_size_t kvc_out_size = sizeof(*token);
 881
 882                 kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_BANK,
 883                                 BANK_PERSONA_TOKEN, kvc_in, kvc_in_size,
 884                                 kvc_out, &kvc_out_size);
 885                 if (kr != KERN_NOT_SUPPORTED
 886                                 // Voucher doesn't have a PERSONA_TOKEN
 887                                 && kr != KERN_INVALID_VALUE
 888                                 // Kernel doesn't understand BANK_PERSONA_TOKEN
 889                                 && kr != KERN_INVALID_ARGUMENT) {
 890                         (void)dispatch_assume_zero(kr);
 891                 }
 892         }
 893         return kr;
 894 }
 895 #endif
 896
 897 uid_t
 898 voucher_get_current_persona(void)
 899 {
 900         uid_t persona_id = PERSONA_ID_NONE;
 901
 902 #if VOUCHER_USE_PERSONA
 903         struct persona_token token;
 904         int err;
 905
 906         if (_voucher_get_current_persona_token(&token) == KERN_SUCCESS) {
 907                 return token.originator.persona_id;
 908         }
 909
 910         // falling back to the process persona if there is no adopted voucher
 911         if (kpersona_get(&persona_id) < 0) {
 912                 err = errno;
 913                 if (err != ESRCH) {
 914                         (void)dispatch_assume_zero(err);
 915                 }
 916         }
 917 #endif
 918         return persona_id;
 919 }
 920
 921 int
 922 voucher_get_current_persona_originator_info(struct proc_persona_info *persona_info)
 923 {
 924 #if VOUCHER_USE_PERSONA
 925         struct persona_token token;
 926         if (_voucher_get_current_persona_token(&token) == KERN_SUCCESS) {
 927                 *persona_info = token.originator;
 928                 return 0;
 929         }
 930 #else
 931         (void)persona_info;
 932 #endif
 933         return -1;
 934 }
 935
 936 int
 937 voucher_get_current_persona_proximate_info(struct proc_persona_info *persona_info)
 938 {
 939 #if VOUCHER_USE_PERSONA
 940         struct persona_token token;
 941         if (_voucher_get_current_persona_token(&token) == KERN_SUCCESS) {
 942                 *persona_info = token.proximate;
 943                 return 0;
 944         }
 945 #else
 946         (void)persona_info;
 947 #endif
 948         return -1;
 949 }
 950 #endif
 951
 952 #pragma mark -
 953 #pragma mark _voucher_init
 954
 955 boolean_t
 956 voucher_mach_msg_set(mach_msg_header_t *msg)
 957 {
 958         return _voucher_mach_msg_set(msg, _voucher_get());
 959 }
 960
 961 void
 962 voucher_mach_msg_clear(mach_msg_header_t *msg)
 963 {
 964         (void)_voucher_mach_msg_clear(msg, false);
 965 }
 966
 967 voucher_mach_msg_state_t
 968 voucher_mach_msg_adopt(mach_msg_header_t *msg)
 969 {
 970         mach_msg_bits_t msgh_bits;
 971         mach_voucher_t kv = _voucher_mach_msg_get(msg, &msgh_bits);
 972         if (!kv) return VOUCHER_MACH_MSG_STATE_UNCHANGED;
 973         voucher_t v = _voucher_create_with_mach_voucher(kv, msgh_bits);
 974         return (voucher_mach_msg_state_t)_voucher_adopt(v);
 975 }
 976
 977 void
 978 voucher_mach_msg_revert(voucher_mach_msg_state_t state)
 979 {
 980         if (state == VOUCHER_MACH_MSG_STATE_UNCHANGED) return;
 981         _voucher_replace((voucher_t)state);
 982 }
 983
 984 #if DISPATCH_USE_LIBKERNEL_VOUCHER_INIT
 985 #include <_libkernel_init.h>
 986
 987 static const struct _libkernel_voucher_functions _voucher_libkernel_functions =
 988 {
 989         .version = 1,
 990         .voucher_mach_msg_set = voucher_mach_msg_set,
 991         .voucher_mach_msg_clear = voucher_mach_msg_clear,
 992         .voucher_mach_msg_adopt = voucher_mach_msg_adopt,
 993         .voucher_mach_msg_revert = voucher_mach_msg_revert,
 994 };
 995
 996 static void
 997 _voucher_libkernel_init(void)
 998 {
 999         kern_return_t kr = __libkernel_voucher_init(&_voucher_libkernel_functions);
1000         dispatch_assert(!kr);
1001 }
1002 #else
1003 #define _voucher_libkernel_init()
1004 #endif
1005
1006 void
1007 voucher_activity_initialize_4libtrace(voucher_activity_hooks_t hooks)
1008 {
1009         if (hooks->vah_version < 3) {
1010                 DISPATCH_CLIENT_CRASH(hooks->vah_version, "unsupported vah_version");
1011         }
1012         if (!os_atomic_cmpxchg(&_voucher_libtrace_hooks, NULL,
1013                         hooks, relaxed)) {
1014                 DISPATCH_CLIENT_CRASH(_voucher_libtrace_hooks,
1015                                 "voucher_activity_initialize_4libtrace called twice");
1016         }
1017 }
1018
1019 void
1020 _voucher_init(void)
1021 {
1022         _voucher_libkernel_init();
1023         unsigned int i;
1024         for (i = 0; i < VL_HASH_SIZE; i++) {
1025                 _voucher_hash_head_init(&_voucher_hash[i]);
1026         }
1027 }
1028
1029 #pragma mark -
1030 #pragma mark voucher_activity_t
1031
1032 DISPATCH_NOINLINE
1033 static uint64_t
1034 _voucher_activity_id_allocate_slow(uint64_t aid)
1035 {
1036         kern_return_t kr;
1037         uint64_t next;
1038
1039         kr = mach_generate_activity_id(mach_task_self(), 1, &next);
1040         if (unlikely(kr)) {
1041                 DISPATCH_CLIENT_CRASH(kr, "Could not generate an activity ID");
1042         }
1043         next *= MACH_ACTIVITY_ID_RANGE_SIZE;
1044         next &= MACH_ACTIVITY_ID_MASK;
1045         if (unlikely(next == 0)) {
1046                 next++;
1047         }
1048
1049         if (unlikely(aid == 0)) {
1050                 if (os_atomic_cmpxchg(&_voucher_aid_next, 0, next + 1, relaxed)) {
1051                         return next;
1052                 }
1053         }
1054         return os_atomic_xchg(&_voucher_aid_next, next, relaxed);
1055 }
1056
1057 DISPATCH_ALWAYS_INLINE
1058 static firehose_activity_id_t
1059 _voucher_activity_id_allocate(firehose_activity_flags_t flags)
1060 {
1061         uint64_t aid, next;
1062         os_atomic_rmw_loop(&_voucher_aid_next, aid, next, relaxed, {
1063                 next = aid + 1;
1064                 if (aid == 0 || next % MACH_ACTIVITY_ID_RANGE_SIZE == 0) {
1065                         os_atomic_rmw_loop_give_up({
1066                                 aid = _voucher_activity_id_allocate_slow(aid);
1067                                 break;
1068                         });
1069                 }
1070         });
1071         return FIREHOSE_ACTIVITY_ID_MAKE(aid, flags);
1072 }
1073
1074 firehose_activity_id_t
1075 voucher_activity_id_allocate(firehose_activity_flags_t flags)
1076 {
1077         return _voucher_activity_id_allocate(flags);
1078 }
1079
1080 #define _voucher_activity_tracepoint_reserve(stamp, stream, pub, priv, privbuf) \
1081                 firehose_buffer_tracepoint_reserve(_firehose_task_buffer, stamp, \
1082                                 stream, pub, priv, privbuf)
1083
1084 #define _voucher_activity_tracepoint_flush(ft, ftid) \
1085                 firehose_buffer_tracepoint_flush(_firehose_task_buffer, ft, ftid)
1086
1087 DISPATCH_NOINLINE
1088 static void
1089 _firehose_task_buffer_init(void *ctx OS_UNUSED)
1090 {
1091         mach_port_t logd_port;
1092
1093         /* Query the uniquepid of the current process */
1094         struct proc_uniqidentifierinfo p_uniqinfo = { };
1095         int info_size = 0;
1096
1097         info_size = proc_pidinfo(getpid(), PROC_PIDUNIQIDENTIFIERINFO, 1,
1098                         &p_uniqinfo, PROC_PIDUNIQIDENTIFIERINFO_SIZE);
1099         if (slowpath(info_size != PROC_PIDUNIQIDENTIFIERINFO_SIZE)) {
1100                 if (info_size == 0) {
1101                         DISPATCH_INTERNAL_CRASH(errno,
1102                                 "Unable to get the unique pid (error)");
1103                 } else {
1104                         DISPATCH_INTERNAL_CRASH(info_size,
1105                                 "Unable to get the unique pid (size)");
1106                 }
1107         }
1108         _voucher_unique_pid = p_uniqinfo.p_uniqueid;
1109
1110
1111         if (!fastpath(_voucher_libtrace_hooks)) {
1112                 if (0) { // <rdar://problem/23393959>
1113                         DISPATCH_CLIENT_CRASH(0,
1114                                         "Activity subsystem isn't initialized yet");
1115                 }
1116                 return;
1117         }
1118         logd_port = _voucher_libtrace_hooks->vah_get_logd_port();
1119         if (logd_port) {
1120                 unsigned long flags = 0;
1121 #if DISPATCH_USE_MEMORYPRESSURE_SOURCE
1122                 if (_dispatch_memory_warn) {
1123                         flags |= FIREHOSE_BUFFER_BANK_FLAG_LOW_MEMORY;
1124                 }
1125 #endif
1126                 // firehose_buffer_create always consumes the send-right
1127                 _firehose_task_buffer = firehose_buffer_create(logd_port,
1128                                 _voucher_unique_pid, flags);
1129                 if (_voucher_libtrace_hooks->vah_version >= 4 &&
1130                                 _voucher_libtrace_hooks->vah_metadata_init) {
1131                         firehose_buffer_t fb = _firehose_task_buffer;
1132                         size_t meta_sz = FIREHOSE_BUFFER_LIBTRACE_HEADER_SIZE;
1133                         void *meta = (void *)((uintptr_t)(&fb->fb_header + 1) - meta_sz);
1134                         _voucher_libtrace_hooks->vah_metadata_init(meta, meta_sz);
1135                 }
1136         }
1137 }
1138
1139 DISPATCH_ALWAYS_INLINE
1140 static inline bool
1141 _voucher_activity_disabled(void)
1142 {
1143         dispatch_once_f(&_firehose_task_buffer_pred,
1144                         NULL, _firehose_task_buffer_init);
1145
1146         firehose_buffer_t fb = _firehose_task_buffer;
1147         if (fastpath(fb)) {
1148                 return slowpath(fb->fb_header.fbh_sendp == MACH_PORT_DEAD);
1149         }
1150         return true;
1151 }
1152
1153 void*
1154 voucher_activity_get_metadata_buffer(size_t *length)
1155 {
1156         if (_voucher_activity_disabled()) {
1157                 *length = 0;
1158                 return NULL;
1159         }
1160
1161         firehose_buffer_header_t fbh = &_firehose_task_buffer->fb_header;
1162
1163         *length = FIREHOSE_BUFFER_LIBTRACE_HEADER_SIZE;
1164         return (void *)((uintptr_t)(fbh + 1) - *length);
1165 }
1166
1167 voucher_t
1168 voucher_activity_create_with_data(firehose_tracepoint_id_t *trace_id,
1169                 voucher_t base, firehose_activity_flags_t flags,
1170                 const void *pubdata, size_t publen)
1171 {
1172         firehose_activity_id_t va_id = 0, current_id = 0, parent_id = 0;
1173         firehose_tracepoint_id_u ftid = { .ftid_value = *trace_id };
1174         uint64_t creator_id = 0;
1175         uint16_t pubsize;
1176         voucher_t ov = _voucher_get();
1177         voucher_t v;
1178
1179         if (os_add_overflow(sizeof(va_id), publen, &pubsize) || pubsize > 128) {
1180                 DISPATCH_CLIENT_CRASH(pubsize, "Absurd publen");
1181         }
1182         if (base == VOUCHER_CURRENT) {
1183                 base = ov;
1184         }
1185
1186         FIREHOSE_TRACE_ID_CLEAR_FLAG(ftid, base, has_unique_pid);
1187         if (ov && (current_id = ov->v_activity)) {
1188                 FIREHOSE_TRACE_ID_SET_FLAG(ftid, base, has_current_aid);
1189                 pubsize += sizeof(firehose_activity_id_t);
1190                 if ((creator_id = ov->v_activity_creator)) {
1191                         FIREHOSE_TRACE_ID_SET_FLAG(ftid, base, has_unique_pid);
1192                         pubsize += sizeof(uint64_t);
1193                 }
1194         }
1195         if (base != VOUCHER_NULL) {
1196                 parent_id = base->v_activity;
1197         }
1198
1199         if (parent_id) {
1200                 FIREHOSE_TRACE_ID_SET_FLAG(ftid, activity, has_other_aid);
1201                 pubsize += sizeof(firehose_activity_id_t);
1202                 flags |= FIREHOSE_ACTIVITY_ID_FLAGS(parent_id);
1203         }
1204
1205         if (firehose_precise_timestamps_enabled()) {
1206                 flags |= firehose_activity_flags_precise_timestamp;
1207         }
1208         voucher_fields_t ignore_fields = VOUCHER_FIELD_ACTIVITY;
1209         v = _voucher_clone(base, ignore_fields);
1210         v->v_activity = va_id = _voucher_activity_id_allocate(flags);
1211         v->v_activity_creator = _voucher_unique_pid;
1212         v->v_parent_activity = parent_id;
1213
1214         if (_voucher_activity_disabled()) {
1215                 goto done;
1216         }
1217
1218         static const firehose_stream_t streams[2] = {
1219                 firehose_stream_metadata,
1220                 firehose_stream_persist,
1221         };
1222         firehose_tracepoint_t ft;
1223         uint64_t stamp = firehose_tracepoint_time(flags);
1224
1225         for (size_t i = 0; i < countof(streams); i++) {
1226                 ft = _voucher_activity_tracepoint_reserve(stamp, streams[i], pubsize,
1227                                 0, NULL);
1228                 if (!fastpath(ft)) continue;
1229
1230                 uint8_t *pubptr = ft->ft_data;
1231                 if (current_id) {
1232                         pubptr = _dispatch_memappend(pubptr, &current_id);
1233                 }
1234                 if (creator_id) {
1235                         pubptr = _dispatch_memappend(pubptr, &creator_id);
1236                 }
1237                 if (parent_id) {
1238                         pubptr = _dispatch_memappend(pubptr, &parent_id);
1239                 }
1240                 pubptr = _dispatch_memappend(pubptr, &va_id);
1241                 pubptr = _dispatch_mempcpy(pubptr, pubdata, publen);
1242                 _voucher_activity_tracepoint_flush(ft, ftid);
1243         }
1244 done:
1245         *trace_id = ftid.ftid_value;
1246         _voucher_trace(CREATE, v, v->v_kvoucher, va_id);
1247         return v;
1248 }
1249
1250 voucher_t
1251 voucher_activity_create_with_location(firehose_tracepoint_id_t *trace_id,
1252                 voucher_t base, firehose_activity_flags_t flags, uint64_t loc)
1253 {
1254         return voucher_activity_create_with_data(trace_id, base, flags,
1255                         &loc, sizeof(loc));
1256 }
1257
1258 #if OS_VOUCHER_ACTIVITY_GENERATE_SWAPS
1259 void
1260 _voucher_activity_swap(firehose_activity_id_t old_id,
1261                 firehose_activity_id_t new_id)
1262 {
1263         if (_voucher_activity_disabled()) return;
1264
1265         firehose_tracepoint_id_u ftid = { .ftid = {
1266                 ._namespace = firehose_tracepoint_namespace_activity,
1267                 ._type = _firehose_tracepoint_type_activity_swap,
1268         } };
1269         uint16_t pubsize = 0;
1270
1271         if (old_id) {
1272                 FIREHOSE_TRACE_ID_SET_FLAG(ftid, base, has_current_aid);
1273                 pubsize += sizeof(firehose_activity_id_t);
1274         }
1275         if (new_id) {
1276                 FIREHOSE_TRACE_ID_SET_FLAG(ftid, activity, has_other_aid);
1277                 pubsize += sizeof(firehose_activity_id_t);
1278         }
1279
1280         firehose_stream_t stream = firehose_stream_metadata;
1281         firehose_tracepoint_t ft;
1282         firehose_activity_flags_t flags = FIREHOSE_ACTIVITY_ID_FLAGS(old_id) |
1283                         FIREHOSE_ACTIVITY_ID_FLAGS(new_id);
1284         uint64_t stamp = firehose_tracepoint_time(flags);
1285
1286         _dispatch_voucher_ktrace_activity_adopt(new_id);
1287
1288         ft = _voucher_activity_tracepoint_reserve(stamp, stream, pubsize, 0, NULL);
1289         if (!fastpath(ft)) return;
1290         uint8_t *pubptr = ft->ft_data;
1291         if (old_id) pubptr = _dispatch_memappend(pubptr, &old_id);
1292         if (new_id) pubptr = _dispatch_memappend(pubptr, &new_id);
1293         _voucher_activity_tracepoint_flush(ft, ftid);
1294 }
1295 #endif
1296
1297 firehose_activity_id_t
1298 voucher_get_activity_id_and_creator(voucher_t v, uint64_t *creator_pid,
1299                 firehose_activity_id_t *parent_id)
1300 {
1301         if (v == VOUCHER_CURRENT) {
1302                 v = _voucher_get();
1303         }
1304         if (v == VOUCHER_NULL) {
1305                 if (creator_pid) *creator_pid = 0;
1306                 if (parent_id) *parent_id = FIREHOSE_ACTIVITY_ID_NULL;
1307                 return FIREHOSE_ACTIVITY_ID_NULL;
1308         }
1309         if (creator_pid) *creator_pid = v->v_activity_creator;
1310         if (parent_id) *parent_id = v->v_parent_activity;
1311         return v->v_activity;
1312 }
1313
1314 firehose_activity_id_t
1315 voucher_get_activity_id(voucher_t v, firehose_activity_id_t *parent_id)
1316 {
1317         return voucher_get_activity_id_and_creator(v, NULL, parent_id);
1318 }
1319
1320 void
1321 voucher_activity_flush(firehose_stream_t stream)
1322 {
1323         if (_voucher_activity_disabled()) return;
1324         firehose_buffer_stream_flush(_firehose_task_buffer, stream);
1325 }
1326
1327 DISPATCH_NOINLINE
1328 firehose_tracepoint_id_t
1329 voucher_activity_trace_v(firehose_stream_t stream,
1330                 firehose_tracepoint_id_t trace_id, uint64_t stamp,
1331                 const struct iovec *iov, size_t publen, size_t privlen)
1332 {
1333         firehose_tracepoint_id_u ftid = { .ftid_value = trace_id };
1334         const uint16_t ft_size = offsetof(struct firehose_tracepoint_s, ft_data);
1335         const size_t _firehose_chunk_payload_size =
1336                         sizeof(((struct firehose_chunk_s *)0)->fc_data);
1337
1338         if (_voucher_activity_disabled()) return 0;
1339
1340         firehose_tracepoint_t ft;
1341         firehose_activity_id_t va_id = 0;
1342         firehose_chunk_t fc;
1343         uint8_t *privptr, *pubptr;
1344         size_t pubsize = publen;
1345         voucher_t ov = _voucher_get();
1346         uint64_t creator_pid;
1347
1348         if ((va_id = _voucher_get_activity_id(ov, &creator_pid))) {
1349                 FIREHOSE_TRACE_ID_SET_FLAG(ftid, base, has_current_aid);
1350                 pubsize += sizeof(va_id);
1351         }
1352         if (FIREHOSE_TRACE_ID_HAS_FLAG(ftid, base, has_unique_pid)) {
1353                 if (creator_pid) {
1354                         pubsize += sizeof(creator_pid);
1355                 } else {
1356                         FIREHOSE_TRACE_ID_CLEAR_FLAG(ftid, base, has_unique_pid);
1357                 }
1358         } else {
1359                 creator_pid = 0;
1360         }
1361
1362         if (privlen) {
1363                 FIREHOSE_TRACE_ID_SET_FLAG(ftid, log, has_private_data);
1364                 pubsize += sizeof(struct firehose_buffer_range_s);
1365         }
1366
1367         if (slowpath(ft_size + pubsize + privlen > _firehose_chunk_payload_size)) {
1368                 DISPATCH_CLIENT_CRASH(ft_size + pubsize + privlen, "Log is too large");
1369         }
1370
1371         ft = _voucher_activity_tracepoint_reserve(stamp, stream, (uint16_t)pubsize,
1372                                 (uint16_t)privlen, &privptr);
1373         if (!fastpath(ft)) return 0;
1374         pubptr = ft->ft_data;
1375         if (va_id) {
1376                 pubptr = _dispatch_memappend(pubptr, &va_id);
1377         }
1378         if (creator_pid) {
1379                 pubptr = _dispatch_memappend(pubptr, &creator_pid);
1380         }
1381         if (privlen) {
1382                 fc = firehose_buffer_chunk_for_address(ft);
1383                 struct firehose_buffer_range_s range = {
1384                         .fbr_offset = (uint16_t)(privptr - fc->fc_start),
1385                         .fbr_length = (uint16_t)privlen,
1386                 };
1387                 pubptr = _dispatch_memappend(pubptr, &range);
1388         }
1389         while (publen > 0) {
1390                 pubptr = _dispatch_mempcpy(pubptr, iov->iov_base, iov->iov_len);
1391                 if (unlikely(os_sub_overflow(publen, iov->iov_len, &publen))) {
1392                         DISPATCH_CLIENT_CRASH(0, "Invalid arguments");
1393                 }
1394                 iov++;
1395         }
1396         while (privlen > 0) {
1397                 privptr = _dispatch_mempcpy(privptr, iov->iov_base, iov->iov_len);
1398                 if (unlikely(os_sub_overflow(privlen, iov->iov_len, &privlen))) {
1399                         DISPATCH_CLIENT_CRASH(0, "Invalid arguments");
1400                 }
1401                 iov++;
1402         }
1403         _voucher_activity_tracepoint_flush(ft, ftid);
1404         return ftid.ftid_value;
1405 }
1406
1407 firehose_tracepoint_id_t
1408 voucher_activity_trace(firehose_stream_t stream,
1409                 firehose_tracepoint_id_t trace_id, uint64_t stamp,
1410                 const void *pubdata, size_t publen)
1411 {
1412         struct iovec iov = { (void *)pubdata, publen };
1413         return voucher_activity_trace_v(stream, trace_id, stamp, &iov, publen, 0);
1414 }
1415
1416 firehose_tracepoint_id_t
1417 voucher_activity_trace_with_private_strings(firehose_stream_t stream,
1418                 firehose_tracepoint_id_t trace_id, uint64_t stamp,
1419                 const void *pubdata, size_t publen,
1420                 const void *privdata, size_t privlen)
1421 {
1422         struct iovec iov[2] = {
1423                 { (void *)pubdata, publen },
1424                 { (void *)privdata, privlen },
1425         };
1426         return voucher_activity_trace_v(stream, trace_id, stamp,
1427                         iov, publen, privlen);
1428 }
1429
1430 #pragma mark -
1431 #pragma mark _voucher_debug
1432
1433 size_t
1434 _voucher_debug(voucher_t v, char* buf, size_t bufsiz)
1435 {
1436         size_t offset = 0;
1437         #define bufprintf(...) \
1438                         offset += dsnprintf(&buf[offset], bufsiz - offset, ##__VA_ARGS__)
1439         bufprintf("voucher[%p] = { xref = %d, ref = %d", v,
1440                         v->os_obj_xref_cnt + 1, v->os_obj_ref_cnt + 1);
1441
1442         if (v->v_kvbase) {
1443                 bufprintf(", base voucher %p", v->v_kvbase);
1444         }
1445         if (v->v_kvoucher) {
1446                 bufprintf(", kvoucher%s 0x%x", v->v_kvoucher == v->v_ipc_kvoucher ?
1447                                 " & ipc kvoucher" : "", v->v_kvoucher);
1448         }
1449         if (v->v_ipc_kvoucher && v->v_ipc_kvoucher != v->v_kvoucher) {
1450                 bufprintf(", ipc kvoucher 0x%x", v->v_ipc_kvoucher);
1451         }
1452         if (v->v_priority) {
1453                 bufprintf(", QOS 0x%x", v->v_priority);
1454         }
1455         if (v->v_activity) {
1456                 bufprintf(", activity 0x%llx (pid: 0x%16llx, parent 0x%llx)",
1457                                 v->v_activity, v->v_activity_creator, v->v_parent_activity);
1458         }
1459         bufprintf(" }");
1460         return offset;
1461 }
1462
1463 #else // VOUCHER_USE_MACH_VOUCHER
1464
1465 #pragma mark -
1466 #pragma mark Simulator / vouchers disabled
1467
1468 #if VOUCHER_ENABLE_RECIPE_OBJECTS
1469 voucher_t
1470 voucher_create(voucher_recipe_t recipe)
1471 {
1472         (void)recipe;
1473         return NULL;
1474 }
1475 #endif // VOUCHER_ENABLE_RECIPE_OBJECTS
1476
1477 voucher_t
1478 voucher_adopt(voucher_t voucher)
1479 {
1480         return voucher;
1481 }
1482
1483 voucher_t
1484 voucher_copy(void)
1485 {
1486         return NULL;
1487 }
1488
1489 voucher_t
1490 voucher_copy_without_importance(void)
1491 {
1492         return NULL;
1493 }
1494
1495 voucher_t
1496 voucher_retain(voucher_t voucher)
1497 {
1498         return voucher;
1499 }
1500
1501 void
1502 voucher_release(voucher_t voucher)
1503 {
1504         (void)voucher;
1505 }
1506
1507 void
1508 voucher_replace_default_voucher(void)
1509 {
1510 }
1511
1512 void
1513 voucher_decrement_importance_count4CF(voucher_t v)
1514 {
1515         (void)v;
1516 }
1517
1518 void
1519 _voucher_thread_cleanup(void *voucher)
1520 {
1521         (void)voucher;
1522 }
1523
1524 void
1525 _voucher_dealloc_mach_voucher(mach_voucher_t kv)
1526 {
1527         (void)kv;
1528 }
1529
1530 mach_voucher_t
1531 _voucher_get_mach_voucher(voucher_t voucher)
1532 {
1533         (void)voucher;
1534         return MACH_VOUCHER_NULL;
1535 }
1536
1537 mach_voucher_t
1538 _voucher_create_mach_voucher_with_priority(voucher_t voucher,
1539                 pthread_priority_t priority)
1540 {
1541         (void)voucher; (void)priority;
1542         return MACH_VOUCHER_NULL;
1543 }
1544
1545 voucher_t
1546 _voucher_create_with_priority_and_mach_voucher(voucher_t voucher,
1547                 pthread_priority_t priority, mach_voucher_t kv)
1548 {
1549         (void)voucher; (void)priority; (void)kv;
1550         return NULL;
1551 }
1552
1553 voucher_t
1554 _voucher_create_accounting_voucher(voucher_t voucher)
1555 {
1556         (void)voucher;
1557         return NULL;
1558 }
1559
1560 #if HAVE_MACH
1561 voucher_t
1562 voucher_create_with_mach_msg(mach_msg_header_t *msg)
1563 {
1564         (void)msg;
1565         return NULL;
1566 }
1567 #endif
1568
1569 #if VOUCHER_ENABLE_GET_MACH_VOUCHER
1570 mach_voucher_t
1571 voucher_get_mach_voucher(voucher_t voucher)
1572 {
1573         (void)voucher;
1574         return 0;
1575 }
1576 #endif // VOUCHER_ENABLE_GET_MACH_VOUCHER
1577
1578 void
1579 _voucher_xref_dispose(voucher_t voucher)
1580 {
1581         (void)voucher;
1582 }
1583
1584 void
1585 _voucher_dispose(voucher_t voucher)
1586 {
1587         (void)voucher;
1588 }
1589
1590 #if VOUCHER_EXPORT_PERSONA_SPI
1591 uid_t
1592 voucher_get_current_persona(void)
1593 {
1594         return PERSONA_ID_NONE;
1595 }
1596
1597 int
1598 voucher_get_current_persona_originator_info(struct proc_persona_info *persona_info)
1599 {
1600         (void)persona_info;
1601         return -1;
1602 }
1603
1604 int
1605 voucher_get_current_persona_proximate_info(struct proc_persona_info *persona_info)
1606 {
1607         (void)persona_info;
1608         return -1;
1609 }
1610 #endif // VOUCHER_EXPORT_PERSONA_SPI
1611
1612 void
1613 _voucher_activity_debug_channel_init(void)
1614 {
1615 }
1616
1617 void
1618 _voucher_atfork_child(void)
1619 {
1620 }
1621
1622 void
1623 _voucher_init(void)
1624 {
1625 }
1626
1627 #if OS_VOUCHER_ACTIVITY_SPI
1628 void*
1629 voucher_activity_get_metadata_buffer(size_t *length)
1630 {
1631     *length = 0;
1632     return NULL;
1633 }
1634
1635 voucher_t
1636 voucher_activity_create(firehose_tracepoint_id_t trace_id,
1637                 voucher_t base, firehose_activity_flags_t flags, uint64_t location)
1638 {
1639         (void)trace_id; (void)base; (void)flags; (void)location;
1640         return NULL;
1641 }
1642
1643 voucher_t
1644 voucher_activity_create_with_location(firehose_tracepoint_id_t *trace_id,
1645                 voucher_t base, firehose_activity_flags_t flags, uint64_t location)
1646 {
1647         (void)trace_id; (void)base; (void)flags; (void)location;
1648         return NULL;
1649 }
1650
1651 firehose_activity_id_t
1652 voucher_get_activity_id(voucher_t voucher, firehose_activity_id_t *parent_id)
1653 {
1654         (void)voucher; (void)parent_id;
1655         return 0;
1656 }
1657
1658 firehose_activity_id_t
1659 voucher_get_activity_id_and_creator(voucher_t voucher, uint64_t *creator_pid,
1660                 firehose_activity_id_t *parent_id)
1661 {
1662         if (creator_pid) *creator_pid = 0;
1663         (void)voucher; (void)parent_id;
1664         return 0;
1665 }
1666
1667 firehose_tracepoint_id_t
1668 voucher_activity_trace(firehose_stream_t stream,
1669                 firehose_tracepoint_id_t trace_id, uint64_t timestamp,
1670                 const void *pubdata, size_t publen)
1671 {
1672         (void)stream; (void)trace_id; (void)timestamp;
1673         (void)pubdata; (void)publen;
1674         return 0;
1675 }
1676
1677 firehose_tracepoint_id_t
1678 voucher_activity_trace_with_private_strings(firehose_stream_t stream,
1679                 firehose_tracepoint_id_t trace_id, uint64_t timestamp,
1680                 const void *pubdata, size_t publen,
1681                 const void *privdata, size_t privlen)
1682 {
1683         (void)stream; (void)trace_id; (void)timestamp;
1684         (void)pubdata; (void)publen; (void)privdata; (void)privlen;
1685         return 0;
1686 }
1687
1688 firehose_tracepoint_id_t
1689 voucher_activity_trace_v(firehose_stream_t stream,
1690                 firehose_tracepoint_id_t trace_id, uint64_t timestamp,
1691                 const struct iovec *iov, size_t publen, size_t privlen)
1692 {
1693         (void)stream; (void)trace_id; (void)timestamp;
1694         (void)iov; (void)publen; (void)privlen;
1695         return 0;
1696 }
1697
1698 void
1699 voucher_activity_flush(firehose_stream_t stream)
1700 {
1701         (void)stream;
1702 }
1703
1704 void
1705 voucher_activity_initialize_4libtrace(voucher_activity_hooks_t hooks)
1706 {
1707         (void)hooks;
1708 }
1709 #endif // OS_VOUCHER_ACTIVITY_SPI
1710
1711 size_t
1712 _voucher_debug(voucher_t v, char* buf, size_t bufsiz)
1713 {
1714         (void)v; (void)buf; (void)bufsiz;
1715         return 0;
1716 }
1717
1718 #endif // VOUCHER_USE_MACH_VOUCHER
1719
1720 #else // DISPATCH_VARIANT_DYLD_STUB
1721
1722 firehose_activity_id_t
1723 voucher_get_activity_id_4dyld(void)
1724 {
1725 #if VOUCHER_USE_MACH_VOUCHER
1726         return _voucher_get_activity_id(_voucher_get(), NULL);
1727 #else
1728         return 0;
1729 #endif
1730 }
1731
1732 #endif // DISPATCH_VARIANT_DYLD_STUB