src/voucher.c

   1 /*
   2  * Copyright (c) 2013-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 #include "internal.h"
  22
  23 #if !defined(VOUCHER_EXPORT_PERSONA_SPI)
  24 #if TARGET_OS_IPHONE
  25 #define VOUCHER_EXPORT_PERSONA_SPI 1
  26 #else
  27 #define VOUCHER_EXPORT_PERSONA_SPI 0
  28 #endif
  29 #endif
  30
  31 #ifndef PERSONA_ID_NONE
  32 #define PERSONA_ID_NONE ((uid_t)-1)
  33 #endif
  34
  35 #if !DISPATCH_VARIANT_DYLD_STUB
  36
  37 #if VOUCHER_USE_MACH_VOUCHER
  38 #if !HAVE_PTHREAD_WORKQUEUE_QOS
  39 #error Unsupported configuration, workqueue QoS support is required
  40 #endif
  41 #include <mach/mach_voucher.h>
  42 #include <sys/proc_info.h>
  43
  44 #define MACH_ACTIVITY_ID_RANGE_SIZE 16
  45 #define MACH_ACTIVITY_ID_MASK ((1ULL << FIREHOSE_ACTIVITY_ID_FLAGS_SHIFT) - 1)
  46 #define FIREHOSE_ACTIVITY_ID_MAKE(aid, flags) \
  47                 FIREHOSE_ACTIVITY_ID_MERGE_FLAGS((aid) & MACH_ACTIVITY_ID_MASK, flags)
  48
  49 static volatile uint64_t _voucher_aid_next;
  50
  51 #pragma mark -
  52 #pragma mark voucher_t
  53
  54 OS_OBJECT_CLASS_DECL(voucher, object);
  55 #if !USE_OBJC
  56 OS_OBJECT_VTABLE_INSTANCE(voucher,
  57                 (void (*)(_os_object_t))_voucher_xref_dispose,
  58                 (void (*)(_os_object_t))_voucher_dispose);
  59 #endif // USE_OBJC
  60 #define VOUCHER_CLASS OS_OBJECT_VTABLE(voucher)
  61
  62 static inline voucher_t
  63 _voucher_alloc(mach_voucher_attr_recipe_size_t extra)
  64 {
  65         voucher_t voucher;
  66         size_t voucher_size = sizeof(voucher_s) + extra;
  67         voucher = (voucher_t)_os_object_alloc_realized(VOUCHER_CLASS, voucher_size);
  68 #if VOUCHER_ENABLE_RECIPE_OBJECTS
  69         voucher->v_recipe_extra_size = extra;
  70         voucher->v_recipe_extra_offset = voucher_size - extra;
  71 #else
  72         dispatch_assert(!extra);
  73 #endif
  74         _dispatch_voucher_debug("alloc", voucher);
  75         return voucher;
  76 }
  77
  78 #if VOUCHER_ENABLE_RECIPE_OBJECTS
  79 voucher_t
  80 voucher_create(voucher_recipe_t recipe)
  81 {
  82         // TODO: capture current activities or current kvoucher ?
  83         mach_voucher_attr_recipe_size_t extra = recipe ? recipe->vr_size : 0;
  84         voucher_t voucher = _voucher_alloc(extra);
  85         if (extra) {
  86                 memcpy(_voucher_extra_recipes(voucher), recipe->vr_data, extra);
  87         }
  88         return voucher;
  89 }
  90 #endif
  91
  92 DISPATCH_ALWAYS_INLINE
  93 static inline voucher_t
  94 _voucher_clone(const voucher_t ov, voucher_fields_t ignore_fields)
  95 {
  96         mach_voucher_attr_recipe_size_t extra = 0;
  97         voucher_t v;
  98
  99         if (ov && !(ignore_fields & VOUCHER_FIELD_EXTRA)) {
 100                 extra = _voucher_extra_size(ov);
 101         }
 102         v = _voucher_alloc(extra);
 103         if (ov) {
 104                 voucher_fields_t fields = ~ignore_fields;
 105                 if ((fields & VOUCHER_FIELD_KVOUCHER) && ov->v_kvoucher) {
 106                         voucher_t kvb = ov->v_kvbase ? ov->v_kvbase : ov;
 107                         v->v_kvbase = _voucher_retain(kvb);
 108                         v->v_kvoucher = kvb->v_kvoucher;
 109                         v->v_kv_has_importance = kvb->v_kv_has_importance;
 110                 }
 111                 if (fields & VOUCHER_FIELD_PRIORITY) {
 112                         v->v_priority = ov->v_priority;
 113                 }
 114                 if (fields & VOUCHER_FIELD_ACTIVITY) {
 115                         v->v_activity = ov->v_activity;
 116                         v->v_activity_creator = ov->v_activity_creator;
 117                         v->v_parent_activity = ov->v_parent_activity;
 118                 }
 119                 if ((fields & VOUCHER_FIELD_EXTRA) && extra) {
 120                         memcpy(_voucher_extra_recipes(v), _voucher_extra_recipes(ov),extra);
 121                 }
 122         }
 123         return v;
 124 }
 125
 126 voucher_t
 127 voucher_adopt(voucher_t voucher)
 128 {
 129         if (voucher == VOUCHER_CURRENT) {
 130                 return _voucher_copy();
 131         }
 132         return _voucher_adopt(voucher);
 133 }
 134
 135 voucher_t
 136 voucher_copy(void)
 137 {
 138         return _voucher_copy();
 139 }
 140
 141 voucher_t
 142 voucher_copy_without_importance(void)
 143 {
 144         return _voucher_copy_without_importance();
 145 }
 146
 147 voucher_t
 148 voucher_retain(voucher_t voucher)
 149 {
 150         return _voucher_retain(voucher);
 151 }
 152
 153 void
 154 voucher_release(voucher_t voucher)
 155 {
 156         return _voucher_release(voucher);
 157 }
 158
 159 void
 160 _voucher_thread_cleanup(void *voucher)
 161 {
 162         // when a thread exits and has a voucher left, the kernel
 163         // will get rid of the voucher kernel object that is set on the thread,
 164         // we only need to release the voucher_t object.
 165         _voucher_release(voucher);
 166 }
 167
 168 DISPATCH_CACHELINE_ALIGN
 169 static TAILQ_HEAD(, voucher_s) _vouchers[VL_HASH_SIZE];
 170 #define _vouchers_head(kv) (&_vouchers[VL_HASH((kv))])
 171 static dispatch_unfair_lock_s _vouchers_lock;
 172 #define _vouchers_lock_lock() _dispatch_unfair_lock_lock(&_vouchers_lock)
 173 #define _vouchers_lock_unlock() _dispatch_unfair_lock_unlock(&_vouchers_lock)
 174
 175 static voucher_t
 176 _voucher_find_and_retain(mach_voucher_t kv)
 177 {
 178         voucher_t v;
 179         if (!kv) return NULL;
 180         _vouchers_lock_lock();
 181         TAILQ_FOREACH(v, _vouchers_head(kv), v_list) {
 182                 if (v->v_ipc_kvoucher == kv) {
 183                         int xref_cnt = os_atomic_inc2o(v, os_obj_xref_cnt, relaxed);
 184                         _dispatch_voucher_debug("retain  -> %d", v, xref_cnt + 1);
 185                         if (slowpath(xref_cnt < 0)) {
 186                                 _dispatch_voucher_debug("over-release", v);
 187                                 _OS_OBJECT_CLIENT_CRASH("Voucher over-release");
 188                         }
 189                         if (xref_cnt == 0) {
 190                                 // resurrection: raced with _voucher_remove
 191                                 (void)os_atomic_inc2o(v, os_obj_ref_cnt, relaxed);
 192                         }
 193                         break;
 194                 }
 195         }
 196         _vouchers_lock_unlock();
 197         return v;
 198 }
 199
 200 static void
 201 _voucher_insert(voucher_t v)
 202 {
 203         mach_voucher_t kv = v->v_ipc_kvoucher;
 204         if (!kv) return;
 205         _vouchers_lock_lock();
 206         if (slowpath(_TAILQ_IS_ENQUEUED(v, v_list))) {
 207                 _dispatch_voucher_debug("corruption", v);
 208                 DISPATCH_CLIENT_CRASH(v->v_list.tqe_prev, "Voucher corruption");
 209         }
 210         TAILQ_INSERT_TAIL(_vouchers_head(kv), v, v_list);
 211         _vouchers_lock_unlock();
 212 }
 213
 214 static void
 215 _voucher_remove(voucher_t v)
 216 {
 217         mach_voucher_t kv = v->v_ipc_kvoucher;
 218         if (!_TAILQ_IS_ENQUEUED(v, v_list)) return;
 219         _vouchers_lock_lock();
 220         if (slowpath(!kv)) {
 221                 _dispatch_voucher_debug("corruption", v);
 222                 DISPATCH_CLIENT_CRASH(0, "Voucher corruption");
 223         }
 224         // check for resurrection race with _voucher_find_and_retain
 225         if (os_atomic_load2o(v, os_obj_xref_cnt, ordered) < 0 &&
 226                         _TAILQ_IS_ENQUEUED(v, v_list)) {
 227                 TAILQ_REMOVE(_vouchers_head(kv), v, v_list);
 228                 _TAILQ_MARK_NOT_ENQUEUED(v, v_list);
 229                 v->v_list.tqe_next = (void*)~0ull;
 230         }
 231         _vouchers_lock_unlock();
 232 }
 233
 234 void
 235 _voucher_dealloc_mach_voucher(mach_voucher_t kv)
 236 {
 237         _dispatch_kvoucher_debug("dealloc", kv);
 238         _dispatch_voucher_debug_machport(kv);
 239         kern_return_t kr = mach_voucher_deallocate(kv);
 240         DISPATCH_VERIFY_MIG(kr);
 241         (void)dispatch_assume_zero(kr);
 242 }
 243
 244 static inline kern_return_t
 245 _voucher_create_mach_voucher(const mach_voucher_attr_recipe_data_t *recipes,
 246                 size_t recipes_size, mach_voucher_t *kvp)
 247 {
 248         kern_return_t kr;
 249         mach_port_t mhp = _dispatch_get_mach_host_port();
 250         mach_voucher_t kv = MACH_VOUCHER_NULL;
 251         mach_voucher_attr_raw_recipe_array_t kvr;
 252         mach_voucher_attr_recipe_size_t kvr_size;
 253         kvr = (mach_voucher_attr_raw_recipe_array_t)recipes;
 254         kvr_size = (mach_voucher_attr_recipe_size_t)recipes_size;
 255         kr = host_create_mach_voucher(mhp, kvr, kvr_size, &kv);
 256         DISPATCH_VERIFY_MIG(kr);
 257         if (!kr) {
 258                 _dispatch_kvoucher_debug("create", kv);
 259                 _dispatch_voucher_debug_machport(kv);
 260         }
 261         *kvp = kv;
 262         return kr;
 263 }
 264
 265 void
 266 _voucher_task_mach_voucher_init(void* ctxt DISPATCH_UNUSED)
 267 {
 268         kern_return_t kr;
 269         mach_voucher_t kv = MACH_VOUCHER_NULL;
 270 #if !VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
 271         static const mach_voucher_attr_recipe_data_t task_create_recipe = {
 272                 .key = MACH_VOUCHER_ATTR_KEY_BANK,
 273                 .command = MACH_VOUCHER_ATTR_BANK_CREATE,
 274         };
 275         kr = _voucher_create_mach_voucher(&task_create_recipe,
 276                         sizeof(task_create_recipe), &kv);
 277         if (slowpath(kr)) {
 278                 DISPATCH_CLIENT_CRASH(kr, "Could not create task mach voucher");
 279         }
 280         _voucher_default_task_mach_voucher = kv;
 281 #endif
 282         _voucher_task_mach_voucher = kv;
 283 }
 284
 285 void
 286 voucher_replace_default_voucher(void)
 287 {
 288         (void)_voucher_get_task_mach_voucher(); // initalize task mach voucher
 289         mach_voucher_t kv, tkv = MACH_VOUCHER_NULL;
 290         voucher_t v = _voucher_get();
 291         if (v && v->v_kvoucher) {
 292                 kern_return_t kr;
 293                 kv = v->v_ipc_kvoucher ? v->v_ipc_kvoucher : v->v_kvoucher;
 294                 const mach_voucher_attr_recipe_data_t task_copy_recipe = {
 295                         .key = MACH_VOUCHER_ATTR_KEY_BANK,
 296                         .command = MACH_VOUCHER_ATTR_COPY,
 297                         .previous_voucher = kv,
 298                 };
 299                 kr = _voucher_create_mach_voucher(&task_copy_recipe,
 300                                 sizeof(task_copy_recipe), &tkv);
 301                 if (dispatch_assume_zero(kr)) {
 302                         tkv = MACH_VOUCHER_NULL;
 303                 }
 304         }
 305         if (!tkv) tkv = _voucher_default_task_mach_voucher;
 306         kv = os_atomic_xchg(&_voucher_task_mach_voucher, tkv, relaxed);
 307         if (kv && kv != _voucher_default_task_mach_voucher) {
 308                 _voucher_dealloc_mach_voucher(kv);
 309         }
 310         _dispatch_voucher_debug("kvoucher[0x%08x] replace default voucher", v, tkv);
 311 }
 312
 313 #define _voucher_mach_recipe_size(payload_size) \
 314         (sizeof(mach_voucher_attr_recipe_data_t) + (payload_size))
 315
 316 #if VOUCHER_USE_MACH_VOUCHER_PRIORITY
 317 #define _voucher_mach_recipe_alloca(v) ((mach_voucher_attr_recipe_t)alloca(\
 318                 _voucher_mach_recipe_size(0) + \
 319                 _voucher_mach_recipe_size(sizeof(ipc_pthread_priority_value_t)) + \
 320                 _voucher_mach_recipe_size(sizeof(_voucher_mach_udata_s)) + \
 321                 _voucher_extra_size(v)))
 322 #else
 323 #define _voucher_mach_recipe_alloca(v) ((mach_voucher_attr_recipe_t)alloca(\
 324                 _voucher_mach_recipe_size(0) + \
 325                 _voucher_mach_recipe_size(sizeof(_voucher_mach_udata_s)) + \
 326                 _voucher_extra_size(v)))
 327 #endif
 328
 329 DISPATCH_ALWAYS_INLINE
 330 static inline mach_voucher_attr_recipe_size_t
 331 _voucher_mach_recipe_init(mach_voucher_attr_recipe_t mvar_buf, voucher_s *v,
 332                 mach_voucher_t kvb, pthread_priority_t pp)
 333 {
 334         mach_voucher_attr_recipe_size_t extra = _voucher_extra_size(v);
 335         mach_voucher_attr_recipe_size_t size = 0;
 336
 337         // normalize to just the QoS class and 0 relative priority
 338         pp &= _PTHREAD_PRIORITY_QOS_CLASS_MASK;
 339         if (pp) pp |= _PTHREAD_PRIORITY_PRIORITY_MASK;
 340
 341         *mvar_buf++ = (mach_voucher_attr_recipe_data_t){
 342                 .key = MACH_VOUCHER_ATTR_KEY_ALL,
 343                 .command = MACH_VOUCHER_ATTR_COPY,
 344                 .previous_voucher = kvb,
 345         };
 346         size += _voucher_mach_recipe_size(0);
 347
 348 #if VOUCHER_USE_MACH_VOUCHER_PRIORITY
 349         if (pp) {
 350                 ipc_pthread_priority_value_t value = (ipc_pthread_priority_value_t)pp;
 351                 *mvar_buf++ = (mach_voucher_attr_recipe_data_t){
 352                         .key = MACH_VOUCHER_ATTR_KEY_PTHPRIORITY,
 353                         .command = MACH_VOUCHER_ATTR_PTHPRIORITY_CREATE,
 354                         .content_size = sizeof(value),
 355                 };
 356                 mvar_buf = _dispatch_memappend(mvar_buf, &value);
 357                 size += _voucher_mach_recipe_size(sizeof(value));
 358         }
 359 #endif // VOUCHER_USE_MACH_VOUCHER_PRIORITY
 360
 361         if ((v && v->v_activity) || pp) {
 362                 _voucher_mach_udata_s *udata_buf;
 363                 unsigned udata_size = 0;
 364
 365                 if (v && v->v_activity) {
 366                         udata_size = offsetof(_voucher_mach_udata_s, _vmu_after_activity);
 367                 } else {
 368                         udata_size = offsetof(_voucher_mach_udata_s, _vmu_after_priority);
 369                 }
 370                 *mvar_buf = (mach_voucher_attr_recipe_data_t){
 371                         .key = MACH_VOUCHER_ATTR_KEY_USER_DATA,
 372                         .command = MACH_VOUCHER_ATTR_USER_DATA_STORE,
 373                         .content_size = udata_size,
 374                 };
 375                 udata_buf = (_voucher_mach_udata_s *)(mvar_buf->content);
 376
 377                 if (v && v->v_activity) {
 378                         *udata_buf = (_voucher_mach_udata_s){
 379                                 .vmu_magic = VOUCHER_MAGIC_V3,
 380                                 .vmu_priority = (_voucher_priority_t)pp,
 381                                 .vmu_activity = v->v_activity,
 382                                 .vmu_activity_pid = v->v_activity_creator,
 383                                 .vmu_parent_activity = v->v_parent_activity,
 384                         };
 385                 } else {
 386                         *udata_buf = (_voucher_mach_udata_s){
 387                                 .vmu_magic = VOUCHER_MAGIC_V3,
 388                                 .vmu_priority = (_voucher_priority_t)pp,
 389                         };
 390                 }
 391
 392                 mvar_buf = (mach_voucher_attr_recipe_t)(mvar_buf->content + udata_size);
 393                 size += _voucher_mach_recipe_size(udata_size);
 394         }
 395
 396         if (extra) {
 397                 memcpy(mvar_buf, _voucher_extra_recipes(v), extra);
 398                 size += extra;
 399         }
 400         return size;
 401 }
 402
 403 mach_voucher_t
 404 _voucher_get_mach_voucher(voucher_t voucher)
 405 {
 406         if (!voucher) return MACH_VOUCHER_NULL;
 407         if (voucher->v_ipc_kvoucher) return voucher->v_ipc_kvoucher;
 408         mach_voucher_t kvb = voucher->v_kvoucher;
 409         if (!kvb) kvb = _voucher_get_task_mach_voucher();
 410         if (!voucher->v_activity && !voucher->v_priority &&
 411                         !_voucher_extra_size(voucher)) {
 412                 return kvb;
 413         }
 414
 415         mach_voucher_attr_recipe_t mvar = _voucher_mach_recipe_alloca(voucher);
 416         mach_voucher_attr_recipe_size_t size;
 417         mach_voucher_t kv, kvo;
 418         kern_return_t kr;
 419
 420         size = _voucher_mach_recipe_init(mvar, voucher, kvb, voucher->v_priority);
 421         kr = _voucher_create_mach_voucher(mvar, size, &kv);
 422         if (dispatch_assume_zero(kr) || !kv){
 423                 return MACH_VOUCHER_NULL;
 424         }
 425         if (!os_atomic_cmpxchgv2o(voucher, v_ipc_kvoucher, MACH_VOUCHER_NULL,
 426                         kv, &kvo, relaxed)) {
 427                 _voucher_dealloc_mach_voucher(kv);
 428                 kv = kvo;
 429         } else {
 430                 if (kv == voucher->v_kvoucher) {
 431                         // if v_kvoucher == v_ipc_kvoucher we keep only one reference
 432                         _voucher_dealloc_mach_voucher(kv);
 433                 }
 434                 _voucher_insert(voucher);
 435                 _dispatch_voucher_debug("kvoucher[0x%08x] create", voucher, kv);
 436         }
 437         return kv;
 438 }
 439
 440 mach_voucher_t
 441 _voucher_create_mach_voucher_with_priority(voucher_t voucher,
 442                 pthread_priority_t priority)
 443 {
 444         if (priority == _voucher_get_priority(voucher)) {
 445                 return MACH_VOUCHER_NULL; // caller will use _voucher_get_mach_voucher
 446         }
 447         kern_return_t kr;
 448         mach_voucher_t kv, kvb = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL;
 449         if (!kvb) kvb = _voucher_get_task_mach_voucher();
 450
 451         mach_voucher_attr_recipe_t mvar = _voucher_mach_recipe_alloca(voucher);
 452         mach_voucher_attr_recipe_size_t size;
 453
 454         size = _voucher_mach_recipe_init(mvar, voucher, kvb, priority);
 455         kr = _voucher_create_mach_voucher(mvar, size, &kv);
 456         if (dispatch_assume_zero(kr) || !kv){
 457                 return MACH_VOUCHER_NULL;
 458         }
 459         _dispatch_kvoucher_debug("create with priority from voucher[%p]", kv,
 460                         voucher);
 461         return kv;
 462 }
 463
 464 static voucher_t
 465 _voucher_create_with_mach_voucher(mach_voucher_t kv, mach_msg_bits_t msgh_bits)
 466 {
 467         if (!kv) return NULL;
 468         kern_return_t kr;
 469         mach_voucher_attr_recipe_t vr;
 470         size_t vr_size;
 471         mach_voucher_attr_recipe_size_t kvr_size = 0;
 472         mach_voucher_attr_content_size_t udata_sz = 0;
 473         _voucher_mach_udata_s *udata = NULL;
 474 #if !VOUCHER_USE_BANK_AUTOREDEEM
 475         mach_voucher_t rkv;
 476         const mach_voucher_attr_recipe_data_t redeem_recipe[] = {
 477                 [0] = {
 478                         .key = MACH_VOUCHER_ATTR_KEY_ALL,
 479                         .command = MACH_VOUCHER_ATTR_COPY,
 480                         .previous_voucher = kv,
 481                 },
 482                 [1] = {
 483                         .key = MACH_VOUCHER_ATTR_KEY_BANK,
 484                         .command = MACH_VOUCHER_ATTR_REDEEM,
 485                 },
 486         };
 487         kr = _voucher_create_mach_voucher(redeem_recipe, sizeof(redeem_recipe),
 488                         &rkv);
 489         if (!dispatch_assume_zero(kr)) {
 490                 _voucher_dealloc_mach_voucher(kv);
 491                 _dispatch_kvoucher_debug("redeemed from 0x%08x", rkv, kv);
 492                 kv = rkv;
 493         } else {
 494                 _dispatch_voucher_debug_machport(kv);
 495         }
 496 #endif
 497         voucher_t v = _voucher_find_and_retain(kv);
 498         if (v) {
 499                 _dispatch_voucher_debug("kvoucher[0x%08x] found", v, kv);
 500                 _voucher_dealloc_mach_voucher(kv);
 501                 return v;
 502         }
 503         vr_size = sizeof(*vr) + sizeof(_voucher_mach_udata_s);
 504         vr = alloca(vr_size);
 505         if (kv) {
 506                 kvr_size = (mach_voucher_attr_recipe_size_t)vr_size;
 507                 kr = mach_voucher_extract_attr_recipe(kv,
 508                                 MACH_VOUCHER_ATTR_KEY_USER_DATA, (void*)vr, &kvr_size);
 509                 DISPATCH_VERIFY_MIG(kr);
 510                 if (!dispatch_assume_zero(kr) && kvr_size >= sizeof(*vr)) {
 511                         udata_sz = vr->content_size;
 512                         udata = (_voucher_mach_udata_s*)vr->content;
 513                         dispatch_assume(udata_sz >= sizeof(_voucher_magic_t));
 514                 }
 515         }
 516         vr = NULL;
 517
 518         v = _voucher_alloc(0);
 519         v->v_ipc_kvoucher = v->v_kvoucher = kv;
 520         v->v_kv_has_importance = !!(msgh_bits & MACH_MSGH_BITS_RAISEIMP);
 521
 522         if (udata_sz >= offsetof(_voucher_mach_udata_s,_vmu_after_priority)){
 523                 if (udata->vmu_magic == VOUCHER_MAGIC_V3) {
 524                         v->v_priority = udata->vmu_priority;
 525                 }
 526         }
 527         bool remove_kv_userdata = false;
 528         if (udata_sz >= offsetof(_voucher_mach_udata_s, _vmu_after_activity)) {
 529 #if !RDAR_25050791
 530                 remove_kv_userdata = true;
 531 #endif
 532                 if (udata->vmu_magic == VOUCHER_MAGIC_V3 && udata->vmu_activity) {
 533                         v->v_activity = udata->vmu_activity;
 534                         v->v_activity_creator = udata->vmu_activity_pid;
 535                         v->v_parent_activity = udata->vmu_parent_activity;
 536                 }
 537         }
 538
 539         if (remove_kv_userdata) {
 540                 mach_voucher_t nkv = MACH_VOUCHER_NULL;
 541                 const mach_voucher_attr_recipe_data_t remove_userdata_recipe[] = {
 542                         [0] = {
 543                                 .key = MACH_VOUCHER_ATTR_KEY_ALL,
 544                                 .command = MACH_VOUCHER_ATTR_COPY,
 545                                 .previous_voucher = kv,
 546                         },
 547                         [1] = {
 548                                 .key = MACH_VOUCHER_ATTR_KEY_USER_DATA,
 549                                 .command = MACH_VOUCHER_ATTR_REMOVE,
 550                         },
 551 #if VOUCHER_USE_MACH_VOUCHER_PRIORITY
 552                         [2] = {
 553                                 .key = MACH_VOUCHER_ATTR_KEY_PTHPRIORITY,
 554                                 .command = MACH_VOUCHER_ATTR_REMOVE,
 555                         },
 556 #endif
 557                 };
 558                 mach_voucher_attr_recipe_size_t size = sizeof(remove_userdata_recipe);
 559
 560                 kr = _voucher_create_mach_voucher(remove_userdata_recipe, size, &nkv);
 561                 if (!dispatch_assume_zero(kr)) {
 562                         _dispatch_voucher_debug("kvoucher[0x%08x] udata removal "
 563                                         "(created 0x%08x)", v, kv, nkv);
 564                         v->v_ipc_kvoucher = MACH_VOUCHER_NULL;
 565                         v->v_kvoucher = nkv;
 566                         v->v_kvbase = _voucher_find_and_retain(nkv);
 567                         if (v->v_kvbase) {
 568                                 _voucher_dealloc_mach_voucher(nkv); // borrow base reference
 569                         }
 570                         _voucher_dealloc_mach_voucher(kv);
 571                         kv = nkv;
 572                 } else {
 573                         _dispatch_voucher_debug_machport(kv);
 574                 }
 575         }
 576
 577         _voucher_insert(v);
 578         _dispatch_voucher_debug("kvoucher[0x%08x] create", v, kv);
 579         return v;
 580 }
 581
 582 voucher_t
 583 _voucher_create_with_priority_and_mach_voucher(voucher_t ov,
 584                 pthread_priority_t priority, mach_voucher_t kv)
 585 {
 586         if (priority == _voucher_get_priority(ov)) {
 587                 if (kv) _voucher_dealloc_mach_voucher(kv);
 588                 return ov ? _voucher_retain(ov) : NULL;
 589         }
 590         voucher_t v = _voucher_find_and_retain(kv);
 591         voucher_fields_t ignore_fields = VOUCHER_FIELD_PRIORITY;
 592
 593         if (v) {
 594                 _dispatch_voucher_debug("kvoucher[0x%08x] find", v, kv);
 595                 _voucher_dealloc_mach_voucher(kv);
 596                 return v;
 597         }
 598
 599         if (kv) ignore_fields |= VOUCHER_FIELD_KVOUCHER;
 600         v = _voucher_clone(ov, ignore_fields);
 601         if (priority) {
 602                 v->v_priority = (_voucher_priority_t)priority;
 603         }
 604         if (kv) {
 605                 v->v_ipc_kvoucher = v->v_kvoucher = kv;
 606                 _voucher_insert(v);
 607                 _dispatch_voucher_debug("kvoucher[0x%08x] create with priority from "
 608                                 "voucher[%p]", v, kv, ov);
 609                 _dispatch_voucher_debug_machport(kv);
 610         }
 611         return v;
 612 }
 613
 614 voucher_t
 615 _voucher_create_without_importance(voucher_t ov)
 616 {
 617         // Nothing to do unless the old voucher has a kernel voucher. If it
 618         // doesn't, it can't have any importance, now or in the future.
 619         if (!ov) return NULL;
 620         if (!ov->v_kvoucher || !ov->v_kv_has_importance) return _voucher_retain(ov);
 621         kern_return_t kr;
 622         mach_voucher_t kv, okv;
 623         // Copy kernel voucher, removing importance.
 624         okv = ov->v_ipc_kvoucher ? ov->v_ipc_kvoucher : ov->v_kvoucher;
 625         const mach_voucher_attr_recipe_data_t importance_remove_recipe[] = {
 626                 [0] = {
 627                         .key = MACH_VOUCHER_ATTR_KEY_ALL,
 628                         .command = MACH_VOUCHER_ATTR_COPY,
 629                         .previous_voucher = okv,
 630                 },
 631                 [1] = {
 632                         .key = MACH_VOUCHER_ATTR_KEY_IMPORTANCE,
 633                         .command = MACH_VOUCHER_ATTR_REMOVE,
 634                 },
 635         };
 636         kr = _voucher_create_mach_voucher(importance_remove_recipe,
 637                         sizeof(importance_remove_recipe), &kv);
 638         if (dispatch_assume_zero(kr) || !kv){
 639                 if (ov->v_ipc_kvoucher) return NULL;
 640                 kv = MACH_VOUCHER_NULL;
 641         }
 642         if (kv == okv) {
 643                 _voucher_dealloc_mach_voucher(kv);
 644                 return _voucher_retain(ov);
 645         }
 646         voucher_t v = _voucher_find_and_retain(kv);
 647         if (v && ov->v_ipc_kvoucher) {
 648                 _dispatch_voucher_debug("kvoucher[0x%08x] find without importance "
 649                                 "from voucher[%p]", v, kv, ov);
 650                 _voucher_dealloc_mach_voucher(kv);
 651                 return v;
 652         }
 653         voucher_t kvbase = v;
 654         voucher_fields_t ignore_fields = VOUCHER_FIELD_KVOUCHER;
 655         v = _voucher_clone(ov, ignore_fields);
 656         v->v_kvoucher = kv;
 657         if (ov->v_ipc_kvoucher) {
 658                 v->v_ipc_kvoucher = kv;
 659                 _voucher_insert(v);
 660         } else if (kvbase) {
 661                 v->v_kvbase = kvbase;
 662                 _voucher_dealloc_mach_voucher(kv); // borrow base reference
 663         }
 664         if (!kvbase) {
 665                 _dispatch_voucher_debug("kvoucher[0x%08x] create without importance "
 666                                 "from voucher[%p]", v, kv, ov);
 667         }
 668         return v;
 669 }
 670
 671 voucher_t
 672 _voucher_create_accounting_voucher(voucher_t ov)
 673 {
 674         // Nothing to do unless the old voucher has a kernel voucher. If it does
 675         // doesn't, it can't have any accounting attributes.
 676         if (!ov || !ov->v_kvoucher) return NULL;
 677         kern_return_t kr = KERN_SUCCESS;
 678         mach_voucher_t okv, kv = MACH_VOUCHER_NULL;
 679         okv = ov->v_ipc_kvoucher ? ov->v_ipc_kvoucher : ov->v_kvoucher;
 680         const mach_voucher_attr_recipe_data_t accounting_copy_recipe = {
 681                 .key = MACH_VOUCHER_ATTR_KEY_BANK,
 682                 .command = MACH_VOUCHER_ATTR_COPY,
 683                 .previous_voucher = okv,
 684         };
 685         kr = _voucher_create_mach_voucher(&accounting_copy_recipe,
 686                         sizeof(accounting_copy_recipe), &kv);
 687         if (dispatch_assume_zero(kr) || !kv){
 688                 return NULL;
 689         }
 690         voucher_t v = _voucher_find_and_retain(kv);
 691         if (v) {
 692                 _dispatch_voucher_debug("kvoucher[0x%08x] find accounting voucher "
 693                                 "from voucher[%p]", v, kv, ov);
 694                 _voucher_dealloc_mach_voucher(kv);
 695                 return v;
 696         }
 697         v = _voucher_alloc(0);
 698         v->v_ipc_kvoucher = v->v_kvoucher = kv;
 699         if (kv == okv) {
 700                 v->v_kvbase = _voucher_retain(ov);
 701                 _voucher_dealloc_mach_voucher(kv); // borrow base reference
 702         }
 703         _voucher_insert(v);
 704         _dispatch_voucher_debug("kvoucher[0x%08x] create accounting voucher "
 705                         "from voucher[%p]", v, kv, ov);
 706         return v;
 707 }
 708
 709 voucher_t
 710 voucher_create_with_mach_msg(mach_msg_header_t *msg)
 711 {
 712         mach_msg_bits_t msgh_bits;
 713         mach_voucher_t kv = _voucher_mach_msg_get(msg, &msgh_bits);
 714         return _voucher_create_with_mach_voucher(kv, msgh_bits);
 715 }
 716
 717 void
 718 voucher_decrement_importance_count4CF(voucher_t v)
 719 {
 720         if (!v || !v->v_kvoucher || !v->v_kv_has_importance) return;
 721         kern_return_t kr;
 722         mach_voucher_t kv = v->v_ipc_kvoucher ? v->v_ipc_kvoucher : v->v_kvoucher;
 723         uint32_t dec = 1;
 724         mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&dec;
 725         mach_voucher_attr_content_size_t kvc_in_size = sizeof(dec);
 726         mach_voucher_attr_content_t kvc_out = NULL;
 727         mach_voucher_attr_content_size_t kvc_out_size = 0;
 728 #if DISPATCH_DEBUG
 729         uint32_t count = UINT32_MAX;
 730         kvc_out = (mach_voucher_attr_content_t)&count;
 731         kvc_out_size = sizeof(count);
 732 #endif
 733         kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_IMPORTANCE,
 734                         MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL, kvc_in, kvc_in_size,
 735                         kvc_out, &kvc_out_size);
 736         DISPATCH_VERIFY_MIG(kr);
 737         if (kr == KERN_INVALID_TASK) return; // non-denap receiver rdar://25643185
 738 #if DISPATCH_DEBUG
 739         _dispatch_voucher_debug("kvoucher[0x%08x] decrement importance count to %u:"
 740                         " %s - 0x%x", v, kv, count, mach_error_string(kr), kr);
 741 #endif
 742         if (slowpath(dispatch_assume_zero(kr) == KERN_FAILURE)) {
 743                 DISPATCH_CLIENT_CRASH(kr, "Voucher importance count underflow");
 744         }
 745 }
 746
 747 #if VOUCHER_ENABLE_GET_MACH_VOUCHER
 748 mach_voucher_t
 749 voucher_get_mach_voucher(voucher_t voucher)
 750 {
 751         return _voucher_get_mach_voucher(voucher);
 752 }
 753 #endif
 754
 755 void
 756 _voucher_xref_dispose(voucher_t voucher)
 757 {
 758         _dispatch_voucher_debug("xref_dispose", voucher);
 759         _voucher_remove(voucher);
 760         return _os_object_release_internal_inline((_os_object_t)voucher);
 761 }
 762
 763 void
 764 _voucher_dispose(voucher_t voucher)
 765 {
 766         _dispatch_voucher_debug("dispose", voucher);
 767         if (slowpath(_TAILQ_IS_ENQUEUED(voucher, v_list))) {
 768                 _dispatch_voucher_debug("corruption", voucher);
 769                 DISPATCH_CLIENT_CRASH(voucher->v_list.tqe_prev, "Voucher corruption");
 770         }
 771         voucher->v_list.tqe_next = DISPATCH_OBJECT_LISTLESS;
 772         if (voucher->v_ipc_kvoucher) {
 773                 if (voucher->v_ipc_kvoucher != voucher->v_kvoucher) {
 774                         _voucher_dealloc_mach_voucher(voucher->v_ipc_kvoucher);
 775                 }
 776                 voucher->v_ipc_kvoucher = MACH_VOUCHER_NULL;
 777         }
 778         if (voucher->v_kvoucher) {
 779                 if (!voucher->v_kvbase) {
 780                         _voucher_dealloc_mach_voucher(voucher->v_kvoucher);
 781                 }
 782                 voucher->v_kvoucher = MACH_VOUCHER_NULL;
 783         }
 784         if (voucher->v_kvbase) {
 785                 _voucher_release(voucher->v_kvbase);
 786                 voucher->v_kvbase = NULL;
 787         }
 788         voucher->v_activity = 0;
 789         voucher->v_activity_creator = 0;
 790         voucher->v_parent_activity = 0;
 791         voucher->v_priority = 0;
 792 #if VOUCHER_ENABLE_RECIPE_OBJECTS
 793         voucher->v_recipe_extra_size = 0;
 794         voucher->v_recipe_extra_offset = 0;
 795 #endif
 796         return _os_object_dealloc((_os_object_t)voucher);
 797 }
 798
 799 static void
 800 _voucher_activity_debug_channel_barrier_nop(void *ctxt DISPATCH_UNUSED)
 801 {
 802 }
 803
 804 void
 805 _voucher_activity_debug_channel_init(void)
 806 {
 807         dispatch_mach_handler_function_t handler = NULL;
 808
 809         if (_voucher_libtrace_hooks) {
 810                 handler = _voucher_libtrace_hooks->vah_debug_channel_handler;
 811         }
 812         if (!handler) return;
 813
 814         dispatch_mach_t dm;
 815         mach_port_t dbgp;
 816         kern_return_t kr;
 817
 818         kr = task_get_debug_control_port(mach_task_self(), &dbgp);
 819         DISPATCH_VERIFY_MIG(kr);
 820         if (kr) {
 821                 DISPATCH_CLIENT_CRASH(kr, "Couldn't get debug control port");
 822         }
 823         if (dbgp) {
 824                 dm = dispatch_mach_create_f("com.apple.debug-channel",
 825                                 DISPATCH_TARGET_QUEUE_DEFAULT, NULL, handler);
 826                 dispatch_mach_connect(dm, dbgp, MACH_PORT_NULL, NULL);
 827                 // will force the DISPATCH_MACH_CONNECTED event
 828                 dispatch_mach_send_barrier_f(dm, NULL,
 829                                 _voucher_activity_debug_channel_barrier_nop);
 830                 _voucher_activity_debug_channel = dm;
 831         }
 832 }
 833
 834 void
 835 _voucher_atfork_child(void)
 836 {
 837         _dispatch_thread_setspecific(dispatch_voucher_key, NULL);
 838         _voucher_task_mach_voucher_pred = 0;
 839         _voucher_task_mach_voucher = MACH_VOUCHER_NULL;
 840 #if !VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
 841         _voucher_default_task_mach_voucher = MACH_PORT_NULL;
 842 #endif
 843         _voucher_aid_next = 0;
 844         _firehose_task_buffer_pred = 0;
 845         _firehose_task_buffer = NULL; // firehose buffer is VM_INHERIT_NONE
 846 }
 847
 848 #if VOUCHER_EXPORT_PERSONA_SPI
 849 #if VOUCHER_USE_PERSONA
 850 static kern_return_t
 851 _voucher_get_current_persona_token(struct persona_token *token)
 852 {
 853         kern_return_t kr = KERN_FAILURE;
 854         voucher_t v = _voucher_get();
 855
 856         if (v && v->v_kvoucher) {
 857                 mach_voucher_t kv = v->v_ipc_kvoucher ?: v->v_kvoucher;
 858                 mach_voucher_attr_content_t kvc_in = NULL;
 859                 mach_voucher_attr_content_size_t kvc_in_size = 0;
 860                 mach_voucher_attr_content_t kvc_out =
 861                         (mach_voucher_attr_content_t)token;
 862                 mach_voucher_attr_content_size_t kvc_out_size = sizeof(*token);
 863
 864                 kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_BANK,
 865                                 BANK_PERSONA_TOKEN, kvc_in, kvc_in_size,
 866                                 kvc_out, &kvc_out_size);
 867                 if (kr != KERN_NOT_SUPPORTED
 868                                 // Voucher doesn't have a PERSONA_TOKEN
 869                                 && kr != KERN_INVALID_VALUE
 870                                 // Kernel doesn't understand BANK_PERSONA_TOKEN
 871                                 && kr != KERN_INVALID_ARGUMENT) {
 872                         (void)dispatch_assume_zero(kr);
 873                 }
 874         }
 875         return kr;
 876 }
 877 #endif
 878
 879 uid_t
 880 voucher_get_current_persona(void)
 881 {
 882         uid_t persona_id = PERSONA_ID_NONE;
 883
 884 #if VOUCHER_USE_PERSONA
 885         struct persona_token token;
 886         int err;
 887
 888         if (_voucher_get_current_persona_token(&token) == KERN_SUCCESS) {
 889                 return token.originator.persona_id;
 890         }
 891
 892         // falling back to the process persona if there is no adopted voucher
 893         if (kpersona_get(&persona_id) < 0) {
 894                 err = errno;
 895                 if (err != ESRCH) {
 896                         (void)dispatch_assume_zero(err);
 897                 }
 898         }
 899 #endif
 900         return persona_id;
 901 }
 902
 903 int
 904 voucher_get_current_persona_originator_info(struct proc_persona_info *persona_info)
 905 {
 906 #if VOUCHER_USE_PERSONA
 907         struct persona_token token;
 908         if (_voucher_get_current_persona_token(&token) == KERN_SUCCESS) {
 909                 *persona_info = token.originator;
 910                 return 0;
 911         }
 912 #else
 913         (void)persona_info;
 914 #endif
 915         return -1;
 916 }
 917
 918 int
 919 voucher_get_current_persona_proximate_info(struct proc_persona_info *persona_info)
 920 {
 921 #if VOUCHER_USE_PERSONA
 922         struct persona_token token;
 923         if (_voucher_get_current_persona_token(&token) == KERN_SUCCESS) {
 924                 *persona_info = token.proximate;
 925                 return 0;
 926         }
 927 #else
 928         (void)persona_info;
 929 #endif
 930         return -1;
 931 }
 932 #endif
 933
 934 #pragma mark -
 935 #pragma mark _voucher_init
 936
 937 boolean_t
 938 voucher_mach_msg_set(mach_msg_header_t *msg)
 939 {
 940         return _voucher_mach_msg_set(msg, _voucher_get());
 941 }
 942
 943 void
 944 voucher_mach_msg_clear(mach_msg_header_t *msg)
 945 {
 946         (void)_voucher_mach_msg_clear(msg, false);
 947 }
 948
 949 voucher_mach_msg_state_t
 950 voucher_mach_msg_adopt(mach_msg_header_t *msg)
 951 {
 952         mach_msg_bits_t msgh_bits;
 953         mach_voucher_t kv = _voucher_mach_msg_get(msg, &msgh_bits);
 954         if (!kv) return VOUCHER_MACH_MSG_STATE_UNCHANGED;
 955         voucher_t v = _voucher_create_with_mach_voucher(kv, msgh_bits);
 956         return (voucher_mach_msg_state_t)_voucher_adopt(v);
 957 }
 958
 959 void
 960 voucher_mach_msg_revert(voucher_mach_msg_state_t state)
 961 {
 962         if (state == VOUCHER_MACH_MSG_STATE_UNCHANGED) return;
 963         _voucher_replace((voucher_t)state);
 964 }
 965
 966 #if DISPATCH_USE_LIBKERNEL_VOUCHER_INIT
 967 #include <_libkernel_init.h>
 968
 969 static const struct _libkernel_voucher_functions _voucher_libkernel_functions =
 970 {
 971         .version = 1,
 972         .voucher_mach_msg_set = voucher_mach_msg_set,
 973         .voucher_mach_msg_clear = voucher_mach_msg_clear,
 974         .voucher_mach_msg_adopt = voucher_mach_msg_adopt,
 975         .voucher_mach_msg_revert = voucher_mach_msg_revert,
 976 };
 977
 978 static void
 979 _voucher_libkernel_init(void)
 980 {
 981         kern_return_t kr = __libkernel_voucher_init(&_voucher_libkernel_functions);
 982         dispatch_assert(!kr);
 983 }
 984 #else
 985 #define _voucher_libkernel_init()
 986 #endif
 987
 988 void
 989 voucher_activity_initialize_4libtrace(voucher_activity_hooks_t hooks)
 990 {
 991         if (hooks->vah_version < 3) {
 992                 DISPATCH_CLIENT_CRASH(hooks->vah_version, "unsupported vah_version");
 993         }
 994         if (!os_atomic_cmpxchg(&_voucher_libtrace_hooks, NULL,
 995                         hooks, relaxed)) {
 996                 DISPATCH_CLIENT_CRASH(_voucher_libtrace_hooks,
 997                                 "voucher_activity_initialize_4libtrace called twice");
 998         }
 999 }
1000
1001 void
1002 _voucher_init(void)
1003 {
1004         _voucher_libkernel_init();
1005         unsigned int i;
1006         for (i = 0; i < VL_HASH_SIZE; i++) {
1007                 TAILQ_INIT(&_vouchers[i]);
1008         }
1009 }
1010
1011 #pragma mark -
1012 #pragma mark voucher_activity_t
1013
1014 DISPATCH_NOINLINE
1015 static uint64_t
1016 _voucher_activity_id_allocate_slow(uint64_t aid)
1017 {
1018         kern_return_t kr;
1019         uint64_t next;
1020
1021         kr = mach_generate_activity_id(mach_task_self(), 1, &next);
1022         if (unlikely(kr)) {
1023                 DISPATCH_CLIENT_CRASH(kr, "Could not generate an activity ID");
1024         }
1025         next *= MACH_ACTIVITY_ID_RANGE_SIZE;
1026         next &= MACH_ACTIVITY_ID_MASK;
1027         if (unlikely(next == 0)) {
1028                 next++;
1029         }
1030
1031         if (unlikely(aid == 0)) {
1032                 if (os_atomic_cmpxchg(&_voucher_aid_next, 0, next + 1, relaxed)) {
1033                         return next;
1034                 }
1035         }
1036         return os_atomic_xchg(&_voucher_aid_next, next, relaxed);
1037 }
1038
1039 DISPATCH_ALWAYS_INLINE
1040 static firehose_activity_id_t
1041 _voucher_activity_id_allocate(firehose_activity_flags_t flags)
1042 {
1043         uint64_t aid, next;
1044         os_atomic_rmw_loop(&_voucher_aid_next, aid, next, relaxed, {
1045                 next = aid + 1;
1046                 if (aid == 0 || next % MACH_ACTIVITY_ID_RANGE_SIZE == 0) {
1047                         os_atomic_rmw_loop_give_up({
1048                                 aid = _voucher_activity_id_allocate_slow(aid);
1049                                 break;
1050                         });
1051                 }
1052         });
1053         return FIREHOSE_ACTIVITY_ID_MAKE(aid, flags);
1054 }
1055
1056 #define _voucher_activity_tracepoint_reserve(stamp, stream, pub, priv, privbuf) \
1057                 firehose_buffer_tracepoint_reserve(_firehose_task_buffer, stamp, \
1058                                 stream, pub, priv, privbuf)
1059
1060 #define _voucher_activity_tracepoint_flush(ft, ftid) \
1061                 firehose_buffer_tracepoint_flush(_firehose_task_buffer, ft, ftid)
1062
1063 DISPATCH_NOINLINE
1064 static void
1065 _firehose_task_buffer_init(void *ctx OS_UNUSED)
1066 {
1067         mach_port_t logd_port;
1068
1069         /* Query the uniquepid of the current process */
1070         struct proc_uniqidentifierinfo p_uniqinfo = { };
1071         int info_size = 0;
1072
1073         info_size = proc_pidinfo(getpid(), PROC_PIDUNIQIDENTIFIERINFO, 1,
1074                         &p_uniqinfo, PROC_PIDUNIQIDENTIFIERINFO_SIZE);
1075         if (slowpath(info_size != PROC_PIDUNIQIDENTIFIERINFO_SIZE)) {
1076                 DISPATCH_INTERNAL_CRASH(info_size, "Unable to get the unique pid");
1077         }
1078         _voucher_unique_pid = p_uniqinfo.p_uniqueid;
1079
1080
1081         if (!fastpath(_voucher_libtrace_hooks)) {
1082                 if (0) { // <rdar://problem/23393959>
1083                         DISPATCH_CLIENT_CRASH(0,
1084                                         "Activity subsystem isn't initialized yet");
1085                 }
1086                 return;
1087         }
1088         logd_port = _voucher_libtrace_hooks->vah_get_logd_port();
1089         if (logd_port) {
1090                 unsigned long flags = 0;
1091 #if DISPATCH_USE_MEMORYPRESSURE_SOURCE
1092                 if (_dispatch_memory_warn) {
1093                         flags |= FIREHOSE_BUFFER_BANK_FLAG_LOW_MEMORY;
1094                 }
1095 #endif
1096                 // firehose_buffer_create always consumes the send-right
1097                 _firehose_task_buffer = firehose_buffer_create(logd_port,
1098                                 _voucher_unique_pid, flags);
1099                 if (_voucher_libtrace_hooks->vah_version >= 4 &&
1100                                 _voucher_libtrace_hooks->vah_metadata_init) {
1101                         firehose_buffer_t fb = _firehose_task_buffer;
1102                         size_t meta_sz = FIREHOSE_BUFFER_LIBTRACE_HEADER_SIZE;
1103                         void *meta = (void *)((uintptr_t)(&fb->fb_header + 1) - meta_sz);
1104                         _voucher_libtrace_hooks->vah_metadata_init(meta, meta_sz);
1105                 }
1106         }
1107 }
1108
1109 DISPATCH_ALWAYS_INLINE
1110 static inline bool
1111 _voucher_activity_disabled(void)
1112 {
1113         dispatch_once_f(&_firehose_task_buffer_pred,
1114                         NULL, _firehose_task_buffer_init);
1115
1116         firehose_buffer_t fb = _firehose_task_buffer;
1117         if (fastpath(fb)) {
1118                 return slowpath(fb->fb_header.fbh_sendp == MACH_PORT_DEAD);
1119         }
1120         return true;
1121 }
1122
1123 void*
1124 voucher_activity_get_metadata_buffer(size_t *length)
1125 {
1126         if (_voucher_activity_disabled()) {
1127                 *length = 0;
1128                 return NULL;
1129         }
1130
1131         firehose_buffer_header_t fbh = &_firehose_task_buffer->fb_header;
1132
1133         *length = FIREHOSE_BUFFER_LIBTRACE_HEADER_SIZE;
1134         return (void *)((uintptr_t)(fbh + 1) - *length);
1135 }
1136
1137 voucher_t
1138 voucher_activity_create_with_data(firehose_tracepoint_id_t *trace_id,
1139                 voucher_t base, firehose_activity_flags_t flags,
1140                 const void *pubdata, size_t publen)
1141 {
1142         firehose_activity_id_t va_id = 0, current_id = 0, parent_id = 0;
1143         firehose_tracepoint_id_u ftid = { .ftid_value = *trace_id };
1144         uint64_t creator_id = 0;
1145         uint16_t pubsize;
1146         voucher_t ov = _voucher_get();
1147         voucher_t v;
1148
1149         if (os_add_overflow(sizeof(va_id), publen, &pubsize) || pubsize > 128) {
1150                 DISPATCH_CLIENT_CRASH(pubsize, "Absurd publen");
1151         }
1152         if (base == VOUCHER_CURRENT) {
1153                 base = ov;
1154         }
1155         if (_voucher_activity_disabled()) {
1156                 *trace_id = 0;
1157                 return base ? _voucher_retain(base) : VOUCHER_NULL;
1158         }
1159
1160         FIREHOSE_TRACE_ID_CLEAR_FLAG(ftid, base, has_unique_pid);
1161         if (ov && (current_id = ov->v_activity)) {
1162                 FIREHOSE_TRACE_ID_SET_FLAG(ftid, base, has_current_aid);
1163                 pubsize += sizeof(firehose_activity_id_t);
1164                 if ((creator_id = ov->v_activity_creator)) {
1165                         FIREHOSE_TRACE_ID_SET_FLAG(ftid, base, has_unique_pid);
1166                         pubsize += sizeof(uint64_t);
1167                 }
1168         }
1169         if (base != VOUCHER_NULL) {
1170                 parent_id = base->v_activity;
1171         }
1172
1173         if (parent_id) {
1174                 FIREHOSE_TRACE_ID_SET_FLAG(ftid, activity, has_other_aid);
1175                 pubsize += sizeof(firehose_activity_id_t);
1176                 flags |= FIREHOSE_ACTIVITY_ID_FLAGS(parent_id);
1177         }
1178
1179         if (firehose_precise_timestamps_enabled()) {
1180                 flags |= firehose_activity_flags_precise_timestamp;
1181         }
1182         voucher_fields_t ignore_fields = VOUCHER_FIELD_ACTIVITY;
1183         v = _voucher_clone(base, ignore_fields);
1184         v->v_activity = va_id = _voucher_activity_id_allocate(flags);
1185         v->v_activity_creator = _voucher_unique_pid;
1186         v->v_parent_activity = parent_id;
1187
1188         static const firehose_stream_t streams[2] = {
1189                 firehose_stream_metadata,
1190                 firehose_stream_persist,
1191         };
1192         firehose_tracepoint_t ft;
1193         uint64_t stamp = firehose_tracepoint_time(flags);
1194
1195         for (size_t i = 0; i < countof(streams); i++) {
1196                 ft = _voucher_activity_tracepoint_reserve(stamp, streams[i], pubsize,
1197                                 0, NULL);
1198                 if (!fastpath(ft)) continue;
1199
1200                 uint8_t *pubptr = ft->ft_data;
1201                 if (current_id) {
1202                         pubptr = _dispatch_memappend(pubptr, &current_id);
1203                 }
1204                 if (creator_id) {
1205                         pubptr = _dispatch_memappend(pubptr, &creator_id);
1206                 }
1207                 if (parent_id) {
1208                         pubptr = _dispatch_memappend(pubptr, &parent_id);
1209                 }
1210                 pubptr = _dispatch_memappend(pubptr, &va_id);
1211                 pubptr = _dispatch_mempcpy(pubptr, pubdata, publen);
1212                 _voucher_activity_tracepoint_flush(ft, ftid);
1213         }
1214         *trace_id = ftid.ftid_value;
1215         return v;
1216 }
1217
1218 voucher_t
1219 voucher_activity_create_with_location(firehose_tracepoint_id_t *trace_id,
1220                 voucher_t base, firehose_activity_flags_t flags, uint64_t loc)
1221 {
1222         return voucher_activity_create_with_data(trace_id, base, flags,
1223                         &loc, sizeof(loc));
1224 }
1225
1226 #if OS_VOUCHER_ACTIVITY_GENERATE_SWAPS
1227 void
1228 _voucher_activity_swap(firehose_activity_id_t old_id,
1229                 firehose_activity_id_t new_id)
1230 {
1231         if (_voucher_activity_disabled()) return;
1232
1233         firehose_tracepoint_id_u ftid = { .ftid = {
1234                 ._namespace = firehose_tracepoint_namespace_activity,
1235                 ._type = _firehose_tracepoint_type_activity_swap,
1236         } };
1237         uint16_t pubsize = 0;
1238
1239         if (old_id) {
1240                 FIREHOSE_TRACE_ID_SET_FLAG(ftid, base, has_current_aid);
1241                 pubsize += sizeof(firehose_activity_id_t);
1242         }
1243         if (new_id) {
1244                 FIREHOSE_TRACE_ID_SET_FLAG(ftid, activity, has_other_aid);
1245                 pubsize += sizeof(firehose_activity_id_t);
1246         }
1247
1248         firehose_stream_t stream = firehose_stream_metadata;
1249         firehose_tracepoint_t ft;
1250         firehose_activity_flags_t flags = FIREHOSE_ACTIVITY_ID_FLAGS(old_id) |
1251                         FIREHOSE_ACTIVITY_ID_FLAGS(new_id);
1252         uint64_t stamp = firehose_tracepoint_time(flags);
1253
1254         _dispatch_voucher_ktrace_activity_adopt(new_id);
1255
1256         ft = _voucher_activity_tracepoint_reserve(stamp, stream, pubsize, 0, NULL);
1257         if (!fastpath(ft)) return;
1258         uint8_t *pubptr = ft->ft_data;
1259         if (old_id) pubptr = _dispatch_memappend(pubptr, &old_id);
1260         if (new_id) pubptr = _dispatch_memappend(pubptr, &new_id);
1261         _voucher_activity_tracepoint_flush(ft, ftid);
1262 }
1263 #endif
1264
1265 firehose_activity_id_t
1266 voucher_get_activity_id_and_creator(voucher_t v, uint64_t *creator_pid,
1267                 firehose_activity_id_t *parent_id)
1268 {
1269         if (v == VOUCHER_CURRENT) {
1270                 v = _voucher_get();
1271         }
1272         if (v == VOUCHER_NULL) {
1273                 if (creator_pid) *creator_pid = 0;
1274                 if (parent_id) *parent_id = FIREHOSE_ACTIVITY_ID_NULL;
1275                 return FIREHOSE_ACTIVITY_ID_NULL;
1276         }
1277         if (creator_pid) *creator_pid = v->v_activity_creator;
1278         if (parent_id) *parent_id = v->v_parent_activity;
1279         return v->v_activity;
1280 }
1281
1282 firehose_activity_id_t
1283 voucher_get_activity_id(voucher_t v, firehose_activity_id_t *parent_id)
1284 {
1285         return voucher_get_activity_id_and_creator(v, NULL, parent_id);
1286 }
1287
1288 void
1289 voucher_activity_flush(firehose_stream_t stream)
1290 {
1291         if (_voucher_activity_disabled()) return;
1292         firehose_buffer_stream_flush(_firehose_task_buffer, stream);
1293 }
1294
1295 DISPATCH_NOINLINE
1296 firehose_tracepoint_id_t
1297 voucher_activity_trace_v(firehose_stream_t stream,
1298                 firehose_tracepoint_id_t trace_id, uint64_t stamp,
1299                 const struct iovec *iov, size_t publen, size_t privlen)
1300 {
1301         firehose_tracepoint_id_u ftid = { .ftid_value = trace_id };
1302         const uint16_t ft_size = offsetof(struct firehose_tracepoint_s, ft_data);
1303         const size_t _firehose_chunk_payload_size =
1304                         sizeof(((struct firehose_chunk_s *)0)->fc_data);
1305
1306         if (_voucher_activity_disabled()) return 0;
1307
1308         firehose_tracepoint_t ft;
1309         firehose_activity_id_t va_id = 0;
1310         firehose_chunk_t fc;
1311         uint8_t *privptr, *pubptr;
1312         size_t pubsize = publen;
1313         voucher_t ov = _voucher_get();
1314         uint64_t creator_pid;
1315
1316         if ((va_id = _voucher_get_activity_id(ov, &creator_pid))) {
1317                 FIREHOSE_TRACE_ID_SET_FLAG(ftid, base, has_current_aid);
1318                 pubsize += sizeof(va_id);
1319         }
1320         if (FIREHOSE_TRACE_ID_HAS_FLAG(ftid, base, has_unique_pid)) {
1321                 if (creator_pid) {
1322                         pubsize += sizeof(creator_pid);
1323                 } else {
1324                         FIREHOSE_TRACE_ID_CLEAR_FLAG(ftid, base, has_unique_pid);
1325                 }
1326         } else {
1327                 creator_pid = 0;
1328         }
1329
1330         if (privlen) {
1331                 FIREHOSE_TRACE_ID_SET_FLAG(ftid, log, has_private_data);
1332                 pubsize += sizeof(struct firehose_buffer_range_s);
1333         }
1334
1335         if (slowpath(ft_size + pubsize + privlen > _firehose_chunk_payload_size)) {
1336                 DISPATCH_CLIENT_CRASH(ft_size + pubsize + privlen, "Log is too large");
1337         }
1338
1339         ft = _voucher_activity_tracepoint_reserve(stamp, stream, (uint16_t)pubsize,
1340                                 (uint16_t)privlen, &privptr);
1341         if (!fastpath(ft)) return 0;
1342         pubptr = ft->ft_data;
1343         if (va_id) {
1344                 pubptr = _dispatch_memappend(pubptr, &va_id);
1345         }
1346         if (creator_pid) {
1347                 pubptr = _dispatch_memappend(pubptr, &creator_pid);
1348         }
1349         if (privlen) {
1350                 fc = firehose_buffer_chunk_for_address(ft);
1351                 struct firehose_buffer_range_s range = {
1352                         .fbr_offset = (uint16_t)(privptr - fc->fc_start),
1353                         .fbr_length = (uint16_t)privlen,
1354                 };
1355                 pubptr = _dispatch_memappend(pubptr, &range);
1356         }
1357         while (publen > 0) {
1358                 pubptr = _dispatch_mempcpy(pubptr, iov->iov_base, iov->iov_len);
1359                 if (unlikely(os_sub_overflow(publen, iov->iov_len, &publen))) {
1360                         DISPATCH_CLIENT_CRASH(0, "Invalid arguments");
1361                 }
1362                 iov++;
1363         }
1364         while (privlen > 0) {
1365                 privptr = _dispatch_mempcpy(privptr, iov->iov_base, iov->iov_len);
1366                 if (unlikely(os_sub_overflow(privlen, iov->iov_len, &privlen))) {
1367                         DISPATCH_CLIENT_CRASH(0, "Invalid arguments");
1368                 }
1369                 iov++;
1370         }
1371         _voucher_activity_tracepoint_flush(ft, ftid);
1372         return ftid.ftid_value;
1373 }
1374
1375 firehose_tracepoint_id_t
1376 voucher_activity_trace(firehose_stream_t stream,
1377                 firehose_tracepoint_id_t trace_id, uint64_t stamp,
1378                 const void *pubdata, size_t publen)
1379 {
1380         struct iovec iov = { (void *)pubdata, publen };
1381         return voucher_activity_trace_v(stream, trace_id, stamp, &iov, publen, 0);
1382 }
1383
1384 firehose_tracepoint_id_t
1385 voucher_activity_trace_with_private_strings(firehose_stream_t stream,
1386                 firehose_tracepoint_id_t trace_id, uint64_t stamp,
1387                 const void *pubdata, size_t publen,
1388                 const void *privdata, size_t privlen)
1389 {
1390         struct iovec iov[2] = {
1391                 { (void *)pubdata, publen },
1392                 { (void *)privdata, privlen },
1393         };
1394         return voucher_activity_trace_v(stream, trace_id, stamp,
1395                         iov, publen, privlen);
1396 }
1397
1398 #pragma mark -
1399 #pragma mark _voucher_debug
1400
1401 size_t
1402 _voucher_debug(voucher_t v, char* buf, size_t bufsiz)
1403 {
1404         size_t offset = 0;
1405         #define bufprintf(...) \
1406                         offset += dsnprintf(&buf[offset], bufsiz - offset, ##__VA_ARGS__)
1407         bufprintf("voucher[%p] = { xrefcnt = 0x%x, refcnt = 0x%x", v,
1408                         v->os_obj_xref_cnt + 1, v->os_obj_ref_cnt + 1);
1409
1410         if (v->v_kvbase) {
1411                 bufprintf(", base voucher %p", v->v_kvbase);
1412         }
1413         if (v->v_kvoucher) {
1414                 bufprintf(", kvoucher%s 0x%x", v->v_kvoucher == v->v_ipc_kvoucher ?
1415                                 " & ipc kvoucher" : "", v->v_kvoucher);
1416         }
1417         if (v->v_ipc_kvoucher && v->v_ipc_kvoucher != v->v_kvoucher) {
1418                 bufprintf(", ipc kvoucher 0x%x", v->v_ipc_kvoucher);
1419         }
1420         if (v->v_priority) {
1421                 bufprintf(", QOS 0x%x", v->v_priority);
1422         }
1423         if (v->v_activity) {
1424                 bufprintf(", activity 0x%llx (pid: 0x%16llx, parent 0x%llx)",
1425                                 v->v_activity, v->v_activity_creator, v->v_parent_activity);
1426         }
1427         bufprintf(" }");
1428         return offset;
1429 }
1430
1431 #else // VOUCHER_USE_MACH_VOUCHER
1432
1433 #pragma mark -
1434 #pragma mark Simulator / vouchers disabled
1435
1436 #if VOUCHER_ENABLE_RECIPE_OBJECTS
1437 voucher_t
1438 voucher_create(voucher_recipe_t recipe)
1439 {
1440         (void)recipe;
1441         return NULL;
1442 }
1443 #endif
1444
1445 voucher_t
1446 voucher_adopt(voucher_t voucher)
1447 {
1448         return voucher;
1449 }
1450
1451 voucher_t
1452 voucher_copy(void)
1453 {
1454         return NULL;
1455 }
1456
1457 voucher_t
1458 voucher_copy_without_importance(void)
1459 {
1460         return NULL;
1461 }
1462
1463 voucher_t
1464 voucher_retain(voucher_t voucher)
1465 {
1466         return voucher;
1467 }
1468
1469 void
1470 voucher_release(voucher_t voucher)
1471 {
1472         (void)voucher;
1473 }
1474
1475 void
1476 voucher_replace_default_voucher(void)
1477 {
1478 }
1479
1480 void
1481 voucher_decrement_importance_count4CF(voucher_t v)
1482 {
1483         (void)v;
1484 }
1485
1486 void
1487 _voucher_thread_cleanup(void *voucher)
1488 {
1489         (void)voucher;
1490 }
1491
1492 void
1493 _voucher_dealloc_mach_voucher(mach_voucher_t kv)
1494 {
1495         (void)kv;
1496 }
1497
1498 mach_voucher_t
1499 _voucher_get_mach_voucher(voucher_t voucher)
1500 {
1501         (void)voucher;
1502         return MACH_VOUCHER_NULL;
1503 }
1504
1505 mach_voucher_t
1506 _voucher_create_mach_voucher_with_priority(voucher_t voucher,
1507                 pthread_priority_t priority)
1508 {
1509         (void)voucher; (void)priority;
1510         return MACH_VOUCHER_NULL;
1511 }
1512
1513 voucher_t
1514 _voucher_create_with_priority_and_mach_voucher(voucher_t voucher,
1515                 pthread_priority_t priority, mach_voucher_t kv)
1516 {
1517         (void)voucher; (void)priority; (void)kv;
1518         return NULL;
1519 }
1520
1521 voucher_t
1522 _voucher_create_accounting_voucher(voucher_t voucher)
1523 {
1524         (void)voucher;
1525         return NULL;
1526 }
1527
1528 voucher_t
1529 voucher_create_with_mach_msg(mach_msg_header_t *msg)
1530 {
1531         (void)msg;
1532         return NULL;
1533 }
1534
1535 #if VOUCHER_ENABLE_GET_MACH_VOUCHER
1536 mach_voucher_t
1537 voucher_get_mach_voucher(voucher_t voucher)
1538 {
1539         (void)voucher;
1540         return 0;
1541 }
1542 #endif
1543
1544 void
1545 _voucher_xref_dispose(voucher_t voucher)
1546 {
1547         (void)voucher;
1548 }
1549
1550 void
1551 _voucher_dispose(voucher_t voucher)
1552 {
1553         (void)voucher;
1554 }
1555
1556 #if VOUCHER_EXPORT_PERSONA_SPI
1557 uid_t
1558 voucher_get_current_persona(void)
1559 {
1560         return PERSONA_ID_NONE;
1561 }
1562
1563 int
1564 voucher_get_current_persona_originator_info(struct proc_persona_info *persona_info)
1565 {
1566         (void)persona_info;
1567         return -1;
1568 }
1569
1570 int
1571 voucher_get_current_persona_proximate_info(struct proc_persona_info *persona_info)
1572 {
1573         (void)persona_info;
1574         return -1;
1575 }
1576 #endif
1577
1578 void
1579 _voucher_activity_debug_channel_init(void)
1580 {
1581 }
1582
1583 void
1584 _voucher_atfork_child(void)
1585 {
1586 }
1587
1588 void
1589 _voucher_init(void)
1590 {
1591 }
1592
1593 #if OS_VOUCHER_ACTIVITY_SPI
1594 void*
1595 voucher_activity_get_metadata_buffer(size_t *length)
1596 {
1597     *length = 0;
1598     return NULL;
1599 }
1600
1601 voucher_t
1602 voucher_activity_create(firehose_tracepoint_id_t trace_id,
1603                 voucher_t base, firehose_activity_flags_t flags, uint64_t location)
1604 {
1605         (void)trace_id; (void)base; (void)flags; (void)location;
1606         return NULL;
1607 }
1608
1609 voucher_t
1610 voucher_activity_create_with_location(firehose_tracepoint_id_t *trace_id,
1611                 voucher_t base, firehose_activity_flags_t flags, uint64_t location)
1612 {
1613         (void)trace_id; (void)base; (void)flags; (void)location;
1614         return NULL;
1615 }
1616
1617 firehose_activity_id_t
1618 voucher_get_activity_id(voucher_t voucher, firehose_activity_id_t *parent_id)
1619 {
1620         (void)voucher; (void)parent_id;
1621         return 0;
1622 }
1623
1624 firehose_activity_id_t
1625 voucher_get_activity_id_and_creator(voucher_t voucher, uint64_t *creator_pid,
1626                 firehose_activity_id_t *parent_id)
1627 {
1628         if (creator_pid) *creator_pid = 0;
1629         (void)voucher; (void)parent_id;
1630         return 0;
1631 }
1632
1633 firehose_tracepoint_id_t
1634 voucher_activity_trace(firehose_stream_t stream,
1635                 firehose_tracepoint_id_t trace_id, uint64_t timestamp,
1636                 const void *pubdata, size_t publen)
1637 {
1638         (void)stream; (void)trace_id; (void)timestamp;
1639         (void)pubdata; (void)publen;
1640         return 0;
1641 }
1642
1643 firehose_tracepoint_id_t
1644 voucher_activity_trace_with_private_strings(firehose_stream_t stream,
1645                 firehose_tracepoint_id_t trace_id, uint64_t timestamp,
1646                 const void *pubdata, size_t publen,
1647                 const void *privdata, size_t privlen)
1648 {
1649         (void)stream; (void)trace_id; (void)timestamp;
1650         (void)pubdata; (void)publen; (void)privdata; (void)privlen;
1651         return 0;
1652 }
1653
1654 firehose_tracepoint_id_t
1655 voucher_activity_trace_v(firehose_stream_t stream,
1656                 firehose_tracepoint_id_t trace_id, uint64_t timestamp,
1657                 const struct iovec *iov, size_t publen, size_t privlen)
1658 {
1659         (void)stream; (void)trace_id; (void)timestamp;
1660         (void)iov; (void)publen; (void)privlen;
1661         return 0;
1662 }
1663
1664 void
1665 voucher_activity_flush(firehose_stream_t stream)
1666 {
1667         (void)stream;
1668 }
1669
1670 void
1671 voucher_activity_initialize_4libtrace(voucher_activity_hooks_t hooks)
1672 {
1673         (void)hooks;
1674 }
1675 #endif // OS_VOUCHER_ACTIVITY_SPI
1676
1677 size_t
1678 _voucher_debug(voucher_t v, char* buf, size_t bufsiz)
1679 {
1680         (void)v; (void)buf; (void)bufsiz;
1681         return 0;
1682 }
1683
1684 #endif // VOUCHER_USE_MACH_VOUCHER
1685
1686 #else // DISPATCH_VARIANT_DYLD_STUB
1687
1688 firehose_activity_id_t
1689 voucher_get_activity_id_4dyld(void)
1690 {
1691 #if VOUCHER_USE_MACH_VOUCHER
1692         return _voucher_get_activity_id(_voucher_get(), NULL);
1693 #else
1694         return 0;
1695 #endif
1696 }
1697
1698 #endif // DISPATCH_VARIANT_DYLD_STUB