libdispatch-703.1.4.tar.gz

[apple/libdispatch.git] / src / voucher.c

/*
 * Copyright (c) 2013-2016 Apple Inc. All rights reserved.
 *
 * @APPLE_APACHE_LICENSE_HEADER_START@
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * @APPLE_APACHE_LICENSE_HEADER_END@
 */

#include "internal.h"

#if !defined(VOUCHER_EXPORT_PERSONA_SPI)
#if TARGET_OS_IPHONE
#define VOUCHER_EXPORT_PERSONA_SPI 1
#else
#define VOUCHER_EXPORT_PERSONA_SPI 0
#endif
#endif

#ifndef PERSONA_ID_NONE
#define PERSONA_ID_NONE ((uid_t)-1)
#endif

#if !DISPATCH_VARIANT_DYLD_STUB

#if VOUCHER_USE_MACH_VOUCHER
#if !HAVE_PTHREAD_WORKQUEUE_QOS
#error Unsupported configuration, workqueue QoS support is required
#endif
#include <mach/mach_voucher.h>
#include <sys/proc_info.h>

#define MACH_ACTIVITY_ID_RANGE_SIZE 16
#define MACH_ACTIVITY_ID_MASK ((1ULL << FIREHOSE_ACTIVITY_ID_FLAGS_SHIFT) - 1)
#define FIREHOSE_ACTIVITY_ID_MAKE(aid, flags) \
                FIREHOSE_ACTIVITY_ID_MERGE_FLAGS((aid) & MACH_ACTIVITY_ID_MASK, flags)

static volatile uint64_t _voucher_aid_next;

#pragma mark -
#pragma mark voucher_t

OS_OBJECT_CLASS_DECL(voucher, object);
#if !USE_OBJC
OS_OBJECT_VTABLE_INSTANCE(voucher,
                (void (*)(_os_object_t))_voucher_xref_dispose,
                (void (*)(_os_object_t))_voucher_dispose);
#endif // USE_OBJC
#define VOUCHER_CLASS OS_OBJECT_VTABLE(voucher)

static inline voucher_t
_voucher_alloc(mach_voucher_attr_recipe_size_t extra)
{
        voucher_t voucher;
        size_t voucher_size = sizeof(voucher_s) + extra;
        voucher = (voucher_t)_os_object_alloc_realized(VOUCHER_CLASS, voucher_size);
#if VOUCHER_ENABLE_RECIPE_OBJECTS
        voucher->v_recipe_extra_size = extra;
        voucher->v_recipe_extra_offset = voucher_size - extra;
#else
        dispatch_assert(!extra);
#endif
        _dispatch_voucher_debug("alloc", voucher);
        return voucher;
}

#if VOUCHER_ENABLE_RECIPE_OBJECTS
voucher_t
voucher_create(voucher_recipe_t recipe)
{
        // TODO: capture current activities or current kvoucher ?
        mach_voucher_attr_recipe_size_t extra = recipe ? recipe->vr_size : 0;
        voucher_t voucher = _voucher_alloc(extra);
        if (extra) {
                memcpy(_voucher_extra_recipes(voucher), recipe->vr_data, extra);
        }
        return voucher;
}
#endif

DISPATCH_ALWAYS_INLINE
static inline voucher_t
_voucher_clone(const voucher_t ov, voucher_fields_t ignore_fields)
{
        mach_voucher_attr_recipe_size_t extra = 0;
        voucher_t v;

        if (ov && !(ignore_fields & VOUCHER_FIELD_EXTRA)) {
                extra = _voucher_extra_size(ov);
        }
        v = _voucher_alloc(extra);
        if (ov) {
                voucher_fields_t fields = ~ignore_fields;
                if ((fields & VOUCHER_FIELD_KVOUCHER) && ov->v_kvoucher) {
                        voucher_t kvb = ov->v_kvbase ? ov->v_kvbase : ov;
                        v->v_kvbase = _voucher_retain(kvb);
                        v->v_kvoucher = kvb->v_kvoucher;
                        v->v_kv_has_importance = kvb->v_kv_has_importance;
                }
                if (fields & VOUCHER_FIELD_PRIORITY) {
                        v->v_priority = ov->v_priority;
                }
                if (fields & VOUCHER_FIELD_ACTIVITY) {
                        v->v_activity = ov->v_activity;
                        v->v_activity_creator = ov->v_activity_creator;
                        v->v_parent_activity = ov->v_parent_activity;
                }
                if ((fields & VOUCHER_FIELD_EXTRA) && extra) {
                        memcpy(_voucher_extra_recipes(v), _voucher_extra_recipes(ov),extra);
                }
        }
        return v;
}

voucher_t
voucher_adopt(voucher_t voucher)
{
        if (voucher == VOUCHER_CURRENT) {
                return _voucher_copy();
        }
        return _voucher_adopt(voucher);
}

voucher_t
voucher_copy(void)
{
        return _voucher_copy();
}

voucher_t
voucher_copy_without_importance(void)
{
        return _voucher_copy_without_importance();
}

voucher_t
voucher_retain(voucher_t voucher)
{
        return _voucher_retain(voucher);
}

void
voucher_release(voucher_t voucher)
{
        return _voucher_release(voucher);
}

void
_voucher_thread_cleanup(void *voucher)
{
        // when a thread exits and has a voucher left, the kernel
        // will get rid of the voucher kernel object that is set on the thread,
        // we only need to release the voucher_t object.
        _voucher_release(voucher);
}

DISPATCH_CACHELINE_ALIGN
static TAILQ_HEAD(, voucher_s) _vouchers[VL_HASH_SIZE];
#define _vouchers_head(kv) (&_vouchers[VL_HASH((kv))])
static dispatch_unfair_lock_s _vouchers_lock;
#define _vouchers_lock_lock() _dispatch_unfair_lock_lock(&_vouchers_lock)
#define _vouchers_lock_unlock() _dispatch_unfair_lock_unlock(&_vouchers_lock)

static voucher_t
_voucher_find_and_retain(mach_voucher_t kv)
{
        voucher_t v;
        if (!kv) return NULL;
        _vouchers_lock_lock();
        TAILQ_FOREACH(v, _vouchers_head(kv), v_list) {
                if (v->v_ipc_kvoucher == kv) {
                        int xref_cnt = os_atomic_inc2o(v, os_obj_xref_cnt, relaxed);
                        _dispatch_voucher_debug("retain  -> %d", v, xref_cnt + 1);
                        if (slowpath(xref_cnt < 0)) {
                                _dispatch_voucher_debug("over-release", v);
                                _OS_OBJECT_CLIENT_CRASH("Voucher over-release");
                        }
                        if (xref_cnt == 0) {
                                // resurrection: raced with _voucher_remove
                                (void)os_atomic_inc2o(v, os_obj_ref_cnt, relaxed);
                        }
                        break;
                }
        }
        _vouchers_lock_unlock();
        return v;
}

static void
_voucher_insert(voucher_t v)
{
        mach_voucher_t kv = v->v_ipc_kvoucher;
        if (!kv) return;
        _vouchers_lock_lock();
        if (slowpath(_TAILQ_IS_ENQUEUED(v, v_list))) {
                _dispatch_voucher_debug("corruption", v);
                DISPATCH_CLIENT_CRASH(v->v_list.tqe_prev, "Voucher corruption");
        }
        TAILQ_INSERT_TAIL(_vouchers_head(kv), v, v_list);
        _vouchers_lock_unlock();
}

static void
_voucher_remove(voucher_t v)
{
        mach_voucher_t kv = v->v_ipc_kvoucher;
        if (!_TAILQ_IS_ENQUEUED(v, v_list)) return;
        _vouchers_lock_lock();
        if (slowpath(!kv)) {
                _dispatch_voucher_debug("corruption", v);
                DISPATCH_CLIENT_CRASH(0, "Voucher corruption");
        }
        // check for resurrection race with _voucher_find_and_retain
        if (os_atomic_load2o(v, os_obj_xref_cnt, ordered) < 0 &&
                        _TAILQ_IS_ENQUEUED(v, v_list)) {
                TAILQ_REMOVE(_vouchers_head(kv), v, v_list);
                _TAILQ_MARK_NOT_ENQUEUED(v, v_list);
                v->v_list.tqe_next = (void*)~0ull;
        }
        _vouchers_lock_unlock();
}

void
_voucher_dealloc_mach_voucher(mach_voucher_t kv)
{
        _dispatch_kvoucher_debug("dealloc", kv);
        _dispatch_voucher_debug_machport(kv);
        kern_return_t kr = mach_voucher_deallocate(kv);
        DISPATCH_VERIFY_MIG(kr);
        (void)dispatch_assume_zero(kr);
}

static inline kern_return_t
_voucher_create_mach_voucher(const mach_voucher_attr_recipe_data_t *recipes,
                size_t recipes_size, mach_voucher_t *kvp)
{
        kern_return_t kr;
        mach_port_t mhp = _dispatch_get_mach_host_port();
        mach_voucher_t kv = MACH_VOUCHER_NULL;
        mach_voucher_attr_raw_recipe_array_t kvr;
        mach_voucher_attr_recipe_size_t kvr_size;
        kvr = (mach_voucher_attr_raw_recipe_array_t)recipes;
        kvr_size = (mach_voucher_attr_recipe_size_t)recipes_size;
        kr = host_create_mach_voucher(mhp, kvr, kvr_size, &kv);
        DISPATCH_VERIFY_MIG(kr);
        if (!kr) {
                _dispatch_kvoucher_debug("create", kv);
                _dispatch_voucher_debug_machport(kv);
        }
        *kvp = kv;
        return kr;
}

void
_voucher_task_mach_voucher_init(void* ctxt DISPATCH_UNUSED)
{
        kern_return_t kr;
        mach_voucher_t kv = MACH_VOUCHER_NULL;
#if !VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
        static const mach_voucher_attr_recipe_data_t task_create_recipe = {
                .key = MACH_VOUCHER_ATTR_KEY_BANK,
                .command = MACH_VOUCHER_ATTR_BANK_CREATE,
        };
        kr = _voucher_create_mach_voucher(&task_create_recipe,
                        sizeof(task_create_recipe), &kv);
        if (slowpath(kr)) {
                DISPATCH_CLIENT_CRASH(kr, "Could not create task mach voucher");
        }
        _voucher_default_task_mach_voucher = kv;
#endif
        _voucher_task_mach_voucher = kv;
}

void
voucher_replace_default_voucher(void)
{
        (void)_voucher_get_task_mach_voucher(); // initalize task mach voucher
        mach_voucher_t kv, tkv = MACH_VOUCHER_NULL;
        voucher_t v = _voucher_get();
        if (v && v->v_kvoucher) {
                kern_return_t kr;
                kv = v->v_ipc_kvoucher ? v->v_ipc_kvoucher : v->v_kvoucher;
                const mach_voucher_attr_recipe_data_t task_copy_recipe = {
                        .key = MACH_VOUCHER_ATTR_KEY_BANK,
                        .command = MACH_VOUCHER_ATTR_COPY,
                        .previous_voucher = kv,
                };
                kr = _voucher_create_mach_voucher(&task_copy_recipe,
                                sizeof(task_copy_recipe), &tkv);
                if (dispatch_assume_zero(kr)) {
                        tkv = MACH_VOUCHER_NULL;
                }
        }
        if (!tkv) tkv = _voucher_default_task_mach_voucher;
        kv = os_atomic_xchg(&_voucher_task_mach_voucher, tkv, relaxed);
        if (kv && kv != _voucher_default_task_mach_voucher) {
                _voucher_dealloc_mach_voucher(kv);
        }
        _dispatch_voucher_debug("kvoucher[0x%08x] replace default voucher", v, tkv);
}

#define _voucher_mach_recipe_size(payload_size) \
        (sizeof(mach_voucher_attr_recipe_data_t) + (payload_size))

#if VOUCHER_USE_MACH_VOUCHER_PRIORITY
#define _voucher_mach_recipe_alloca(v) ((mach_voucher_attr_recipe_t)alloca(\
                _voucher_mach_recipe_size(0) + \
                _voucher_mach_recipe_size(sizeof(ipc_pthread_priority_value_t)) + \
                _voucher_mach_recipe_size(sizeof(_voucher_mach_udata_s)) + \
                _voucher_extra_size(v)))
#else
#define _voucher_mach_recipe_alloca(v) ((mach_voucher_attr_recipe_t)alloca(\
                _voucher_mach_recipe_size(0) + \
                _voucher_mach_recipe_size(sizeof(_voucher_mach_udata_s)) + \
                _voucher_extra_size(v)))
#endif

DISPATCH_ALWAYS_INLINE
static inline mach_voucher_attr_recipe_size_t
_voucher_mach_recipe_init(mach_voucher_attr_recipe_t mvar_buf, voucher_s *v,
                mach_voucher_t kvb, pthread_priority_t pp)
{
        mach_voucher_attr_recipe_size_t extra = _voucher_extra_size(v);
        mach_voucher_attr_recipe_size_t size = 0;

        // normalize to just the QoS class and 0 relative priority
        pp &= _PTHREAD_PRIORITY_QOS_CLASS_MASK;
        if (pp) pp |= _PTHREAD_PRIORITY_PRIORITY_MASK;

        *mvar_buf++ = (mach_voucher_attr_recipe_data_t){
                .key = MACH_VOUCHER_ATTR_KEY_ALL,
                .command = MACH_VOUCHER_ATTR_COPY,
                .previous_voucher = kvb,
        };
        size += _voucher_mach_recipe_size(0);

#if VOUCHER_USE_MACH_VOUCHER_PRIORITY
        if (pp) {
                ipc_pthread_priority_value_t value = (ipc_pthread_priority_value_t)pp;
                *mvar_buf++ = (mach_voucher_attr_recipe_data_t){
                        .key = MACH_VOUCHER_ATTR_KEY_PTHPRIORITY,
                        .command = MACH_VOUCHER_ATTR_PTHPRIORITY_CREATE,
                        .content_size = sizeof(value),
                };
                mvar_buf = _dispatch_memappend(mvar_buf, &value);
                size += _voucher_mach_recipe_size(sizeof(value));
        }
#endif // VOUCHER_USE_MACH_VOUCHER_PRIORITY

        if ((v && v->v_activity) || pp) {
                _voucher_mach_udata_s *udata_buf;
                unsigned udata_size = 0;

                if (v && v->v_activity) {
                        udata_size = offsetof(_voucher_mach_udata_s, _vmu_after_activity);
                } else {
                        udata_size = offsetof(_voucher_mach_udata_s, _vmu_after_priority);
                }
                *mvar_buf = (mach_voucher_attr_recipe_data_t){
                        .key = MACH_VOUCHER_ATTR_KEY_USER_DATA,
                        .command = MACH_VOUCHER_ATTR_USER_DATA_STORE,
                        .content_size = udata_size,
                };
                udata_buf = (_voucher_mach_udata_s *)(mvar_buf->content);

                if (v && v->v_activity) {
                        *udata_buf = (_voucher_mach_udata_s){
                                .vmu_magic = VOUCHER_MAGIC_V3,
                                .vmu_priority = (_voucher_priority_t)pp,
                                .vmu_activity = v->v_activity,
                                .vmu_activity_pid = v->v_activity_creator,
                                .vmu_parent_activity = v->v_parent_activity,
                        };
                } else {
                        *udata_buf = (_voucher_mach_udata_s){
                                .vmu_magic = VOUCHER_MAGIC_V3,
                                .vmu_priority = (_voucher_priority_t)pp,
                        };
                }

                mvar_buf = (mach_voucher_attr_recipe_t)(mvar_buf->content + udata_size);
                size += _voucher_mach_recipe_size(udata_size);
        }

        if (extra) {
                memcpy(mvar_buf, _voucher_extra_recipes(v), extra);
                size += extra;
        }
        return size;
}

mach_voucher_t
_voucher_get_mach_voucher(voucher_t voucher)
{
        if (!voucher) return MACH_VOUCHER_NULL;
        if (voucher->v_ipc_kvoucher) return voucher->v_ipc_kvoucher;
        mach_voucher_t kvb = voucher->v_kvoucher;
        if (!kvb) kvb = _voucher_get_task_mach_voucher();
        if (!voucher->v_activity && !voucher->v_priority &&
                        !_voucher_extra_size(voucher)) {
                return kvb;
        }

        mach_voucher_attr_recipe_t mvar = _voucher_mach_recipe_alloca(voucher);
        mach_voucher_attr_recipe_size_t size;
        mach_voucher_t kv, kvo;
        kern_return_t kr;

        size = _voucher_mach_recipe_init(mvar, voucher, kvb, voucher->v_priority);
        kr = _voucher_create_mach_voucher(mvar, size, &kv);
        if (dispatch_assume_zero(kr) || !kv){
                return MACH_VOUCHER_NULL;
        }
        if (!os_atomic_cmpxchgv2o(voucher, v_ipc_kvoucher, MACH_VOUCHER_NULL,
                        kv, &kvo, relaxed)) {
                _voucher_dealloc_mach_voucher(kv);
                kv = kvo;
        } else {
                if (kv == voucher->v_kvoucher) {
                        // if v_kvoucher == v_ipc_kvoucher we keep only one reference
                        _voucher_dealloc_mach_voucher(kv);
                }
                _voucher_insert(voucher);
                _dispatch_voucher_debug("kvoucher[0x%08x] create", voucher, kv);
        }
        return kv;
}

mach_voucher_t
_voucher_create_mach_voucher_with_priority(voucher_t voucher,
                pthread_priority_t priority)
{
        if (priority == _voucher_get_priority(voucher)) {
                return MACH_VOUCHER_NULL; // caller will use _voucher_get_mach_voucher
        }
        kern_return_t kr;
        mach_voucher_t kv, kvb = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL;
        if (!kvb) kvb = _voucher_get_task_mach_voucher();

        mach_voucher_attr_recipe_t mvar = _voucher_mach_recipe_alloca(voucher);
        mach_voucher_attr_recipe_size_t size;

        size = _voucher_mach_recipe_init(mvar, voucher, kvb, priority);
        kr = _voucher_create_mach_voucher(mvar, size, &kv);
        if (dispatch_assume_zero(kr) || !kv){
                return MACH_VOUCHER_NULL;
        }
        _dispatch_kvoucher_debug("create with priority from voucher[%p]", kv,
                        voucher);
        return kv;
}

static voucher_t
_voucher_create_with_mach_voucher(mach_voucher_t kv, mach_msg_bits_t msgh_bits)
{
        if (!kv) return NULL;
        kern_return_t kr;
        mach_voucher_attr_recipe_t vr;
        size_t vr_size;
        mach_voucher_attr_recipe_size_t kvr_size = 0;
        mach_voucher_attr_content_size_t udata_sz = 0;
        _voucher_mach_udata_s *udata = NULL;
#if !VOUCHER_USE_BANK_AUTOREDEEM
        mach_voucher_t rkv;
        const mach_voucher_attr_recipe_data_t redeem_recipe[] = {
                [0] = {
                        .key = MACH_VOUCHER_ATTR_KEY_ALL,
                        .command = MACH_VOUCHER_ATTR_COPY,
                        .previous_voucher = kv,
                },
                [1] = {
                        .key = MACH_VOUCHER_ATTR_KEY_BANK,
                        .command = MACH_VOUCHER_ATTR_REDEEM,
                },
        };
        kr = _voucher_create_mach_voucher(redeem_recipe, sizeof(redeem_recipe),
                        &rkv);
        if (!dispatch_assume_zero(kr)) {
                _voucher_dealloc_mach_voucher(kv);
                _dispatch_kvoucher_debug("redeemed from 0x%08x", rkv, kv);
                kv = rkv;
        } else {
                _dispatch_voucher_debug_machport(kv);
        }
#endif
        voucher_t v = _voucher_find_and_retain(kv);
        if (v) {
                _dispatch_voucher_debug("kvoucher[0x%08x] found", v, kv);
                _voucher_dealloc_mach_voucher(kv);
                return v;
        }
        vr_size = sizeof(*vr) + sizeof(_voucher_mach_udata_s);
        vr = alloca(vr_size);
        if (kv) {
                kvr_size = (mach_voucher_attr_recipe_size_t)vr_size;
                kr = mach_voucher_extract_attr_recipe(kv,
                                MACH_VOUCHER_ATTR_KEY_USER_DATA, (void*)vr, &kvr_size);
                DISPATCH_VERIFY_MIG(kr);
                if (!dispatch_assume_zero(kr) && kvr_size >= sizeof(*vr)) {
                        udata_sz = vr->content_size;
                        udata = (_voucher_mach_udata_s*)vr->content;
                        dispatch_assume(udata_sz >= sizeof(_voucher_magic_t));
                }
        }
        vr = NULL;

        v = _voucher_alloc(0);
        v->v_ipc_kvoucher = v->v_kvoucher = kv;
        v->v_kv_has_importance = !!(msgh_bits & MACH_MSGH_BITS_RAISEIMP);

        if (udata_sz >= offsetof(_voucher_mach_udata_s,_vmu_after_priority)){
                if (udata->vmu_magic == VOUCHER_MAGIC_V3) {
                        v->v_priority = udata->vmu_priority;
                }
        }
        bool remove_kv_userdata = false;
        if (udata_sz >= offsetof(_voucher_mach_udata_s, _vmu_after_activity)) {
#if !RDAR_25050791
                remove_kv_userdata = true;
#endif
                if (udata->vmu_magic == VOUCHER_MAGIC_V3 && udata->vmu_activity) {
                        v->v_activity = udata->vmu_activity;
                        v->v_activity_creator = udata->vmu_activity_pid;
                        v->v_parent_activity = udata->vmu_parent_activity;
                }
        }

        if (remove_kv_userdata) {
                mach_voucher_t nkv = MACH_VOUCHER_NULL;
                const mach_voucher_attr_recipe_data_t remove_userdata_recipe[] = {
                        [0] = {
                                .key = MACH_VOUCHER_ATTR_KEY_ALL,
                                .command = MACH_VOUCHER_ATTR_COPY,
                                .previous_voucher = kv,
                        },
                        [1] = {
                                .key = MACH_VOUCHER_ATTR_KEY_USER_DATA,
                                .command = MACH_VOUCHER_ATTR_REMOVE,
                        },
#if VOUCHER_USE_MACH_VOUCHER_PRIORITY
                        [2] = {
                                .key = MACH_VOUCHER_ATTR_KEY_PTHPRIORITY,
                                .command = MACH_VOUCHER_ATTR_REMOVE,
                        },
#endif
                };
                mach_voucher_attr_recipe_size_t size = sizeof(remove_userdata_recipe);

                kr = _voucher_create_mach_voucher(remove_userdata_recipe, size, &nkv);
                if (!dispatch_assume_zero(kr)) {
                        _dispatch_voucher_debug("kvoucher[0x%08x] udata removal "
                                        "(created 0x%08x)", v, kv, nkv);
                        v->v_ipc_kvoucher = MACH_VOUCHER_NULL;
                        v->v_kvoucher = nkv;
                        v->v_kvbase = _voucher_find_and_retain(nkv);
                        if (v->v_kvbase) {
                                _voucher_dealloc_mach_voucher(nkv); // borrow base reference
                        }
                        _voucher_dealloc_mach_voucher(kv);
                        kv = nkv;
                } else {
                        _dispatch_voucher_debug_machport(kv);
                }
        }

        _voucher_insert(v);
        _dispatch_voucher_debug("kvoucher[0x%08x] create", v, kv);
        return v;
}

voucher_t
_voucher_create_with_priority_and_mach_voucher(voucher_t ov,
                pthread_priority_t priority, mach_voucher_t kv)
{
        if (priority == _voucher_get_priority(ov)) {
                if (kv) _voucher_dealloc_mach_voucher(kv);
                return ov ? _voucher_retain(ov) : NULL;
        }
        voucher_t v = _voucher_find_and_retain(kv);
        voucher_fields_t ignore_fields = VOUCHER_FIELD_PRIORITY;

        if (v) {
                _dispatch_voucher_debug("kvoucher[0x%08x] find", v, kv);
                _voucher_dealloc_mach_voucher(kv);
                return v;
        }

        if (kv) ignore_fields |= VOUCHER_FIELD_KVOUCHER;
        v = _voucher_clone(ov, ignore_fields);
        if (priority) {
                v->v_priority = (_voucher_priority_t)priority;
        }
        if (kv) {
                v->v_ipc_kvoucher = v->v_kvoucher = kv;
                _voucher_insert(v);
                _dispatch_voucher_debug("kvoucher[0x%08x] create with priority from "
                                "voucher[%p]", v, kv, ov);
                _dispatch_voucher_debug_machport(kv);
        }
        return v;
}

voucher_t
_voucher_create_without_importance(voucher_t ov)
{
        // Nothing to do unless the old voucher has a kernel voucher. If it
        // doesn't, it can't have any importance, now or in the future.
        if (!ov) return NULL;
        if (!ov->v_kvoucher || !ov->v_kv_has_importance) return _voucher_retain(ov);
        kern_return_t kr;
        mach_voucher_t kv, okv;
        // Copy kernel voucher, removing importance.
        okv = ov->v_ipc_kvoucher ? ov->v_ipc_kvoucher : ov->v_kvoucher;
        const mach_voucher_attr_recipe_data_t importance_remove_recipe[] = {
                [0] = {
                        .key = MACH_VOUCHER_ATTR_KEY_ALL,
                        .command = MACH_VOUCHER_ATTR_COPY,
                        .previous_voucher = okv,
                },
                [1] = {
                        .key = MACH_VOUCHER_ATTR_KEY_IMPORTANCE,
                        .command = MACH_VOUCHER_ATTR_REMOVE,
                },
        };
        kr = _voucher_create_mach_voucher(importance_remove_recipe,
                        sizeof(importance_remove_recipe), &kv);
        if (dispatch_assume_zero(kr) || !kv){
                if (ov->v_ipc_kvoucher) return NULL;
                kv = MACH_VOUCHER_NULL;
        }
        if (kv == okv) {
                _voucher_dealloc_mach_voucher(kv);
                return _voucher_retain(ov);
        }
        voucher_t v = _voucher_find_and_retain(kv);
        if (v && ov->v_ipc_kvoucher) {
                _dispatch_voucher_debug("kvoucher[0x%08x] find without importance "
                                "from voucher[%p]", v, kv, ov);
                _voucher_dealloc_mach_voucher(kv);
                return v;
        }
        voucher_t kvbase = v;
        voucher_fields_t ignore_fields = VOUCHER_FIELD_KVOUCHER;
        v = _voucher_clone(ov, ignore_fields);
        v->v_kvoucher = kv;
        if (ov->v_ipc_kvoucher) {
                v->v_ipc_kvoucher = kv;
                _voucher_insert(v);
        } else if (kvbase) {
                v->v_kvbase = kvbase;
                _voucher_dealloc_mach_voucher(kv); // borrow base reference
        }
        if (!kvbase) {
                _dispatch_voucher_debug("kvoucher[0x%08x] create without importance "
                                "from voucher[%p]", v, kv, ov);
        }
        return v;
}

voucher_t
_voucher_create_accounting_voucher(voucher_t ov)
{
        // Nothing to do unless the old voucher has a kernel voucher. If it does
        // doesn't, it can't have any accounting attributes.
        if (!ov || !ov->v_kvoucher) return NULL;
        kern_return_t kr = KERN_SUCCESS;
        mach_voucher_t okv, kv = MACH_VOUCHER_NULL;
        okv = ov->v_ipc_kvoucher ? ov->v_ipc_kvoucher : ov->v_kvoucher;
        const mach_voucher_attr_recipe_data_t accounting_copy_recipe = {
                .key = MACH_VOUCHER_ATTR_KEY_BANK,
                .command = MACH_VOUCHER_ATTR_COPY,
                .previous_voucher = okv,
        };
        kr = _voucher_create_mach_voucher(&accounting_copy_recipe,
                        sizeof(accounting_copy_recipe), &kv);
        if (dispatch_assume_zero(kr) || !kv){
                return NULL;
        }
        voucher_t v = _voucher_find_and_retain(kv);
        if (v) {
                _dispatch_voucher_debug("kvoucher[0x%08x] find accounting voucher "
                                "from voucher[%p]", v, kv, ov);
                _voucher_dealloc_mach_voucher(kv);
                return v;
        }
        v = _voucher_alloc(0);
        v->v_ipc_kvoucher = v->v_kvoucher = kv;
        if (kv == okv) {
                v->v_kvbase = _voucher_retain(ov);
                _voucher_dealloc_mach_voucher(kv); // borrow base reference
        }
        _voucher_insert(v);
        _dispatch_voucher_debug("kvoucher[0x%08x] create accounting voucher "
                        "from voucher[%p]", v, kv, ov);
        return v;
}

voucher_t
voucher_create_with_mach_msg(mach_msg_header_t *msg)
{
        mach_msg_bits_t msgh_bits;
        mach_voucher_t kv = _voucher_mach_msg_get(msg, &msgh_bits);
        return _voucher_create_with_mach_voucher(kv, msgh_bits);
}

void
voucher_decrement_importance_count4CF(voucher_t v)
{
        if (!v || !v->v_kvoucher || !v->v_kv_has_importance) return;
        kern_return_t kr;
        mach_voucher_t kv = v->v_ipc_kvoucher ? v->v_ipc_kvoucher : v->v_kvoucher;
        uint32_t dec = 1;
        mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&dec;
        mach_voucher_attr_content_size_t kvc_in_size = sizeof(dec);
        mach_voucher_attr_content_t kvc_out = NULL;
        mach_voucher_attr_content_size_t kvc_out_size = 0;
#if DISPATCH_DEBUG
        uint32_t count = UINT32_MAX;
        kvc_out = (mach_voucher_attr_content_t)&count;
        kvc_out_size = sizeof(count);
#endif
        kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_IMPORTANCE,
                        MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL, kvc_in, kvc_in_size,
                        kvc_out, &kvc_out_size);
        DISPATCH_VERIFY_MIG(kr);
        if (kr == KERN_INVALID_TASK) return; // non-denap receiver rdar://25643185
#if DISPATCH_DEBUG
        _dispatch_voucher_debug("kvoucher[0x%08x] decrement importance count to %u:"
                        " %s - 0x%x", v, kv, count, mach_error_string(kr), kr);
#endif
        if (slowpath(dispatch_assume_zero(kr) == KERN_FAILURE)) {
                DISPATCH_CLIENT_CRASH(kr, "Voucher importance count underflow");
        }
}

#if VOUCHER_ENABLE_GET_MACH_VOUCHER
mach_voucher_t
voucher_get_mach_voucher(voucher_t voucher)
{
        return _voucher_get_mach_voucher(voucher);
}
#endif

void
_voucher_xref_dispose(voucher_t voucher)
{
        _dispatch_voucher_debug("xref_dispose", voucher);
        _voucher_remove(voucher);
        return _os_object_release_internal_inline((_os_object_t)voucher);
}

void
_voucher_dispose(voucher_t voucher)
{
        _dispatch_voucher_debug("dispose", voucher);
        if (slowpath(_TAILQ_IS_ENQUEUED(voucher, v_list))) {
                _dispatch_voucher_debug("corruption", voucher);
                DISPATCH_CLIENT_CRASH(voucher->v_list.tqe_prev, "Voucher corruption");
        }
        voucher->v_list.tqe_next = DISPATCH_OBJECT_LISTLESS;
        if (voucher->v_ipc_kvoucher) {
                if (voucher->v_ipc_kvoucher != voucher->v_kvoucher) {
                        _voucher_dealloc_mach_voucher(voucher->v_ipc_kvoucher);
                }
                voucher->v_ipc_kvoucher = MACH_VOUCHER_NULL;
        }
        if (voucher->v_kvoucher) {
                if (!voucher->v_kvbase) {
                        _voucher_dealloc_mach_voucher(voucher->v_kvoucher);
                }
                voucher->v_kvoucher = MACH_VOUCHER_NULL;
        }
        if (voucher->v_kvbase) {
                _voucher_release(voucher->v_kvbase);
                voucher->v_kvbase = NULL;
        }
        voucher->v_activity = 0;
        voucher->v_activity_creator = 0;
        voucher->v_parent_activity = 0;
        voucher->v_priority = 0;
#if VOUCHER_ENABLE_RECIPE_OBJECTS
        voucher->v_recipe_extra_size = 0;
        voucher->v_recipe_extra_offset = 0;
#endif
        return _os_object_dealloc((_os_object_t)voucher);
}

static void
_voucher_activity_debug_channel_barrier_nop(void *ctxt DISPATCH_UNUSED)
{
}

void
_voucher_activity_debug_channel_init(void)
{
        dispatch_mach_handler_function_t handler = NULL;

        if (_voucher_libtrace_hooks && _voucher_libtrace_hooks->vah_version >= 2) {
                handler = _voucher_libtrace_hooks->vah_debug_channel_handler;
        }

        if (!handler) return;

        dispatch_mach_t dm;
        mach_port_t dbgp;
        kern_return_t kr;

        kr = task_get_debug_control_port(mach_task_self(), &dbgp);
        DISPATCH_VERIFY_MIG(kr);
        if (kr) {
                DISPATCH_CLIENT_CRASH(kr, "Couldn't get debug control port");
        }
        if (dbgp) {
                dm = dispatch_mach_create_f("com.apple.debug-channel",
                                DISPATCH_TARGET_QUEUE_DEFAULT, NULL, handler);
                dispatch_mach_connect(dm, dbgp, MACH_PORT_NULL, NULL);
                // will force the DISPATCH_MACH_CONNECTED event
                dispatch_mach_send_barrier_f(dm, NULL,
                                _voucher_activity_debug_channel_barrier_nop);
                _voucher_activity_debug_channel = dm;
        }
}

void
_voucher_atfork_child(void)
{
        _dispatch_thread_setspecific(dispatch_voucher_key, NULL);
        _voucher_task_mach_voucher_pred = 0;
        _voucher_task_mach_voucher = MACH_VOUCHER_NULL;
#if !VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
        _voucher_default_task_mach_voucher = MACH_PORT_NULL;
#endif
        _voucher_aid_next = 0;
        _firehose_task_buffer_pred = 0;
        _firehose_task_buffer = NULL; // firehose buffer is VM_INHERIT_NONE
}

#if VOUCHER_EXPORT_PERSONA_SPI
#if VOUCHER_USE_PERSONA
static kern_return_t
_voucher_get_current_persona_token(struct persona_token *token)
{
        kern_return_t kr = KERN_FAILURE;
        voucher_t v = _voucher_get();

        if (v && v->v_kvoucher) {
                mach_voucher_t kv = v->v_ipc_kvoucher ?: v->v_kvoucher;
                mach_voucher_attr_content_t kvc_in = NULL;
                mach_voucher_attr_content_size_t kvc_in_size = 0;
                mach_voucher_attr_content_t kvc_out =
                        (mach_voucher_attr_content_t)token;
                mach_voucher_attr_content_size_t kvc_out_size = sizeof(*token);

                kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_BANK,
                                BANK_PERSONA_TOKEN, kvc_in, kvc_in_size,
                                kvc_out, &kvc_out_size);
                if (kr != KERN_NOT_SUPPORTED
                                // Voucher doesn't have a PERSONA_TOKEN
                                && kr != KERN_INVALID_VALUE
                                // Kernel doesn't understand BANK_PERSONA_TOKEN
                                && kr != KERN_INVALID_ARGUMENT) {
                        (void)dispatch_assume_zero(kr);
                }
        }
        return kr;
}
#endif

uid_t
voucher_get_current_persona(void)
{
        uid_t persona_id = PERSONA_ID_NONE;

#if VOUCHER_USE_PERSONA
        struct persona_token token;
        int err;

        if (_voucher_get_current_persona_token(&token) == KERN_SUCCESS) {
                return token.originator.persona_id;
        }

        // falling back to the process persona if there is no adopted voucher
        if (kpersona_get(&persona_id) < 0) {
                err = errno;
                if (err != ESRCH) {
                        (void)dispatch_assume_zero(err);
                }
        }
#endif
        return persona_id;
}

int
voucher_get_current_persona_originator_info(struct proc_persona_info *persona_info)
{
#if VOUCHER_USE_PERSONA
        struct persona_token token;
        if (_voucher_get_current_persona_token(&token) == KERN_SUCCESS) {
                *persona_info = token.originator;
                return 0;
        }
#else
        (void)persona_info;
#endif
        return -1;
}

int
voucher_get_current_persona_proximate_info(struct proc_persona_info *persona_info)
{
#if VOUCHER_USE_PERSONA
        struct persona_token token;
        if (_voucher_get_current_persona_token(&token) == KERN_SUCCESS) {
                *persona_info = token.proximate;
                return 0;
        }
#else
        (void)persona_info;
#endif
        return -1;
}
#endif

#pragma mark -
#pragma mark _voucher_init

boolean_t
voucher_mach_msg_set(mach_msg_header_t *msg)
{
        return _voucher_mach_msg_set(msg, _voucher_get());
}

void
voucher_mach_msg_clear(mach_msg_header_t *msg)
{
        (void)_voucher_mach_msg_clear(msg, false);
}

voucher_mach_msg_state_t
voucher_mach_msg_adopt(mach_msg_header_t *msg)
{
        mach_msg_bits_t msgh_bits;
        mach_voucher_t kv = _voucher_mach_msg_get(msg, &msgh_bits);
        if (!kv) return VOUCHER_MACH_MSG_STATE_UNCHANGED;
        voucher_t v = _voucher_create_with_mach_voucher(kv, msgh_bits);
        return (voucher_mach_msg_state_t)_voucher_adopt(v);
}

void
voucher_mach_msg_revert(voucher_mach_msg_state_t state)
{
        if (state == VOUCHER_MACH_MSG_STATE_UNCHANGED) return;
        _voucher_replace((voucher_t)state);
}

#if DISPATCH_USE_LIBKERNEL_VOUCHER_INIT
#include <_libkernel_init.h>

static const struct _libkernel_voucher_functions _voucher_libkernel_functions =
{
        .version = 1,
        .voucher_mach_msg_set = voucher_mach_msg_set,
        .voucher_mach_msg_clear = voucher_mach_msg_clear,
        .voucher_mach_msg_adopt = voucher_mach_msg_adopt,
        .voucher_mach_msg_revert = voucher_mach_msg_revert,
};

static void
_voucher_libkernel_init(void)
{
        kern_return_t kr = __libkernel_voucher_init(&_voucher_libkernel_functions);
        dispatch_assert(!kr);
}
#else
#define _voucher_libkernel_init()
#endif

void
voucher_activity_initialize_4libtrace(voucher_activity_hooks_t hooks)
{
        if (!os_atomic_cmpxchg(&_voucher_libtrace_hooks, NULL,
                        hooks, relaxed)) {
                DISPATCH_CLIENT_CRASH(_voucher_libtrace_hooks,
                                "voucher_activity_initialize_4libtrace called twice");
        }
}

void
_voucher_init(void)
{
        _voucher_libkernel_init();
        unsigned int i;
        for (i = 0; i < VL_HASH_SIZE; i++) {
                TAILQ_INIT(&_vouchers[i]);
        }
}

#pragma mark -
#pragma mark voucher_activity_t

DISPATCH_NOINLINE
static uint64_t
_voucher_activity_id_allocate_slow(uint64_t aid)
{
        kern_return_t kr;
        uint64_t next;

        kr = mach_generate_activity_id(mach_task_self(), 1, &next);
        if (unlikely(kr)) {
                DISPATCH_CLIENT_CRASH(kr, "Could not generate an activity ID");
        }
        next *= MACH_ACTIVITY_ID_RANGE_SIZE;
        next &= MACH_ACTIVITY_ID_MASK;
        if (unlikely(next == 0)) {
                next++;
        }

        if (unlikely(aid == 0)) {
                if (os_atomic_cmpxchg(&_voucher_aid_next, 0, next + 1, relaxed)) {
                        return next;
                }
        }
        return os_atomic_xchg(&_voucher_aid_next, next, relaxed);
}

DISPATCH_ALWAYS_INLINE
static firehose_activity_id_t
_voucher_activity_id_allocate(firehose_activity_flags_t flags)
{
        uint64_t aid, next;
        os_atomic_rmw_loop(&_voucher_aid_next, aid, next, relaxed, {
                next = aid + 1;
                if (aid == 0 || next % MACH_ACTIVITY_ID_RANGE_SIZE == 0) {
                        os_atomic_rmw_loop_give_up({
                                aid = _voucher_activity_id_allocate_slow(aid);
                                break;
                        });
                }
        });
        return FIREHOSE_ACTIVITY_ID_MAKE(aid, flags);
}

#define _voucher_activity_tracepoint_reserve(stamp, stream, pub, priv, privbuf) \
                firehose_buffer_tracepoint_reserve(_firehose_task_buffer, stamp, \
                                stream, pub, priv, privbuf)

#define _voucher_activity_tracepoint_flush(ft, ftid) \
                firehose_buffer_tracepoint_flush(_firehose_task_buffer, ft, ftid)

DISPATCH_NOINLINE
static void
_firehose_task_buffer_init(void *ctx OS_UNUSED)
{
        mach_port_t logd_port;

        /* Query the uniquepid of the current process */
        struct proc_uniqidentifierinfo p_uniqinfo = { };
        int info_size = 0;

        info_size = proc_pidinfo(getpid(), PROC_PIDUNIQIDENTIFIERINFO, 1,
                        &p_uniqinfo, PROC_PIDUNIQIDENTIFIERINFO_SIZE);
        if (slowpath(info_size != PROC_PIDUNIQIDENTIFIERINFO_SIZE)) {
                DISPATCH_INTERNAL_CRASH(info_size, "Unable to get the unique pid");
        }
        _voucher_unique_pid = p_uniqinfo.p_uniqueid;


        if (!fastpath(_voucher_libtrace_hooks)) {
                if (0) { // <rdar://problem/23393959>
                        DISPATCH_CLIENT_CRASH(0,
                                        "Activity subsystem isn't initialized yet");
                }
                return;
        }
        logd_port = _voucher_libtrace_hooks->vah_get_logd_port();
        if (logd_port) {
                unsigned long flags = 0;
#if DISPATCH_USE_MEMORYPRESSURE_SOURCE
                if (_dispatch_memory_warn) {
                        flags |= FIREHOSE_BUFFER_BANK_FLAG_LOW_MEMORY;
                }
#endif
                // firehose_buffer_create always consumes the send-right
                _firehose_task_buffer = firehose_buffer_create(logd_port,
                                _voucher_unique_pid, flags);
        }
}

DISPATCH_ALWAYS_INLINE
static inline bool
_voucher_activity_disabled(void)
{
        dispatch_once_f(&_firehose_task_buffer_pred,
                        NULL, _firehose_task_buffer_init);

        firehose_buffer_t fb = _firehose_task_buffer;
        if (fastpath(fb)) {
                return slowpath(fb->fb_header.fbh_sendp == MACH_PORT_DEAD);
        }
        return true;
}

void*
voucher_activity_get_metadata_buffer(size_t *length)
{
        if (_voucher_activity_disabled()) {
                *length = 0;
                return NULL;
        }

        firehose_buffer_header_t fbh = &_firehose_task_buffer->fb_header;

        *length = FIREHOSE_BUFFER_LIBTRACE_HEADER_SIZE;
        return (void *)((uintptr_t)(fbh + 1) - *length);
}

voucher_t
voucher_activity_create(firehose_tracepoint_id_t trace_id,
                voucher_t base, firehose_activity_flags_t flags, uint64_t location)
{
        return voucher_activity_create_with_location(&trace_id, base, flags, location);
}

voucher_t
voucher_activity_create_with_location(firehose_tracepoint_id_t *trace_id,
                voucher_t base, firehose_activity_flags_t flags, uint64_t location)
{
        firehose_activity_id_t va_id = 0, current_id = 0, parent_id = 0;
        firehose_tracepoint_id_u ftid = { .ftid_value = *trace_id };
        uint16_t pubsize = sizeof(va_id) + sizeof(location);
        uint64_t creator_id = 0;
        voucher_t ov = _voucher_get();
        voucher_t v;

        if (base == VOUCHER_CURRENT) {
                base = ov;
        }
        if (_voucher_activity_disabled()) {
                *trace_id = 0;
                return base ? _voucher_retain(base) : VOUCHER_NULL;
        }

        FIREHOSE_TRACE_ID_CLEAR_FLAG(ftid, base, has_unique_pid);
        if (ov && (current_id = ov->v_activity)) {
                FIREHOSE_TRACE_ID_SET_FLAG(ftid, base, has_current_aid);
                pubsize += sizeof(firehose_activity_id_t);
                if ((creator_id = ov->v_activity_creator)) {
                        FIREHOSE_TRACE_ID_SET_FLAG(ftid, base, has_unique_pid);
                        pubsize += sizeof(uint64_t);
                }
        }
        if (base != VOUCHER_NULL) {
                parent_id = base->v_activity;
        }

        if (parent_id) {
                FIREHOSE_TRACE_ID_SET_FLAG(ftid, activity, has_other_aid);
                pubsize += sizeof(firehose_activity_id_t);
                flags |= FIREHOSE_ACTIVITY_ID_FLAGS(parent_id);
        }

        if (firehose_precise_timestamps_enabled()) {
                flags |= firehose_activity_flags_precise_timestamp;
        }
        voucher_fields_t ignore_fields = VOUCHER_FIELD_ACTIVITY;
        v = _voucher_clone(base, ignore_fields);
        v->v_activity = va_id = _voucher_activity_id_allocate(flags);
        v->v_activity_creator = _voucher_unique_pid;
        v->v_parent_activity = parent_id;

        static const firehose_stream_t streams[2] = {
                firehose_stream_metadata,
                firehose_stream_persist,
        };
        firehose_tracepoint_t ft;
        uint64_t stamp = firehose_tracepoint_time(flags);

        for (size_t i = 0; i < countof(streams); i++) {
                ft = _voucher_activity_tracepoint_reserve(stamp, streams[i], pubsize,
                                0, NULL);
                if (!fastpath(ft)) continue;

                uint8_t *pubptr = ft->ft_data;
                if (current_id) {
                        pubptr = _dispatch_memappend(pubptr, &current_id);
                }
                if (creator_id) {
                        pubptr = _dispatch_memappend(pubptr, &creator_id);
                }
                if (parent_id) {
                        pubptr = _dispatch_memappend(pubptr, &parent_id);
                }
                pubptr = _dispatch_memappend(pubptr, &va_id);
                pubptr = _dispatch_memappend(pubptr, &location);
                _voucher_activity_tracepoint_flush(ft, ftid);
        }
        *trace_id = ftid.ftid_value;
        return v;
}

void
_voucher_activity_swap(firehose_activity_id_t old_id,
                firehose_activity_id_t new_id)
{
        if (_voucher_activity_disabled()) return;

        firehose_tracepoint_id_u ftid = { .ftid = {
                ._namespace = firehose_tracepoint_namespace_activity,
                ._type = _firehose_tracepoint_type_activity_swap,
        } };
        uint16_t pubsize = 0;

        if (old_id) {
                FIREHOSE_TRACE_ID_SET_FLAG(ftid, base, has_current_aid);
                pubsize += sizeof(firehose_activity_id_t);
        }
        if (new_id) {
                FIREHOSE_TRACE_ID_SET_FLAG(ftid, activity, has_other_aid);
                pubsize += sizeof(firehose_activity_id_t);
        }

        firehose_stream_t stream = firehose_stream_metadata;
        firehose_tracepoint_t ft;
        firehose_activity_flags_t flags = FIREHOSE_ACTIVITY_ID_FLAGS(old_id) |
                        FIREHOSE_ACTIVITY_ID_FLAGS(new_id);
        uint64_t stamp = firehose_tracepoint_time(flags);

        _dispatch_voucher_ktrace_activity_adopt(new_id);

        ft = _voucher_activity_tracepoint_reserve(stamp, stream, pubsize, 0, NULL);
        if (!fastpath(ft)) return;
        uint8_t *pubptr = ft->ft_data;
        if (old_id) pubptr = _dispatch_memappend(pubptr, &old_id);
        if (new_id) pubptr = _dispatch_memappend(pubptr, &new_id);
        _voucher_activity_tracepoint_flush(ft, ftid);
}

firehose_activity_id_t
voucher_get_activity_id_and_creator(voucher_t v, uint64_t *creator_pid,
                firehose_activity_id_t *parent_id)
{
        if (v == VOUCHER_CURRENT) {
                v = _voucher_get();
        }
        if (v == VOUCHER_NULL) {
                if (creator_pid) *creator_pid = 0;
                if (parent_id) *parent_id = FIREHOSE_ACTIVITY_ID_NULL;
                return FIREHOSE_ACTIVITY_ID_NULL;
        }
        if (creator_pid) *creator_pid = v->v_activity_creator;
        if (parent_id) *parent_id = v->v_parent_activity;
        return v->v_activity;
}

firehose_activity_id_t
voucher_get_activity_id(voucher_t v, firehose_activity_id_t *parent_id)
{
        return voucher_get_activity_id_and_creator(v, NULL, parent_id);
}

void
voucher_activity_flush(firehose_stream_t stream)
{
        if (_voucher_activity_disabled()) return;
        firehose_buffer_stream_flush(_firehose_task_buffer, stream);
}

DISPATCH_ALWAYS_INLINE
static inline firehose_tracepoint_id_t
_voucher_activity_trace(firehose_stream_t stream,
                firehose_tracepoint_id_u ftid, uint64_t stamp,
                const void *pubdata, size_t publen,
                const void *privdata, size_t privlen)
{
        const uint16_t ft_size = offsetof(struct firehose_tracepoint_s, ft_data);
        const size_t _firehose_chunk_payload_size =
                        sizeof(((struct firehose_buffer_chunk_s *)0)->fbc_data);

        if (_voucher_activity_disabled()) return 0;

        firehose_tracepoint_t ft;
        firehose_activity_id_t va_id = 0;
        firehose_buffer_chunk_t fbc;
        uint8_t *privptr, *pubptr;
        size_t pubsize = publen;
        voucher_t ov = _voucher_get();
        uint64_t creator_pid;

        if ((va_id = _voucher_get_activity_id(ov, &creator_pid))) {
                FIREHOSE_TRACE_ID_SET_FLAG(ftid, base, has_current_aid);
                pubsize += sizeof(va_id);
        }
        if (FIREHOSE_TRACE_ID_HAS_FLAG(ftid, base, has_unique_pid)) {
                if (creator_pid) {
                        pubsize += sizeof(creator_pid);
                } else {
                        FIREHOSE_TRACE_ID_CLEAR_FLAG(ftid, base, has_unique_pid);
                }
        } else {
                creator_pid = 0;
        }

        if (privlen) {
                FIREHOSE_TRACE_ID_SET_FLAG(ftid, log, has_private_data);
                pubsize += sizeof(struct firehose_buffer_range_s);
        }

        if (slowpath(ft_size + pubsize + privlen > _firehose_chunk_payload_size)) {
                DISPATCH_CLIENT_CRASH(ft_size + pubsize + privlen, "Log is too large");
        }

        ft = _voucher_activity_tracepoint_reserve(stamp, stream, (uint16_t)pubsize,
                                (uint16_t)privlen, &privptr);
        if (!fastpath(ft)) return 0;
        pubptr = ft->ft_data;
        if (va_id) {
                pubptr = _dispatch_memappend(pubptr, &va_id);
        }
        if (creator_pid) {
                pubptr = _dispatch_memappend(pubptr, &creator_pid);
        }
        if (privlen) {
                fbc = firehose_buffer_chunk_for_address(ft);
                struct firehose_buffer_range_s range = {
                        .fbr_offset = (uint16_t)(privptr - fbc->fbc_start),
                        .fbr_length = (uint16_t)privlen,
                };
                pubptr = _dispatch_memappend(pubptr, &range);
                _dispatch_mempcpy(privptr, privdata, privlen);
        }
        _dispatch_mempcpy(pubptr, pubdata, publen);
        _voucher_activity_tracepoint_flush(ft, ftid);
        return ftid.ftid_value;
}

firehose_tracepoint_id_t
voucher_activity_trace(firehose_stream_t stream,
                firehose_tracepoint_id_t trace_id, uint64_t timestamp,
                const void *pubdata, size_t publen)
{
        firehose_tracepoint_id_u ftid = { .ftid_value = trace_id };
        return _voucher_activity_trace(stream, ftid, timestamp, pubdata, publen,
                        NULL, 0);
}

firehose_tracepoint_id_t
voucher_activity_trace_with_private_strings(firehose_stream_t stream,
                firehose_tracepoint_id_t trace_id, uint64_t timestamp,
                const void *pubdata, size_t publen,
                const void *privdata, size_t privlen)
{
        firehose_tracepoint_id_u ftid = { .ftid_value = trace_id };
        return _voucher_activity_trace(stream, ftid, timestamp,
                        pubdata, publen, privdata, privlen);
}

#pragma mark -
#pragma mark _voucher_debug

size_t
_voucher_debug(voucher_t v, char* buf, size_t bufsiz)
{
        size_t offset = 0;
        #define bufprintf(...) \
                        offset += dsnprintf(&buf[offset], bufsiz - offset, ##__VA_ARGS__)
        bufprintf("voucher[%p] = { xrefcnt = 0x%x, refcnt = 0x%x", v,
                        v->os_obj_xref_cnt + 1, v->os_obj_ref_cnt + 1);

        if (v->v_kvbase) {
                bufprintf(", base voucher %p", v->v_kvbase);
        }
        if (v->v_kvoucher) {
                bufprintf(", kvoucher%s 0x%x", v->v_kvoucher == v->v_ipc_kvoucher ?
                                " & ipc kvoucher" : "", v->v_kvoucher);
        }
        if (v->v_ipc_kvoucher && v->v_ipc_kvoucher != v->v_kvoucher) {
                bufprintf(", ipc kvoucher 0x%x", v->v_ipc_kvoucher);
        }
        if (v->v_priority) {
                bufprintf(", QOS 0x%x", v->v_priority);
        }
        if (v->v_activity) {
                bufprintf(", activity 0x%llx (pid: 0x%16llx, parent 0x%llx)",
                                v->v_activity, v->v_activity_creator, v->v_parent_activity);
        }
        bufprintf(" }");
        return offset;
}

#else // VOUCHER_USE_MACH_VOUCHER

#pragma mark -
#pragma mark Simulator / vouchers disabled

#if VOUCHER_ENABLE_RECIPE_OBJECTS
voucher_t
voucher_create(voucher_recipe_t recipe)
{
        (void)recipe;
        return NULL;
}
#endif

voucher_t
voucher_adopt(voucher_t voucher)
{
        return voucher;
}

voucher_t
voucher_copy(void)
{
        return NULL;
}

voucher_t
voucher_copy_without_importance(void)
{
        return NULL;
}

voucher_t
voucher_retain(voucher_t voucher)
{
        return voucher;
}

void
voucher_release(voucher_t voucher)
{
        (void)voucher;
}

void
voucher_replace_default_voucher(void)
{
}

void
voucher_decrement_importance_count4CF(voucher_t v)
{
        (void)v;
}

void
_voucher_thread_cleanup(void *voucher)
{
        (void)voucher;
}

void
_voucher_dealloc_mach_voucher(mach_voucher_t kv)
{
        (void)kv;
}

mach_voucher_t
_voucher_get_mach_voucher(voucher_t voucher)
{
        (void)voucher;
        return MACH_VOUCHER_NULL;
}

mach_voucher_t
_voucher_create_mach_voucher_with_priority(voucher_t voucher,
                pthread_priority_t priority)
{
        (void)voucher; (void)priority;
        return MACH_VOUCHER_NULL;
}

voucher_t
_voucher_create_with_priority_and_mach_voucher(voucher_t voucher,
                pthread_priority_t priority, mach_voucher_t kv)
{
        (void)voucher; (void)priority; (void)kv;
        return NULL;
}

voucher_t
_voucher_create_accounting_voucher(voucher_t voucher)
{
        (void)voucher;
        return NULL;
}

voucher_t
voucher_create_with_mach_msg(mach_msg_header_t *msg)
{
        (void)msg;
        return NULL;
}

#if VOUCHER_ENABLE_GET_MACH_VOUCHER
mach_voucher_t
voucher_get_mach_voucher(voucher_t voucher)
{
        (void)voucher;
        return 0;
}
#endif

void
_voucher_xref_dispose(voucher_t voucher)
{
        (void)voucher;
}

void
_voucher_dispose(voucher_t voucher)
{
        (void)voucher;
}

#if VOUCHER_EXPORT_PERSONA_SPI
uid_t
voucher_get_current_persona(void)
{
        return PERSONA_ID_NONE;
}

int
voucher_get_current_persona_originator_info(struct proc_persona_info *persona_info)
{
        (void)persona_info;
        return -1;
}

int
voucher_get_current_persona_proximate_info(struct proc_persona_info *persona_info)
{
        (void)persona_info;
        return -1;
}
#endif

void
_voucher_activity_debug_channel_init(void)
{
}

void
_voucher_atfork_child(void)
{
}

void
_voucher_init(void)
{
}

void*
voucher_activity_get_metadata_buffer(size_t *length)
{
    *length = 0;
    return NULL;
}

voucher_t
voucher_activity_create(firehose_tracepoint_id_t trace_id,
                voucher_t base, firehose_activity_flags_t flags, uint64_t location)
{
        (void)trace_id; (void)base; (void)flags; (void)location;
        return NULL;
}

voucher_t
voucher_activity_create_with_location(firehose_tracepoint_id_t *trace_id,
                voucher_t base, firehose_activity_flags_t flags, uint64_t location)
{
        (void)trace_id; (void)base; (void)flags; (void)location;
        return NULL;
}

firehose_activity_id_t
voucher_get_activity_id(voucher_t voucher, firehose_activity_id_t *parent_id)
{
        (void)voucher; (void)parent_id;
        return 0;
}

firehose_activity_id_t
voucher_get_activity_id_and_creator(voucher_t voucher, uint64_t *creator_pid,
                firehose_activity_id_t *parent_id)
{
        if (creator_pid) *creator_pid = 0;
        (void)voucher; (void)parent_id;
        return 0;
}

firehose_tracepoint_id_t
voucher_activity_trace(firehose_stream_t stream,
                firehose_tracepoint_id_t trace_id, uint64_t timestamp,
                const void *pubdata, size_t publen)
{
        (void)stream; (void)trace_id; (void)timestamp;
        (void)pubdata; (void)publen;
        return 0;
}

firehose_tracepoint_id_t
voucher_activity_trace_with_private_strings(firehose_stream_t stream,
                firehose_tracepoint_id_t trace_id, uint64_t timestamp,
                const void *pubdata, size_t publen,
                const void *privdata, size_t privlen)
{
        (void)stream; (void)trace_id; (void)timestamp;
        (void)pubdata; (void)publen; (void)privdata; (void)privlen;
        return 0;
}

void
voucher_activity_flush(firehose_stream_t stream)
{
        (void)stream;
}

void
voucher_activity_initialize_4libtrace(voucher_activity_hooks_t hooks)
{
        (void)hooks;
}

size_t
_voucher_debug(voucher_t v, char* buf, size_t bufsiz)
{
        (void)v; (void)buf; (void)bufsiz;
        return 0;
}

#endif // VOUCHER_USE_MACH_VOUCHER

#else // DISPATCH_VARIANT_DYLD_STUB

firehose_activity_id_t
voucher_get_activity_id_4dyld(void)
{
#if VOUCHER_USE_MACH_VOUCHER
        return _voucher_get_activity_id(_voucher_get(), NULL);
#else
        return 0;
#endif
}

#endif // DISPATCH_VARIANT_DYLD_STUB