X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/490019cf9519204c5fb36b2fba54ceb983bb6b72..a991bd8d3e7fe02dbca0644054bab73c5b75324a:/osfmk/ipc/ipc_voucher.c diff --git a/osfmk/ipc/ipc_voucher.c b/osfmk/ipc/ipc_voucher.c index f914dec6f..4a753c737 100644 --- a/osfmk/ipc/ipc_voucher.c +++ b/osfmk/ipc/ipc_voucher.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Apple Inc. All rights reserved. + * Copyright (c) 2013-2020 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -27,6 +27,7 @@ */ #include +#include #include #include #include @@ -42,14 +43,23 @@ #include #include #include +#include /* * Sysctl variable; enable and disable tracing of voucher contents */ uint32_t ipc_voucher_trace_contents = 0; -static zone_t ipc_voucher_zone; -static zone_t ipc_voucher_attr_control_zone; +static SECURITY_READ_ONLY_LATE(zone_t) ipc_voucher_zone; +static ZONE_DECLARE(ipc_voucher_attr_control_zone, "ipc voucher attr controls", + sizeof(struct ipc_voucher_attr_control), ZC_NOENCRYPT | ZC_ZFREE_CLEARMEM); + +ZONE_INIT(&ipc_voucher_zone, "ipc vouchers", sizeof(struct ipc_voucher), + ZC_NOENCRYPT | ZC_ZFREE_CLEARMEM | ZC_NOSEQUESTER, + ZONE_ID_IPC_VOUCHERS, NULL); + +#define voucher_require(v) \ + zone_id_require(ZONE_ID_IPC_VOUCHERS, sizeof(struct ipc_voucher), v) /* * Voucher hash table @@ -58,18 +68,16 @@ static zone_t ipc_voucher_attr_control_zone; #define IV_HASH_BUCKET(x) ((x) % IV_HASH_BUCKETS) static queue_head_t ivht_bucket[IV_HASH_BUCKETS]; -static lck_spin_t ivht_lock_data; +static LCK_SPIN_DECLARE_ATTR(ivht_lock_data, &ipc_lck_grp, &ipc_lck_attr); static uint32_t ivht_count = 0; -#define ivht_lock_init() \ - lck_spin_init(&ivht_lock_data, &ipc_lck_grp, &ipc_lck_attr) #define ivht_lock_destroy() \ lck_spin_destroy(&ivht_lock_data, &ipc_lck_grp) -#define ivht_lock() \ - lck_spin_lock(&ivht_lock_data) -#define ivht_lock_try() \ - lck_spin_try_lock(&ivht_lock_data) -#define ivht_unlock() \ +#define ivht_lock() \ + lck_spin_lock_grp(&ivht_lock_data, &ipc_lck_grp) +#define ivht_lock_try() \ + lck_spin_try_lock_grp(&ivht_lock_data, &ipc_lck_grp) +#define ivht_unlock() \ lck_spin_unlock(&ivht_lock_data) /* @@ -81,40 +89,35 @@ static uint32_t ivht_count = 0; */ static iv_index_t ivgt_keys_in_use = MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN; static ipc_voucher_global_table_element iv_global_table[MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN]; -static lck_spin_t ivgt_lock_data; +static LCK_SPIN_DECLARE_ATTR(ivgt_lock_data, &ipc_lck_grp, &ipc_lck_attr); -#define ivgt_lock_init() \ - lck_spin_init(&ivgt_lock_data, &ipc_lck_grp, &ipc_lck_attr) #define ivgt_lock_destroy() \ lck_spin_destroy(&ivgt_lock_data, &ipc_lck_grp) -#define ivgt_lock() \ - lck_spin_lock(&ivgt_lock_data) -#define ivgt_lock_try() \ - lck_spin_try_lock(&ivgt_lock_data) -#define ivgt_unlock() \ +#define ivgt_lock() \ + lck_spin_lock_grp(&ivgt_lock_data, &ipc_lck_grp) +#define ivgt_lock_try() \ + lck_spin_try_lock_grp(&ivgt_lock_data, &ipc_lck_grp) +#define ivgt_unlock() \ lck_spin_unlock(&ivgt_lock_data) ipc_voucher_t iv_alloc(iv_index_t entries); void iv_dealloc(ipc_voucher_t iv, boolean_t unhash); -static inline iv_refs_t +os_refgrp_decl(static, iv_refgrp, "voucher", NULL); +os_refgrp_decl(static, ivac_refgrp, "voucher attribute control", NULL); + +static inline void iv_reference(ipc_voucher_t iv) { - iv_refs_t refs; - - refs = hw_atomic_add(&iv->iv_refs, 1); - return refs; + os_ref_retain(&iv->iv_refs); } static inline void iv_release(ipc_voucher_t iv) { - iv_refs_t refs; - - assert(0 < iv->iv_refs); - refs = hw_atomic_sub(&iv->iv_refs, 1); - if (0 == refs) + if (os_ref_release(&iv->iv_refs) == 0) { iv_dealloc(iv, TRUE); + } } /* @@ -156,32 +159,33 @@ static inline iv_index_t iv_key_to_index(mach_voucher_attr_key_t key) { if (MACH_VOUCHER_ATTR_KEY_ALL == key || - MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN < key) + MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN < key) { return IV_UNUSED_KEYINDEX; + } return (iv_index_t)key - 1; } static inline mach_voucher_attr_key_t iv_index_to_key(iv_index_t key_index) { - if (MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN > key_index) + if (MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN > key_index) { return iv_global_table[key_index].ivgte_key; + } return MACH_VOUCHER_ATTR_KEY_NONE; - } static void ivace_release(iv_index_t key_index, iv_index_t value_index); -static void ivace_lookup_values(iv_index_t key_index, iv_index_t value_index, - mach_voucher_attr_value_handle_array_t values, - mach_voucher_attr_value_handle_array_size_t *count); +static void ivace_lookup_values(iv_index_t key_index, iv_index_t value_index, + mach_voucher_attr_value_handle_array_t values, + mach_voucher_attr_value_handle_array_size_t *count); static iv_index_t iv_lookup(ipc_voucher_t, iv_index_t); - + static void ivgt_lookup(iv_index_t, - boolean_t, - ipc_voucher_attr_manager_t *, - ipc_voucher_attr_control_t *); + boolean_t, + ipc_voucher_attr_manager_t *, + ipc_voucher_attr_control_t *); static kern_return_t ipc_voucher_prepare_processing_recipe( @@ -192,41 +196,16 @@ ipc_voucher_prepare_processing_recipe( ipc_voucher_attr_manager_flags flags, int *need_processing); -#if defined(MACH_VOUCHER_ATTR_KEY_USER_DATA) || defined(MACH_VOUCHER_ATTR_KEY_TEST) -void user_data_attr_manager_init(void); -#endif - -void +__startup_func +static void ipc_voucher_init(void) { - natural_t ipc_voucher_max = (task_max + thread_max) * 2; - natural_t attr_manager_max = MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN; - iv_index_t i; - - ipc_voucher_zone = zinit(sizeof(struct ipc_voucher), - ipc_voucher_max * sizeof(struct ipc_voucher), - sizeof(struct ipc_voucher), - "ipc vouchers"); - zone_change(ipc_voucher_zone, Z_NOENCRYPT, TRUE); - - ipc_voucher_attr_control_zone = zinit(sizeof(struct ipc_voucher_attr_control), - attr_manager_max * sizeof(struct ipc_voucher_attr_control), - sizeof(struct ipc_voucher_attr_control), - "ipc voucher attr controls"); - zone_change(ipc_voucher_attr_control_zone, Z_NOENCRYPT, TRUE); - /* initialize voucher hash */ - ivht_lock_init(); - for (i = 0; i < IV_HASH_BUCKETS; i++) + for (iv_index_t i = 0; i < IV_HASH_BUCKETS; i++) { queue_init(&ivht_bucket[i]); - - /* initialize global table locking */ - ivgt_lock_init(); - -#if defined(MACH_VOUCHER_ATTR_KEY_USER_DATA) || defined(MACH_VOUCHER_ATTR_KEY_TEST) - user_data_attr_manager_init(); -#endif + } } +STARTUP(MACH_IPC, STARTUP_RANK_FIRST, ipc_voucher_init); ipc_voucher_t iv_alloc(iv_index_t entries) @@ -236,10 +215,11 @@ iv_alloc(iv_index_t entries) iv = (ipc_voucher_t)zalloc(ipc_voucher_zone); - if (IV_NULL == iv) + if (IV_NULL == iv) { return IV_NULL; - - iv->iv_refs = 1; + } + + os_ref_init(&iv->iv_refs, &iv_refgrp); iv->iv_sum = 0; iv->iv_hash = 0; iv->iv_port = IP_NULL; @@ -261,10 +241,11 @@ iv_alloc(iv_index_t entries) } /* initialize the table entries */ - for (i=0; i < iv->iv_table_size; i++) + for (i = 0; i < iv->iv_table_size; i++) { iv->iv_table[i] = IV_UNUSED_VALINDEX; - - return (iv); + } + + return iv; } /* @@ -276,9 +257,9 @@ iv_alloc(iv_index_t entries) * they are immutable once references are distributed. */ static void -iv_set(ipc_voucher_t iv, - iv_index_t key_index, - iv_index_t value_index) +iv_set(ipc_voucher_t iv, + iv_index_t key_index, + iv_index_t value_index) { assert(key_index < iv->iv_table_size); iv->iv_table[key_index] = value_index; @@ -295,17 +276,18 @@ iv_dealloc(ipc_voucher_t iv, boolean_t unhash) */ if (unhash) { ivht_lock(); - assert(0 == iv->iv_refs); + assert(os_ref_get_count(&iv->iv_refs) == 0); assert(IV_HASH_BUCKETS > iv->iv_hash); queue_remove(&ivht_bucket[iv->iv_hash], iv, ipc_voucher_t, iv_hash_link); ivht_count--; ivht_unlock(); - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_VOUCHER_DESTROY) | DBG_FUNC_NONE, - VM_KERNEL_ADDRPERM((uintptr_t)iv), 0, ivht_count, 0, 0); - - } else - assert(0 == --iv->iv_refs); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_VOUCHER_DESTROY) | DBG_FUNC_NONE, + VM_KERNEL_ADDRPERM((uintptr_t)iv), 0, ivht_count, 0, 0); + } else { + os_ref_count_t cnt __assert_only = os_ref_release(&iv->iv_refs); + assert(cnt == 0); + } /* * if a port was allocated for this voucher, @@ -314,7 +296,7 @@ iv_dealloc(ipc_voucher_t iv, boolean_t unhash) * is gone. We can just discard it now. */ if (IP_VALID(port)) { - assert(ip_active(port)); + require_ip_active(port); assert(port->ip_srights == 0); ipc_port_dealloc_kernel(port); @@ -327,10 +309,11 @@ iv_dealloc(ipc_voucher_t iv, boolean_t unhash) iv_set(iv, i, ~0); #endif } - - if (iv->iv_table != iv->iv_inline_table) - kfree(iv->iv_table, - iv->iv_table_size * sizeof(*iv->iv_table)); + + if (iv->iv_table != iv->iv_inline_table) { + kfree(iv->iv_table, + iv->iv_table_size * sizeof(*iv->iv_table)); + } zfree(ipc_voucher_zone, iv); } @@ -346,8 +329,9 @@ iv_dealloc(ipc_voucher_t iv, boolean_t unhash) static inline iv_index_t iv_lookup(ipc_voucher_t iv, iv_index_t key_index) { - if (key_index < iv->iv_table_size) + if (key_index < iv->iv_table_size) { return iv->iv_table[key_index]; + } return IV_UNUSED_VALINDEX; } @@ -366,22 +350,35 @@ iv_lookup(ipc_voucher_t iv, iv_index_t key_index) */ uintptr_t unsafe_convert_port_to_voucher( - ipc_port_t port) + ipc_port_t port) { if (IP_VALID(port)) { - uintptr_t voucher = (uintptr_t) port->ip_kobject; + /* vouchers never labeled (they get transformed before use) */ + if (ip_is_kolabeled(port)) { + return (uintptr_t)IV_NULL; + } /* * No need to lock because we have a reference on the * port, and if it is a true voucher port, that reference * keeps the voucher bound to the port (and active). */ - if (ip_kotype(port) == IKOT_VOUCHER) - return (voucher); + if (ip_kotype(port) == IKOT_VOUCHER) { + return (uintptr_t)port->ip_kobject; + } } return (uintptr_t)IV_NULL; } +static ipc_voucher_t +ip_get_voucher(ipc_port_t port) +{ + ipc_voucher_t voucher = (ipc_voucher_t)ip_get_kobject(port); + require_ip_active(port); + voucher_require(voucher); + return voucher; +} + /* * Routine: convert_port_to_voucher * Purpose: @@ -394,23 +391,17 @@ unsafe_convert_port_to_voucher( */ ipc_voucher_t convert_port_to_voucher( - ipc_port_t port) + ipc_port_t port) { - if (IP_VALID(port)) { - ipc_voucher_t voucher = (ipc_voucher_t) port->ip_kobject; - + if (IP_VALID(port) && ip_kotype(port) == IKOT_VOUCHER) { /* * No need to lock because we have a reference on the * port, and if it is a true voucher port, that reference * keeps the voucher bound to the port (and active). */ - if (ip_kotype(port) != IKOT_VOUCHER) - return IV_NULL; - - assert(ip_active(port)); - + ipc_voucher_t voucher = ip_get_voucher(port); ipc_voucher_reference(voucher); - return (voucher); + return voucher; } return IV_NULL; } @@ -426,7 +417,7 @@ convert_port_to_voucher( ipc_voucher_t convert_port_name_to_voucher( - mach_port_name_t voucher_name) + mach_port_name_t voucher_name) { ipc_voucher_t iv; kern_return_t kr; @@ -434,8 +425,9 @@ convert_port_name_to_voucher( if (MACH_PORT_VALID(voucher_name)) { kr = ipc_port_translate_send(current_space(), voucher_name, &port); - if (KERN_SUCCESS != kr) + if (KERN_SUCCESS != kr) { return IV_NULL; + } iv = convert_port_to_voucher(port); ip_unlock(port); @@ -448,20 +440,19 @@ convert_port_name_to_voucher( void ipc_voucher_reference(ipc_voucher_t voucher) { - iv_refs_t refs; - - if (IPC_VOUCHER_NULL == voucher) + if (IPC_VOUCHER_NULL == voucher) { return; + } - refs = iv_reference(voucher); - assert(1 < refs); + iv_reference(voucher); } void ipc_voucher_release(ipc_voucher_t voucher) { - if (IPC_VOUCHER_NULL != voucher) + if (IPC_VOUCHER_NULL != voucher) { iv_release(voucher); + } } /* @@ -469,26 +460,18 @@ ipc_voucher_release(ipc_voucher_t voucher) * Purpose: * Called whenever the Mach port system detects no-senders * on the voucher port. - * - * Each time the send-right count goes positive, a no-senders - * notification is armed (and a voucher reference is donated). - * So, each notification that comes in must release a voucher - * reference. If more send rights have been added since it - * fired (asynchronously), they will be protected by a different - * reference hold. */ void ipc_voucher_notify(mach_msg_header_t *msg) { mach_no_senders_notification_t *notification = (void *)msg; ipc_port_t port = notification->not_header.msgh_remote_port; - ipc_voucher_t iv; + ipc_voucher_t voucher = ip_get_voucher(port); - assert(ip_active(port)); assert(IKOT_VOUCHER == ip_kotype(port)); - iv = (ipc_voucher_t)port->ip_kobject; - ipc_voucher_release(iv); + /* consume the reference donated by convert_voucher_to_port */ + ipc_voucher_release(voucher); } /* @@ -497,47 +480,22 @@ ipc_voucher_notify(mach_msg_header_t *msg) ipc_port_t convert_voucher_to_port(ipc_voucher_t voucher) { - ipc_port_t port, send; - - if (IV_NULL == voucher) - return (IP_NULL); - - assert(0 < voucher->iv_refs); - - /* create a port if needed */ - port = voucher->iv_port; - if (!IP_VALID(port)) { - port = ipc_port_alloc_kernel(); - assert(IP_VALID(port)); - ipc_kobject_set_atomically(port, (ipc_kobject_t) voucher, IKOT_VOUCHER); - - /* If we lose the race, deallocate and pick up the other guy's port */ - if (!OSCompareAndSwapPtr(IP_NULL, port, &voucher->iv_port)) { - ipc_port_dealloc_kernel(port); - port = voucher->iv_port; - assert(ip_kotype(port) == IKOT_VOUCHER); - assert(port->ip_kobject == (ipc_kobject_t)voucher); - } + if (IV_NULL == voucher) { + return IP_NULL; } - - ip_lock(port); - assert(ip_active(port)); - send = ipc_port_make_send_locked(port); - if (1 == port->ip_srights) { - ipc_port_t old_notify; + voucher_require(voucher); + assert(os_ref_get_count(&voucher->iv_refs) > 0); - /* transfer our ref to the port, and arm the no-senders notification */ - assert(IP_NULL == port->ip_nsrequest); - ipc_port_nsrequest(port, port->ip_mscount, ipc_port_make_sonce_locked(port), &old_notify); - /* port unlocked */ - assert(IP_NULL == old_notify); - } else { - /* piggyback on the existing port reference, so consume ours */ - ip_unlock(port); + /* + * make a send right and donate our reference for ipc_voucher_notify + * if this is the first send right + */ + if (!ipc_kobject_make_send_lazy_alloc_port(&voucher->iv_port, + (ipc_kobject_t)voucher, IKOT_VOUCHER, false, 0)) { ipc_voucher_release(voucher); } - return (send); + return voucher->iv_port; } #define ivace_reset_data(ivace_elem, next_index) { \ @@ -573,32 +531,33 @@ ivac_alloc(iv_index_t key_index) ivac = (ipc_voucher_attr_control_t)zalloc(ipc_voucher_attr_control_zone); - if (IVAC_NULL == ivac) + if (IVAC_NULL == ivac) { return IVAC_NULL; - - ivac->ivac_refs = 1; + } + + os_ref_init(&ivac->ivac_refs, &ivac_refgrp); ivac->ivac_is_growing = FALSE; ivac->ivac_port = IP_NULL; /* start with just the inline table */ - table = (ivac_entry_t) kalloc(IVAC_ENTRIES_MIN * sizeof(ivac_entry)); + table = (ivac_entry_t) kalloc(IVAC_ENTRIES_MIN * sizeof(ivac_entry)); ivac->ivac_table = table; ivac->ivac_table_size = IVAC_ENTRIES_MIN; ivac->ivac_init_table_size = IVAC_ENTRIES_MIN; for (i = 0; i < ivac->ivac_table_size; i++) { - ivace_reset_data(&table[i], i+1); + ivace_reset_data(&table[i], i + 1); } /* the default table entry is never on freelist */ table[0].ivace_next = IV_HASH_END; table[0].ivace_free = FALSE; - table[i-1].ivace_next = IV_FREELIST_END; + table[i - 1].ivace_next = IV_FREELIST_END; ivac->ivac_freelist = 1; ivac_lock_init(ivac); ivac->ivac_key_index = key_index; - return (ivac); + return ivac; } - + void ivac_dealloc(ipc_voucher_attr_control_t ivac) @@ -614,7 +573,7 @@ ivac_dealloc(ipc_voucher_attr_control_t ivac) * that the reference count is still zero. */ ivgt_lock(); - if (ivac->ivac_refs > 0) { + if (os_ref_get_count(&ivac->ivac_refs) > 0) { ivgt_unlock(); return; } @@ -629,8 +588,9 @@ ivac_dealloc(ipc_voucher_attr_control_t ivac) ivgt_unlock(); /* release the reference held on the resource manager */ - if (IVAM_NULL != ivam) + if (IVAM_NULL != ivam) { (ivam->ivam_release)(ivam); + } /* * if a port was allocated for this voucher, @@ -639,7 +599,7 @@ ivac_dealloc(ipc_voucher_attr_control_t ivac) * is gone. We can just discard it now. */ if (IP_VALID(port)) { - assert(ip_active(port)); + require_ip_active(port); assert(port->ip_srights == 0); ipc_port_dealloc_kernel(port); @@ -651,9 +611,11 @@ ivac_dealloc(ipc_voucher_attr_control_t ivac) * table. */ #ifdef MACH_DEBUG - for (i = 0; i < ivac->ivac_table_size; i++) - if (ivac->ivac_table[i].ivace_refs != 0) + for (i = 0; i < ivac->ivac_table_size; i++) { + if (ivac->ivac_table[i].ivace_refs != 0) { panic("deallocing a resource manager with live refs to its attr values\n"); + } + } #endif kfree(ivac->ivac_table, ivac->ivac_table_size * sizeof(*ivac->ivac_table)); ivac_lock_destroy(ivac); @@ -683,10 +645,10 @@ ipc_voucher_attr_control_release(ipc_voucher_attr_control_t control) */ ipc_voucher_attr_control_t convert_port_to_voucher_attr_control( - ipc_port_t port) + ipc_port_t port) { if (IP_VALID(port)) { - ipc_voucher_attr_control_t ivac = (ipc_voucher_attr_control_t) port->ip_kobject; + ipc_voucher_attr_control_t ivac = (ipc_voucher_attr_control_t) ip_get_kobject(port); /* * No need to lock because we have a reference on the @@ -694,17 +656,24 @@ convert_port_to_voucher_attr_control( * that reference keeps the voucher bound to the port * (and active). */ - if (ip_kotype(port) != IKOT_VOUCHER_ATTR_CONTROL) + if (ip_kotype(port) != IKOT_VOUCHER_ATTR_CONTROL) { return IVAC_NULL; + } + require_ip_active(port); - assert(ip_active(port)); - + zone_require(ipc_voucher_attr_control_zone, ivac); ivac_reference(ivac); - return (ivac); + return ivac; } return IVAC_NULL; } +/* + * Routine: ipc_voucher_notify + * Purpose: + * Called whenever the Mach port system detects no-senders + * on the voucher attr control port. + */ void ipc_voucher_attr_control_notify(mach_msg_header_t *msg) { @@ -712,18 +681,12 @@ ipc_voucher_attr_control_notify(mach_msg_header_t *msg) ipc_port_t port = notification->not_header.msgh_remote_port; ipc_voucher_attr_control_t ivac; + require_ip_active(port); assert(IKOT_VOUCHER_ATTR_CONTROL == ip_kotype(port)); - ip_lock(port); - assert(ip_active(port)); - /* if no new send rights, drop a control reference */ - if (port->ip_mscount == notification->not_count) { - ivac = (ipc_voucher_attr_control_t)port->ip_kobject; - ip_unlock(port); - - ivac_release(ivac); - } - ip_unlock(port); + /* release the reference donated by convert_voucher_attr_control_to_port */ + ivac = (ipc_voucher_attr_control_t)ip_get_kobject(port); + ivac_release(ivac); } /* @@ -732,46 +695,21 @@ ipc_voucher_attr_control_notify(mach_msg_header_t *msg) ipc_port_t convert_voucher_attr_control_to_port(ipc_voucher_attr_control_t control) { - ipc_port_t port, send; - - if (IVAC_NULL == control) - return (IP_NULL); - - /* create a port if needed */ - port = control->ivac_port; - if (!IP_VALID(port)) { - port = ipc_port_alloc_kernel(); - assert(IP_VALID(port)); - if (OSCompareAndSwapPtr(IP_NULL, port, &control->ivac_port)) { - ip_lock(port); - ipc_kobject_set_atomically(port, (ipc_kobject_t) control, IKOT_VOUCHER_ATTR_CONTROL); - } else { - ipc_port_dealloc_kernel(port); - port = control->ivac_port; - ip_lock(port); - assert(ip_kotype(port) == IKOT_VOUCHER_ATTR_CONTROL); - assert(port->ip_kobject == (ipc_kobject_t)control); - } - } else - ip_lock(port); - - assert(ip_active(port)); - send = ipc_port_make_send_locked(port); + if (IVAC_NULL == control) { + return IP_NULL; + } - if (1 == port->ip_srights) { - ipc_port_t old_notify; + zone_require(ipc_voucher_attr_control_zone, control); - /* transfer our ref to the port, and arm the no-senders notification */ - assert(IP_NULL == port->ip_nsrequest); - ipc_port_nsrequest(port, port->ip_mscount, ipc_port_make_sonce_locked(port), &old_notify); - assert(IP_NULL == old_notify); - ip_unlock(port); - } else { - /* piggyback on the existing port reference, so consume ours */ - ip_unlock(port); + /* + * make a send right and donate our reference for + * ipc_voucher_attr_control_notify if this is the first send right + */ + if (!ipc_kobject_make_send_lazy_alloc_port(&control->ivac_port, + (ipc_kobject_t)control, IKOT_VOUCHER_ATTR_CONTROL, false, 0)) { ivac_release(control); } - return (send); + return control->ivac_port; } /* @@ -779,10 +717,10 @@ convert_voucher_attr_control_to_port(ipc_voucher_attr_control_t control) */ static void ivace_lookup_values( - iv_index_t key_index, - iv_index_t value_index, - mach_voucher_attr_value_handle_array_t values, - mach_voucher_attr_value_handle_array_size_t *count) + iv_index_t key_index, + iv_index_t value_index, + mach_voucher_attr_value_handle_array_t values, + mach_voucher_attr_value_handle_array_size_t *count) { ipc_voucher_attr_control_t ivac; ivac_entry_t ivace; @@ -846,19 +784,19 @@ ivac_grow_table(ipc_voucher_attr_control_t ivac) assert(new_size < IVAC_ENTRIES_MAX); new_table = kalloc(sizeof(ivac_entry) * new_size); - if (!new_table){ + if (!new_table) { panic("Failed to grow ivac table to size %d\n", new_size); return; } /* setup the free list for new entries */ for (i = old_size; i < new_size; i++) { - ivace_reset_data(&new_table[i], i+1); + ivace_reset_data(&new_table[i], i + 1); } ivac_lock(ivac); - - for (i = 0; i < ivac->ivac_table_size; i++){ + + for (i = 0; i < ivac->ivac_table_size; i++) { ivace_copy_data(&ivac->ivac_table[i], &new_table[i]); } @@ -866,14 +804,14 @@ ivac_grow_table(ipc_voucher_attr_control_t ivac) ivac->ivac_table = new_table; ivac->ivac_table_size = new_size; - + /* adding new free entries at head of freelist */ ivac->ivac_table[new_size - 1].ivace_next = ivac->ivac_freelist; ivac->ivac_freelist = old_size; ivac->ivac_is_growing = 0; ivac_wakeup(ivac); - if (old_table){ + if (old_table) { ivac_unlock(ivac); kfree(old_table, old_size * sizeof(ivac_entry)); ivac_lock(ivac); @@ -889,14 +827,15 @@ ivac_grow_table(ipc_voucher_attr_control_t ivac) */ static void ivace_reference_by_index( - iv_index_t key_index, - iv_index_t val_index) + iv_index_t key_index, + iv_index_t val_index) { ipc_voucher_attr_control_t ivac; ivac_entry_t ivace; - if (IV_UNUSED_VALINDEX == val_index) + if (IV_UNUSED_VALINDEX == val_index) { return; + } ivgt_lookup(key_index, FALSE, NULL, &ivac); assert(IVAC_NULL != ivac); @@ -927,8 +866,8 @@ ivace_reference_by_index( */ static iv_index_t ivace_reference_by_value( - ipc_voucher_attr_control_t ivac, - mach_voucher_attr_value_handle_t value, + ipc_voucher_attr_control_t ivac, + mach_voucher_attr_value_handle_t value, mach_voucher_attr_value_flags_t flag) { ivac_entry_t ivace = IVACE_NULL; @@ -939,7 +878,7 @@ ivace_reference_by_value( return IV_UNUSED_VALINDEX; } - ivac_lock(ivac); + ivac_lock(ivac); restart: hash_index = IV_HASH_VAL(ivac->ivac_init_table_size, value); index = ivac->ivac_table[hash_index].ivace_index; @@ -948,15 +887,16 @@ restart: ivace = &ivac->ivac_table[index]; assert(!ivace->ivace_free); - if (ivace->ivace_value == value) + if (ivace->ivace_value == value) { break; + } assert(ivace->ivace_next != index); index = ivace->ivace_next; } /* found it? */ - if (index != IV_HASH_END) { + if (index != IV_HASH_END) { /* only add reference on non-persistent value */ if (!ivace->ivace_persist) { ivace->ivace_refs++; @@ -1001,11 +941,12 @@ restart: * Release a reference on the given pair. * * Conditions: called with nothing locked, as it may cause - * callouts and/or messaging to the resource + * callouts and/or messaging to the resource * manager. */ -static void ivace_release( - iv_index_t key_index, +static void +ivace_release( + iv_index_t key_index, iv_index_t value_index) { ipc_voucher_attr_control_t ivac; @@ -1018,8 +959,9 @@ static void ivace_release( kern_return_t kr; /* cant release the default value */ - if (IV_UNUSED_VALINDEX == value_index) + if (IV_UNUSED_VALINDEX == value_index) { return; + } ivgt_lookup(key_index, FALSE, &ivam, &ivac); assert(IVAC_NULL != ivac); @@ -1059,7 +1001,7 @@ static void ivace_release( ivace->ivace_releasing = TRUE; value = ivace->ivace_value; - redrive: +redrive: assert(value == ivace->ivace_value); assert(!ivace->ivace_free); made = ivace->ivace_made; @@ -1081,20 +1023,20 @@ static void ivace_release( * re-drive the release. */ if (ivace->ivace_made != made) { - assert(made < ivace->ivace_made); - - if (KERN_SUCCESS == kr) + if (KERN_SUCCESS == kr) { ivace->ivace_made -= made; + } - if (0 == ivace->ivace_refs) + if (0 == ivace->ivace_refs) { goto redrive; + } ivace->ivace_releasing = FALSE; ivac_unlock(ivac); return; } else { /* - * If the manager returned FAILURE, someone took a + * If the manager returned FAILURE, someone took a * reference on the value but have not updated the ivace, * release the lock and return since thread who got * the new reference will update the ivace and will have @@ -1158,16 +1100,17 @@ static void ivace_release( */ static void ivgt_lookup(iv_index_t key_index, - boolean_t take_reference, - ipc_voucher_attr_manager_t *manager, - ipc_voucher_attr_control_t *control) + boolean_t take_reference, + ipc_voucher_attr_manager_t *manager, + ipc_voucher_attr_control_t *control) { ipc_voucher_attr_control_t ivac; if (key_index < MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN) { ivgt_lock(); - if (NULL != manager) + if (NULL != manager) { *manager = iv_global_table[key_index].ivgte_manager; + } ivac = iv_global_table[key_index].ivgte_control; if (IVAC_NULL != ivac) { assert(key_index == ivac->ivac_key_index); @@ -1177,18 +1120,21 @@ ivgt_lookup(iv_index_t key_index, } } ivgt_unlock(); - if (NULL != control) + if (NULL != control) { *control = ivac; + } } else { - if (NULL != manager) + if (NULL != manager) { *manager = IVAM_NULL; - if (NULL != control) + } + if (NULL != control) { *control = IVAC_NULL; + } } } /* - * Routine: ipc_replace_voucher_value + * Routine: ipc_replace_voucher_value * Purpose: * Replace the value with the results of * running the supplied command through the resource @@ -1199,11 +1145,11 @@ ivgt_lookup(iv_index_t key_index, */ static kern_return_t ipc_replace_voucher_value( - ipc_voucher_t voucher, - mach_voucher_attr_key_t key, - mach_voucher_attr_recipe_command_t command, - ipc_voucher_t prev_voucher, - mach_voucher_attr_content_t content, + ipc_voucher_t voucher, + mach_voucher_attr_key_t key, + mach_voucher_attr_recipe_command_t command, + ipc_voucher_t prev_voucher, + mach_voucher_attr_content_t content, mach_voucher_attr_content_size_t content_size) { mach_voucher_attr_value_handle_t previous_vals[MACH_VOUCHER_ATTR_VALUE_MAX_NESTED]; @@ -1218,15 +1164,16 @@ ipc_replace_voucher_value( iv_index_t val_index; iv_index_t key_index; kern_return_t kr; - + /* * Get the manager for this key_index. * Returns a reference on the control. */ key_index = iv_key_to_index(key); ivgt_lookup(key_index, TRUE, &ivam, &ivac); - if (IVAM_NULL == ivam) + if (IVAM_NULL == ivam) { return KERN_INVALID_ARGUMENT; + } /* save the current value stored in the forming voucher */ save_val_index = iv_lookup(voucher, key_index); @@ -1238,26 +1185,27 @@ ipc_replace_voucher_value( * in the forming voucher. */ prev_val_index = (IV_NULL != prev_voucher) ? - iv_lookup(prev_voucher, key_index) : - save_val_index; + iv_lookup(prev_voucher, key_index) : + save_val_index; ivace_lookup_values(key_index, prev_val_index, - previous_vals, &previous_vals_count); + previous_vals, &previous_vals_count); /* Call out to resource manager to get new value */ new_value_voucher = IV_NULL; kr = (ivam->ivam_get_value)( - ivam, key, command, - previous_vals, previous_vals_count, - content, content_size, - &new_value, &new_flag, &new_value_voucher); + ivam, key, command, + previous_vals, previous_vals_count, + content, content_size, + &new_value, &new_flag, &new_value_voucher); if (KERN_SUCCESS != kr) { ivac_release(ivac); return kr; } /* TODO: value insertion from returned voucher */ - if (IV_NULL != new_value_voucher) + if (IV_NULL != new_value_voucher) { iv_release(new_value_voucher); + } /* * Find or create a slot in the table associated @@ -1275,12 +1223,12 @@ ipc_replace_voucher_value( * as was there before. */ ivace_release(key_index, save_val_index); - + return KERN_SUCCESS; } /* - * Routine: ipc_directly_replace_voucher_value + * Routine: ipc_directly_replace_voucher_value * Purpose: * Replace the value with the value-handle * supplied directly by the attribute manager. @@ -1291,24 +1239,25 @@ ipc_replace_voucher_value( */ static kern_return_t ipc_directly_replace_voucher_value( - ipc_voucher_t voucher, - mach_voucher_attr_key_t key, - mach_voucher_attr_value_handle_t new_value) + ipc_voucher_t voucher, + mach_voucher_attr_key_t key, + mach_voucher_attr_value_handle_t new_value) { ipc_voucher_attr_manager_t ivam; ipc_voucher_attr_control_t ivac; iv_index_t save_val_index; iv_index_t val_index; iv_index_t key_index; - + /* * Get the manager for this key_index. * Returns a reference on the control. */ key_index = iv_key_to_index(key); ivgt_lookup(key_index, TRUE, &ivam, &ivac); - if (IVAM_NULL == ivam) + if (IVAM_NULL == ivam) { return KERN_INVALID_ARGUMENT; + } /* save the current value stored in the forming voucher */ save_val_index = iv_lookup(voucher, key_index); @@ -1320,7 +1269,7 @@ ipc_directly_replace_voucher_value( * we find a matching existing value. */ val_index = ivace_reference_by_value(ivac, new_value, - MACH_VOUCHER_ATTR_VALUE_FLAGS_NONE); + MACH_VOUCHER_ATTR_VALUE_FLAGS_NONE); iv_set(voucher, key_index, val_index); /* @@ -1330,26 +1279,25 @@ ipc_directly_replace_voucher_value( * as was there before. */ ivace_release(key_index, save_val_index); - + return KERN_SUCCESS; } static kern_return_t ipc_execute_voucher_recipe_command( - ipc_voucher_t voucher, - mach_voucher_attr_key_t key, - mach_voucher_attr_recipe_command_t command, - ipc_voucher_t prev_iv, - mach_voucher_attr_content_t content, - mach_voucher_attr_content_size_t content_size, - boolean_t key_priv) + ipc_voucher_t voucher, + mach_voucher_attr_key_t key, + mach_voucher_attr_recipe_command_t command, + ipc_voucher_t prev_iv, + mach_voucher_attr_content_t content, + mach_voucher_attr_content_size_t content_size, + boolean_t key_priv) { iv_index_t prev_val_index; iv_index_t val_index; kern_return_t kr; switch (command) { - /* * MACH_VOUCHER_ATTR_COPY * Copy the attribute(s) from the previous voucher to the new @@ -1358,22 +1306,24 @@ ipc_execute_voucher_recipe_command( * voucher. */ case MACH_VOUCHER_ATTR_COPY: - + /* no recipe data on a copy */ - if (0 < content_size) + if (0 < content_size) { return KERN_INVALID_ARGUMENT; + } /* nothing to copy from? - done */ - if (IV_NULL == prev_iv) + if (IV_NULL == prev_iv) { return KERN_SUCCESS; + } if (MACH_VOUCHER_ATTR_KEY_ALL == key) { iv_index_t limit, j; /* reconcile possible difference in voucher sizes */ limit = (prev_iv->iv_table_size < voucher->iv_table_size) ? - prev_iv->iv_table_size : - voucher->iv_table_size; + prev_iv->iv_table_size : + voucher->iv_table_size; /* wildcard matching */ for (j = 0; j < limit; j++) { @@ -1391,8 +1341,9 @@ ipc_execute_voucher_recipe_command( /* copy just one key */ key_index = iv_key_to_index(key); - if (ivgt_keys_in_use < key_index) + if (ivgt_keys_in_use < key_index) { return KERN_INVALID_ARGUMENT; + } /* release old value being replaced */ val_index = iv_lookup(voucher, key_index); @@ -1415,16 +1366,17 @@ ipc_execute_voucher_recipe_command( */ case MACH_VOUCHER_ATTR_REMOVE: /* no recipe data on a remove */ - if (0 < content_size) + if (0 < content_size) { return KERN_INVALID_ARGUMENT; + } if (MACH_VOUCHER_ATTR_KEY_ALL == key) { iv_index_t limit, j; /* reconcile possible difference in voucher sizes */ limit = (IV_NULL == prev_iv) ? voucher->iv_table_size : - ((prev_iv->iv_table_size < voucher->iv_table_size) ? - prev_iv->iv_table_size : voucher->iv_table_size); + ((prev_iv->iv_table_size < voucher->iv_table_size) ? + prev_iv->iv_table_size : voucher->iv_table_size); /* wildcard matching */ for (j = 0; j < limit; j++) { @@ -1433,8 +1385,9 @@ ipc_execute_voucher_recipe_command( /* If not matched in previous, skip */ if (IV_NULL != prev_iv) { prev_val_index = iv_lookup(prev_iv, j); - if (val_index != prev_val_index) + if (val_index != prev_val_index) { continue; + } } /* release and clear */ ivace_release(j, val_index); @@ -1445,16 +1398,18 @@ ipc_execute_voucher_recipe_command( /* copy just one key */ key_index = iv_key_to_index(key); - if (ivgt_keys_in_use < key_index) + if (ivgt_keys_in_use < key_index) { return KERN_INVALID_ARGUMENT; + } val_index = iv_lookup(voucher, key_index); /* If not matched in previous, skip */ if (IV_NULL != prev_iv) { prev_val_index = iv_lookup(prev_iv, key_index); - if (val_index != prev_val_index) + if (val_index != prev_val_index) { break; + } } /* release and clear */ @@ -1473,17 +1428,19 @@ ipc_execute_voucher_recipe_command( if (key_priv) { mach_voucher_attr_value_handle_t new_value; - if (sizeof(mach_voucher_attr_value_handle_t) != content_size) + if (sizeof(mach_voucher_attr_value_handle_t) != content_size) { return KERN_INVALID_ARGUMENT; - + } + new_value = *(mach_voucher_attr_value_handle_t *)(void *)content; kr = ipc_directly_replace_voucher_value(voucher, - key, - new_value); - if (KERN_SUCCESS != kr) + key, new_value); + if (KERN_SUCCESS != kr) { return kr; - } else + } + } else { return KERN_INVALID_CAPABILITY; + } break; /* @@ -1491,19 +1448,20 @@ ipc_execute_voucher_recipe_command( * Redeem the attribute(s) from the previous voucher for a possibly * new value in the new voucher. A wildcard key is an acceptable value, * indicating a desire to redeem all the values. - */ + */ case MACH_VOUCHER_ATTR_REDEEM: if (MACH_VOUCHER_ATTR_KEY_ALL == key) { iv_index_t limit, j; /* reconcile possible difference in voucher sizes */ - if (IV_NULL != prev_iv) + if (IV_NULL != prev_iv) { limit = (prev_iv->iv_table_size < voucher->iv_table_size) ? - prev_iv->iv_table_size : - voucher->iv_table_size; - else + prev_iv->iv_table_size : + voucher->iv_table_size; + } else { limit = voucher->iv_table_size; + } /* wildcard matching */ for (j = 0; j < limit; j++) { @@ -1512,22 +1470,24 @@ ipc_execute_voucher_recipe_command( j_key = iv_index_to_key(j); /* skip non-existent managers */ - if (MACH_VOUCHER_ATTR_KEY_NONE == j_key) + if (MACH_VOUCHER_ATTR_KEY_NONE == j_key) { continue; + } /* get the new value from redeem (skip empty previous) */ kr = ipc_replace_voucher_value(voucher, - j_key, - command, - prev_iv, - content, - content_size); - if (KERN_SUCCESS != kr) + j_key, + command, + prev_iv, + content, + content_size); + if (KERN_SUCCESS != kr) { return kr; + } } break; } - /* fall thru for single key redemption */ + OS_FALLTHROUGH; /* fall thru for single key redemption */ /* * DEFAULT: @@ -1537,13 +1497,14 @@ ipc_execute_voucher_recipe_command( */ default: kr = ipc_replace_voucher_value(voucher, - key, - command, - prev_iv, - content, - content_size); - if (KERN_SUCCESS != kr) + key, + command, + prev_iv, + content, + content_size); + if (KERN_SUCCESS != kr) { return kr; + } break; } @@ -1551,7 +1512,7 @@ ipc_execute_voucher_recipe_command( } /* - * Routine: iv_checksum + * Routine: iv_checksum * Purpose: * Compute the voucher sum. This is more position- * relevant than many other checksums - important for @@ -1565,13 +1526,13 @@ iv_checksum(ipc_voucher_t voucher, boolean_t *emptyp) boolean_t empty = TRUE; if (0 < voucher->iv_table_size) { iv_index_t i = voucher->iv_table_size - 1; - + do { iv_index_t v = voucher->iv_table[i]; - c = c << 3 | c >> (32 - 3); /* rotate */ - c = ~c; /* invert */ + c = c << 3 | c >> (32 - 3); /* rotate */ + c = ~c; /* invert */ if (0 < v) { - c += v; /* add in */ + c += v; /* add in */ empty = FALSE; } } while (0 < i--); @@ -1581,7 +1542,7 @@ iv_checksum(ipc_voucher_t voucher, boolean_t *emptyp) } /* - * Routine: iv_dedup + * Routine: iv_dedup * Purpose: * See if the set of values represented by this new voucher * already exist in another voucher. If so return a reference @@ -1596,7 +1557,7 @@ static ipc_voucher_t iv_dedup(ipc_voucher_t new_iv) { boolean_t empty; - iv_index_t sum; + iv_index_t sum; iv_index_t hash; ipc_voucher_t iv; @@ -1615,40 +1576,41 @@ iv_dedup(ipc_voucher_t new_iv) assert(iv->iv_hash == hash); /* if not already deallocating and sums match... */ - if (0 < iv->iv_refs && iv->iv_sum == sum) { - iv_refs_t refs; + if ((os_ref_get_count(&iv->iv_refs) > 0) && (iv->iv_sum == sum)) { iv_index_t i; assert(iv->iv_table_size <= new_iv->iv_table_size); - + /* and common entries match... */ - for (i = 0; i < iv->iv_table_size; i++) - if (iv->iv_table[i] != new_iv->iv_table[i]) + for (i = 0; i < iv->iv_table_size; i++) { + if (iv->iv_table[i] != new_iv->iv_table[i]) { break; - if (i < iv->iv_table_size) + } + } + if (i < iv->iv_table_size) { continue; + } /* and all extra entries in new one are unused... */ - while (i < new_iv->iv_table_size) - if (new_iv->iv_table[i++] != IV_UNUSED_VALINDEX) + while (i < new_iv->iv_table_size) { + if (new_iv->iv_table[i++] != IV_UNUSED_VALINDEX) { break; - if (i < new_iv->iv_table_size) + } + } + if (i < new_iv->iv_table_size) { continue; + } /* ... we found a match... */ /* can we get a ref before it hits 0 * - * This is thread safe. The reference is just an atomic - * add. If the reference count is zero when we adjust it, - * no other thread can have a reference to the voucher. + * This is thread safe. If the reference count is zero before we + * adjust it, no other thread can have a reference to the voucher. * The dealloc code requires holding the ivht_lock, so * the voucher cannot be yanked out from under us. */ - refs = iv_reference(iv); - if (1 == refs) { - /* drats! going away. Put back to zero */ - iv->iv_refs = 0; + if (!os_ref_retain_try(&iv->iv_refs)) { continue; } @@ -1700,7 +1662,7 @@ iv_dedup(ipc_voucher_t new_iv) #define PAYLOAD_PER_TRACEPOINT (4 * sizeof(uintptr_t)) #define PAYLOAD_SIZE 1024 - _Static_assert(PAYLOAD_SIZE % PAYLOAD_PER_TRACEPOINT == 0, "size invariant violated"); + static_assert(PAYLOAD_SIZE % PAYLOAD_PER_TRACEPOINT == 0, "size invariant violated"); mach_voucher_attr_raw_recipe_array_size_t payload_size = PAYLOAD_SIZE; uintptr_t payload[PAYLOAD_SIZE / sizeof(uintptr_t)]; @@ -1718,28 +1680,25 @@ iv_dedup(ipc_voucher_t new_iv) size_t remainder = payload_size % PAYLOAD_PER_TRACEPOINT; if (remainder) { bzero((uint8_t*)payload + payload_size, - PAYLOAD_PER_TRACEPOINT - remainder); + PAYLOAD_PER_TRACEPOINT - remainder); } } - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_VOUCHER_CREATE) | DBG_FUNC_NONE, - voucher_addr, - new_iv->iv_table_size, ivht_count, payload_size, 0); + KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_VOUCHER_CREATE), + voucher_addr, new_iv->iv_table_size, ivht_count, + payload_size); uintptr_t index = 0; while (attr_tracepoints_needed--) { - KERNEL_DEBUG_CONSTANT1(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_VOUCHER_CREATE_ATTR_DATA) | DBG_FUNC_NONE, - payload[index], - payload[index+1], - payload[index+2], - payload[index+3], - voucher_addr); + KDBG(MACHDBG_CODE(DBG_MACH_IPC, + MACH_IPC_VOUCHER_CREATE_ATTR_DATA), payload[index], + payload[index + 1], payload[index + 2], + payload[index + 3]); index += 4; } } else { - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_VOUCHER_CREATE) | DBG_FUNC_NONE, - voucher_addr, - new_iv->iv_table_size, ivht_count, 0, 0); + KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_VOUCHER_CREATE), + voucher_addr, new_iv->iv_table_size, ivht_count); } } #endif /* KDEBUG_LEVEL >= KDEBUG_LEVEL_STANDARD */ @@ -1748,7 +1707,7 @@ iv_dedup(ipc_voucher_t new_iv) } /* - * Routine: ipc_create_mach_voucher + * Routine: ipc_create_mach_voucher * Purpose: * Create a new mach voucher and initialize it with the * value(s) created by having the appropriate resource @@ -1761,9 +1720,9 @@ iv_dedup(ipc_voucher_t new_iv) */ kern_return_t ipc_create_mach_voucher( - ipc_voucher_attr_raw_recipe_array_t recipes, - ipc_voucher_attr_raw_recipe_array_size_t recipe_size, - ipc_voucher_t *new_voucher) + ipc_voucher_attr_raw_recipe_array_t recipes, + ipc_voucher_attr_raw_recipe_array_size_t recipe_size, + ipc_voucher_t *new_voucher) { ipc_voucher_attr_recipe_t sub_recipe; ipc_voucher_attr_recipe_size_t recipe_used = 0; @@ -1778,12 +1737,12 @@ ipc_create_mach_voucher( /* allocate a voucher */ voucher = iv_alloc(ivgt_keys_in_use); - if (IV_NULL == voucher) + if (IV_NULL == voucher) { return KERN_RESOURCE_SHORTAGE; + } /* iterate over the recipe items */ while (0 < recipe_size - recipe_used) { - if (recipe_size - recipe_used < sizeof(*sub_recipe)) { kr = KERN_INVALID_ARGUMENT; break; @@ -1798,14 +1757,15 @@ ipc_create_mach_voucher( recipe_used += sizeof(*sub_recipe) + sub_recipe->content_size; kr = ipc_execute_voucher_recipe_command(voucher, - sub_recipe->key, - sub_recipe->command, - sub_recipe->previous_voucher, - sub_recipe->content, - sub_recipe->content_size, - FALSE); - if (KERN_SUCCESS != kr) + sub_recipe->key, + sub_recipe->command, + sub_recipe->previous_voucher, + sub_recipe->content, + sub_recipe->content_size, + FALSE); + if (KERN_SUCCESS != kr) { break; + } } if (KERN_SUCCESS == kr) { @@ -1818,7 +1778,7 @@ ipc_create_mach_voucher( } /* - * Routine: ipc_voucher_attr_control_create_mach_voucher + * Routine: ipc_voucher_attr_control_create_mach_voucher * Purpose: * Create a new mach voucher and initialize it with the * value(s) created by having the appropriate resource @@ -1839,10 +1799,10 @@ ipc_create_mach_voucher( */ kern_return_t ipc_voucher_attr_control_create_mach_voucher( - ipc_voucher_attr_control_t control, - ipc_voucher_attr_raw_recipe_array_t recipes, - ipc_voucher_attr_raw_recipe_array_size_t recipe_size, - ipc_voucher_t *new_voucher) + ipc_voucher_attr_control_t control, + ipc_voucher_attr_raw_recipe_array_t recipes, + ipc_voucher_attr_raw_recipe_array_size_t recipe_size, + ipc_voucher_t *new_voucher) { mach_voucher_attr_key_t control_key; ipc_voucher_attr_recipe_t sub_recipe; @@ -1850,8 +1810,9 @@ ipc_voucher_attr_control_create_mach_voucher( ipc_voucher_t voucher = IV_NULL; kern_return_t kr = KERN_SUCCESS; - if (IPC_VOUCHER_ATTR_CONTROL_NULL == control) + if (IPC_VOUCHER_ATTR_CONTROL_NULL == control) { return KERN_INVALID_CAPABILITY; + } /* if nothing to do ... */ if (0 == recipe_size) { @@ -1861,14 +1822,14 @@ ipc_voucher_attr_control_create_mach_voucher( /* allocate new voucher */ voucher = iv_alloc(ivgt_keys_in_use); - if (IV_NULL == voucher) + if (IV_NULL == voucher) { return KERN_RESOURCE_SHORTAGE; + } control_key = iv_index_to_key(control->ivac_key_index); /* iterate over the recipe items */ while (0 < recipe_size - recipe_used) { - if (recipe_size - recipe_used < sizeof(*sub_recipe)) { kr = KERN_INVALID_ARGUMENT; break; @@ -1883,14 +1844,15 @@ ipc_voucher_attr_control_create_mach_voucher( recipe_used += sizeof(*sub_recipe) + sub_recipe->content_size; kr = ipc_execute_voucher_recipe_command(voucher, - sub_recipe->key, - sub_recipe->command, - sub_recipe->previous_voucher, - sub_recipe->content, - sub_recipe->content_size, - (sub_recipe->key == control_key)); - if (KERN_SUCCESS != kr) + sub_recipe->key, + sub_recipe->command, + sub_recipe->previous_voucher, + sub_recipe->content, + sub_recipe->content_size, + (sub_recipe->key == control_key)); + if (KERN_SUCCESS != kr) { break; + } } if (KERN_SUCCESS == kr) { @@ -1903,7 +1865,7 @@ ipc_voucher_attr_control_create_mach_voucher( } /* - * ipc_register_well_known_mach_voucher_attr_manager + * ipc_register_well_known_mach_voucher_attr_manager * * Register the resource manager responsible for a given key value. */ @@ -1911,23 +1873,26 @@ kern_return_t ipc_register_well_known_mach_voucher_attr_manager( ipc_voucher_attr_manager_t manager, mach_voucher_attr_value_handle_t default_value, - mach_voucher_attr_key_t key, + mach_voucher_attr_key_t key, ipc_voucher_attr_control_t *control) { ipc_voucher_attr_control_t new_control; iv_index_t key_index; iv_index_t hash_index; - if (IVAM_NULL == manager) + if (IVAM_NULL == manager) { return KERN_INVALID_ARGUMENT; + } key_index = iv_key_to_index(key); - if (IV_UNUSED_KEYINDEX == key_index) + if (IV_UNUSED_KEYINDEX == key_index) { return KERN_INVALID_ARGUMENT; + } new_control = ivac_alloc(key_index); - if (IVAC_NULL == new_control) + if (IVAC_NULL == new_control) { return KERN_RESOURCE_SHORTAGE; + } /* insert the default value into slot 0 */ new_control->ivac_table[IV_UNUSED_VALINDEX].ivace_value = default_value; @@ -1954,7 +1919,7 @@ ipc_register_well_known_mach_voucher_attr_manager( new_control->ivac_table[hash_index].ivace_index = IV_UNUSED_VALINDEX; ivgt_unlock(); - + /* return the reference on the new cache control to the caller */ *control = new_control; @@ -1962,7 +1927,7 @@ ipc_register_well_known_mach_voucher_attr_manager( } /* - * Routine: mach_voucher_extract_attr_content + * Routine: mach_voucher_extract_attr_content * Purpose: * Extract the content for a given pair. * @@ -1977,10 +1942,10 @@ ipc_register_well_known_mach_voucher_attr_manager( */ kern_return_t mach_voucher_extract_attr_content( - ipc_voucher_t voucher, - mach_voucher_attr_key_t key, - mach_voucher_attr_content_t content, - mach_voucher_attr_content_size_t *in_out_size) + ipc_voucher_t voucher, + mach_voucher_attr_key_t key, + mach_voucher_attr_content_t content, + mach_voucher_attr_content_size_t *in_out_size) { mach_voucher_attr_value_handle_t vals[MACH_VOUCHER_ATTR_VALUE_MAX_NESTED]; mach_voucher_attr_value_handle_array_size_t vals_count; @@ -1991,8 +1956,9 @@ mach_voucher_extract_attr_content( kern_return_t kr; - if (IV_NULL == voucher) + if (IV_NULL == voucher) { return KERN_INVALID_ARGUMENT; + } key_index = iv_key_to_index(key); @@ -2018,20 +1984,18 @@ mach_voucher_extract_attr_content( * for this value_index. */ ivace_lookup_values(key_index, value_index, - vals, &vals_count); + vals, &vals_count); assert(0 < vals_count); /* callout to manager */ - - kr = (manager->ivam_extract_content)(manager, key, - vals, vals_count, - &command, - content, in_out_size); + + kr = (manager->ivam_extract_content)(manager, key, + vals, vals_count, &command, content, in_out_size); return kr; } /* - * Routine: mach_voucher_extract_attr_recipe + * Routine: mach_voucher_extract_attr_recipe * Purpose: * Extract a recipe for a given pair. * @@ -2046,10 +2010,10 @@ mach_voucher_extract_attr_content( */ kern_return_t mach_voucher_extract_attr_recipe( - ipc_voucher_t voucher, - mach_voucher_attr_key_t key, - mach_voucher_attr_raw_recipe_t raw_recipe, - mach_voucher_attr_raw_recipe_size_t *in_out_size) + ipc_voucher_t voucher, + mach_voucher_attr_key_t key, + mach_voucher_attr_raw_recipe_t raw_recipe, + mach_voucher_attr_raw_recipe_size_t *in_out_size) { mach_voucher_attr_value_handle_t vals[MACH_VOUCHER_ATTR_VALUE_MAX_NESTED]; mach_voucher_attr_value_handle_array_size_t vals_count; @@ -2060,8 +2024,9 @@ mach_voucher_extract_attr_recipe( kern_return_t kr; - if (IV_NULL == voucher) + if (IV_NULL == voucher) { return KERN_INVALID_ARGUMENT; + } key_index = iv_key_to_index(key); @@ -2071,8 +2036,9 @@ mach_voucher_extract_attr_recipe( return KERN_SUCCESS; } - if (*in_out_size < sizeof(*recipe)) + if (*in_out_size < sizeof(*recipe)) { return KERN_NO_SPACE; + } recipe = (mach_voucher_attr_recipe_t)(void *)raw_recipe; recipe->key = key; @@ -2096,17 +2062,17 @@ mach_voucher_extract_attr_recipe( * for this value_index. */ ivace_lookup_values(key_index, value_index, - vals, &vals_count); + vals, &vals_count); assert(0 < vals_count); /* callout to manager */ - kr = (manager->ivam_extract_content)(manager, key, - vals, vals_count, - &recipe->command, - recipe->content, &recipe->content_size); + kr = (manager->ivam_extract_content)(manager, key, + vals, vals_count, + &recipe->command, + recipe->content, &recipe->content_size); if (KERN_SUCCESS == kr) { - assert(*in_out_size - sizeof(*recipe) >= recipe->content_size); - *in_out_size = sizeof(*recipe) + recipe->content_size; + assert(*in_out_size - sizeof(*recipe) >= recipe->content_size); + *in_out_size = sizeof(*recipe) + recipe->content_size; } return kr; @@ -2115,27 +2081,28 @@ mach_voucher_extract_attr_recipe( /* - * Routine: mach_voucher_extract_all_attr_recipes + * Routine: mach_voucher_extract_all_attr_recipes * Purpose: * Extract all the (non-default) contents for a given voucher, - * building up a recipe that could be provided to a future + * building up a recipe that could be provided to a future * voucher creation call. - * Conditions: + * Conditions: * Nothing locked (may invoke user-space). * Caller holds a reference on the supplied voucher. */ kern_return_t mach_voucher_extract_all_attr_recipes( - ipc_voucher_t voucher, - mach_voucher_attr_raw_recipe_array_t recipes, - mach_voucher_attr_raw_recipe_array_size_t *in_out_size) + ipc_voucher_t voucher, + mach_voucher_attr_raw_recipe_array_t recipes, + mach_voucher_attr_raw_recipe_array_size_t *in_out_size) { mach_voucher_attr_recipe_size_t recipe_size = *in_out_size; mach_voucher_attr_recipe_size_t recipe_used = 0; iv_index_t key_index; - if (IV_NULL == voucher) + if (IV_NULL == voucher) { return KERN_INVALID_ARGUMENT; + } for (key_index = 0; key_index < voucher->iv_table_size; key_index++) { mach_voucher_attr_value_handle_t vals[MACH_VOUCHER_ATTR_VALUE_MAX_NESTED]; @@ -2149,11 +2116,13 @@ mach_voucher_extract_all_attr_recipes( /* don't output anything for a default value */ value_index = iv_lookup(voucher, key_index); - if (IV_UNUSED_VALINDEX == value_index) + if (IV_UNUSED_VALINDEX == value_index) { continue; + } - if (recipe_size - recipe_used < sizeof(*recipe)) + if (recipe_size - recipe_used < sizeof(*recipe)) { return KERN_NO_SPACE; + } /* * Get the manager for this key_index. The @@ -2175,7 +2144,7 @@ mach_voucher_extract_all_attr_recipes( * for this value_index. */ ivace_lookup_values(key_index, value_index, - vals, &vals_count); + vals, &vals_count); assert(0 < vals_count); key = iv_index_to_key(key_index); @@ -2185,12 +2154,13 @@ mach_voucher_extract_all_attr_recipes( recipe->content_size = content_size; /* callout to manager */ - kr = (manager->ivam_extract_content)(manager, key, - vals, vals_count, - &recipe->command, - recipe->content, &recipe->content_size); - if (KERN_SUCCESS != kr) + kr = (manager->ivam_extract_content)(manager, key, + vals, vals_count, + &recipe->command, + recipe->content, &recipe->content_size); + if (KERN_SUCCESS != kr) { return kr; + } assert(recipe->content_size <= content_size); recipe_used += sizeof(*recipe) + recipe->content_size; @@ -2201,10 +2171,10 @@ mach_voucher_extract_all_attr_recipes( } /* - * Routine: mach_voucher_debug_info + * Routine: mach_voucher_debug_info * Purpose: * Extract all the (non-default) contents for a given mach port name, - * building up a recipe that could be provided to a future + * building up a recipe that could be provided to a future * voucher creation call. * Conditions: * Nothing locked (may invoke user-space). @@ -2213,32 +2183,37 @@ mach_voucher_extract_all_attr_recipes( #if !(DEVELOPMENT || DEBUG) kern_return_t mach_voucher_debug_info( - ipc_space_t __unused space, - mach_port_name_t __unused voucher_name, - mach_voucher_attr_raw_recipe_array_t __unused recipes, - mach_voucher_attr_raw_recipe_array_size_t __unused *in_out_size) + ipc_space_t __unused space, + mach_port_name_t __unused voucher_name, + mach_voucher_attr_raw_recipe_array_t __unused recipes, + mach_voucher_attr_raw_recipe_array_size_t __unused *in_out_size) { return KERN_NOT_SUPPORTED; } #else kern_return_t mach_voucher_debug_info( - ipc_space_t space, - mach_port_name_t voucher_name, - mach_voucher_attr_raw_recipe_array_t recipes, - mach_voucher_attr_raw_recipe_array_size_t *in_out_size) + ipc_space_t space, + mach_port_name_t voucher_name, + mach_voucher_attr_raw_recipe_array_t recipes, + mach_voucher_attr_raw_recipe_array_size_t *in_out_size) { ipc_voucher_t voucher = IPC_VOUCHER_NULL; kern_return_t kr; ipc_port_t port = MACH_PORT_NULL; + if (space == IS_NULL) { + return KERN_INVALID_TASK; + } + if (!MACH_PORT_VALID(voucher_name)) { return KERN_INVALID_ARGUMENT; } kr = ipc_port_translate_send(space, voucher_name, &port); - if (KERN_SUCCESS != kr) + if (KERN_SUCCESS != kr) { return KERN_INVALID_ARGUMENT; + } voucher = convert_port_to_voucher(port); ip_unlock(port); @@ -2254,7 +2229,7 @@ mach_voucher_debug_info( #endif /* - * Routine: mach_voucher_attr_command + * Routine: mach_voucher_attr_command * Purpose: * Invoke an attribute-specific command through this voucher. * @@ -2267,13 +2242,13 @@ mach_voucher_debug_info( */ kern_return_t mach_voucher_attr_command( - ipc_voucher_t voucher, - mach_voucher_attr_key_t key, - mach_voucher_attr_command_t command, - mach_voucher_attr_content_t in_content, - mach_voucher_attr_content_size_t in_content_size, - mach_voucher_attr_content_t out_content, - mach_voucher_attr_content_size_t *out_content_size) + ipc_voucher_t voucher, + mach_voucher_attr_key_t key, + mach_voucher_attr_command_t command, + mach_voucher_attr_content_t in_content, + mach_voucher_attr_content_size_t in_content_size, + mach_voucher_attr_content_t out_content, + mach_voucher_attr_content_size_t *out_content_size) { mach_voucher_attr_value_handle_t vals[MACH_VOUCHER_ATTR_VALUE_MAX_NESTED]; mach_voucher_attr_value_handle_array_size_t vals_count; @@ -2284,8 +2259,9 @@ mach_voucher_attr_command( kern_return_t kr; - if (IV_NULL == voucher) + if (IV_NULL == voucher) { return KERN_INVALID_ARGUMENT; + } key_index = iv_key_to_index(key); @@ -2310,14 +2286,14 @@ mach_voucher_attr_command( */ value_index = iv_lookup(voucher, key_index); ivace_lookup_values(key_index, value_index, - vals, &vals_count); + vals, &vals_count); /* callout to manager */ - kr = (manager->ivam_command)(manager, key, - vals, vals_count, - command, - in_content, in_content_size, - out_content, out_content_size); + kr = (manager->ivam_command)(manager, key, + vals, vals_count, + command, + in_content, in_content_size, + out_content, out_content_size); /* release reference on control */ ivac_release(control); @@ -2326,7 +2302,7 @@ mach_voucher_attr_command( } /* - * Routine: mach_voucher_attr_control_get_values + * Routine: mach_voucher_attr_control_get_values * Purpose: * For a given voucher, get the value handle associated with the * specified attribute manager. @@ -2340,26 +2316,29 @@ mach_voucher_attr_control_get_values( { iv_index_t key_index, value_index; - if (IPC_VOUCHER_ATTR_CONTROL_NULL == control) + if (IPC_VOUCHER_ATTR_CONTROL_NULL == control) { return KERN_INVALID_CAPABILITY; + } - if (IV_NULL == voucher) + if (IV_NULL == voucher) { return KERN_INVALID_ARGUMENT; + } - if (0 == *in_out_size) + if (0 == *in_out_size) { return KERN_SUCCESS; + } key_index = control->ivac_key_index; - assert(0 < voucher->iv_refs); + assert(os_ref_get_count(&voucher->iv_refs) > 0); value_index = iv_lookup(voucher, key_index); ivace_lookup_values(key_index, value_index, - out_values, in_out_size); + out_values, in_out_size); return KERN_SUCCESS; } /* - * Routine: mach_voucher_attr_control_create_mach_voucher + * Routine: mach_voucher_attr_control_create_mach_voucher * Purpose: * Create a new mach voucher and initialize it by processing the * supplied recipe(s). @@ -2387,8 +2366,9 @@ mach_voucher_attr_control_create_mach_voucher( ipc_voucher_t voucher = IV_NULL; kern_return_t kr = KERN_SUCCESS; - if (IPC_VOUCHER_ATTR_CONTROL_NULL == control) + if (IPC_VOUCHER_ATTR_CONTROL_NULL == control) { return KERN_INVALID_CAPABILITY; + } /* if nothing to do ... */ if (0 == recipe_size) { @@ -2398,8 +2378,9 @@ mach_voucher_attr_control_create_mach_voucher( /* allocate new voucher */ voucher = iv_alloc(ivgt_keys_in_use); - if (IV_NULL == voucher) + if (IV_NULL == voucher) { return KERN_RESOURCE_SHORTAGE; + } control_key = iv_index_to_key(control->ivac_key_index); @@ -2428,16 +2409,17 @@ mach_voucher_attr_control_create_mach_voucher( } kr = ipc_execute_voucher_recipe_command(voucher, - sub_recipe->key, - sub_recipe->command, - prev_iv, - sub_recipe->content, - sub_recipe->content_size, - (sub_recipe->key == control_key)); + sub_recipe->key, + sub_recipe->command, + prev_iv, + sub_recipe->content, + sub_recipe->content_size, + (sub_recipe->key == control_key)); ipc_voucher_release(prev_iv); - if (KERN_SUCCESS != kr) + if (KERN_SUCCESS != kr) { break; + } } if (KERN_SUCCESS == kr) { @@ -2450,7 +2432,7 @@ mach_voucher_attr_control_create_mach_voucher( } /* - * Routine: host_create_mach_voucher + * Routine: host_create_mach_voucher * Purpose: * Create a new mach voucher and initialize it by processing the * supplied recipe(s). @@ -2474,8 +2456,9 @@ host_create_mach_voucher( ipc_voucher_t voucher = IV_NULL; kern_return_t kr = KERN_SUCCESS; - if (host == HOST_NULL) + if (host == HOST_NULL) { return KERN_INVALID_ARGUMENT; + } /* if nothing to do ... */ if (0 == recipe_size) { @@ -2485,8 +2468,9 @@ host_create_mach_voucher( /* allocate new voucher */ voucher = iv_alloc(ivgt_keys_in_use); - if (IV_NULL == voucher) + if (IV_NULL == voucher) { return KERN_RESOURCE_SHORTAGE; + } /* iterate over the recipe items */ while (0 < recipe_size - recipe_used) { @@ -2513,16 +2497,17 @@ host_create_mach_voucher( } kr = ipc_execute_voucher_recipe_command(voucher, - sub_recipe->key, - sub_recipe->command, - prev_iv, - sub_recipe->content, - sub_recipe->content_size, - FALSE); + sub_recipe->key, + sub_recipe->command, + prev_iv, + sub_recipe->content, + sub_recipe->content_size, + FALSE); ipc_voucher_release(prev_iv); - if (KERN_SUCCESS != kr) + if (KERN_SUCCESS != kr) { break; + } } if (KERN_SUCCESS == kr) { @@ -2535,10 +2520,10 @@ host_create_mach_voucher( } /* - * Routine: host_register_well_known_mach_voucher_attr_manager + * Routine: host_register_well_known_mach_voucher_attr_manager * Purpose: * Register the user-level resource manager responsible for a given - * key value. + * key value. * Conditions: * The manager port passed in has to be converted/wrapped * in an ipc_voucher_attr_manager_t structure and then call the @@ -2548,14 +2533,15 @@ host_create_mach_voucher( */ kern_return_t host_register_well_known_mach_voucher_attr_manager( - host_t host, + host_t host, mach_voucher_attr_manager_t __unused manager, mach_voucher_attr_value_handle_t __unused default_value, - mach_voucher_attr_key_t __unused key, + mach_voucher_attr_key_t __unused key, ipc_voucher_attr_control_t __unused *control) { - if (HOST_NULL == host) + if (HOST_NULL == host) { return KERN_INVALID_HOST; + } #if 1 return KERN_NOT_SUPPORTED; @@ -2574,18 +2560,19 @@ host_register_well_known_mach_voucher_attr_manager( proxy = mvam_alloc(manager); kr = ipc_register_well_known_mach_voucher_attr_manager(&proxy->mvam_manager, - default_value, - key, - control); - if (KERN_SUCCESS != kr) + default_value, + key, + control); + if (KERN_SUCCESS != kr) { mvam_release(proxy); + } return kr; #endif } /* - * Routine: host_register_mach_voucher_attr_manager + * Routine: host_register_mach_voucher_attr_manager * Purpose: * Register the user-space resource manager and return a * dynamically allocated key. @@ -2596,18 +2583,62 @@ host_register_well_known_mach_voucher_attr_manager( */ kern_return_t host_register_mach_voucher_attr_manager( - host_t host, + host_t host, mach_voucher_attr_manager_t __unused manager, mach_voucher_attr_value_handle_t __unused default_value, - mach_voucher_attr_key_t __unused *key, + mach_voucher_attr_key_t __unused *key, ipc_voucher_attr_control_t __unused *control) { - if (HOST_NULL == host) + if (HOST_NULL == host) { return KERN_INVALID_HOST; + } return KERN_NOT_SUPPORTED; } +/* + * Routine: ipc_get_pthpriority_from_kmsg_voucher + * Purpose: + * Get the canonicalized pthread priority from the voucher attached in the kmsg. + */ +kern_return_t +ipc_get_pthpriority_from_kmsg_voucher( + ipc_kmsg_t kmsg, + ipc_pthread_priority_value_t *canonicalize_priority_value) +{ + ipc_voucher_t pthread_priority_voucher; + mach_voucher_attr_raw_recipe_size_t content_size = + sizeof(mach_voucher_attr_recipe_data_t) + sizeof(ipc_pthread_priority_value_t); + uint8_t content_data[content_size]; + mach_voucher_attr_recipe_t cur_content; + kern_return_t kr = KERN_SUCCESS; + + if (!IP_VALID(kmsg->ikm_voucher)) { + return KERN_FAILURE; + } + + pthread_priority_voucher = ip_get_voucher(kmsg->ikm_voucher); + kr = mach_voucher_extract_attr_recipe(pthread_priority_voucher, + MACH_VOUCHER_ATTR_KEY_PTHPRIORITY, + content_data, + &content_size); + if (kr != KERN_SUCCESS) { + return kr; + } + + /* return KERN_INVALID_VALUE for default value */ + if (content_size < sizeof(mach_voucher_attr_recipe_t)) { + return KERN_INVALID_VALUE; + } + + cur_content = (mach_voucher_attr_recipe_t) (void *) &content_data[0]; + assert(cur_content->content_size == sizeof(ipc_pthread_priority_value_t)); + memcpy(canonicalize_priority_value, cur_content->content, sizeof(ipc_pthread_priority_value_t)); + + return KERN_SUCCESS; +} + + /* * Routine: ipc_voucher_send_preprocessing * Purpose: @@ -2619,8 +2650,8 @@ void ipc_voucher_send_preprocessing(ipc_kmsg_t kmsg) { uint8_t recipes[(MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN + 1) * sizeof(ipc_voucher_attr_recipe_data_t)]; - ipc_voucher_attr_raw_recipe_array_size_t recipe_size = (MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN + 1) * - sizeof(ipc_voucher_attr_recipe_data_t); + ipc_voucher_attr_raw_recipe_array_size_t recipe_size = (MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN + 1) * + sizeof(ipc_voucher_attr_recipe_data_t); ipc_voucher_t pre_processed_voucher; ipc_voucher_t voucher_to_send; kern_return_t kr; @@ -2631,12 +2662,12 @@ ipc_voucher_send_preprocessing(ipc_kmsg_t kmsg) } /* setup recipe for preprocessing of all the attributes. */ - pre_processed_voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject; + pre_processed_voucher = ip_get_voucher(kmsg->ikm_voucher); kr = ipc_voucher_prepare_processing_recipe(pre_processed_voucher, - (mach_voucher_attr_raw_recipe_array_t)recipes, - &recipe_size, MACH_VOUCHER_ATTR_SEND_PREPROCESS, - IVAM_FLAGS_SUPPORT_SEND_PREPROCESS, &need_preprocessing); + (mach_voucher_attr_raw_recipe_array_t)recipes, + &recipe_size, MACH_VOUCHER_ATTR_SEND_PREPROCESS, + IVAM_FLAGS_SUPPORT_SEND_PREPROCESS, &need_preprocessing); assert(KERN_SUCCESS == kr); /* @@ -2644,8 +2675,8 @@ ipc_voucher_send_preprocessing(ipc_kmsg_t kmsg) */ if (need_preprocessing) { kr = ipc_create_mach_voucher(recipes, - recipe_size, - &voucher_to_send); + recipe_size, + &voucher_to_send); assert(KERN_SUCCESS == kr); ipc_port_release_send(kmsg->ikm_voucher); kmsg->ikm_voucher = convert_voucher_to_port(voucher_to_send); @@ -2663,29 +2694,29 @@ ipc_voucher_send_preprocessing(ipc_kmsg_t kmsg) */ void ipc_voucher_receive_postprocessing( - ipc_kmsg_t kmsg, - mach_msg_option_t option) + ipc_kmsg_t kmsg, + mach_msg_option_t option) { uint8_t recipes[(MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN + 1) * sizeof(ipc_voucher_attr_recipe_data_t)]; - ipc_voucher_attr_raw_recipe_array_size_t recipe_size = (MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN + 1) * - sizeof(ipc_voucher_attr_recipe_data_t); + ipc_voucher_attr_raw_recipe_array_size_t recipe_size = (MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN + 1) * + sizeof(ipc_voucher_attr_recipe_data_t); ipc_voucher_t recv_voucher; ipc_voucher_t sent_voucher; kern_return_t kr; int need_postprocessing = FALSE; if ((option & MACH_RCV_VOUCHER) == 0 || (!IP_VALID(kmsg->ikm_voucher)) || - current_task() == kernel_task) { + current_task() == kernel_task) { return; } /* setup recipe for auto redeem of all the attributes. */ - sent_voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject; + sent_voucher = ip_get_voucher(kmsg->ikm_voucher); kr = ipc_voucher_prepare_processing_recipe(sent_voucher, - (mach_voucher_attr_raw_recipe_array_t)recipes, - &recipe_size, MACH_VOUCHER_ATTR_AUTO_REDEEM, - IVAM_FLAGS_SUPPORT_RECEIVE_POSTPROCESS, &need_postprocessing); + (mach_voucher_attr_raw_recipe_array_t)recipes, + &recipe_size, MACH_VOUCHER_ATTR_AUTO_REDEEM, + IVAM_FLAGS_SUPPORT_RECEIVE_POSTPROCESS, &need_postprocessing); assert(KERN_SUCCESS == kr); @@ -2694,8 +2725,8 @@ ipc_voucher_receive_postprocessing( */ if (need_postprocessing) { kr = ipc_create_mach_voucher(recipes, - recipe_size, - &recv_voucher); + recipe_size, + &recv_voucher); assert(KERN_SUCCESS == kr); /* swap the voucher port (and set voucher bits in case it didn't already exist) */ kmsg->ikm_header->msgh_bits |= (MACH_MSG_TYPE_MOVE_SEND << 16); @@ -2724,13 +2755,15 @@ ipc_voucher_prepare_processing_recipe( ipc_voucher_attr_raw_recipe_array_size_t recipe_used = 0; iv_index_t key_index; ipc_voucher_attr_recipe_t recipe; - - if (IV_NULL == voucher) + + if (IV_NULL == voucher) { return KERN_INVALID_ARGUMENT; - + } + /* Setup a recipe to copy all attributes. */ - if (recipe_size < sizeof(*recipe)) - return KERN_NO_SPACE; + if (recipe_size < sizeof(*recipe)) { + return KERN_NO_SPACE; + } *need_processing = FALSE; recipe = (ipc_voucher_attr_recipe_t)(void *)&recipes[recipe_used]; @@ -2747,14 +2780,16 @@ ipc_voucher_prepare_processing_recipe( /* don't output anything for a default value */ value_index = iv_lookup(voucher, key_index); - if (IV_UNUSED_VALINDEX == value_index) + if (IV_UNUSED_VALINDEX == value_index) { continue; + } - if (recipe_size - recipe_used < sizeof(*recipe)) + if (recipe_size - recipe_used < sizeof(*recipe)) { return KERN_NO_SPACE; + } recipe = (ipc_voucher_attr_recipe_t)(void *)&recipes[recipe_used]; - + /* * Get the manager for this key_index. The * existence of a non-default value for this @@ -2768,9 +2803,10 @@ ipc_voucher_prepare_processing_recipe( } /* Check if the supported flag is set in the manager */ - if ((manager->ivam_flags & flags) == 0) + if ((manager->ivam_flags & flags) == 0) { continue; - + } + key = iv_index_to_key(key_index); recipe->key = key; @@ -2786,20 +2822,61 @@ ipc_voucher_prepare_processing_recipe( return KERN_SUCCESS; } +/* + * Activity id Generation + */ +uint64_t voucher_activity_id; + +#define generate_activity_id(x) \ + ((uint64_t)OSAddAtomic64((x), (int64_t *)&voucher_activity_id)) + +/* + * Routine: mach_init_activity_id + * Purpose: + * Initialize voucher activity id. + */ +void +mach_init_activity_id(void) +{ + voucher_activity_id = 1; +} + +/* + * Routine: mach_generate_activity_id + * Purpose: + * Generate a system wide voucher activity id. + */ +kern_return_t +mach_generate_activity_id( + struct mach_generate_activity_id_args *args) +{ + uint64_t activity_id; + kern_return_t kr = KERN_SUCCESS; + + if (args->count <= 0 || args->count > MACH_ACTIVITY_ID_COUNT_MAX) { + return KERN_INVALID_ARGUMENT; + } + + activity_id = generate_activity_id(args->count); + kr = copyout(&activity_id, args->activity_id, sizeof(activity_id)); + + return kr; +} + #if defined(MACH_VOUCHER_ATTR_KEY_USER_DATA) || defined(MACH_VOUCHER_ATTR_KEY_TEST) /* * Build-in a simple User Data Resource Manager */ -#define USER_DATA_MAX_DATA (16*1024) +#define USER_DATA_MAX_DATA (16*1024) struct user_data_value_element { - mach_voucher_attr_value_reference_t e_made; - mach_voucher_attr_content_size_t e_size; - iv_index_t e_sum; - iv_index_t e_hash; - queue_chain_t e_hash_link; - uint8_t e_data[]; + mach_voucher_attr_value_reference_t e_made; + mach_voucher_attr_content_size_t e_size; + iv_index_t e_sum; + iv_index_t e_hash; + queue_chain_t e_hash_link; + uint8_t e_data[]; }; typedef struct user_data_value_element *user_data_element_t; @@ -2811,80 +2888,78 @@ typedef struct user_data_value_element *user_data_element_t; #define USER_DATA_HASH_BUCKET(x) ((x) % USER_DATA_HASH_BUCKETS) static queue_head_t user_data_bucket[USER_DATA_HASH_BUCKETS]; -static lck_spin_t user_data_lock_data; +static LCK_SPIN_DECLARE_ATTR(user_data_lock_data, &ipc_lck_grp, &ipc_lck_attr); -#define user_data_lock_init() \ - lck_spin_init(&user_data_lock_data, &ipc_lck_grp, &ipc_lck_attr) #define user_data_lock_destroy() \ lck_spin_destroy(&user_data_lock_data, &ipc_lck_grp) -#define user_data_lock() \ - lck_spin_lock(&user_data_lock_data) -#define user_data_lock_try() \ - lck_spin_try_lock(&user_data_lock_data) -#define user_data_unlock() \ +#define user_data_lock() \ + lck_spin_lock_grp(&user_data_lock_data, &ipc_lck_grp) +#define user_data_lock_try() \ + lck_spin_try_lock_grp(&user_data_lock_data, &ipc_lck_grp) +#define user_data_unlock() \ lck_spin_unlock(&user_data_lock_data) static kern_return_t user_data_release_value( - ipc_voucher_attr_manager_t manager, - mach_voucher_attr_key_t key, - mach_voucher_attr_value_handle_t value, - mach_voucher_attr_value_reference_t sync); + ipc_voucher_attr_manager_t manager, + mach_voucher_attr_key_t key, + mach_voucher_attr_value_handle_t value, + mach_voucher_attr_value_reference_t sync); static kern_return_t user_data_get_value( - ipc_voucher_attr_manager_t manager, - mach_voucher_attr_key_t key, - mach_voucher_attr_recipe_command_t command, - mach_voucher_attr_value_handle_array_t prev_values, - mach_voucher_attr_value_handle_array_size_t prev_value_count, - mach_voucher_attr_content_t content, - mach_voucher_attr_content_size_t content_size, - mach_voucher_attr_value_handle_t *out_value, - mach_voucher_attr_value_flags_t *out_flags, - ipc_voucher_t *out_value_voucher); + ipc_voucher_attr_manager_t manager, + mach_voucher_attr_key_t key, + mach_voucher_attr_recipe_command_t command, + mach_voucher_attr_value_handle_array_t prev_values, + mach_voucher_attr_value_handle_array_size_t prev_value_count, + mach_voucher_attr_content_t content, + mach_voucher_attr_content_size_t content_size, + mach_voucher_attr_value_handle_t *out_value, + mach_voucher_attr_value_flags_t *out_flags, + ipc_voucher_t *out_value_voucher); static kern_return_t user_data_extract_content( - ipc_voucher_attr_manager_t manager, - mach_voucher_attr_key_t key, - mach_voucher_attr_value_handle_array_t values, - mach_voucher_attr_value_handle_array_size_t value_count, - mach_voucher_attr_recipe_command_t *out_command, - mach_voucher_attr_content_t out_content, - mach_voucher_attr_content_size_t *in_out_content_size); + ipc_voucher_attr_manager_t manager, + mach_voucher_attr_key_t key, + mach_voucher_attr_value_handle_array_t values, + mach_voucher_attr_value_handle_array_size_t value_count, + mach_voucher_attr_recipe_command_t *out_command, + mach_voucher_attr_content_t out_content, + mach_voucher_attr_content_size_t *in_out_content_size); static kern_return_t user_data_command( - ipc_voucher_attr_manager_t manager, - mach_voucher_attr_key_t key, - mach_voucher_attr_value_handle_array_t values, - mach_msg_type_number_t value_count, - mach_voucher_attr_command_t command, - mach_voucher_attr_content_t in_content, - mach_voucher_attr_content_size_t in_content_size, - mach_voucher_attr_content_t out_content, - mach_voucher_attr_content_size_t *out_content_size); + ipc_voucher_attr_manager_t manager, + mach_voucher_attr_key_t key, + mach_voucher_attr_value_handle_array_t values, + mach_msg_type_number_t value_count, + mach_voucher_attr_command_t command, + mach_voucher_attr_content_t in_content, + mach_voucher_attr_content_size_t in_content_size, + mach_voucher_attr_content_t out_content, + mach_voucher_attr_content_size_t *out_content_size); static void user_data_release( - ipc_voucher_attr_manager_t manager); - -struct ipc_voucher_attr_manager user_data_manager = { - .ivam_release_value = user_data_release_value, - .ivam_get_value = user_data_get_value, - .ivam_extract_content = user_data_extract_content, - .ivam_command = user_data_command, - .ivam_release = user_data_release, - .ivam_flags = IVAM_FLAGS_NONE, + ipc_voucher_attr_manager_t manager); + +const struct ipc_voucher_attr_manager user_data_manager = { + .ivam_release_value = user_data_release_value, + .ivam_get_value = user_data_get_value, + .ivam_extract_content = user_data_extract_content, + .ivam_command = user_data_command, + .ivam_release = user_data_release, + .ivam_flags = IVAM_FLAGS_NONE, }; ipc_voucher_attr_control_t user_data_control; ipc_voucher_attr_control_t test_control; #if defined(MACH_VOUCHER_ATTR_KEY_USER_DATA) && defined(MACH_VOUCHER_ATTR_KEY_TEST) -#define USER_DATA_ASSERT_KEY(key) \ - assert(MACH_VOUCHER_ATTR_KEY_USER_DATA == (key) || \ +#define USER_DATA_ASSERT_KEY(key) \ + assert(MACH_VOUCHER_ATTR_KEY_USER_DATA == (key) || \ MACH_VOUCHER_ATTR_KEY_TEST == (key)); #elif defined(MACH_VOUCHER_ATTR_KEY_USER_DATA) #define USER_DATA_ASSERT_KEY(key) assert(MACH_VOUCHER_ATTR_KEY_USER_DATA == (key)) @@ -2893,7 +2968,7 @@ ipc_voucher_attr_control_t test_control; #endif /* - * Routine: user_data_release_value + * Routine: user_data_release_value * Purpose: * Release a made reference on a specific value managed by * this voucher attribute manager. @@ -2903,15 +2978,15 @@ ipc_voucher_attr_control_t test_control; */ static kern_return_t user_data_release_value( - ipc_voucher_attr_manager_t __assert_only manager, - mach_voucher_attr_key_t __assert_only key, - mach_voucher_attr_value_handle_t value, - mach_voucher_attr_value_reference_t sync) + ipc_voucher_attr_manager_t __assert_only manager, + mach_voucher_attr_key_t __assert_only key, + mach_voucher_attr_value_handle_t value, + mach_voucher_attr_value_reference_t sync) { user_data_element_t elem; iv_index_t hash; - assert (&user_data_manager == manager); + assert(&user_data_manager == manager); USER_DATA_ASSERT_KEY(key); elem = (user_data_element_t)value; @@ -2931,28 +3006,28 @@ user_data_release_value( } /* - * Routine: user_data_checksum + * Routine: user_data_checksum * Purpose: * Provide a rudimentary checksum for the data presented * to these voucher attribute managers. */ static iv_index_t user_data_checksum( - mach_voucher_attr_content_t content, - mach_voucher_attr_content_size_t content_size) + mach_voucher_attr_content_t content, + mach_voucher_attr_content_size_t content_size) { mach_voucher_attr_content_size_t i; iv_index_t cksum = 0; - for(i = 0; i < content_size; i++, content++) { + for (i = 0; i < content_size; i++, content++) { cksum = (cksum << 8) ^ (cksum + *(unsigned char *)content); } - return (~cksum); + return ~cksum; } /* - * Routine: user_data_dedup + * Routine: user_data_dedup * Purpose: * See if the content represented by this request already exists * in another user data element. If so return a made reference @@ -2965,10 +3040,10 @@ user_data_checksum( */ static user_data_element_t user_data_dedup( - mach_voucher_attr_content_t content, - mach_voucher_attr_content_size_t content_size) + mach_voucher_attr_content_t content, + mach_voucher_attr_content_size_t content_size) { - iv_index_t sum; + iv_index_t sum; iv_index_t hash; user_data_element_t elem; user_data_element_t alloc = NULL; @@ -2976,7 +3051,7 @@ user_data_dedup( sum = user_data_checksum(content, content_size); hash = USER_DATA_HASH_BUCKET(sum); - retry: +retry: user_data_lock(); queue_iterate(&user_data_bucket[hash], elem, user_data_element_t, e_hash_link) { assert(elem->e_hash == hash); @@ -2986,19 +3061,23 @@ user_data_dedup( iv_index_t i; /* and all data matches */ - for (i = 0; i < content_size; i++) - if (elem->e_data[i] != content[i]) + for (i = 0; i < content_size; i++) { + if (elem->e_data[i] != content[i]) { break; - if (i < content_size) + } + } + if (i < content_size) { continue; + } /* ... we found a match... */ elem->e_made++; user_data_unlock(); - if (NULL != alloc) + if (NULL != alloc) { kfree(alloc, sizeof(*alloc) + content_size); + } return elem; } @@ -3024,20 +3103,20 @@ user_data_dedup( static kern_return_t user_data_get_value( - ipc_voucher_attr_manager_t __assert_only manager, - mach_voucher_attr_key_t __assert_only key, - mach_voucher_attr_recipe_command_t command, - mach_voucher_attr_value_handle_array_t prev_values, - mach_voucher_attr_value_handle_array_size_t prev_value_count, - mach_voucher_attr_content_t content, - mach_voucher_attr_content_size_t content_size, - mach_voucher_attr_value_handle_t *out_value, - mach_voucher_attr_value_flags_t *out_flags, - ipc_voucher_t *out_value_voucher) + ipc_voucher_attr_manager_t __assert_only manager, + mach_voucher_attr_key_t __assert_only key, + mach_voucher_attr_recipe_command_t command, + mach_voucher_attr_value_handle_array_t prev_values, + mach_voucher_attr_value_handle_array_size_t prev_value_count, + mach_voucher_attr_content_t content, + mach_voucher_attr_content_size_t content_size, + mach_voucher_attr_value_handle_t *out_value, + mach_voucher_attr_value_flags_t *out_flags, + ipc_voucher_t *out_value_voucher) { user_data_element_t elem; - assert (&user_data_manager == manager); + assert(&user_data_manager == manager); USER_DATA_ASSERT_KEY(key); /* never an out voucher */ @@ -3045,15 +3124,18 @@ user_data_get_value( *out_flags = MACH_VOUCHER_ATTR_VALUE_FLAGS_NONE; switch (command) { - case MACH_VOUCHER_ATTR_REDEEM: /* redeem of previous values is the value */ if (0 < prev_value_count) { elem = (user_data_element_t)prev_values[0]; + + user_data_lock(); assert(0 < elem->e_made); elem->e_made++; - *out_value = prev_values[0]; + user_data_unlock(); + + *out_value = (mach_voucher_attr_value_handle_t)elem; return KERN_SUCCESS; } @@ -3062,8 +3144,9 @@ user_data_get_value( return KERN_SUCCESS; case MACH_VOUCHER_ATTR_USER_DATA_STORE: - if (USER_DATA_MAX_DATA < content_size) + if (USER_DATA_MAX_DATA < content_size) { return KERN_RESOURCE_SHORTAGE; + } /* empty is the default */ if (0 == content_size) { @@ -3083,28 +3166,29 @@ user_data_get_value( static kern_return_t user_data_extract_content( - ipc_voucher_attr_manager_t __assert_only manager, - mach_voucher_attr_key_t __assert_only key, - mach_voucher_attr_value_handle_array_t values, - mach_voucher_attr_value_handle_array_size_t value_count, - mach_voucher_attr_recipe_command_t *out_command, - mach_voucher_attr_content_t out_content, - mach_voucher_attr_content_size_t *in_out_content_size) + ipc_voucher_attr_manager_t __assert_only manager, + mach_voucher_attr_key_t __assert_only key, + mach_voucher_attr_value_handle_array_t values, + mach_voucher_attr_value_handle_array_size_t value_count, + mach_voucher_attr_recipe_command_t *out_command, + mach_voucher_attr_content_t out_content, + mach_voucher_attr_content_size_t *in_out_content_size) { mach_voucher_attr_content_size_t size = 0; user_data_element_t elem; unsigned int i; - assert (&user_data_manager == manager); + assert(&user_data_manager == manager); USER_DATA_ASSERT_KEY(key); /* concatenate the stored data items */ - for (i = 0; i < value_count ; i++) { + for (i = 0; i < value_count && *in_out_content_size > 0; i++) { elem = (user_data_element_t)values[i]; assert(USER_DATA_MAX_DATA >= elem->e_size); - if (size + elem->e_size > *in_out_content_size) + if (size + elem->e_size > *in_out_content_size) { return KERN_NO_SPACE; + } memcpy(&out_content[size], elem->e_data, elem->e_size); size += elem->e_size; @@ -3116,70 +3200,62 @@ user_data_extract_content( static kern_return_t user_data_command( - ipc_voucher_attr_manager_t __assert_only manager, - mach_voucher_attr_key_t __assert_only key, - mach_voucher_attr_value_handle_array_t __unused values, - mach_msg_type_number_t __unused value_count, - mach_voucher_attr_command_t __unused command, - mach_voucher_attr_content_t __unused in_content, - mach_voucher_attr_content_size_t __unused in_content_size, - mach_voucher_attr_content_t __unused out_content, - mach_voucher_attr_content_size_t __unused *out_content_size) -{ - assert (&user_data_manager == manager); + ipc_voucher_attr_manager_t __assert_only manager, + mach_voucher_attr_key_t __assert_only key, + mach_voucher_attr_value_handle_array_t __unused values, + mach_msg_type_number_t __unused value_count, + mach_voucher_attr_command_t __unused command, + mach_voucher_attr_content_t __unused in_content, + mach_voucher_attr_content_size_t __unused in_content_size, + mach_voucher_attr_content_t __unused out_content, + mach_voucher_attr_content_size_t __unused *out_content_size) +{ + assert(&user_data_manager == manager); USER_DATA_ASSERT_KEY(key); return KERN_FAILURE; } static void user_data_release( - ipc_voucher_attr_manager_t manager) + ipc_voucher_attr_manager_t manager) { - if (manager != &user_data_manager) + if (manager != &user_data_manager) { return; + } panic("Voucher user-data manager released"); } -static int user_data_manager_inited = 0; - -void -user_data_attr_manager_init() +__startup_func +static void +user_data_attr_manager_init(void) { kern_return_t kr; -#if defined(MACH_VOUCHER_ATTR_KEY_USER_DATA) - if ((user_data_manager_inited & 0x1) != 0x1) { - kr = ipc_register_well_known_mach_voucher_attr_manager(&user_data_manager, - (mach_voucher_attr_value_handle_t)0, - MACH_VOUCHER_ATTR_KEY_USER_DATA, - &user_data_control); - if (KERN_SUCCESS != kr) - printf("Voucher user-data manager register(USER-DATA) returned %d", kr); - else - user_data_manager_inited |= 0x1; +#if defined(MACH_VOUCHER_ATTR_KEY_USER_DATA) + kr = ipc_register_well_known_mach_voucher_attr_manager(&user_data_manager, + (mach_voucher_attr_value_handle_t)0, + MACH_VOUCHER_ATTR_KEY_USER_DATA, + &user_data_control); + if (KERN_SUCCESS != kr) { + printf("Voucher user-data manager register(USER-DATA) returned %d", kr); } #endif #if defined(MACH_VOUCHER_ATTR_KEY_TEST) - if ((user_data_manager_inited & 0x2) != 0x2) { - kr = ipc_register_well_known_mach_voucher_attr_manager(&user_data_manager, - (mach_voucher_attr_value_handle_t)0, - MACH_VOUCHER_ATTR_KEY_TEST, - &test_control); - if (KERN_SUCCESS != kr) - printf("Voucher user-data manager register(TEST) returned %d", kr); - else - user_data_manager_inited |= 0x2; + kr = ipc_register_well_known_mach_voucher_attr_manager(&user_data_manager, + (mach_voucher_attr_value_handle_t)0, + MACH_VOUCHER_ATTR_KEY_TEST, + &test_control); + if (KERN_SUCCESS != kr) { + printf("Voucher user-data manager register(TEST) returned %d", kr); } #endif #if defined(MACH_VOUCHER_ATTR_KEY_USER_DATA) || defined(MACH_VOUCHER_ATTR_KEY_TEST) - int i; - - for (i=0; i < USER_DATA_HASH_BUCKETS; i++) + for (int i = 0; i < USER_DATA_HASH_BUCKETS; i++) { queue_init(&user_data_bucket[i]); - - user_data_lock_init(); + } #endif } +STARTUP(MACH_IPC, STARTUP_RANK_FIRST, user_data_attr_manager_init); -#endif /* MACH_DEBUG */ +#endif /* MACH_VOUCHER_ATTR_KEY_USER_DATA || MACH_VOUCHER_ATTR_KEY_TEST */