]> git.saurik.com Git - apple/xnu.git/blobdiff - osfmk/ipc/ipc_voucher.c
xnu-7195.101.1.tar.gz
[apple/xnu.git] / osfmk / ipc / ipc_voucher.c
index ff6da560511d4cc38cd6355b6303a715f2593cf2..f8673f3252f7a4f55457d069e9e5fb64d1517885 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013 Apple Inc. All rights reserved.
+ * Copyright (c) 2013-2020 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  */
 uint32_t ipc_voucher_trace_contents = 0;
 
-static zone_t ipc_voucher_zone;
-static zone_t ipc_voucher_attr_control_zone;
+static SECURITY_READ_ONLY_LATE(zone_t) ipc_voucher_zone;
+static ZONE_DECLARE(ipc_voucher_attr_control_zone, "ipc voucher attr controls",
+    sizeof(struct ipc_voucher_attr_control), ZC_NOENCRYPT | ZC_ZFREE_CLEARMEM);
+
+ZONE_INIT(&ipc_voucher_zone, "ipc vouchers", sizeof(struct ipc_voucher),
+    ZC_NOENCRYPT | ZC_ZFREE_CLEARMEM | ZC_NOSEQUESTER,
+    ZONE_ID_IPC_VOUCHERS, NULL);
+
+#define voucher_require(v) \
+       zone_id_require(ZONE_ID_IPC_VOUCHERS, sizeof(struct ipc_voucher), v)
 
 /*
  * Voucher hash table
@@ -60,11 +68,9 @@ static zone_t ipc_voucher_attr_control_zone;
 #define IV_HASH_BUCKET(x) ((x) % IV_HASH_BUCKETS)
 
 static queue_head_t ivht_bucket[IV_HASH_BUCKETS];
-static lck_spin_t ivht_lock_data;
+static LCK_SPIN_DECLARE_ATTR(ivht_lock_data, &ipc_lck_grp, &ipc_lck_attr);
 static uint32_t ivht_count = 0;
 
-#define ivht_lock_init() \
-       lck_spin_init(&ivht_lock_data, &ipc_lck_grp, &ipc_lck_attr)
 #define ivht_lock_destroy() \
        lck_spin_destroy(&ivht_lock_data, &ipc_lck_grp)
 #define ivht_lock() \
@@ -83,10 +89,8 @@ static uint32_t ivht_count = 0;
  */
 static iv_index_t ivgt_keys_in_use = MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN;
 static ipc_voucher_global_table_element iv_global_table[MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN];
-static lck_spin_t ivgt_lock_data;
+static LCK_SPIN_DECLARE_ATTR(ivgt_lock_data, &ipc_lck_grp, &ipc_lck_attr);
 
-#define ivgt_lock_init() \
-       lck_spin_init(&ivgt_lock_data, &ipc_lck_grp, &ipc_lck_attr)
 #define ivgt_lock_destroy() \
        lck_spin_destroy(&ivgt_lock_data, &ipc_lck_grp)
 #define ivgt_lock() \
@@ -192,42 +196,16 @@ ipc_voucher_prepare_processing_recipe(
        ipc_voucher_attr_manager_flags flags,
        int *need_processing);
 
-#if defined(MACH_VOUCHER_ATTR_KEY_USER_DATA) || defined(MACH_VOUCHER_ATTR_KEY_TEST)
-void user_data_attr_manager_init(void);
-#endif
-
-void
+__startup_func
+static void
 ipc_voucher_init(void)
 {
-       natural_t ipc_voucher_max = (task_max + thread_max) * 2;
-       natural_t attr_manager_max = MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN;
-       iv_index_t i;
-
-       ipc_voucher_zone = zinit(sizeof(struct ipc_voucher),
-           ipc_voucher_max * sizeof(struct ipc_voucher),
-           sizeof(struct ipc_voucher),
-           "ipc vouchers");
-       zone_change(ipc_voucher_zone, Z_NOENCRYPT, TRUE);
-
-       ipc_voucher_attr_control_zone = zinit(sizeof(struct ipc_voucher_attr_control),
-           attr_manager_max * sizeof(struct ipc_voucher_attr_control),
-           sizeof(struct ipc_voucher_attr_control),
-           "ipc voucher attr controls");
-       zone_change(ipc_voucher_attr_control_zone, Z_NOENCRYPT, TRUE);
-
        /* initialize voucher hash */
-       ivht_lock_init();
-       for (i = 0; i < IV_HASH_BUCKETS; i++) {
+       for (iv_index_t i = 0; i < IV_HASH_BUCKETS; i++) {
                queue_init(&ivht_bucket[i]);
        }
-
-       /* initialize global table locking */
-       ivgt_lock_init();
-
-#if defined(MACH_VOUCHER_ATTR_KEY_USER_DATA) || defined(MACH_VOUCHER_ATTR_KEY_TEST)
-       user_data_attr_manager_init();
-#endif
 }
+STARTUP(MACH_IPC, STARTUP_RANK_FIRST, ipc_voucher_init);
 
 ipc_voucher_t
 iv_alloc(iv_index_t entries)
@@ -318,7 +296,7 @@ iv_dealloc(ipc_voucher_t iv, boolean_t unhash)
         * is gone.  We can just discard it now.
         */
        if (IP_VALID(port)) {
-               assert(ip_active(port));
+               require_ip_active(port);
                assert(port->ip_srights == 0);
 
                ipc_port_dealloc_kernel(port);
@@ -375,7 +353,10 @@ unsafe_convert_port_to_voucher(
        ipc_port_t      port)
 {
        if (IP_VALID(port)) {
-               uintptr_t voucher = (uintptr_t) port->ip_kobject;
+               /* vouchers never labeled (they get transformed before use) */
+               if (ip_is_kolabeled(port)) {
+                       return (uintptr_t)IV_NULL;
+               }
 
                /*
                 * No need to lock because we have a reference on the
@@ -383,12 +364,21 @@ unsafe_convert_port_to_voucher(
                 * keeps the voucher bound to the port (and active).
                 */
                if (ip_kotype(port) == IKOT_VOUCHER) {
-                       return voucher;
+                       return (uintptr_t)ipc_kobject_get(port);
                }
        }
        return (uintptr_t)IV_NULL;
 }
 
+static ipc_voucher_t
+ip_get_voucher(ipc_port_t port)
+{
+       ipc_voucher_t voucher = (ipc_voucher_t)ip_get_kobject(port);
+       require_ip_active(port);
+       voucher_require(voucher);
+       return voucher;
+}
+
 /*
  *     Routine:        convert_port_to_voucher
  *     Purpose:
@@ -403,20 +393,13 @@ ipc_voucher_t
 convert_port_to_voucher(
        ipc_port_t      port)
 {
-       if (IP_VALID(port)) {
-               ipc_voucher_t voucher = (ipc_voucher_t) port->ip_kobject;
-
+       if (IP_VALID(port) && ip_kotype(port) == IKOT_VOUCHER) {
                /*
                 * No need to lock because we have a reference on the
                 * port, and if it is a true voucher port, that reference
                 * keeps the voucher bound to the port (and active).
                 */
-               if (ip_kotype(port) != IKOT_VOUCHER) {
-                       return IV_NULL;
-               }
-
-               assert(ip_active(port));
-
+               ipc_voucher_t voucher = ip_get_voucher(port);
                ipc_voucher_reference(voucher);
                return voucher;
        }
@@ -477,26 +460,18 @@ ipc_voucher_release(ipc_voucher_t voucher)
  * Purpose:
  *     Called whenever the Mach port system detects no-senders
  *     on the voucher port.
- *
- *     Each time the send-right count goes positive, a no-senders
- *     notification is armed (and a voucher reference is donated).
- *     So, each notification that comes in must release a voucher
- *     reference.  If more send rights have been added since it
- *     fired (asynchronously), they will be protected by a different
- *     reference hold.
  */
 void
 ipc_voucher_notify(mach_msg_header_t *msg)
 {
        mach_no_senders_notification_t *notification = (void *)msg;
        ipc_port_t port = notification->not_header.msgh_remote_port;
-       ipc_voucher_t iv;
+       ipc_voucher_t voucher = ip_get_voucher(port);
 
-       assert(ip_active(port));
        assert(IKOT_VOUCHER == ip_kotype(port));
-       iv = (ipc_voucher_t)port->ip_kobject;
 
-       ipc_voucher_release(iv);
+       /* consume the reference donated by convert_voucher_to_port */
+       ipc_voucher_release(voucher);
 }
 
 /*
@@ -505,48 +480,22 @@ ipc_voucher_notify(mach_msg_header_t *msg)
 ipc_port_t
 convert_voucher_to_port(ipc_voucher_t voucher)
 {
-       ipc_port_t      port, send;
-
        if (IV_NULL == voucher) {
                return IP_NULL;
        }
 
+       voucher_require(voucher);
        assert(os_ref_get_count(&voucher->iv_refs) > 0);
 
-       /* create a port if needed */
-       port = voucher->iv_port;
-       if (!IP_VALID(port)) {
-               port = ipc_port_alloc_kernel();
-               assert(IP_VALID(port));
-               ipc_kobject_set_atomically(port, (ipc_kobject_t) voucher, IKOT_VOUCHER);
-
-               /* If we lose the race, deallocate and pick up the other guy's port */
-               if (!OSCompareAndSwapPtr(IP_NULL, port, &voucher->iv_port)) {
-                       ipc_port_dealloc_kernel(port);
-                       port = voucher->iv_port;
-                       assert(ip_kotype(port) == IKOT_VOUCHER);
-                       assert(port->ip_kobject == (ipc_kobject_t)voucher);
-               }
-       }
-
-       ip_lock(port);
-       assert(ip_active(port));
-       send = ipc_port_make_send_locked(port);
-
-       if (1 == port->ip_srights) {
-               ipc_port_t old_notify;
-
-               /* transfer our ref to the port, and arm the no-senders notification */
-               assert(IP_NULL == port->ip_nsrequest);
-               ipc_port_nsrequest(port, port->ip_mscount, ipc_port_make_sonce_locked(port), &old_notify);
-               /* port unlocked */
-               assert(IP_NULL == old_notify);
-       } else {
-               /* piggyback on the existing port reference, so consume ours */
-               ip_unlock(port);
+       /*
+        * make a send right and donate our reference for ipc_voucher_notify
+        * if this is the first send right
+        */
+       if (!ipc_kobject_make_send_lazy_alloc_port(&voucher->iv_port,
+           (ipc_kobject_t)voucher, IKOT_VOUCHER, IPC_KOBJECT_ALLOC_NONE, false, 0)) {
                ipc_voucher_release(voucher);
        }
-       return send;
+       return voucher->iv_port;
 }
 
 #define ivace_reset_data(ivace_elem, next_index) {       \
@@ -650,7 +599,7 @@ ivac_dealloc(ipc_voucher_attr_control_t ivac)
         * is gone.  We can just discard it now.
         */
        if (IP_VALID(port)) {
-               assert(ip_active(port));
+               require_ip_active(port);
                assert(port->ip_srights == 0);
 
                ipc_port_dealloc_kernel(port);
@@ -699,7 +648,7 @@ convert_port_to_voucher_attr_control(
        ipc_port_t      port)
 {
        if (IP_VALID(port)) {
-               ipc_voucher_attr_control_t ivac = (ipc_voucher_attr_control_t) port->ip_kobject;
+               ipc_voucher_attr_control_t ivac = (ipc_voucher_attr_control_t) ip_get_kobject(port);
 
                /*
                 * No need to lock because we have a reference on the
@@ -710,15 +659,21 @@ convert_port_to_voucher_attr_control(
                if (ip_kotype(port) != IKOT_VOUCHER_ATTR_CONTROL) {
                        return IVAC_NULL;
                }
+               require_ip_active(port);
 
-               assert(ip_active(port));
-
+               zone_require(ipc_voucher_attr_control_zone, ivac);
                ivac_reference(ivac);
                return ivac;
        }
        return IVAC_NULL;
 }
 
+/*
+ * Routine:    ipc_voucher_notify
+ * Purpose:
+ *     Called whenever the Mach port system detects no-senders
+ *     on the voucher attr control port.
+ */
 void
 ipc_voucher_attr_control_notify(mach_msg_header_t *msg)
 {
@@ -726,19 +681,12 @@ ipc_voucher_attr_control_notify(mach_msg_header_t *msg)
        ipc_port_t port = notification->not_header.msgh_remote_port;
        ipc_voucher_attr_control_t ivac;
 
+       require_ip_active(port);
        assert(IKOT_VOUCHER_ATTR_CONTROL == ip_kotype(port));
-       ip_lock(port);
-       assert(ip_active(port));
-
-       /* if no new send rights, drop a control reference */
-       if (port->ip_mscount == notification->not_count) {
-               ivac = (ipc_voucher_attr_control_t)port->ip_kobject;
-               ip_unlock(port);
 
-               ivac_release(ivac);
-       } else {
-               ip_unlock(port);
-       }
+       /* release the reference donated by convert_voucher_attr_control_to_port */
+       ivac = (ipc_voucher_attr_control_t)ip_get_kobject(port);
+       ivac_release(ivac);
 }
 
 /*
@@ -747,48 +695,21 @@ ipc_voucher_attr_control_notify(mach_msg_header_t *msg)
 ipc_port_t
 convert_voucher_attr_control_to_port(ipc_voucher_attr_control_t control)
 {
-       ipc_port_t      port, send;
-
        if (IVAC_NULL == control) {
                return IP_NULL;
        }
 
-       /* create a port if needed */
-       port = control->ivac_port;
-       if (!IP_VALID(port)) {
-               port = ipc_port_alloc_kernel();
-               assert(IP_VALID(port));
-               if (OSCompareAndSwapPtr(IP_NULL, port, &control->ivac_port)) {
-                       ip_lock(port);
-                       ipc_kobject_set_atomically(port, (ipc_kobject_t) control, IKOT_VOUCHER_ATTR_CONTROL);
-               } else {
-                       ipc_port_dealloc_kernel(port);
-                       port = control->ivac_port;
-                       ip_lock(port);
-                       assert(ip_kotype(port) == IKOT_VOUCHER_ATTR_CONTROL);
-                       assert(port->ip_kobject == (ipc_kobject_t)control);
-               }
-       } else {
-               ip_lock(port);
-       }
-
-       assert(ip_active(port));
-       send = ipc_port_make_send_locked(port);
+       zone_require(ipc_voucher_attr_control_zone, control);
 
-       if (1 == port->ip_srights) {
-               ipc_port_t old_notify;
-
-               /* transfer our ref to the port, and arm the no-senders notification */
-               assert(IP_NULL == port->ip_nsrequest);
-               ipc_port_nsrequest(port, port->ip_mscount, ipc_port_make_sonce_locked(port), &old_notify);
-               assert(IP_NULL == old_notify);
-               /* ipc_port_nsrequest unlocks the port */
-       } else {
-               /* piggyback on the existing port reference, so consume ours */
-               ip_unlock(port);
+       /*
+        * make a send right and donate our reference for
+        * ipc_voucher_attr_control_notify if this is the first send right
+        */
+       if (!ipc_kobject_make_send_lazy_alloc_port(&control->ivac_port,
+           (ipc_kobject_t)control, IKOT_VOUCHER_ATTR_CONTROL, IPC_KOBJECT_ALLOC_NONE, false, 0)) {
                ivac_release(control);
        }
-       return send;
+       return control->ivac_port;
 }
 
 /*
@@ -1213,7 +1134,7 @@ ivgt_lookup(iv_index_t key_index,
 }
 
 /*
- *     Routine:        ipc_replace_voucher_value
+ *     Routine:        ipc_replace_voucher_value
  *     Purpose:
  *             Replace the <voucher, key> value with the results of
  *             running the supplied command through the resource
@@ -1307,7 +1228,7 @@ ipc_replace_voucher_value(
 }
 
 /*
- *     Routine:        ipc_directly_replace_voucher_value
+ *     Routine:        ipc_directly_replace_voucher_value
  *     Purpose:
  *             Replace the <voucher, key> value with the value-handle
  *             supplied directly by the attribute manager.
@@ -1513,8 +1434,7 @@ ipc_execute_voucher_recipe_command(
 
                        new_value = *(mach_voucher_attr_value_handle_t *)(void *)content;
                        kr = ipc_directly_replace_voucher_value(voucher,
-                           key,
-                           new_value);
+                           key, new_value);
                        if (KERN_SUCCESS != kr) {
                                return kr;
                        }
@@ -1567,7 +1487,7 @@ ipc_execute_voucher_recipe_command(
                        }
                        break;
                }
-       /* fall thru for single key redemption */
+               OS_FALLTHROUGH; /* fall thru for single key redemption */
 
        /*
         * DEFAULT:
@@ -1592,7 +1512,7 @@ ipc_execute_voucher_recipe_command(
 }
 
 /*
- *     Routine:        iv_checksum
+ *     Routine:        iv_checksum
  *     Purpose:
  *             Compute the voucher sum.  This is more position-
  *             relevant than many other checksums - important for
@@ -1622,7 +1542,7 @@ iv_checksum(ipc_voucher_t voucher, boolean_t *emptyp)
 }
 
 /*
- *     Routine:        iv_dedup
+ *     Routine:        iv_dedup
  *     Purpose:
  *             See if the set of values represented by this new voucher
  *             already exist in another voucher.  If so return a reference
@@ -1787,7 +1707,7 @@ iv_dedup(ipc_voucher_t new_iv)
 }
 
 /*
- *     Routine:        ipc_create_mach_voucher
+ *     Routine:        ipc_create_mach_voucher
  *     Purpose:
  *             Create a new mach voucher and initialize it with the
  *             value(s) created by having the appropriate resource
@@ -1858,7 +1778,7 @@ ipc_create_mach_voucher(
 }
 
 /*
- *     Routine:        ipc_voucher_attr_control_create_mach_voucher
+ *     Routine:        ipc_voucher_attr_control_create_mach_voucher
  *     Purpose:
  *             Create a new mach voucher and initialize it with the
  *             value(s) created by having the appropriate resource
@@ -1945,7 +1865,7 @@ ipc_voucher_attr_control_create_mach_voucher(
 }
 
 /*
- *      ipc_register_well_known_mach_voucher_attr_manager
+ *     ipc_register_well_known_mach_voucher_attr_manager
  *
  *     Register the resource manager responsible for a given key value.
  */
@@ -2007,7 +1927,7 @@ ipc_register_well_known_mach_voucher_attr_manager(
 }
 
 /*
- *      Routine:       mach_voucher_extract_attr_content
+ *     Routine:        mach_voucher_extract_attr_content
  *     Purpose:
  *             Extract the content for a given <voucher, key> pair.
  *
@@ -2070,14 +1990,12 @@ mach_voucher_extract_attr_content(
        /* callout to manager */
 
        kr = (manager->ivam_extract_content)(manager, key,
-           vals, vals_count,
-           &command,
-           content, in_out_size);
+           vals, vals_count, &command, content, in_out_size);
        return kr;
 }
 
 /*
- *      Routine:       mach_voucher_extract_attr_recipe
+ *     Routine:        mach_voucher_extract_attr_recipe
  *     Purpose:
  *             Extract a recipe for a given <voucher, key> pair.
  *
@@ -2163,7 +2081,7 @@ mach_voucher_extract_attr_recipe(
 
 
 /*
- *     Routine:        mach_voucher_extract_all_attr_recipes
+ *     Routine:        mach_voucher_extract_all_attr_recipes
  *     Purpose:
  *             Extract all the (non-default) contents for a given voucher,
  *             building up a recipe that could be provided to a future
@@ -2253,7 +2171,7 @@ mach_voucher_extract_all_attr_recipes(
 }
 
 /*
- *     Routine:        mach_voucher_debug_info
+ *     Routine:        mach_voucher_debug_info
  *     Purpose:
  *             Extract all the (non-default) contents for a given mach port name,
  *             building up a recipe that could be provided to a future
@@ -2284,6 +2202,10 @@ mach_voucher_debug_info(
        kern_return_t kr;
        ipc_port_t port = MACH_PORT_NULL;
 
+       if (space == IS_NULL) {
+               return KERN_INVALID_TASK;
+       }
+
        if (!MACH_PORT_VALID(voucher_name)) {
                return KERN_INVALID_ARGUMENT;
        }
@@ -2307,7 +2229,7 @@ mach_voucher_debug_info(
 #endif
 
 /*
- *      Routine:       mach_voucher_attr_command
+ *     Routine:        mach_voucher_attr_command
  *     Purpose:
  *             Invoke an attribute-specific command through this voucher.
  *
@@ -2380,7 +2302,7 @@ mach_voucher_attr_command(
 }
 
 /*
- *      Routine:       mach_voucher_attr_control_get_values
+ *     Routine:        mach_voucher_attr_control_get_values
  *     Purpose:
  *             For a given voucher, get the value handle associated with the
  *             specified attribute manager.
@@ -2416,7 +2338,7 @@ mach_voucher_attr_control_get_values(
 }
 
 /*
- *      Routine:       mach_voucher_attr_control_create_mach_voucher
+ *     Routine:        mach_voucher_attr_control_create_mach_voucher
  *     Purpose:
  *             Create a new mach voucher and initialize it by processing the
  *             supplied recipe(s).
@@ -2510,7 +2432,7 @@ mach_voucher_attr_control_create_mach_voucher(
 }
 
 /*
- *      Routine:       host_create_mach_voucher
+ *     Routine:        host_create_mach_voucher
  *     Purpose:
  *             Create a new mach voucher and initialize it by processing the
  *             supplied recipe(s).
@@ -2598,10 +2520,10 @@ host_create_mach_voucher(
 }
 
 /*
- *      Routine:       host_register_well_known_mach_voucher_attr_manager
+ *     Routine:        host_register_well_known_mach_voucher_attr_manager
  *     Purpose:
  *             Register the user-level resource manager responsible for a given
- *              key value.
+ *             key value.
  *     Conditions:
  *             The manager port passed in has to be converted/wrapped
  *             in an ipc_voucher_attr_manager_t structure and then call the
@@ -2650,7 +2572,7 @@ host_register_well_known_mach_voucher_attr_manager(
 }
 
 /*
- *      Routine:       host_register_mach_voucher_attr_manager
+ *     Routine:        host_register_mach_voucher_attr_manager
  *     Purpose:
  *             Register the user-space resource manager and return a
  *             dynamically allocated key.
@@ -2695,7 +2617,7 @@ ipc_get_pthpriority_from_kmsg_voucher(
                return KERN_FAILURE;
        }
 
-       pthread_priority_voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject;
+       pthread_priority_voucher = ip_get_voucher(kmsg->ikm_voucher);
        kr = mach_voucher_extract_attr_recipe(pthread_priority_voucher,
            MACH_VOUCHER_ATTR_KEY_PTHPRIORITY,
            content_data,
@@ -2740,7 +2662,7 @@ ipc_voucher_send_preprocessing(ipc_kmsg_t kmsg)
        }
 
        /* setup recipe for preprocessing of all the attributes. */
-       pre_processed_voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject;
+       pre_processed_voucher = ip_get_voucher(kmsg->ikm_voucher);
 
        kr = ipc_voucher_prepare_processing_recipe(pre_processed_voucher,
            (mach_voucher_attr_raw_recipe_array_t)recipes,
@@ -2789,7 +2711,7 @@ ipc_voucher_receive_postprocessing(
        }
 
        /* setup recipe for auto redeem of all the attributes. */
-       sent_voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject;
+       sent_voucher = ip_get_voucher(kmsg->ikm_voucher);
 
        kr = ipc_voucher_prepare_processing_recipe(sent_voucher,
            (mach_voucher_attr_raw_recipe_array_t)recipes,
@@ -2954,7 +2876,7 @@ struct user_data_value_element {
        iv_index_t                              e_sum;
        iv_index_t                              e_hash;
        queue_chain_t                           e_hash_link;
-       uint8_t                                 e_data[];
+       uint8_t                                *e_data;
 };
 
 typedef struct user_data_value_element *user_data_element_t;
@@ -2966,10 +2888,8 @@ typedef struct user_data_value_element *user_data_element_t;
 #define USER_DATA_HASH_BUCKET(x) ((x) % USER_DATA_HASH_BUCKETS)
 
 static queue_head_t user_data_bucket[USER_DATA_HASH_BUCKETS];
-static lck_spin_t user_data_lock_data;
+static LCK_SPIN_DECLARE_ATTR(user_data_lock_data, &ipc_lck_grp, &ipc_lck_attr);
 
-#define user_data_lock_init() \
-       lck_spin_init(&user_data_lock_data, &ipc_lck_grp, &ipc_lck_attr)
 #define user_data_lock_destroy() \
        lck_spin_destroy(&user_data_lock_data, &ipc_lck_grp)
 #define user_data_lock() \
@@ -3025,7 +2945,7 @@ static void
 user_data_release(
        ipc_voucher_attr_manager_t              manager);
 
-struct ipc_voucher_attr_manager user_data_manager = {
+const struct ipc_voucher_attr_manager user_data_manager = {
        .ivam_release_value =   user_data_release_value,
        .ivam_get_value =       user_data_get_value,
        .ivam_extract_content = user_data_extract_content,
@@ -3047,8 +2967,15 @@ ipc_voucher_attr_control_t test_control;
 #define USER_DATA_ASSERT_KEY(key) assert(MACH_VOUCHER_ATTR_KEY_TEST == (key))
 #endif
 
+static void
+user_data_value_element_free(user_data_element_t elem)
+{
+       kheap_free(KHEAP_DATA_BUFFERS, elem->e_data, elem->e_size);
+       kfree(elem, sizeof(struct user_data_value_element));
+}
+
 /*
- *     Routine:        user_data_release_value
+ *     Routine:        user_data_release_value
  *     Purpose:
  *             Release a made reference on a specific value managed by
  *             this voucher attribute manager.
@@ -3076,7 +3003,7 @@ user_data_release_value(
        if (sync == elem->e_made) {
                queue_remove(&user_data_bucket[hash], elem, user_data_element_t, e_hash_link);
                user_data_unlock();
-               kfree(elem, sizeof(*elem) + elem->e_size);
+               user_data_value_element_free(elem);
                return KERN_SUCCESS;
        }
        assert(sync < elem->e_made);
@@ -3086,7 +3013,7 @@ user_data_release_value(
 }
 
 /*
- *     Routine:        user_data_checksum
+ *     Routine:        user_data_checksum
  *     Purpose:
  *             Provide a rudimentary checksum for the data presented
  *             to these voucher attribute managers.
@@ -3107,7 +3034,7 @@ user_data_checksum(
 }
 
 /*
- *     Routine:        user_data_dedup
+ *     Routine:        user_data_dedup
  *     Purpose:
  *             See if the content represented by this request already exists
  *             in another user data element.  If so return a made reference
@@ -3156,7 +3083,7 @@ retry:
                        user_data_unlock();
 
                        if (NULL != alloc) {
-                               kfree(alloc, sizeof(*alloc) + content_size);
+                               user_data_value_element_free(alloc);
                        }
 
                        return elem;
@@ -3166,11 +3093,12 @@ retry:
        if (NULL == alloc) {
                user_data_unlock();
 
-               alloc = (user_data_element_t)kalloc(sizeof(*alloc) + content_size);
+               alloc = kalloc(sizeof(struct user_data_value_element));
                alloc->e_made = 1;
                alloc->e_size = content_size;
                alloc->e_sum = sum;
                alloc->e_hash = hash;
+               alloc->e_data = kheap_alloc(KHEAP_DATA_BUFFERS, content_size, Z_WAITOK | Z_NOFAIL);
                memcpy(alloc->e_data, content, content_size);
                goto retry;
        }
@@ -3209,9 +3137,13 @@ user_data_get_value(
                /* redeem of previous values is the value */
                if (0 < prev_value_count) {
                        elem = (user_data_element_t)prev_values[0];
+
+                       user_data_lock();
                        assert(0 < elem->e_made);
                        elem->e_made++;
-                       *out_value = prev_values[0];
+                       user_data_unlock();
+
+                       *out_value = (mach_voucher_attr_value_handle_t)elem;
                        return KERN_SUCCESS;
                }
 
@@ -3302,48 +3234,36 @@ user_data_release(
        panic("Voucher user-data manager released");
 }
 
-static int user_data_manager_inited = 0;
-
-void
-user_data_attr_manager_init()
+__startup_func
+static void
+user_data_attr_manager_init(void)
 {
        kern_return_t kr;
 
 #if defined(MACH_VOUCHER_ATTR_KEY_USER_DATA)
-       if ((user_data_manager_inited & 0x1) != 0x1) {
-               kr = ipc_register_well_known_mach_voucher_attr_manager(&user_data_manager,
-                   (mach_voucher_attr_value_handle_t)0,
-                   MACH_VOUCHER_ATTR_KEY_USER_DATA,
-                   &user_data_control);
-               if (KERN_SUCCESS != kr) {
-                       printf("Voucher user-data manager register(USER-DATA) returned %d", kr);
-               } else {
-                       user_data_manager_inited |= 0x1;
-               }
+       kr = ipc_register_well_known_mach_voucher_attr_manager(&user_data_manager,
+           (mach_voucher_attr_value_handle_t)0,
+           MACH_VOUCHER_ATTR_KEY_USER_DATA,
+           &user_data_control);
+       if (KERN_SUCCESS != kr) {
+               printf("Voucher user-data manager register(USER-DATA) returned %d", kr);
        }
 #endif
 #if defined(MACH_VOUCHER_ATTR_KEY_TEST)
-       if ((user_data_manager_inited & 0x2) != 0x2) {
-               kr = ipc_register_well_known_mach_voucher_attr_manager(&user_data_manager,
-                   (mach_voucher_attr_value_handle_t)0,
-                   MACH_VOUCHER_ATTR_KEY_TEST,
-                   &test_control);
-               if (KERN_SUCCESS != kr) {
-                       printf("Voucher user-data manager register(TEST) returned %d", kr);
-               } else {
-                       user_data_manager_inited |= 0x2;
-               }
+       kr = ipc_register_well_known_mach_voucher_attr_manager(&user_data_manager,
+           (mach_voucher_attr_value_handle_t)0,
+           MACH_VOUCHER_ATTR_KEY_TEST,
+           &test_control);
+       if (KERN_SUCCESS != kr) {
+               printf("Voucher user-data manager register(TEST) returned %d", kr);
        }
 #endif
 #if defined(MACH_VOUCHER_ATTR_KEY_USER_DATA) || defined(MACH_VOUCHER_ATTR_KEY_TEST)
-       int i;
-
-       for (i = 0; i < USER_DATA_HASH_BUCKETS; i++) {
+       for (int i = 0; i < USER_DATA_HASH_BUCKETS; i++) {
                queue_init(&user_data_bucket[i]);
        }
-
-       user_data_lock_init();
 #endif
 }
+STARTUP(MACH_IPC, STARTUP_RANK_FIRST, user_data_attr_manager_init);
 
-#endif /* MACH_DEBUG */
+#endif /* MACH_VOUCHER_ATTR_KEY_USER_DATA || MACH_VOUCHER_ATTR_KEY_TEST */