X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/0a7de7458d150b5d4dffc935ba399be265ef0a1a..refs/heads/master:/osfmk/ipc/ipc_voucher.c

diff --git a/osfmk/ipc/ipc_voucher.c b/osfmk/ipc/ipc_voucher.c
index ff6da5605..f8673f325 100644
--- a/osfmk/ipc/ipc_voucher.c
+++ b/osfmk/ipc/ipc_voucher.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013 Apple Inc. All rights reserved.
+ * Copyright (c) 2013-2020 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
@@ -50,8 +50,16 @@
  */
 uint32_t ipc_voucher_trace_contents = 0;
 
-static zone_t ipc_voucher_zone;
-static zone_t ipc_voucher_attr_control_zone;
+static SECURITY_READ_ONLY_LATE(zone_t) ipc_voucher_zone;
+static ZONE_DECLARE(ipc_voucher_attr_control_zone, "ipc voucher attr controls",
+    sizeof(struct ipc_voucher_attr_control), ZC_NOENCRYPT | ZC_ZFREE_CLEARMEM);
+
+ZONE_INIT(&ipc_voucher_zone, "ipc vouchers", sizeof(struct ipc_voucher),
+    ZC_NOENCRYPT | ZC_ZFREE_CLEARMEM | ZC_NOSEQUESTER,
+    ZONE_ID_IPC_VOUCHERS, NULL);
+
+#define voucher_require(v) \
+	zone_id_require(ZONE_ID_IPC_VOUCHERS, sizeof(struct ipc_voucher), v)
 
 /*
  * Voucher hash table
@@ -60,11 +68,9 @@ static zone_t ipc_voucher_attr_control_zone;
 #define IV_HASH_BUCKET(x) ((x) % IV_HASH_BUCKETS)
 
 static queue_head_t ivht_bucket[IV_HASH_BUCKETS];
-static lck_spin_t ivht_lock_data;
+static LCK_SPIN_DECLARE_ATTR(ivht_lock_data, &ipc_lck_grp, &ipc_lck_attr);
 static uint32_t ivht_count = 0;
 
-#define ivht_lock_init() \
-	lck_spin_init(&ivht_lock_data, &ipc_lck_grp, &ipc_lck_attr)
 #define ivht_lock_destroy() \
 	lck_spin_destroy(&ivht_lock_data, &ipc_lck_grp)
 #define ivht_lock() \
@@ -83,10 +89,8 @@ static uint32_t ivht_count = 0;
  */
 static iv_index_t ivgt_keys_in_use = MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN;
 static ipc_voucher_global_table_element iv_global_table[MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN];
-static lck_spin_t ivgt_lock_data;
+static LCK_SPIN_DECLARE_ATTR(ivgt_lock_data, &ipc_lck_grp, &ipc_lck_attr);
 
-#define ivgt_lock_init() \
-	lck_spin_init(&ivgt_lock_data, &ipc_lck_grp, &ipc_lck_attr)
 #define ivgt_lock_destroy() \
 	lck_spin_destroy(&ivgt_lock_data, &ipc_lck_grp)
 #define ivgt_lock() \
@@ -192,42 +196,16 @@ ipc_voucher_prepare_processing_recipe(
 	ipc_voucher_attr_manager_flags flags,
 	int *need_processing);
 
-#if defined(MACH_VOUCHER_ATTR_KEY_USER_DATA) || defined(MACH_VOUCHER_ATTR_KEY_TEST)
-void user_data_attr_manager_init(void);
-#endif
-
-void
+__startup_func
+static void
 ipc_voucher_init(void)
 {
-	natural_t ipc_voucher_max = (task_max + thread_max) * 2;
-	natural_t attr_manager_max = MACH_VOUCHER_ATTR_KEY_NUM_WELL_KNOWN;
-	iv_index_t i;
-
-	ipc_voucher_zone = zinit(sizeof(struct ipc_voucher),
-	    ipc_voucher_max * sizeof(struct ipc_voucher),
-	    sizeof(struct ipc_voucher),
-	    "ipc vouchers");
-	zone_change(ipc_voucher_zone, Z_NOENCRYPT, TRUE);
-
-	ipc_voucher_attr_control_zone = zinit(sizeof(struct ipc_voucher_attr_control),
-	    attr_manager_max * sizeof(struct ipc_voucher_attr_control),
-	    sizeof(struct ipc_voucher_attr_control),
-	    "ipc voucher attr controls");
-	zone_change(ipc_voucher_attr_control_zone, Z_NOENCRYPT, TRUE);
-
 	/* initialize voucher hash */
-	ivht_lock_init();
-	for (i = 0; i < IV_HASH_BUCKETS; i++) {
+	for (iv_index_t i = 0; i < IV_HASH_BUCKETS; i++) {
 		queue_init(&ivht_bucket[i]);
 	}
-
-	/* initialize global table locking */
-	ivgt_lock_init();
-
-#if defined(MACH_VOUCHER_ATTR_KEY_USER_DATA) || defined(MACH_VOUCHER_ATTR_KEY_TEST)
-	user_data_attr_manager_init();
-#endif
 }
+STARTUP(MACH_IPC, STARTUP_RANK_FIRST, ipc_voucher_init);
 
 ipc_voucher_t
 iv_alloc(iv_index_t entries)
@@ -318,7 +296,7 @@ iv_dealloc(ipc_voucher_t iv, boolean_t unhash)
 	 * is gone.  We can just discard it now.
 	 */
 	if (IP_VALID(port)) {
-		assert(ip_active(port));
+		require_ip_active(port);
 		assert(port->ip_srights == 0);
 
 		ipc_port_dealloc_kernel(port);
@@ -375,7 +353,10 @@ unsafe_convert_port_to_voucher(
 	ipc_port_t      port)
 {
 	if (IP_VALID(port)) {
-		uintptr_t voucher = (uintptr_t) port->ip_kobject;
+		/* vouchers never labeled (they get transformed before use) */
+		if (ip_is_kolabeled(port)) {
+			return (uintptr_t)IV_NULL;
+		}
 
 		/*
 		 * No need to lock because we have a reference on the
@@ -383,12 +364,21 @@ unsafe_convert_port_to_voucher(
 		 * keeps the voucher bound to the port (and active).
 		 */
 		if (ip_kotype(port) == IKOT_VOUCHER) {
-			return voucher;
+			return (uintptr_t)ipc_kobject_get(port);
 		}
 	}
 	return (uintptr_t)IV_NULL;
 }
 
+static ipc_voucher_t
+ip_get_voucher(ipc_port_t port)
+{
+	ipc_voucher_t voucher = (ipc_voucher_t)ip_get_kobject(port);
+	require_ip_active(port);
+	voucher_require(voucher);
+	return voucher;
+}
+
 /*
  *	Routine:	convert_port_to_voucher
  *	Purpose:
@@ -403,20 +393,13 @@ ipc_voucher_t
 convert_port_to_voucher(
 	ipc_port_t      port)
 {
-	if (IP_VALID(port)) {
-		ipc_voucher_t voucher = (ipc_voucher_t) port->ip_kobject;
-
+	if (IP_VALID(port) && ip_kotype(port) == IKOT_VOUCHER) {
 		/*
 		 * No need to lock because we have a reference on the
 		 * port, and if it is a true voucher port, that reference
 		 * keeps the voucher bound to the port (and active).
 		 */
-		if (ip_kotype(port) != IKOT_VOUCHER) {
-			return IV_NULL;
-		}
-
-		assert(ip_active(port));
-
+		ipc_voucher_t voucher = ip_get_voucher(port);
 		ipc_voucher_reference(voucher);
 		return voucher;
 	}
@@ -477,26 +460,18 @@ ipc_voucher_release(ipc_voucher_t voucher)
  * Purpose:
  *	Called whenever the Mach port system detects no-senders
  *	on the voucher port.
- *
- *	Each time the send-right count goes positive, a no-senders
- *	notification is armed (and a voucher reference is donated).
- *	So, each notification that comes in must release a voucher
- *	reference.  If more send rights have been added since it
- *	fired (asynchronously), they will be protected by a different
- *	reference hold.
  */
 void
 ipc_voucher_notify(mach_msg_header_t *msg)
 {
 	mach_no_senders_notification_t *notification = (void *)msg;
 	ipc_port_t port = notification->not_header.msgh_remote_port;
-	ipc_voucher_t iv;
+	ipc_voucher_t voucher = ip_get_voucher(port);
 
-	assert(ip_active(port));
 	assert(IKOT_VOUCHER == ip_kotype(port));
-	iv = (ipc_voucher_t)port->ip_kobject;
 
-	ipc_voucher_release(iv);
+	/* consume the reference donated by convert_voucher_to_port */
+	ipc_voucher_release(voucher);
 }
 
 /*
@@ -505,48 +480,22 @@ ipc_voucher_notify(mach_msg_header_t *msg)
 ipc_port_t
 convert_voucher_to_port(ipc_voucher_t voucher)
 {
-	ipc_port_t      port, send;
-
 	if (IV_NULL == voucher) {
 		return IP_NULL;
 	}
 
+	voucher_require(voucher);
 	assert(os_ref_get_count(&voucher->iv_refs) > 0);
 
-	/* create a port if needed */
-	port = voucher->iv_port;
-	if (!IP_VALID(port)) {
-		port = ipc_port_alloc_kernel();
-		assert(IP_VALID(port));
-		ipc_kobject_set_atomically(port, (ipc_kobject_t) voucher, IKOT_VOUCHER);
-
-		/* If we lose the race, deallocate and pick up the other guy's port */
-		if (!OSCompareAndSwapPtr(IP_NULL, port, &voucher->iv_port)) {
-			ipc_port_dealloc_kernel(port);
-			port = voucher->iv_port;
-			assert(ip_kotype(port) == IKOT_VOUCHER);
-			assert(port->ip_kobject == (ipc_kobject_t)voucher);
-		}
-	}
-
-	ip_lock(port);
-	assert(ip_active(port));
-	send = ipc_port_make_send_locked(port);
-
-	if (1 == port->ip_srights) {
-		ipc_port_t old_notify;
-
-		/* transfer our ref to the port, and arm the no-senders notification */
-		assert(IP_NULL == port->ip_nsrequest);
-		ipc_port_nsrequest(port, port->ip_mscount, ipc_port_make_sonce_locked(port), &old_notify);
-		/* port unlocked */
-		assert(IP_NULL == old_notify);
-	} else {
-		/* piggyback on the existing port reference, so consume ours */
-		ip_unlock(port);
+	/*
+	 * make a send right and donate our reference for ipc_voucher_notify
+	 * if this is the first send right
+	 */
+	if (!ipc_kobject_make_send_lazy_alloc_port(&voucher->iv_port,
+	    (ipc_kobject_t)voucher, IKOT_VOUCHER, IPC_KOBJECT_ALLOC_NONE, false, 0)) {
 		ipc_voucher_release(voucher);
 	}
-	return send;
+	return voucher->iv_port;
 }
 
 #define ivace_reset_data(ivace_elem, next_index) {       \
@@ -650,7 +599,7 @@ ivac_dealloc(ipc_voucher_attr_control_t ivac)
 	 * is gone.  We can just discard it now.
 	 */
 	if (IP_VALID(port)) {
-		assert(ip_active(port));
+		require_ip_active(port);
 		assert(port->ip_srights == 0);
 
 		ipc_port_dealloc_kernel(port);
@@ -699,7 +648,7 @@ convert_port_to_voucher_attr_control(
 	ipc_port_t      port)
 {
 	if (IP_VALID(port)) {
-		ipc_voucher_attr_control_t ivac = (ipc_voucher_attr_control_t) port->ip_kobject;
+		ipc_voucher_attr_control_t ivac = (ipc_voucher_attr_control_t) ip_get_kobject(port);
 
 		/*
 		 * No need to lock because we have a reference on the
@@ -710,15 +659,21 @@ convert_port_to_voucher_attr_control(
 		if (ip_kotype(port) != IKOT_VOUCHER_ATTR_CONTROL) {
 			return IVAC_NULL;
 		}
+		require_ip_active(port);
 
-		assert(ip_active(port));
-
+		zone_require(ipc_voucher_attr_control_zone, ivac);
 		ivac_reference(ivac);
 		return ivac;
 	}
 	return IVAC_NULL;
 }
 
+/*
+ * Routine:	ipc_voucher_notify
+ * Purpose:
+ *	Called whenever the Mach port system detects no-senders
+ *	on the voucher attr control port.
+ */
 void
 ipc_voucher_attr_control_notify(mach_msg_header_t *msg)
 {
@@ -726,19 +681,12 @@ ipc_voucher_attr_control_notify(mach_msg_header_t *msg)
 	ipc_port_t port = notification->not_header.msgh_remote_port;
 	ipc_voucher_attr_control_t ivac;
 
+	require_ip_active(port);
 	assert(IKOT_VOUCHER_ATTR_CONTROL == ip_kotype(port));
-	ip_lock(port);
-	assert(ip_active(port));
-
-	/* if no new send rights, drop a control reference */
-	if (port->ip_mscount == notification->not_count) {
-		ivac = (ipc_voucher_attr_control_t)port->ip_kobject;
-		ip_unlock(port);
 
-		ivac_release(ivac);
-	} else {
-		ip_unlock(port);
-	}
+	/* release the reference donated by convert_voucher_attr_control_to_port */
+	ivac = (ipc_voucher_attr_control_t)ip_get_kobject(port);
+	ivac_release(ivac);
 }
 
 /*
@@ -747,48 +695,21 @@ ipc_voucher_attr_control_notify(mach_msg_header_t *msg)
 ipc_port_t
 convert_voucher_attr_control_to_port(ipc_voucher_attr_control_t control)
 {
-	ipc_port_t      port, send;
-
 	if (IVAC_NULL == control) {
 		return IP_NULL;
 	}
 
-	/* create a port if needed */
-	port = control->ivac_port;
-	if (!IP_VALID(port)) {
-		port = ipc_port_alloc_kernel();
-		assert(IP_VALID(port));
-		if (OSCompareAndSwapPtr(IP_NULL, port, &control->ivac_port)) {
-			ip_lock(port);
-			ipc_kobject_set_atomically(port, (ipc_kobject_t) control, IKOT_VOUCHER_ATTR_CONTROL);
-		} else {
-			ipc_port_dealloc_kernel(port);
-			port = control->ivac_port;
-			ip_lock(port);
-			assert(ip_kotype(port) == IKOT_VOUCHER_ATTR_CONTROL);
-			assert(port->ip_kobject == (ipc_kobject_t)control);
-		}
-	} else {
-		ip_lock(port);
-	}
-
-	assert(ip_active(port));
-	send = ipc_port_make_send_locked(port);
+	zone_require(ipc_voucher_attr_control_zone, control);
 
-	if (1 == port->ip_srights) {
-		ipc_port_t old_notify;
-
-		/* transfer our ref to the port, and arm the no-senders notification */
-		assert(IP_NULL == port->ip_nsrequest);
-		ipc_port_nsrequest(port, port->ip_mscount, ipc_port_make_sonce_locked(port), &old_notify);
-		assert(IP_NULL == old_notify);
-		/* ipc_port_nsrequest unlocks the port */
-	} else {
-		/* piggyback on the existing port reference, so consume ours */
-		ip_unlock(port);
+	/*
+	 * make a send right and donate our reference for
+	 * ipc_voucher_attr_control_notify if this is the first send right
+	 */
+	if (!ipc_kobject_make_send_lazy_alloc_port(&control->ivac_port,
+	    (ipc_kobject_t)control, IKOT_VOUCHER_ATTR_CONTROL, IPC_KOBJECT_ALLOC_NONE, false, 0)) {
 		ivac_release(control);
 	}
-	return send;
+	return control->ivac_port;
 }
 
 /*
@@ -1213,7 +1134,7 @@ ivgt_lookup(iv_index_t key_index,
 }
 
 /*
- *	Routine:        ipc_replace_voucher_value
+ *	Routine:	ipc_replace_voucher_value
  *	Purpose:
  *		Replace the <voucher, key> value with the results of
  *		running the supplied command through the resource
@@ -1307,7 +1228,7 @@ ipc_replace_voucher_value(
 }
 
 /*
- *	Routine:        ipc_directly_replace_voucher_value
+ *	Routine:	ipc_directly_replace_voucher_value
  *	Purpose:
  *		Replace the <voucher, key> value with the value-handle
  *		supplied directly by the attribute manager.
@@ -1513,8 +1434,7 @@ ipc_execute_voucher_recipe_command(
 
 			new_value = *(mach_voucher_attr_value_handle_t *)(void *)content;
 			kr = ipc_directly_replace_voucher_value(voucher,
-			    key,
-			    new_value);
+			    key, new_value);
 			if (KERN_SUCCESS != kr) {
 				return kr;
 			}
@@ -1567,7 +1487,7 @@ ipc_execute_voucher_recipe_command(
 			}
 			break;
 		}
-	/* fall thru for single key redemption */
+		OS_FALLTHROUGH; /* fall thru for single key redemption */
 
 	/*
 	 * DEFAULT:
@@ -1592,7 +1512,7 @@ ipc_execute_voucher_recipe_command(
 }
 
 /*
- *	Routine:        iv_checksum
+ *	Routine:	iv_checksum
  *	Purpose:
  *		Compute the voucher sum.  This is more position-
  *		relevant than many other checksums - important for
@@ -1622,7 +1542,7 @@ iv_checksum(ipc_voucher_t voucher, boolean_t *emptyp)
 }
 
 /*
- *	Routine:        iv_dedup
+ *	Routine:	iv_dedup
  *	Purpose:
  *		See if the set of values represented by this new voucher
  *		already exist in another voucher.  If so return a reference
@@ -1787,7 +1707,7 @@ iv_dedup(ipc_voucher_t new_iv)
 }
 
 /*
- *	Routine:        ipc_create_mach_voucher
+ *	Routine:	ipc_create_mach_voucher
  *	Purpose:
  *		Create a new mach voucher and initialize it with the
  *		value(s) created by having the appropriate resource
@@ -1858,7 +1778,7 @@ ipc_create_mach_voucher(
 }
 
 /*
- *	Routine:        ipc_voucher_attr_control_create_mach_voucher
+ *	Routine:	ipc_voucher_attr_control_create_mach_voucher
  *	Purpose:
  *		Create a new mach voucher and initialize it with the
  *		value(s) created by having the appropriate resource
@@ -1945,7 +1865,7 @@ ipc_voucher_attr_control_create_mach_voucher(
 }
 
 /*
- *      ipc_register_well_known_mach_voucher_attr_manager
+ *	ipc_register_well_known_mach_voucher_attr_manager
  *
  *	Register the resource manager responsible for a given key value.
  */
@@ -2007,7 +1927,7 @@ ipc_register_well_known_mach_voucher_attr_manager(
 }
 
 /*
- *      Routine:	mach_voucher_extract_attr_content
+ *	Routine:	mach_voucher_extract_attr_content
  *	Purpose:
  *		Extract the content for a given <voucher, key> pair.
  *
@@ -2070,14 +1990,12 @@ mach_voucher_extract_attr_content(
 	/* callout to manager */
 
 	kr = (manager->ivam_extract_content)(manager, key,
-	    vals, vals_count,
-	    &command,
-	    content, in_out_size);
+	    vals, vals_count, &command, content, in_out_size);
 	return kr;
 }
 
 /*
- *      Routine:	mach_voucher_extract_attr_recipe
+ *	Routine:	mach_voucher_extract_attr_recipe
  *	Purpose:
  *		Extract a recipe for a given <voucher, key> pair.
  *
@@ -2163,7 +2081,7 @@ mach_voucher_extract_attr_recipe(
 
 
 /*
- *	Routine:        mach_voucher_extract_all_attr_recipes
+ *	Routine:	mach_voucher_extract_all_attr_recipes
  *	Purpose:
  *		Extract all the (non-default) contents for a given voucher,
  *		building up a recipe that could be provided to a future
@@ -2253,7 +2171,7 @@ mach_voucher_extract_all_attr_recipes(
 }
 
 /*
- *	Routine:        mach_voucher_debug_info
+ *	Routine:	mach_voucher_debug_info
  *	Purpose:
  *		Extract all the (non-default) contents for a given mach port name,
  *		building up a recipe that could be provided to a future
@@ -2284,6 +2202,10 @@ mach_voucher_debug_info(
 	kern_return_t kr;
 	ipc_port_t port = MACH_PORT_NULL;
 
+	if (space == IS_NULL) {
+		return KERN_INVALID_TASK;
+	}
+
 	if (!MACH_PORT_VALID(voucher_name)) {
 		return KERN_INVALID_ARGUMENT;
 	}
@@ -2307,7 +2229,7 @@ mach_voucher_debug_info(
 #endif
 
 /*
- *      Routine:	mach_voucher_attr_command
+ *	Routine:	mach_voucher_attr_command
  *	Purpose:
  *		Invoke an attribute-specific command through this voucher.
  *
@@ -2380,7 +2302,7 @@ mach_voucher_attr_command(
 }
 
 /*
- *      Routine:	mach_voucher_attr_control_get_values
+ *	Routine:	mach_voucher_attr_control_get_values
  *	Purpose:
  *		For a given voucher, get the value handle associated with the
  *		specified attribute manager.
@@ -2416,7 +2338,7 @@ mach_voucher_attr_control_get_values(
 }
 
 /*
- *      Routine:	mach_voucher_attr_control_create_mach_voucher
+ *	Routine:	mach_voucher_attr_control_create_mach_voucher
  *	Purpose:
  *		Create a new mach voucher and initialize it by processing the
  *		supplied recipe(s).
@@ -2510,7 +2432,7 @@ mach_voucher_attr_control_create_mach_voucher(
 }
 
 /*
- *      Routine:	host_create_mach_voucher
+ *	Routine:	host_create_mach_voucher
  *	Purpose:
  *		Create a new mach voucher and initialize it by processing the
  *		supplied recipe(s).
@@ -2598,10 +2520,10 @@ host_create_mach_voucher(
 }
 
 /*
- *      Routine:	host_register_well_known_mach_voucher_attr_manager
+ *	Routine:	host_register_well_known_mach_voucher_attr_manager
  *	Purpose:
  *		Register the user-level resource manager responsible for a given
- *              key value.
+ *		key value.
  *	Conditions:
  *		The manager port passed in has to be converted/wrapped
  *		in an ipc_voucher_attr_manager_t structure and then call the
@@ -2650,7 +2572,7 @@ host_register_well_known_mach_voucher_attr_manager(
 }
 
 /*
- *      Routine:	host_register_mach_voucher_attr_manager
+ *	Routine:	host_register_mach_voucher_attr_manager
  *	Purpose:
  *		Register the user-space resource manager and return a
  *		dynamically allocated key.
@@ -2695,7 +2617,7 @@ ipc_get_pthpriority_from_kmsg_voucher(
 		return KERN_FAILURE;
 	}
 
-	pthread_priority_voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject;
+	pthread_priority_voucher = ip_get_voucher(kmsg->ikm_voucher);
 	kr = mach_voucher_extract_attr_recipe(pthread_priority_voucher,
 	    MACH_VOUCHER_ATTR_KEY_PTHPRIORITY,
 	    content_data,
@@ -2740,7 +2662,7 @@ ipc_voucher_send_preprocessing(ipc_kmsg_t kmsg)
 	}
 
 	/* setup recipe for preprocessing of all the attributes. */
-	pre_processed_voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject;
+	pre_processed_voucher = ip_get_voucher(kmsg->ikm_voucher);
 
 	kr = ipc_voucher_prepare_processing_recipe(pre_processed_voucher,
 	    (mach_voucher_attr_raw_recipe_array_t)recipes,
@@ -2789,7 +2711,7 @@ ipc_voucher_receive_postprocessing(
 	}
 
 	/* setup recipe for auto redeem of all the attributes. */
-	sent_voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject;
+	sent_voucher = ip_get_voucher(kmsg->ikm_voucher);
 
 	kr = ipc_voucher_prepare_processing_recipe(sent_voucher,
 	    (mach_voucher_attr_raw_recipe_array_t)recipes,
@@ -2954,7 +2876,7 @@ struct user_data_value_element {
 	iv_index_t                              e_sum;
 	iv_index_t                              e_hash;
 	queue_chain_t                           e_hash_link;
-	uint8_t                                 e_data[];
+	uint8_t                                *e_data;
 };
 
 typedef struct user_data_value_element *user_data_element_t;
@@ -2966,10 +2888,8 @@ typedef struct user_data_value_element *user_data_element_t;
 #define USER_DATA_HASH_BUCKET(x) ((x) % USER_DATA_HASH_BUCKETS)
 
 static queue_head_t user_data_bucket[USER_DATA_HASH_BUCKETS];
-static lck_spin_t user_data_lock_data;
+static LCK_SPIN_DECLARE_ATTR(user_data_lock_data, &ipc_lck_grp, &ipc_lck_attr);
 
-#define user_data_lock_init() \
-	lck_spin_init(&user_data_lock_data, &ipc_lck_grp, &ipc_lck_attr)
 #define user_data_lock_destroy() \
 	lck_spin_destroy(&user_data_lock_data, &ipc_lck_grp)
 #define user_data_lock() \
@@ -3025,7 +2945,7 @@ static void
 user_data_release(
 	ipc_voucher_attr_manager_t              manager);
 
-struct ipc_voucher_attr_manager user_data_manager = {
+const struct ipc_voucher_attr_manager user_data_manager = {
 	.ivam_release_value =   user_data_release_value,
 	.ivam_get_value =       user_data_get_value,
 	.ivam_extract_content = user_data_extract_content,
@@ -3047,8 +2967,15 @@ ipc_voucher_attr_control_t test_control;
 #define USER_DATA_ASSERT_KEY(key) assert(MACH_VOUCHER_ATTR_KEY_TEST == (key))
 #endif
 
+static void
+user_data_value_element_free(user_data_element_t elem)
+{
+	kheap_free(KHEAP_DATA_BUFFERS, elem->e_data, elem->e_size);
+	kfree(elem, sizeof(struct user_data_value_element));
+}
+
 /*
- *	Routine:        user_data_release_value
+ *	Routine:	user_data_release_value
  *	Purpose:
  *		Release a made reference on a specific value managed by
  *		this voucher attribute manager.
@@ -3076,7 +3003,7 @@ user_data_release_value(
 	if (sync == elem->e_made) {
 		queue_remove(&user_data_bucket[hash], elem, user_data_element_t, e_hash_link);
 		user_data_unlock();
-		kfree(elem, sizeof(*elem) + elem->e_size);
+		user_data_value_element_free(elem);
 		return KERN_SUCCESS;
 	}
 	assert(sync < elem->e_made);
@@ -3086,7 +3013,7 @@ user_data_release_value(
 }
 
 /*
- *	Routine:        user_data_checksum
+ *	Routine:	user_data_checksum
  *	Purpose:
  *		Provide a rudimentary checksum for the data presented
  *		to these voucher attribute managers.
@@ -3107,7 +3034,7 @@ user_data_checksum(
 }
 
 /*
- *	Routine:        user_data_dedup
+ *	Routine:	user_data_dedup
  *	Purpose:
  *		See if the content represented by this request already exists
  *		in another user data element.  If so return a made reference
@@ -3156,7 +3083,7 @@ retry:
 			user_data_unlock();
 
 			if (NULL != alloc) {
-				kfree(alloc, sizeof(*alloc) + content_size);
+				user_data_value_element_free(alloc);
 			}
 
 			return elem;
@@ -3166,11 +3093,12 @@ retry:
 	if (NULL == alloc) {
 		user_data_unlock();
 
-		alloc = (user_data_element_t)kalloc(sizeof(*alloc) + content_size);
+		alloc = kalloc(sizeof(struct user_data_value_element));
 		alloc->e_made = 1;
 		alloc->e_size = content_size;
 		alloc->e_sum = sum;
 		alloc->e_hash = hash;
+		alloc->e_data = kheap_alloc(KHEAP_DATA_BUFFERS, content_size, Z_WAITOK | Z_NOFAIL);
 		memcpy(alloc->e_data, content, content_size);
 		goto retry;
 	}
@@ -3209,9 +3137,13 @@ user_data_get_value(
 		/* redeem of previous values is the value */
 		if (0 < prev_value_count) {
 			elem = (user_data_element_t)prev_values[0];
+
+			user_data_lock();
 			assert(0 < elem->e_made);
 			elem->e_made++;
-			*out_value = prev_values[0];
+			user_data_unlock();
+
+			*out_value = (mach_voucher_attr_value_handle_t)elem;
 			return KERN_SUCCESS;
 		}
 
@@ -3302,48 +3234,36 @@ user_data_release(
 	panic("Voucher user-data manager released");
 }
 
-static int user_data_manager_inited = 0;
-
-void
-user_data_attr_manager_init()
+__startup_func
+static void
+user_data_attr_manager_init(void)
 {
 	kern_return_t kr;
 
 #if defined(MACH_VOUCHER_ATTR_KEY_USER_DATA)
-	if ((user_data_manager_inited & 0x1) != 0x1) {
-		kr = ipc_register_well_known_mach_voucher_attr_manager(&user_data_manager,
-		    (mach_voucher_attr_value_handle_t)0,
-		    MACH_VOUCHER_ATTR_KEY_USER_DATA,
-		    &user_data_control);
-		if (KERN_SUCCESS != kr) {
-			printf("Voucher user-data manager register(USER-DATA) returned %d", kr);
-		} else {
-			user_data_manager_inited |= 0x1;
-		}
+	kr = ipc_register_well_known_mach_voucher_attr_manager(&user_data_manager,
+	    (mach_voucher_attr_value_handle_t)0,
+	    MACH_VOUCHER_ATTR_KEY_USER_DATA,
+	    &user_data_control);
+	if (KERN_SUCCESS != kr) {
+		printf("Voucher user-data manager register(USER-DATA) returned %d", kr);
 	}
 #endif
 #if defined(MACH_VOUCHER_ATTR_KEY_TEST)
-	if ((user_data_manager_inited & 0x2) != 0x2) {
-		kr = ipc_register_well_known_mach_voucher_attr_manager(&user_data_manager,
-		    (mach_voucher_attr_value_handle_t)0,
-		    MACH_VOUCHER_ATTR_KEY_TEST,
-		    &test_control);
-		if (KERN_SUCCESS != kr) {
-			printf("Voucher user-data manager register(TEST) returned %d", kr);
-		} else {
-			user_data_manager_inited |= 0x2;
-		}
+	kr = ipc_register_well_known_mach_voucher_attr_manager(&user_data_manager,
+	    (mach_voucher_attr_value_handle_t)0,
+	    MACH_VOUCHER_ATTR_KEY_TEST,
+	    &test_control);
+	if (KERN_SUCCESS != kr) {
+		printf("Voucher user-data manager register(TEST) returned %d", kr);
 	}
 #endif
 #if defined(MACH_VOUCHER_ATTR_KEY_USER_DATA) || defined(MACH_VOUCHER_ATTR_KEY_TEST)
-	int i;
-
-	for (i = 0; i < USER_DATA_HASH_BUCKETS; i++) {
+	for (int i = 0; i < USER_DATA_HASH_BUCKETS; i++) {
 		queue_init(&user_data_bucket[i]);
 	}
-
-	user_data_lock_init();
 #endif
 }
+STARTUP(MACH_IPC, STARTUP_RANK_FIRST, user_data_attr_manager_init);
 
-#endif /* MACH_DEBUG */
+#endif /* MACH_VOUCHER_ATTR_KEY_USER_DATA || MACH_VOUCHER_ATTR_KEY_TEST */