--- /dev/null
+/*
+ * Copyright (c) 2012-2013 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <atm/atm_internal.h>
+#include <mach/mach_types.h>
+#include <mach/kern_return.h>
+#include <ipc/ipc_port.h>
+#include <mach/mach_vm.h>
+#include <mach/vm_map.h>
+#include <vm/vm_map.h>
+#include <atm/atm_notification.h>
+#include <mach/host_priv.h>
+#include <mach/host_special_ports.h>
+#include <kern/host.h>
+#include <kern/kalloc.h>
+
+#define MAX_ATM_VALUES (2 * 4096)
+#define MAX_TRACE_BUFFER_SIZE (0x40000000) /* Restrict to 1GB per task */
+#define MAX_MAILBOX_SIZE (8 * 4096)
+
+#define ATM_VALUE_TO_HANDLE(x) (CAST_DOWN(atm_voucher_id_t, (x)))
+#define HANDLE_TO_ATM_VALUE(x) (CAST_DOWN(atm_value_t, (x)))
+
+#define ATM_MAX_HASH_TABLE_SIZE (256)
+#define AID_HASH_MASK (0xFF)
+#define AID_TO_HASH(x) ((x) & (AID_HASH_MASK))
+
+#define ATM_LIST_DEAD_MAX 15
+
+#define AID_ARRAY_COUNT_MAX (256)
+
+struct atm_value_hash atm_value_hash_table[ATM_MAX_HASH_TABLE_SIZE];
+extern int maxproc;
+
+/* Global flag to disable ATM. ATM get value and memory registration will return error. */
+boolean_t disable_atm = FALSE;
+
+#if DEVELOPMENT || DEBUG
+queue_head_t atm_descriptors_list;
+queue_head_t atm_values_list;
+#endif
+
+ipc_voucher_attr_control_t voucher_attr_control; /* communication channel from ATM to voucher system */
+static zone_t atm_value_zone, atm_descriptors_zone, atm_link_objects_zone;
+
+static aid_t get_aid();
+static atm_value_t atm_value_alloc_init();
+static void atm_value_dealloc(atm_value_t atm_value);
+static void atm_hash_table_init();
+static void atm_value_hash_table_insert(atm_value_t new_atm_value);
+static void atm_value_hash_table_delete(atm_value_t atm_value);
+static atm_value_t get_atm_value_from_aid(aid_t aid);
+static void atm_value_get_ref(atm_value_t atm_value);
+static kern_return_t atm_listener_insert(atm_value_t atm_value, atm_task_descriptor_t task_descriptor, mailbox_offset_t mailbox_offset);
+static void atm_listener_delete_all(atm_value_t atm_value);
+static atm_task_descriptor_t atm_task_descriptor_alloc_init(mach_port_t trace_buffer,uint64_t buffer_size, void *mailbox_addr, uint64_t mailbox_array_size, __assert_only task_t task);
+static void atm_descriptor_get_reference(atm_task_descriptor_t task_descriptor);
+static void atm_task_descriptor_dealloc(atm_task_descriptor_t task_descriptor);
+static mach_atm_subaid_t atm_get_min_sub_aid(atm_value_t atm_value);
+static void
+atm_get_min_sub_aid_array(aid_t *aid_array, mach_atm_subaid_t *subaid_array, uint32_t count) __unused;
+static kern_return_t atm_value_unregister(atm_value_t atm_value, atm_task_descriptor_t task_descriptor, mailbox_offset_t mailbox_offset);
+static kern_return_t atm_listener_delete(atm_value_t atm_value, atm_task_descriptor_t task_descriptor, mailbox_offset_t mailbox_offset);
+static void atm_link_get_reference(atm_link_object_t link_object);
+static void atm_link_dealloc(atm_link_object_t link_object);
+kern_return_t atm_invoke_collection(atm_value_t atm_value, uint64_t sub_activity_id, uint32_t flags);
+kern_return_t atm_send_user_notification(aid_t aid, uint64_t subaid, mach_port_t *buffers_array, uint64_t *sizes_array, mach_msg_type_number_t count, uint32_t flags);
+
+kern_return_t
+atm_release_value(
+ ipc_voucher_attr_manager_t __assert_only manager,
+ mach_voucher_attr_key_t __assert_only key,
+ mach_voucher_attr_value_handle_t value,
+ mach_voucher_attr_value_reference_t sync);
+
+kern_return_t
+atm_get_value(
+ ipc_voucher_attr_manager_t __assert_only manager,
+ mach_voucher_attr_key_t __assert_only key,
+ mach_voucher_attr_recipe_command_t command,
+ mach_voucher_attr_value_handle_array_t prev_values,
+ mach_msg_type_number_t __assert_only prev_value_count,
+ mach_voucher_attr_content_t recipe,
+ mach_voucher_attr_content_size_t recipe_size,
+ mach_voucher_attr_value_handle_t *out_value,
+ ipc_voucher_t *out_value_voucher);
+
+kern_return_t
+atm_extract_content(
+ ipc_voucher_attr_manager_t __assert_only manager,
+ mach_voucher_attr_key_t __assert_only key,
+ mach_voucher_attr_value_handle_array_t values,
+ mach_msg_type_number_t value_count,
+ mach_voucher_attr_recipe_command_t *out_command,
+ mach_voucher_attr_content_t out_recipe,
+ mach_voucher_attr_content_size_t *in_out_recipe_size);
+
+kern_return_t
+atm_command(
+ ipc_voucher_attr_manager_t __assert_only manager,
+ mach_voucher_attr_key_t __assert_only key,
+ mach_voucher_attr_value_handle_array_t values,
+ mach_msg_type_number_t value_count,
+ mach_voucher_attr_command_t command,
+ mach_voucher_attr_content_t in_content,
+ mach_voucher_attr_content_size_t in_content_size,
+ mach_voucher_attr_content_t out_content,
+ mach_voucher_attr_content_size_t *in_out_content_size);
+
+void
+atm_release(ipc_voucher_attr_manager_t __assert_only manager);
+
+/*
+ * communication channel from voucher system to ATM
+ */
+struct ipc_voucher_attr_manager atm_manager = {
+ .ivam_release_value = atm_release_value,
+ .ivam_get_value = atm_get_value,
+ .ivam_extract_content = atm_extract_content,
+ .ivam_command = atm_command,
+ .ivam_release = atm_release,
+};
+
+#if DEVELOPMENT || DEBUG
+decl_lck_mtx_data(, atm_descriptors_list_lock);
+decl_lck_mtx_data(, atm_values_list_lock);
+
+lck_grp_t atm_dev_lock_grp;
+lck_attr_t atm_dev_lock_attr;
+lck_grp_attr_t atm_dev_lock_grp_attr;
+#endif
+
+extern vm_map_t kernel_map;
+/*
+ * Global aid. Incremented on each get_aid.
+ */
+aid_t global_aid;
+
+/*
+ * Lock group attributes for atm sub system.
+ */
+lck_grp_t atm_lock_grp;
+lck_attr_t atm_lock_attr;
+lck_grp_attr_t atm_lock_grp_attr;
+
+
+/*
+ * Routine: atm_init
+ * Purpose: Initialize the atm subsystem.
+ * Returns: None.
+ */
+void
+atm_init()
+{
+ kern_return_t kr = KERN_SUCCESS;
+ char temp_buf[20];
+
+ /* Disable atm if disable_atm present in device-tree properties or in boot-args */
+ if ((PE_get_default("kern.disable_atm", temp_buf, sizeof(temp_buf))) ||
+ (PE_parse_boot_argn("-disable_atm", temp_buf, sizeof(temp_buf)))) {
+ disable_atm = TRUE;
+ }
+
+ /* setup zones for descriptors, values and link objects */
+ atm_value_zone = zinit(sizeof(struct atm_value),
+ MAX_ATM_VALUES * sizeof(struct atm_value),
+ sizeof(struct atm_value),
+ "atm_values");
+
+ atm_descriptors_zone = zinit(sizeof(struct atm_task_descriptor),
+ MAX_ATM_VALUES * sizeof(struct atm_task_descriptor),
+ sizeof(struct atm_task_descriptor),
+ "atm_task_descriptors");
+
+ atm_link_objects_zone = zinit(sizeof(struct atm_link_object),
+ MAX_ATM_VALUES * sizeof(struct atm_link_object),
+ sizeof(struct atm_link_object),
+ "atm_link_objects");
+
+ /* Initialize atm lock group and lock attributes. */
+ lck_grp_attr_setdefault(&atm_lock_grp_attr);
+ lck_grp_init(&atm_lock_grp, "atm_lock", &atm_lock_grp_attr);
+ lck_attr_setdefault(&atm_lock_attr);
+
+ global_aid = 1;
+ atm_hash_table_init();
+
+#if DEVELOPMENT || DEBUG
+ /* Initialize global atm development lock group and lock attributes. */
+ lck_grp_attr_setdefault(&atm_dev_lock_grp_attr);
+ lck_grp_init(&atm_dev_lock_grp, "atm_dev_lock", &atm_dev_lock_grp_attr);
+ lck_attr_setdefault(&atm_dev_lock_attr);
+
+ lck_mtx_init(&atm_descriptors_list_lock, &atm_dev_lock_grp, &atm_dev_lock_attr);
+ lck_mtx_init(&atm_values_list_lock, &atm_dev_lock_grp, &atm_dev_lock_attr);
+
+ queue_init(&atm_descriptors_list);
+ queue_init(&atm_values_list);
+#endif
+
+ /* Register the atm manager with the Vouchers sub system. */
+ kr = ipc_register_well_known_mach_voucher_attr_manager(
+ &atm_manager,
+ 0,
+ MACH_VOUCHER_ATTR_KEY_ATM,
+ &voucher_attr_control);
+ if (kr != KERN_SUCCESS )
+ panic("ATM subsystem initialization failed");
+
+ kprintf("ATM subsystem is initialized\n");
+ return ;
+}
+
+
+/*
+ * ATM Resource Manager Routines.
+ */
+
+
+/*
+ * Routine: atm_release_value
+ * Purpose: Release a value, if sync matches the sync count in value.
+ * Returns: KERN_SUCCESS: on Successful deletion.
+ * KERN_FAILURE: if sync value does not matches.
+ */
+kern_return_t
+atm_release_value(
+ ipc_voucher_attr_manager_t __assert_only manager,
+ mach_voucher_attr_key_t __assert_only key,
+ mach_voucher_attr_value_handle_t value,
+ mach_voucher_attr_value_reference_t sync)
+{
+ atm_value_t atm_value = ATM_VALUE_NULL;
+
+ assert(MACH_VOUCHER_ATTR_KEY_ATM == key);
+ assert(manager == &atm_manager);
+
+ atm_value = HANDLE_TO_ATM_VALUE(value);
+ if (atm_value == VAM_DEFAULT_VALUE) {
+ /* Return success for default value */
+ return KERN_SUCCESS;
+ }
+
+ if (atm_value->sync != sync) {
+ return KERN_FAILURE;
+ }
+
+ /* Deallocate the atm value. */
+ atm_value_hash_table_delete(atm_value);
+ atm_value_dealloc(atm_value);
+ return KERN_SUCCESS;
+}
+
+
+/*
+ * Routine: atm_get_value
+ */
+kern_return_t
+atm_get_value(
+ ipc_voucher_attr_manager_t __assert_only manager,
+ mach_voucher_attr_key_t __assert_only key,
+ mach_voucher_attr_recipe_command_t command,
+ mach_voucher_attr_value_handle_array_t prev_values,
+ mach_msg_type_number_t __assert_only prev_value_count,
+ mach_voucher_attr_content_t __unused recipe,
+ mach_voucher_attr_content_size_t __unused recipe_size,
+ mach_voucher_attr_value_handle_t *out_value,
+ ipc_voucher_t *out_value_voucher)
+{
+ atm_value_t atm_value = ATM_VALUE_NULL;
+ mach_voucher_attr_value_handle_t atm_handle;
+ atm_task_descriptor_t task_descriptor = ATM_TASK_DESCRIPTOR_NULL;
+ task_t task;
+ mailbox_offset_t mailbox_offset;
+ natural_t i;
+ kern_return_t kr = KERN_SUCCESS;
+
+ assert(MACH_VOUCHER_ATTR_KEY_ATM == key);
+ assert(manager == &atm_manager);
+
+ /* never an out voucher */
+ *out_value_voucher = IPC_VOUCHER_NULL;
+
+ if (disable_atm)
+ return KERN_NOT_SUPPORTED;
+
+ switch (command) {
+
+ case MACH_VOUCHER_ATTR_ATM_REGISTER:
+
+ for (i = 0; i < prev_value_count; i++) {
+ atm_handle = prev_values[i];
+ atm_value = HANDLE_TO_ATM_VALUE(atm_handle);
+
+ if (atm_value == VAM_DEFAULT_VALUE)
+ continue;
+
+ task = current_task();
+ task_descriptor = task->atm_context;
+ if (task_descriptor != ATM_TASK_DESCRIPTOR_NULL) {
+ if (recipe_size != sizeof(mailbox_offset_t)) {
+ kr = KERN_INVALID_ARGUMENT;
+ break;
+ }
+ memcpy(&mailbox_offset, recipe, sizeof(mailbox_offset_t));
+ if (mailbox_offset > task_descriptor->mailbox_array_size) {
+ kr = KERN_INVALID_ARGUMENT;
+ break;
+ }
+
+ kr = atm_listener_insert(atm_value, task_descriptor, mailbox_offset);
+ if (kr != KERN_SUCCESS) {
+ break;
+ }
+ }
+
+ /* Increment sync value. */
+ lck_mtx_lock(&atm_value->listener_lock);
+ atm_value->sync++;
+ lck_mtx_unlock(&atm_value->listener_lock);
+
+ *out_value = atm_handle;
+ return kr;
+ }
+
+ *out_value = ATM_VALUE_TO_HANDLE(VAM_DEFAULT_VALUE);
+ break;
+
+ case MACH_VOUCHER_ATTR_ATM_CREATE:
+
+ /* Allocate a new atm value. */
+ atm_value = atm_value_alloc_init();
+ atm_value_hash_table_insert(atm_value);
+
+ if (atm_value == ATM_VALUE_NULL) {
+ return KERN_RESOURCE_SHORTAGE;
+ }
+
+ *out_value = ATM_VALUE_TO_HANDLE(atm_value);
+ break;
+
+ case MACH_VOUCHER_ATTR_ATM_NULL:
+ default:
+ kr = KERN_INVALID_ARGUMENT;
+ break;
+ }
+
+ return kr;
+}
+
+
+/*
+ * Routine: atm_extract_content
+ * Purpose: Extract a set of aid from an array of voucher values.
+ * Returns: KERN_SUCCESS: on Success.
+ * KERN_FAILURE: one of the value is not present in the hash.
+ * KERN_NO_SPACE: insufficeint buffer provided to fill an array of aid.
+ */
+kern_return_t
+atm_extract_content(
+ ipc_voucher_attr_manager_t __assert_only manager,
+ mach_voucher_attr_key_t __assert_only key,
+ mach_voucher_attr_value_handle_array_t values,
+ mach_msg_type_number_t value_count,
+ mach_voucher_attr_recipe_command_t *out_command,
+ mach_voucher_attr_content_t out_recipe,
+ mach_voucher_attr_content_size_t *in_out_recipe_size)
+{
+ atm_value_t atm_value;
+ mach_voucher_attr_value_handle_t atm_handle;
+ natural_t i;
+
+ assert(MACH_VOUCHER_ATTR_KEY_ATM == key);
+ assert(manager == &atm_manager);
+
+ for (i = 0; i < value_count; i++) {
+ atm_handle = values[i];
+ atm_value = HANDLE_TO_ATM_VALUE(atm_handle);
+ if (atm_value == VAM_DEFAULT_VALUE)
+ continue;
+
+ if (( sizeof(aid_t)) > *in_out_recipe_size) {
+ *in_out_recipe_size = 0;
+ return KERN_NO_SPACE;
+ }
+
+ memcpy(&out_recipe[0], &atm_value->aid, sizeof(aid_t));
+ *out_command = MACH_VOUCHER_ATTR_ATM_NULL;
+ *in_out_recipe_size = sizeof(aid_t);
+ return KERN_SUCCESS;
+ }
+
+ *in_out_recipe_size = 0;
+ return KERN_SUCCESS;
+}
+
+/*
+ * Routine: atm_command
+ * Purpose: Execute a command against a set of ATM values.
+ * Returns: KERN_SUCCESS: On successful execution of command.
+ KERN_FAILURE: On failure.
+ */
+kern_return_t
+atm_command(
+ ipc_voucher_attr_manager_t __assert_only manager,
+ mach_voucher_attr_key_t __assert_only key,
+ mach_voucher_attr_value_handle_array_t values,
+ mach_msg_type_number_t value_count,
+ mach_voucher_attr_command_t command,
+ mach_voucher_attr_content_t in_content,
+ mach_voucher_attr_content_size_t in_content_size,
+ mach_voucher_attr_content_t out_content,
+ mach_voucher_attr_content_size_t *out_content_size)
+{
+ assert(MACH_VOUCHER_ATTR_KEY_ATM == key);
+ assert(manager == &atm_manager);
+ atm_value_t atm_value = ATM_VALUE_NULL;
+ natural_t i = 0;
+ aid_t *aid_array = NULL;
+ mach_atm_subaid_t *subaid_array = NULL;
+ uint32_t aid_array_count = 0;
+ atm_task_descriptor_t task_descriptor = ATM_TASK_DESCRIPTOR_NULL;
+ task_t task;
+ uint32_t collection_flags = ATM_ACTION_LOGFAIL;
+ kern_return_t kr = KERN_SUCCESS;
+
+ switch (command) {
+ case ATM_ACTION_COLLECT:
+ collection_flags = ATM_ACTION_COLLECT;
+ /* Fall through */
+
+ case ATM_ACTION_LOGFAIL: {
+ mach_atm_subaid_t sub_aid = 0;
+
+ /* find the first non-default atm_value */
+ for (i = 0; i < value_count; i++) {
+ atm_value = HANDLE_TO_ATM_VALUE(values[i]);
+ if (atm_value != VAM_DEFAULT_VALUE)
+ break;
+ }
+
+ /* if we are not able to find any atm values
+ * in stack then this call was made in error
+ */
+ if (atm_value == NULL) {
+ return KERN_FAILURE;
+ }
+ if (in_content == NULL || in_content_size < sizeof(mach_atm_subaid_t) ){
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ sub_aid = *(mach_atm_subaid_t *)(void *)in_content;
+ *out_content_size = 0;
+ kr = atm_invoke_collection(atm_value, sub_aid, collection_flags);
+ break;
+ }
+
+ case ATM_FIND_MIN_SUB_AID:
+ if ((in_content_size/sizeof(aid_t)) > (*out_content_size/sizeof(mach_atm_subaid_t)))
+ return KERN_FAILURE;
+
+ aid_array_count = in_content_size / sizeof(aid_t);
+ if (aid_array_count > AID_ARRAY_COUNT_MAX)
+ return KERN_FAILURE;
+
+ aid_array = (aid_t *) kalloc(aid_array_count * sizeof(aid_t));
+ if (aid_array == NULL)
+ return KERN_NO_SPACE;
+
+ subaid_array = (mach_atm_subaid_t *) kalloc(aid_array_count * sizeof(mach_atm_subaid_t));
+ if (subaid_array == NULL) {
+ kfree(aid_array, aid_array_count * sizeof(aid_t));
+ return KERN_NO_SPACE;
+ }
+
+ memcpy(aid_array, in_content, aid_array_count * sizeof(aid_t));
+ atm_get_min_sub_aid_array(aid_array, subaid_array, aid_array_count);
+
+ memcpy(out_content, subaid_array, aid_array_count * sizeof(mach_atm_subaid_t));
+ *out_content_size = aid_array_count * sizeof(mach_atm_subaid_t);
+
+ kfree(aid_array, aid_array_count * sizeof(aid_t));
+ kfree(subaid_array, aid_array_count * sizeof(mach_atm_subaid_t));
+ kr = KERN_SUCCESS;
+
+ break;
+
+ case ATM_ACTION_UNREGISTER:
+ /* find the first non-default atm_value */
+ for (i = 0; i < value_count; i++) {
+ atm_value = HANDLE_TO_ATM_VALUE(values[i]);
+ if (atm_value != VAM_DEFAULT_VALUE)
+ break;
+ }
+
+ /* if we are not able to find any atm values
+ * in stack then this call was made in error
+ */
+ if (atm_value == NULL) {
+ return KERN_FAILURE;
+ }
+ if (in_content == NULL || in_content_size != sizeof(mailbox_offset_t)){
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ mailbox_offset_t mailbox_offset;
+ memcpy(&mailbox_offset, in_content, sizeof(mailbox_offset_t));
+ task = current_task();
+ task_descriptor = task->atm_context;
+
+ kr = atm_value_unregister(atm_value, task_descriptor, mailbox_offset);
+
+ break;
+
+ default:
+ kr = KERN_INVALID_ARGUMENT;
+ break;
+ }
+
+ return kr;
+}
+
+
+void
+atm_release(
+ ipc_voucher_attr_manager_t __assert_only manager)
+{
+ assert(manager == &atm_manager);
+}
+
+
+/*
+ * Routine: atm_invoke_collection
+ * Purpose: Sends a notification with array of memory buffer.
+ * Note: may block till user daemon responds.
+ */
+kern_return_t
+atm_invoke_collection(
+ atm_value_t atm_value,
+ subaid_t sub_activity_id,
+ uint32_t flags)
+{
+ aid_t aid = atm_value->aid;
+ kern_return_t kr = KERN_SUCCESS;
+ uint32_t array_count = 0, i = 0, requestor_index = 0;
+ uint64_t *sizes_array = NULL;
+ atm_link_object_t link_object = NULL;
+ mach_port_t *mem_array = NULL;
+ boolean_t need_swap_first = FALSE;
+ atm_task_descriptor_t requesting_descriptor = current_task()->atm_context;
+
+ lck_mtx_lock(&atm_value->listener_lock);
+ array_count = atm_value->listener_count;
+ lck_mtx_unlock(&atm_value->listener_lock);
+
+ if (array_count == 0){
+ return KERN_SUCCESS;
+ }
+
+ mem_array = kalloc(sizeof(mach_port_t) * array_count);
+ if (mem_array == NULL){
+ return KERN_NO_SPACE;
+ }
+
+ sizes_array = kalloc(sizeof(uint64_t) * array_count);
+ if (sizes_array == NULL){
+ kfree(mem_array, sizeof(mach_port_t) * array_count);
+ return KERN_NO_SPACE;
+ }
+
+ lck_mtx_lock(&atm_value->listener_lock);
+ queue_iterate(&atm_value->listeners, link_object, atm_link_object_t, listeners_element) {
+ if (i >= array_count){
+ break;
+ }
+
+ if (!need_swap_first && requesting_descriptor == link_object->descriptor){
+ assert(requesting_descriptor != NULL);
+ requestor_index = i;
+ need_swap_first = TRUE;
+ }
+
+ sizes_array[i] = link_object->descriptor->trace_buffer_size;
+ mem_array[i] = ipc_port_copy_send(link_object->descriptor->trace_buffer);
+ if (!IPC_PORT_VALID(mem_array[i])){
+ mem_array[i] = NULL;
+ }
+ i++;
+ }
+ lck_mtx_unlock(&atm_value->listener_lock);
+
+ /*
+ * Swap the position of requesting task ahead, diagnostics can
+ * process its buffers the first.
+ */
+ if (need_swap_first && requestor_index != 0){
+ assert(requestor_index < array_count);
+ mach_port_t tmp_port = mem_array[0];
+ uint64_t tmp_size = sizes_array[0];
+ mem_array[0] = mem_array[requestor_index];
+ sizes_array[0] = sizes_array[requestor_index];
+ mem_array[requestor_index] = tmp_port;
+ sizes_array[requestor_index] = tmp_size;
+ }
+
+ if (i > 0) {
+ kr = atm_send_user_notification(aid, sub_activity_id, mem_array, sizes_array, i, flags);
+ }
+
+ kfree(mem_array, sizeof(mach_port_t) * array_count);
+ kfree(sizes_array, sizeof(uint64_t) * array_count);
+
+ return kr;
+}
+
+/*
+ * Routine: atm_send_user_notification
+ * Purpose: Make an upcall to user space daemon if its listening for atm notifications.
+ * Returns: KERN_SUCCESS for successful delivery.
+ * KERN_FAILURE if port is dead or NULL.
+ */
+kern_return_t
+atm_send_user_notification(
+ aid_t aid,
+ subaid_t subaid,
+ mach_port_t *buffers_array,
+ uint64_t *sizes_array,
+ mach_msg_type_number_t count,
+ uint32_t flags)
+{
+ mach_port_t user_port;
+ int error;
+ error = host_get_atm_notification_port(host_priv_self(), &user_port);
+ if ((error != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) {
+ return KERN_FAILURE;
+ }
+
+ return atm_collect_trace_info(user_port, aid, subaid, flags, buffers_array, count, sizes_array, count);
+}
+
+/*
+ * Routine: atm_send_proc_inspect_notification
+ * Purpose: Make an upcall to user space daemon if its listening for trace
+ * notifications for per process inspection.
+ * Returns: KERN_SUCCESS for successful delivery.
+ * KERN_FAILURE if port is dead or NULL.
+ */
+
+kern_return_t
+atm_send_proc_inspect_notification(
+ task_t task,
+ int32_t traced_pid,
+ uint64_t traced_uniqueid)
+{
+ mach_port_t user_port = MACH_PORT_NULL;
+ mach_port_t memory_port = MACH_PORT_NULL;
+ atm_task_descriptor_t task_descriptor = ATM_TASK_DESCRIPTOR_NULL;
+ uint64_t buffer_size = 0;
+ int error;
+
+ /* look for the requested memory in target task */
+ if (!task)
+ return KERN_INVALID_TASK;
+
+ task_lock(task);
+ if (task->atm_context){
+ task_descriptor = task->atm_context;
+ atm_descriptor_get_reference(task_descriptor);
+ }
+ task_unlock(task);
+
+ if (task_descriptor == ATM_TASK_DESCRIPTOR_NULL){
+ return KERN_FAILURE;
+ }
+
+ memory_port = ipc_port_copy_send(task_descriptor->trace_buffer);
+ buffer_size = task_descriptor->trace_buffer_size;
+ atm_task_descriptor_dealloc(task_descriptor);
+
+ /* get the communication port */
+ error = host_get_atm_notification_port(host_priv_self(), &user_port);
+ if ((error != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) {
+ ipc_port_release_send(memory_port);
+ return KERN_FAILURE;
+ }
+
+ return atm_inspect_process_buffer(user_port, traced_pid, traced_uniqueid, buffer_size, memory_port);
+}
+
+/*
+ * Routine: atm_value_alloc_init
+ * Purpose: Allocates an atm value struct and initialize it.
+ * Returns: atm_value_t: On Success with a sync count on atm_value.
+ * ATM_VALUE_NULL: On failure.
+ */
+static atm_value_t
+atm_value_alloc_init()
+{
+ atm_value_t new_atm_value = ATM_VALUE_NULL;
+
+ new_atm_value = (atm_value_t) zalloc(atm_value_zone);
+ if (new_atm_value == ATM_VALUE_NULL)
+ panic("Ran out of ATM values structure.\n\n");
+
+ new_atm_value->aid = get_aid();
+ queue_init(&new_atm_value->listeners);
+ new_atm_value->sync = 1;
+ new_atm_value->listener_count = 0;
+ new_atm_value->reference_count = 1;
+ lck_mtx_init(&new_atm_value->listener_lock, &atm_lock_grp, &atm_lock_attr);
+
+#if DEVELOPMENT || DEBUG
+ lck_mtx_lock(&atm_values_list_lock);
+ queue_enter(&atm_values_list, new_atm_value, atm_value_t, value_elt);
+ lck_mtx_unlock(&atm_values_list_lock);
+#endif
+ return new_atm_value;
+}
+
+
+/*
+ * Routine: get_aid
+ * Purpose: Increment the global aid counter and return it.
+ * Returns: aid
+ */
+static aid_t
+get_aid()
+{
+ aid_t aid;
+ aid = (aid_t)OSIncrementAtomic64((SInt64 *)&global_aid);
+ return aid;
+}
+
+
+/*
+ * Routine: atm_value_dealloc
+ * Purpose: Drops the reference on atm value and deallocates.
+ * Deletes all the listeners on deallocation.
+ * Returns: None.
+ */
+static void
+atm_value_dealloc(atm_value_t atm_value)
+{
+ lck_mtx_lock(&atm_value->listener_lock);
+
+ atm_value->reference_count--;
+ assert(atm_value->reference_count >= 0);
+
+ if (atm_value->reference_count > 0) {
+ lck_mtx_unlock(&atm_value->listener_lock);
+ return;
+ }
+
+ lck_mtx_unlock(&atm_value->listener_lock);
+
+ /* Free up the atm value and also remove all the listeners. */
+ atm_listener_delete_all(atm_value);
+
+ lck_mtx_destroy(&atm_value->listener_lock, &atm_lock_grp);
+
+#if DEVELOPMENT || DEBUG
+ lck_mtx_lock(&atm_values_list_lock);
+ queue_remove(&atm_values_list, atm_value, atm_value_t, value_elt);
+ lck_mtx_unlock(&atm_values_list_lock);
+#endif
+ zfree(atm_value_zone, atm_value);
+ return;
+}
+
+
+/*
+ * Routine: atm_hash_table_init
+ * Purpose: Initialize the atm aid hash table.
+ * Returns: None.
+ */
+static void
+atm_hash_table_init()
+{
+ int i;
+
+ for (i = 0; i < ATM_MAX_HASH_TABLE_SIZE; i++) {
+ queue_init(&atm_value_hash_table[i].hash_list);
+ lck_mtx_init(&atm_value_hash_table[i].hash_list_lock, &atm_lock_grp, &atm_lock_attr);
+ }
+}
+
+
+/*
+ * Routine: atm_value_hash_table_insert
+ * Purpose: Insert an atm value in the hash table.
+ * Returns: None.
+ */
+static void
+atm_value_hash_table_insert(atm_value_t new_atm_value)
+{
+ int hash_index;
+ atm_value_hash_t hash_list_head;
+ aid_t aid = new_atm_value->aid;
+
+ hash_index = AID_TO_HASH(aid);
+ hash_list_head = &atm_value_hash_table[hash_index];
+
+ lck_mtx_lock(&hash_list_head->hash_list_lock);
+ queue_enter(&hash_list_head->hash_list, new_atm_value, atm_value_t, vid_hash_elt);
+ lck_mtx_unlock(&hash_list_head->hash_list_lock);
+}
+
+
+/*
+ * Routine: atm_value_hash_table_delete
+ * Purpose: Delete the atm value from the hash table.
+ * Returns: None.
+ */
+static void
+atm_value_hash_table_delete(atm_value_t atm_value)
+{
+ int hash_index;
+ atm_value_hash_t hash_list_head;
+ aid_t aid = atm_value->aid;
+
+ hash_index = AID_TO_HASH(aid);
+ hash_list_head = &atm_value_hash_table[hash_index];
+
+ lck_mtx_lock(&hash_list_head->hash_list_lock);
+ queue_remove(&hash_list_head->hash_list, atm_value, atm_value_t, vid_hash_elt);
+ lck_mtx_unlock(&hash_list_head->hash_list_lock);
+}
+
+
+/*
+ * Routine: get_atm_value_from_aid
+ * Purpose: Search a given aid in atm value hash table and
+ * return the atm value stucture.
+ * Returns: atm value structure if aid found.
+ * ATM_VALUE_NULL: If aid not found in atm value hash table.
+ */
+static atm_value_t
+get_atm_value_from_aid(aid_t aid)
+{
+ int hash_index;
+ atm_value_hash_t hash_list_head;
+ atm_value_t next;
+
+ hash_index = AID_TO_HASH(aid);
+ hash_list_head = &atm_value_hash_table[hash_index];
+
+ /* Lock the atm list and search for the aid. */
+ lck_mtx_lock(&hash_list_head->hash_list_lock);
+
+ queue_iterate(&hash_list_head->hash_list, next, atm_value_t, vid_hash_elt) {
+ if (next->aid == aid) {
+ /*
+ * Aid found. Incerease ref count and return
+ * the atm value structure.
+ */
+ atm_value_get_ref(next);
+ lck_mtx_unlock(&hash_list_head->hash_list_lock);
+ return (next);
+ }
+ }
+ lck_mtx_unlock(&hash_list_head->hash_list_lock);
+ return ATM_VALUE_NULL;
+}
+
+
+/*
+ * Routine: atm_value_get_ref
+ * Purpose: Get a reference on atm value.
+ * Returns: None.
+ */
+static void
+atm_value_get_ref(atm_value_t atm_value)
+{
+ lck_mtx_lock(&atm_value->listener_lock);
+ atm_value->reference_count++;
+ lck_mtx_unlock(&atm_value->listener_lock);
+}
+
+
+/*
+ * Routine: atm_listener_insert
+ * Purpose: Insert a listener to an atm value.
+ * Returns: KERN_SUCCESS on success.
+ * KERN_FAILURE if the task is already present as a listener.
+ */
+static kern_return_t
+atm_listener_insert(
+ atm_value_t atm_value,
+ atm_task_descriptor_t task_descriptor,
+ mailbox_offset_t mailbox_offset)
+{
+ atm_link_object_t new_link_object;
+ atm_link_object_t next;
+ void *mailbox = (void *)((char *)task_descriptor->mailbox_kernel_addr + mailbox_offset);
+
+ new_link_object = (atm_link_object_t) zalloc(atm_link_objects_zone);
+ new_link_object->descriptor = task_descriptor;
+ new_link_object->reference_count = 1;
+ new_link_object->flags = 0;
+ new_link_object->mailbox = mailbox;
+
+ /* Get a reference on the task descriptor */
+ atm_descriptor_get_reference(task_descriptor);
+
+ /* Check if the task mailbox is already on the listener list */
+ lck_mtx_lock(&atm_value->listener_lock);
+ queue_iterate(&atm_value->listeners, next, atm_link_object_t, listeners_element) {
+ if (next->descriptor == task_descriptor) {
+ /*
+ * Replace the mailbox with the new one, the old mailbox is anyways on unregister path.
+ * There is a race when get_min_sub_aid would cache the mailbox, and this function will
+ * replace it. It would just behave as if the get value call happened after get_min_sub_aid
+ * was already completed.
+ */
+ next->mailbox = mailbox;
+ lck_mtx_unlock(&atm_value->listener_lock);
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_GETVALUE_INFO, (ATM_VALUE_REPLACED))) | DBG_FUNC_NONE,
+ atm_value, atm_value->aid, mailbox_offset, 0, 0);
+
+ /* Drop the extra reference on task descriptor taken by this function. */
+ atm_task_descriptor_dealloc(task_descriptor);
+ zfree(atm_link_objects_zone, new_link_object);
+ return KERN_SUCCESS;
+ }
+ }
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_GETVALUE_INFO, (ATM_VALUE_ADDED))) | DBG_FUNC_NONE,
+ atm_value, atm_value->aid, mailbox_offset, 0, 0);
+
+ queue_enter(&atm_value->listeners, new_link_object, atm_link_object_t, listeners_element);
+ atm_value->listener_count++;
+ lck_mtx_unlock(&atm_value->listener_lock);
+ return KERN_SUCCESS;
+}
+
+
+/*
+ * Routine: atm_listener_delete_all
+ * Purpose: Deletes all the listeners for an atm value.
+ * Returns: None.
+ */
+static void
+atm_listener_delete_all(atm_value_t atm_value)
+{
+ atm_link_object_t next;
+
+ while(!queue_empty(&atm_value->listeners)) {
+ queue_remove_first(&atm_value->listeners, next, atm_link_object_t, listeners_element);
+
+ /* Drops the reference on the link object */
+ atm_link_dealloc(next);
+ }
+}
+
+
+/*
+ * Routine: atm_listener_delete
+ * Purpose: Deletes a listerner for an atm value.
+ * Returns: KERN_SUCCESS on successful unregister.
+ * KERN_INVALID_VALUE on finding a different mailbox.
+ * KERN_FAILURE on failure.
+ */
+static kern_return_t
+atm_listener_delete(
+ atm_value_t atm_value,
+ atm_task_descriptor_t task_descriptor,
+ mailbox_offset_t mailbox_offset)
+{
+ queue_head_t free_listeners;
+ atm_link_object_t next, elem;
+ void *mailbox = (void *)((char *)task_descriptor->mailbox_kernel_addr + mailbox_offset);
+ kern_return_t kr = KERN_FAILURE;
+
+ queue_init(&free_listeners);
+
+ lck_mtx_lock(&atm_value->listener_lock);
+
+ next = (atm_link_object_t)(void *) queue_first(&atm_value->listeners);
+ while (!queue_end(&atm_value->listeners, (queue_entry_t)next)) {
+ elem = next;
+ next = (atm_link_object_t)(void *) queue_next(&next->listeners_element);
+
+ if (elem->descriptor == task_descriptor) {
+ if (elem->mailbox == mailbox) {
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_UNREGISTER_INFO,
+ (ATM_VALUE_UNREGISTERED))) | DBG_FUNC_NONE,
+ atm_value, atm_value->aid, mailbox_offset, 0, 0);
+ queue_remove(&atm_value->listeners, elem, atm_link_object_t, listeners_element);
+ queue_enter(&free_listeners, elem, atm_link_object_t, listeners_element);
+ atm_value->listener_count--;
+ kr = KERN_SUCCESS;
+ break;
+ } else {
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_UNREGISTER_INFO,
+ (ATM_VALUE_DIFF_MAILBOX))) | DBG_FUNC_NONE,
+ atm_value, atm_value->aid, 0, 0, 0);
+ kr = KERN_INVALID_VALUE;
+ break;
+ }
+ }
+ }
+ lck_mtx_unlock(&atm_value->listener_lock);
+
+ while(!queue_empty(&free_listeners)) {
+ queue_remove_first(&free_listeners, next, atm_link_object_t, listeners_element);
+
+ /* Drops the reference on the link object */
+ atm_link_dealloc(next);
+ }
+ return kr;
+}
+
+
+/*
+ * Routine: atm_descriptor_alloc_init
+ * Purpose: Allocate an atm task descriptor and initialize it and takes a reference.
+ * Returns: atm task descriptor: On success.
+ * NULL: on error.
+ */
+static atm_task_descriptor_t
+atm_task_descriptor_alloc_init(
+ mach_port_t trace_buffer,
+ uint64_t buffer_size,
+ void * mailbox_addr,
+ uint64_t mailbox_array_size,
+ task_t __assert_only task)
+{
+ atm_task_descriptor_t new_task_descriptor;
+
+ new_task_descriptor = (atm_task_descriptor_t) zalloc(atm_descriptors_zone);
+
+ new_task_descriptor->trace_buffer = trace_buffer;
+ new_task_descriptor->trace_buffer_size = buffer_size;
+ new_task_descriptor->mailbox_array_size = mailbox_array_size;
+ new_task_descriptor->mailbox_kernel_addr = mailbox_addr;
+ new_task_descriptor->reference_count = 1;
+ new_task_descriptor->flags = 0;
+ lck_mtx_init(&new_task_descriptor->lock, &atm_lock_grp, &atm_lock_attr);
+
+#if DEVELOPMENT || DEBUG
+ new_task_descriptor->task = task;
+ lck_mtx_lock(&atm_descriptors_list_lock);
+ queue_enter(&atm_descriptors_list, new_task_descriptor, atm_task_descriptor_t, descriptor_elt);
+ lck_mtx_unlock(&atm_descriptors_list_lock);
+#endif
+
+ return new_task_descriptor;
+}
+
+
+/*
+ * Routine: atm_descriptor_get_reference
+ * Purpose: Get a reference count on task descriptor.
+ * Returns: None.
+ */
+static void
+atm_descriptor_get_reference(atm_task_descriptor_t task_descriptor)
+{
+ lck_mtx_lock(&task_descriptor->lock);
+ task_descriptor->reference_count++;
+ lck_mtx_unlock(&task_descriptor->lock);
+}
+
+
+/*
+ * Routine: atm_task_descriptor_dealloc
+ * Prupose: Drops the reference on atm descriptor.
+ * Returns: None.
+ */
+static void
+atm_task_descriptor_dealloc(atm_task_descriptor_t task_descriptor)
+{
+ lck_mtx_lock(&task_descriptor->lock);
+ task_descriptor->reference_count--;
+ assert(task_descriptor->reference_count >= 0);
+ if (task_descriptor->reference_count > 0) {
+ lck_mtx_unlock(&task_descriptor->lock);
+ return;
+ }
+
+#if DEVELOPMENT || DEBUG
+ lck_mtx_lock(&atm_descriptors_list_lock);
+ queue_remove(&atm_descriptors_list, task_descriptor, atm_task_descriptor_t, descriptor_elt);
+ lck_mtx_unlock(&atm_descriptors_list_lock);
+#endif
+ mach_vm_deallocate(kernel_map, (mach_vm_address_t)task_descriptor->mailbox_kernel_addr,
+ task_descriptor->mailbox_array_size);
+ task_descriptor->mailbox_kernel_addr = NULL;
+ task_descriptor->mailbox_array_size = 0;
+ /* release the send right for the named memory entry */
+ ipc_port_release_send(task_descriptor->trace_buffer);
+ lck_mtx_unlock(&task_descriptor->lock);
+ lck_mtx_destroy(&task_descriptor->lock, &atm_lock_grp);
+ zfree(atm_descriptors_zone, task_descriptor);
+ return;
+}
+
+
+/*
+ * Routine: atm_link_get_reference
+ * Purpose: Get a reference count on atm link object.
+ * Returns: None.
+ */
+static void
+atm_link_get_reference(atm_link_object_t link_object)
+{
+ atm_link_object_reference_internal(link_object);
+}
+
+
+/*
+ * Routine: atm_link_dealloc
+ * Prupose: Drops the reference on link object.
+ * Returns: None.
+ */
+static void
+atm_link_dealloc(atm_link_object_t link_object)
+{
+ if (0 < atm_link_object_release_internal(link_object)) {
+ return;
+ }
+
+ assert(link_object->reference_count == 0);
+
+ /* Drop the reference on atm task descriptor. */
+ atm_task_descriptor_dealloc(link_object->descriptor);
+ zfree(atm_link_objects_zone, link_object);
+}
+
+
+/*
+ * Routine: atm_register_trace_memory
+ * Purpose: Registers trace memory for a task.
+ * Returns: KERN_SUCCESS: on Success.
+ * KERN_FAILURE: on Error.
+ */
+kern_return_t
+atm_register_trace_memory(
+ task_t task,
+ uint64_t trace_buffer_address,
+ uint64_t buffer_size,
+ uint64_t mailbox_array_size)
+{
+ atm_task_descriptor_t task_descriptor;
+ mach_port_t trace_buffer = MACH_PORT_NULL;
+ mach_vm_offset_t mailbox_kernel_ptr = 0;
+ kern_return_t kr = KERN_SUCCESS;
+
+ if (disable_atm)
+ return KERN_NOT_SUPPORTED;
+
+ if (task != current_task())
+ return KERN_INVALID_ARGUMENT;
+
+ if (task->atm_context != NULL
+ || (void *)trace_buffer_address == NULL
+ || buffer_size == 0
+ || (buffer_size & PAGE_MASK) != 0
+ || buffer_size > MAX_TRACE_BUFFER_SIZE
+ || mailbox_array_size == 0
+ || mailbox_array_size >= buffer_size
+ || mailbox_array_size > MAX_MAILBOX_SIZE
+ || mailbox_array_size & PAGE_MIN_MASK) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ vm_map_t map = current_map();
+ memory_object_size_t mo_size = (memory_object_size_t) buffer_size;
+ kr = mach_make_memory_entry_64(map,
+ &mo_size,
+ (mach_vm_offset_t)trace_buffer_address,
+ VM_PROT_READ,
+ &trace_buffer,
+ NULL);
+ if (kr != KERN_SUCCESS)
+ return kr;
+
+ kr = mach_vm_map(kernel_map,
+ &mailbox_kernel_ptr,
+ mailbox_array_size,
+ 0,
+ VM_FLAGS_ANYWHERE,
+ trace_buffer,
+ 0,
+ FALSE,
+ VM_PROT_READ,
+ VM_PROT_READ,
+ VM_INHERIT_NONE
+ );
+
+ if (kr != KERN_SUCCESS){
+ ipc_port_release_send(trace_buffer);
+ return kr;
+ }
+
+ task_descriptor = atm_task_descriptor_alloc_init(trace_buffer, buffer_size, (void *)mailbox_kernel_ptr, mailbox_array_size, task);
+ if (task_descriptor == ATM_TASK_DESCRIPTOR_NULL) {
+ ipc_port_release_send(trace_buffer);
+ mach_vm_deallocate(kernel_map, (mach_vm_address_t)mailbox_kernel_ptr, mailbox_array_size);
+ return KERN_NO_SPACE;
+ }
+
+ task_lock(task);
+ if (task->atm_context == NULL) {
+ task->atm_context = task_descriptor;
+ kr = KERN_SUCCESS;
+ } else {
+ kr = KERN_FAILURE;
+ }
+ task_unlock(task);
+
+ if (kr != KERN_SUCCESS) {
+ /* undo the mapping and allocations since we failed to hook descriptor to task */
+ atm_task_descriptor_dealloc(task_descriptor);
+ }
+ return KERN_SUCCESS;
+}
+
+
+/*
+ * Routine: atm_get_min_sub_aid_array
+ * Purpose: For an array of aid, lookup the atm value and fill the minimum subaid.
+ * Returns: None.
+ */
+static void
+atm_get_min_sub_aid_array(
+ aid_t *aid_array,
+ mach_atm_subaid_t *subaid_array,
+ uint32_t count)
+{
+ atm_value_t atm_value;
+ uint32_t i;
+
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_SUBAID_INFO, (ATM_MIN_CALLED))) | DBG_FUNC_START,
+ 0, 0, 0, 0, 0);
+
+ for (i = 0; i < count; i++) {
+ atm_value = get_atm_value_from_aid(aid_array[i]);
+ if (atm_value == ATM_VALUE_NULL) {
+ subaid_array[i] = ATM_SUBAID32_MAX;
+ continue;
+ }
+ subaid_array[i] = atm_get_min_sub_aid(atm_value);
+ atm_value_dealloc(atm_value);
+ }
+
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_SUBAID_INFO, (ATM_MIN_CALLED))) | DBG_FUNC_END,
+ count, 0, 0, 0, 0);
+
+}
+
+
+/*
+ * Routine: atm_get_min_sub_aid
+ * Purpose: Walk the list of listeners and get the min sub-aid for an activity id.
+ * Returns: Minimum sub-aid to keep.
+ * Note: Unlock the listener lock before accessing the mailbox, since it may page fault and
+ * might take long time. Also cleans the listeners list for the tasks which are dead
+ * and atm_task_descriptors do not hold any useful data.
+ */
+static mach_atm_subaid_t
+atm_get_min_sub_aid(atm_value_t atm_value)
+{
+ int32_t i = 0, j, freed_count = 0, dead_but_not_freed = 0;
+ int32_t listener_count;
+ atm_subaid32_t min_subaid = ATM_SUBAID32_MAX, subaid, max_subaid;
+ atm_link_object_t *link_object_array = NULL;
+ atm_link_object_t next, elem;
+ queue_head_t free_listeners;
+
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_SUBAID_INFO, (ATM_MIN_LINK_LIST))) | DBG_FUNC_START,
+ 0, 0, 0, 0, 0);
+
+ lck_mtx_lock(&atm_value->listener_lock);
+ listener_count = atm_value->listener_count;
+ lck_mtx_unlock(&atm_value->listener_lock);
+
+ /* separate memory access from locked iterate since memory read may fault */
+ link_object_array = (atm_link_object_t *) kalloc(sizeof(atm_link_object_t) * listener_count);
+ if (link_object_array == NULL) {
+ return 0;
+ }
+
+ /* Iterate the list and take a ref on link objects and store it in an array */
+ lck_mtx_lock(&atm_value->listener_lock);
+ queue_iterate(&atm_value->listeners, next, atm_link_object_t, listeners_element) {
+ /* Additional listener are added between the allocation of array and iterating the list */
+ if (i >= listener_count)
+ break;
+
+ /* Get a ref on the link object */
+ atm_link_get_reference(next);
+ link_object_array[i] = (atm_link_object_t)next;
+ i++;
+ }
+ lck_mtx_unlock(&atm_value->listener_lock);
+ j = i;
+
+ /* Iterate the array to find the min */
+ for (i = 0; i < j; i++) {
+ /* Ignore the min value of the dead processes. */
+ if (link_object_array[i]->descriptor->flags == ATM_TASK_DEAD)
+ continue;
+ /* Dereference the mailbox to get the min subaid */
+ subaid = *((atm_subaid32_t *)link_object_array[i]->mailbox);
+ if (subaid < min_subaid)
+ min_subaid = subaid;
+ }
+
+ /*
+ * Mark the link object that can be freed, and release the ref on the link object
+ * Mark the link object of dead task free after the dead task descriptor count
+ * increases than ATM_LIST_DEAD_MAX.
+ */
+ for (i = j - 1; i >= 0; i--) {
+ if (link_object_array[i]->descriptor->flags == ATM_TASK_DEAD) {
+ if (dead_but_not_freed > ATM_LIST_DEAD_MAX) {
+ link_object_array[i]->flags = ATM_LINK_REMOVE;
+ freed_count++;
+ } else {
+ max_subaid = *(((atm_subaid32_t *)link_object_array[i]->mailbox) + 1);
+ if (max_subaid < min_subaid) {
+ link_object_array[i]->flags = ATM_LINK_REMOVE;
+ freed_count++;
+ } else {
+ dead_but_not_freed++;
+ }
+ }
+ }
+ atm_link_dealloc(link_object_array[i]);
+ link_object_array[i] = NULL;
+ }
+
+ /* Check if the number of live entries in list is less than maxproc */
+ assert((j - (freed_count + dead_but_not_freed)) <= maxproc);
+
+ kfree(link_object_array, (sizeof(atm_link_object_t) * listener_count));
+
+ /* Remove the marked link objects from the list */
+ lck_mtx_lock(&atm_value->listener_lock);
+
+ queue_init(&free_listeners);
+ next = (atm_link_object_t)(void *) queue_first(&atm_value->listeners);
+ while (!queue_end(&atm_value->listeners, (queue_entry_t)next)) {
+ elem = next;
+ next = (atm_link_object_t)(void *) queue_next(&next->listeners_element);
+
+ if (elem->flags == ATM_LINK_REMOVE) {
+ queue_remove(&atm_value->listeners, elem, atm_link_object_t, listeners_element);
+ queue_enter(&free_listeners, elem, atm_link_object_t, listeners_element);
+ atm_value->listener_count--;
+ }
+ }
+ lck_mtx_unlock(&atm_value->listener_lock);
+
+ /* Free the link objects */
+ while(!queue_empty(&free_listeners)) {
+ queue_remove_first(&free_listeners, next, atm_link_object_t, listeners_element);
+
+ /* Drops the reference on the link object */
+ atm_link_dealloc(next);
+ }
+
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_SUBAID_INFO, (ATM_MIN_LINK_LIST))) | DBG_FUNC_END,
+ j, freed_count, dead_but_not_freed, 0, 0);
+
+ /* explicitly upgrade uint32_t to 64 bit mach size */
+ return CAST_DOWN(mach_atm_subaid_t, min_subaid);
+}
+
+
+/*
+ * Routine: atm_value_unregister
+ * Purpose: Unregisters a process from an activity id.
+ * Returns: KERN_SUCCESS on successful unregister.
+ * KERN_INVALID_VALUE on finding a diff mailbox.
+ * KERN_FAILURE on failure.
+ */
+static kern_return_t
+atm_value_unregister(
+ atm_value_t atm_value,
+ atm_task_descriptor_t task_descriptor,
+ mailbox_offset_t mailbox_offset)
+{
+ kern_return_t kr;
+
+ if (task_descriptor == ATM_TASK_DESCRIPTOR_NULL)
+ return KERN_INVALID_ARGUMENT;
+ if (mailbox_offset > task_descriptor->mailbox_array_size)
+ return KERN_INVALID_ARGUMENT;
+
+ kr = atm_listener_delete(atm_value, task_descriptor, mailbox_offset);
+ return kr;
+}
+
+void
+atm_task_descriptor_destroy(atm_task_descriptor_t task_descriptor)
+{
+ /* Mark the task dead in the task descriptor to make task descriptor eligible for cleanup. */
+ lck_mtx_lock(&task_descriptor->lock);
+ task_descriptor->flags = ATM_TASK_DEAD;
+ lck_mtx_unlock(&task_descriptor->lock);
+
+ atm_task_descriptor_dealloc(task_descriptor);
+}
projects / apple / xnu.git / blobdiff