X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/fe8ab488e9161c46dd9885d58fc52996dc0249ff..e8c3f78193f1895ea514044358b93b1add9322f3:/osfmk/ipc/ipc_kmsg.c?ds=sidebyside diff --git a/osfmk/ipc/ipc_kmsg.c b/osfmk/ipc/ipc_kmsg.c index 1005a37d9..7955cf45d 100644 --- a/osfmk/ipc/ipc_kmsg.c +++ b/osfmk/ipc/ipc_kmsg.c @@ -89,10 +89,12 @@ #include #include #include -#include #include #include #include +#include + +#include #include @@ -112,9 +114,17 @@ #include #include #include +#if MACH_FLIPC +#include +#include +#endif + +#include #include +#include + #include #ifdef ppc @@ -397,7 +407,7 @@ mm_copy_options_string64( name = "VIRTUAL"; break; case MACH_MSG_OVERWRITE: - name = "OVERWRITE"; + name = "OVERWRITE(DEPRECATED)"; break; case MACH_MSG_ALLOCATE: name = "ALLOCATE"; @@ -538,6 +548,286 @@ MACRO_BEGIN \ } \ MACRO_END +#define KMSG_TRACE_FLAG_TRACED 0x000001 +#define KMSG_TRACE_FLAG_COMPLEX 0x000002 +#define KMSG_TRACE_FLAG_OOLMEM 0x000004 +#define KMSG_TRACE_FLAG_VCPY 0x000008 +#define KMSG_TRACE_FLAG_PCPY 0x000010 +#define KMSG_TRACE_FLAG_SND64 0x000020 +#define KMSG_TRACE_FLAG_RAISEIMP 0x000040 +#define KMSG_TRACE_FLAG_APP_SRC 0x000080 +#define KMSG_TRACE_FLAG_APP_DST 0x000100 +#define KMSG_TRACE_FLAG_DAEMON_SRC 0x000200 +#define KMSG_TRACE_FLAG_DAEMON_DST 0x000400 +#define KMSG_TRACE_FLAG_DST_NDFLTQ 0x000800 +#define KMSG_TRACE_FLAG_SRC_NDFLTQ 0x001000 +#define KMSG_TRACE_FLAG_DST_SONCE 0x002000 +#define KMSG_TRACE_FLAG_SRC_SONCE 0x004000 +#define KMSG_TRACE_FLAG_CHECKIN 0x008000 +#define KMSG_TRACE_FLAG_ONEWAY 0x010000 +#define KMSG_TRACE_FLAG_IOKIT 0x020000 +#define KMSG_TRACE_FLAG_SNDRCV 0x040000 +#define KMSG_TRACE_FLAG_DSTQFULL 0x080000 +#define KMSG_TRACE_FLAG_VOUCHER 0x100000 +#define KMSG_TRACE_FLAG_TIMER 0x200000 +#define KMSG_TRACE_FLAG_SEMA 0x400000 +#define KMSG_TRACE_FLAG_DTMPOWNER 0x800000 + +#define KMSG_TRACE_FLAGS_MASK 0xffffff +#define KMSG_TRACE_FLAGS_SHIFT 8 + +#define KMSG_TRACE_PORTS_MASK 0xff +#define KMSG_TRACE_PORTS_SHIFT 0 + +#if (KDEBUG_LEVEL >= KDEBUG_LEVEL_STANDARD) +#include +extern boolean_t kdebug_debugid_enabled(uint32_t debugid); + +void ipc_kmsg_trace_send(ipc_kmsg_t kmsg, + mach_msg_option_t option) +{ + task_t send_task = TASK_NULL; + ipc_port_t dst_port, src_port; + boolean_t is_task_64bit; + mach_msg_header_t *msg; + mach_msg_trailer_t *trailer; + + int kotype = 0; + uint32_t msg_size = 0; + uint32_t msg_flags = KMSG_TRACE_FLAG_TRACED; + uint32_t num_ports = 0; + uint32_t send_pid, dst_pid; + + /* + * check to see not only if ktracing is enabled, but if we will + * _actually_ emit the KMSG_INFO tracepoint. This saves us a + * significant amount of processing (and a port lock hold) in + * the non-tracing case. + */ + if (__probable((kdebug_enable & KDEBUG_TRACE) == 0)) + return; + if (!kdebug_debugid_enabled(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO))) + return; + + msg = kmsg->ikm_header; + + dst_port = (ipc_port_t)(msg->msgh_remote_port); + if (!IPC_PORT_VALID(dst_port)) + return; + + /* + * Message properties / options + */ + if ((option & (MACH_SEND_MSG|MACH_RCV_MSG)) == (MACH_SEND_MSG|MACH_RCV_MSG)) + msg_flags |= KMSG_TRACE_FLAG_SNDRCV; + + if (msg->msgh_id >= is_iokit_subsystem.start && + msg->msgh_id < is_iokit_subsystem.end + 100) + msg_flags |= KMSG_TRACE_FLAG_IOKIT; + /* magic XPC checkin message id (XPC_MESSAGE_ID_CHECKIN) from libxpc */ + else if (msg->msgh_id == 0x77303074u /* w00t */) + msg_flags |= KMSG_TRACE_FLAG_CHECKIN; + + if (msg->msgh_bits & MACH_MSGH_BITS_RAISEIMP) + msg_flags |= KMSG_TRACE_FLAG_RAISEIMP; + + if (unsafe_convert_port_to_voucher(kmsg->ikm_voucher)) + msg_flags |= KMSG_TRACE_FLAG_VOUCHER; + + /* + * Sending task / port + */ + send_task = current_task(); + send_pid = task_pid(send_task); + + if (send_pid != 0) { + if (task_is_daemon(send_task)) + msg_flags |= KMSG_TRACE_FLAG_DAEMON_SRC; + else if (task_is_app(send_task)) + msg_flags |= KMSG_TRACE_FLAG_APP_SRC; + } + + is_task_64bit = (send_task->map->max_offset > VM_MAX_ADDRESS); + if (is_task_64bit) + msg_flags |= KMSG_TRACE_FLAG_SND64; + + src_port = (ipc_port_t)(msg->msgh_local_port); + if (src_port) { + if (src_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT) + msg_flags |= KMSG_TRACE_FLAG_SRC_NDFLTQ; + switch (MACH_MSGH_BITS_LOCAL(msg->msgh_bits)) { + case MACH_MSG_TYPE_MOVE_SEND_ONCE: + msg_flags |= KMSG_TRACE_FLAG_SRC_SONCE; + break; + default: + break; + } + } else { + msg_flags |= KMSG_TRACE_FLAG_ONEWAY; + } + + + /* + * Destination task / port + */ + ip_lock(dst_port); + if (!ip_active(dst_port)) { + /* dst port is being torn down */ + dst_pid = (uint32_t)0xfffffff0; + } else if (dst_port->ip_tempowner) { + msg_flags |= KMSG_TRACE_FLAG_DTMPOWNER; + if (IIT_NULL != dst_port->ip_imp_task) + dst_pid = task_pid(dst_port->ip_imp_task->iit_task); + else + dst_pid = (uint32_t)0xfffffff1; + } else if (dst_port->ip_receiver_name == MACH_PORT_NULL) { + /* dst_port is otherwise in-transit */ + dst_pid = (uint32_t)0xfffffff2; + } else { + if (dst_port->ip_receiver == ipc_space_kernel) { + dst_pid = 0; + } else { + ipc_space_t dst_space; + dst_space = dst_port->ip_receiver; + if (dst_space && is_active(dst_space)) { + dst_pid = task_pid(dst_space->is_task); + if (task_is_daemon(dst_space->is_task)) + msg_flags |= KMSG_TRACE_FLAG_DAEMON_DST; + else if (task_is_app(dst_space->is_task)) + msg_flags |= KMSG_TRACE_FLAG_APP_DST; + } else { + /* receiving task is being torn down */ + dst_pid = (uint32_t)0xfffffff3; + } + } + } + + if (dst_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT) + msg_flags |= KMSG_TRACE_FLAG_DST_NDFLTQ; + if (imq_full(&dst_port->ip_messages)) + msg_flags |= KMSG_TRACE_FLAG_DSTQFULL; + + kotype = ip_kotype(dst_port); + + ip_unlock(dst_port); + + switch (kotype) { + case IKOT_SEMAPHORE: + msg_flags |= KMSG_TRACE_FLAG_SEMA; + break; + case IKOT_TIMER: + case IKOT_CLOCK: + msg_flags |= KMSG_TRACE_FLAG_TIMER; + break; + case IKOT_MASTER_DEVICE: + case IKOT_IOKIT_CONNECT: + case IKOT_IOKIT_OBJECT: + case IKOT_IOKIT_IDENT: + msg_flags |= KMSG_TRACE_FLAG_IOKIT; + break; + default: + break; + } + + switch(MACH_MSGH_BITS_REMOTE(msg->msgh_bits)) { + case MACH_MSG_TYPE_PORT_SEND_ONCE: + msg_flags |= KMSG_TRACE_FLAG_DST_SONCE; + break; + default: + break; + } + + + /* + * Message size / content + */ + msg_size = msg->msgh_size - sizeof(mach_msg_header_t); + + if (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) { + mach_msg_body_t *msg_body; + mach_msg_descriptor_t *kern_dsc; + int dsc_count; + + msg_flags |= KMSG_TRACE_FLAG_COMPLEX; + + msg_body = (mach_msg_body_t *)(kmsg->ikm_header + 1); + dsc_count = (int)msg_body->msgh_descriptor_count; + kern_dsc = (mach_msg_descriptor_t *)(msg_body + 1); + + /* this is gross: see ipc_kmsg_copyin_body()... */ + if (!is_task_64bit) + msg_size -= (dsc_count * 12); + + for (int i = 0; i < dsc_count; i++) { + switch (kern_dsc[i].type.type) { + case MACH_MSG_PORT_DESCRIPTOR: + num_ports++; + if (is_task_64bit) + msg_size -= 12; + break; + case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: + case MACH_MSG_OOL_DESCRIPTOR: { + mach_msg_ool_descriptor_t *dsc; + dsc = (mach_msg_ool_descriptor_t *)&kern_dsc[i]; + msg_flags |= KMSG_TRACE_FLAG_OOLMEM; + msg_size += dsc->size; + if ((dsc->size >= MSG_OOL_SIZE_SMALL) && + (dsc->copy == MACH_MSG_PHYSICAL_COPY) && + !dsc->deallocate) + msg_flags |= KMSG_TRACE_FLAG_PCPY; + else if (dsc->size <= MSG_OOL_SIZE_SMALL) + msg_flags |= KMSG_TRACE_FLAG_PCPY; + else + msg_flags |= KMSG_TRACE_FLAG_VCPY; + if (is_task_64bit) + msg_size -= 16; + } break; + case MACH_MSG_OOL_PORTS_DESCRIPTOR: { + mach_msg_ool_ports_descriptor_t *dsc; + dsc = (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i]; + num_ports += dsc->count; + if (is_task_64bit) + msg_size -= 16; + } break; + default: + break; + } + } + } + + /* + * Trailer contents + */ + trailer = (mach_msg_trailer_t *)((vm_offset_t)msg + + round_msg((vm_offset_t)msg->msgh_size)); + if (trailer->msgh_trailer_size <= sizeof(mach_msg_security_trailer_t)) { + extern security_token_t KERNEL_SECURITY_TOKEN; + mach_msg_security_trailer_t *strailer; + strailer = (mach_msg_security_trailer_t *)trailer; + /* + * verify the sender PID: replies from the kernel often look + * like self-talk because the sending port is not reset. + */ + if (memcmp(&strailer->msgh_sender, + &KERNEL_SECURITY_TOKEN, + sizeof(KERNEL_SECURITY_TOKEN)) == 0) { + send_pid = 0; + msg_flags &= ~(KMSG_TRACE_FLAG_APP_SRC | KMSG_TRACE_FLAG_DAEMON_SRC); + } + } + + KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, + (uintptr_t)send_pid, + (uintptr_t)dst_pid, + (uintptr_t)msg_size, + (uintptr_t)( + ((msg_flags & KMSG_TRACE_FLAGS_MASK) << KMSG_TRACE_FLAGS_SHIFT) | + ((num_ports & KMSG_TRACE_PORTS_MASK) << KMSG_TRACE_PORTS_SHIFT) + ) + ); +} +#endif + /* zone for cached ipc_kmsg_t structures */ zone_t ipc_kmsg_zone; @@ -561,9 +851,10 @@ void ipc_kmsg_clean_partial( vm_size_t length); mach_msg_return_t ipc_kmsg_copyin_body( - ipc_kmsg_t kmsg, - ipc_space_t space, - vm_map_t map); + ipc_kmsg_t kmsg, + ipc_space_t space, + vm_map_t map, + mach_msg_option_t *optionp); /* * We keep a per-processor cache of kernel message buffers. @@ -624,21 +915,6 @@ ipc_kmsg_alloc( max_expanded_size = IKM_SAVED_MSG_SIZE; /* round up for ikm_cache */ if (max_expanded_size == IKM_SAVED_MSG_SIZE) { - struct ikm_cache *cache; - unsigned int i; - - disable_preemption(); - cache = &PROCESSOR_DATA(current_processor(), ikm_cache); - if ((i = cache->avail) > 0) { - assert(i <= IKM_STASH); - kmsg = cache->entries[--i]; - cache->avail = i; - enable_preemption(); - ikm_check_init(kmsg, max_expanded_size); - ikm_set_header(kmsg, msg_and_trailer_size); - return (kmsg); - } - enable_preemption(); kmsg = (ipc_kmsg_t)zalloc(ipc_kmsg_zone); } else { kmsg = (ipc_kmsg_t)kalloc(ikm_plus_overhead(max_expanded_size)); @@ -696,22 +972,7 @@ ipc_kmsg_free( ip_release(port); /* May be last reference */ } - /* - * Peek and see if it has to go back in the cache. - */ if (kmsg->ikm_size == IKM_SAVED_MSG_SIZE) { - struct ikm_cache *cache; - unsigned int i; - - disable_preemption(); - cache = &PROCESSOR_DATA(current_processor(), ikm_cache); - if ((i = cache->avail) < IKM_STASH) { - cache->entries[i] = kmsg; - cache->avail = i + 1; - enable_preemption(); - return; - } - enable_preemption(); zfree(ipc_kmsg_zone, kmsg); return; } @@ -730,7 +991,100 @@ ipc_kmsg_enqueue( ipc_kmsg_queue_t queue, ipc_kmsg_t kmsg) { - ipc_kmsg_enqueue_macro(queue, kmsg); + ipc_kmsg_t first = queue->ikmq_base; + ipc_kmsg_t last; + + if (first == IKM_NULL) { + queue->ikmq_base = kmsg; + kmsg->ikm_next = kmsg; + kmsg->ikm_prev = kmsg; + } else { + last = first->ikm_prev; + kmsg->ikm_next = first; + kmsg->ikm_prev = last; + first->ikm_prev = kmsg; + last->ikm_next = kmsg; + } +} + +/* + * Routine: ipc_kmsg_enqueue_qos + * Purpose: + * Enqueue a kmsg, propagating qos + * overrides towards the head of the queue. + * + * Returns: + * whether the head of the queue had + * it's override-qos adjusted because + * of this insertion. + */ + +boolean_t +ipc_kmsg_enqueue_qos( + ipc_kmsg_queue_t queue, + ipc_kmsg_t kmsg) +{ + ipc_kmsg_t first = queue->ikmq_base; + ipc_kmsg_t prev; + mach_msg_priority_t override; + + if (first == IKM_NULL) { + /* insert a first message */ + queue->ikmq_base = kmsg; + kmsg->ikm_next = kmsg; + kmsg->ikm_prev = kmsg; + return TRUE; + } + + /* insert at the tail */ + prev = first->ikm_prev; + kmsg->ikm_next = first; + kmsg->ikm_prev = prev; + first->ikm_prev = kmsg; + prev->ikm_next = kmsg; + + /* apply QoS overrides towards the head */ + override = kmsg->ikm_qos_override; + while (prev != kmsg && + override > prev->ikm_qos_override) { + prev->ikm_qos_override = override; + prev = prev->ikm_prev; + } + + /* did we adjust everything? */ + return (prev == kmsg); +} + +/* + * Routine: ipc_kmsg_override_qos + * Purpose: + * Update the override for a given kmsg already + * enqueued, propagating qos override adjustments + * towards the head of the queue. + * + * Returns: + * whether the head of the queue had + * it's override-qos adjusted because + * of this insertion. + */ + +boolean_t +ipc_kmsg_override_qos( + ipc_kmsg_queue_t queue, + ipc_kmsg_t kmsg, + mach_msg_priority_t override) +{ + ipc_kmsg_t first = queue->ikmq_base; + ipc_kmsg_t cur = kmsg; + + /* apply QoS overrides towards the head */ + while (override > cur->ikm_qos_override) { + cur->ikm_qos_override = override; + if (cur == first) + return TRUE; + cur = cur->ikm_prev; + } + return FALSE; } /* @@ -748,7 +1102,7 @@ ipc_kmsg_dequeue( first = ipc_kmsg_queue_first(queue); if (first != IKM_NULL) - ipc_kmsg_rmqueue_first_macro(queue, first); + ipc_kmsg_rmqueue(queue, first); return first; } @@ -777,6 +1131,12 @@ ipc_kmsg_rmqueue( queue->ikmq_base = IKM_NULL; } else { + if (__improbable(next->ikm_prev != kmsg || prev->ikm_next != kmsg)) { + panic("ipc_kmsg_rmqueue: inconsistent prev/next pointers. " + "(prev->next: %p, next->prev: %p, kmsg: %p)", + prev->ikm_next, next->ikm_prev, kmsg); + } + if (queue->ikmq_base == kmsg) queue->ikmq_base = next; @@ -1083,6 +1443,9 @@ ipc_kmsg_set_prealloc( assert(kmsg->ikm_prealloc == IP_NULL); kmsg->ikm_prealloc = IP_NULL; + + assert(port_send_turnstile(port) == TURNSTILE_NULL); + kmsg->ikm_turnstile = TURNSTILE_NULL; IP_SET_PREALLOC(port, kmsg); } @@ -1098,10 +1461,12 @@ ipc_kmsg_clear_prealloc( ipc_kmsg_t kmsg, ipc_port_t port) { - assert(kmsg->ikm_prealloc == port); - - kmsg->ikm_prealloc = IP_NULL; + /* take the mqueue lock since the turnstile is protected under it */ + imq_lock(&port->ip_messages); + IP_CLEAR_PREALLOC(port, kmsg); + set_port_send_turnstile(port, kmsg->ikm_turnstile); + imq_unlock(&port->ip_messages); } /* @@ -1166,6 +1531,14 @@ ipc_kmsg_get( if (copyinmsg(msg_addr, (char *)&legacy_base, len_copied)) return MACH_SEND_INVALID_DATA; + /* + * If the message claims to be complex, it must at least + * have the length of a "base" message (header + dsc_count). + */ + if (len_copied < sizeof(mach_msg_legacy_base_t) && + (legacy_base.header.msgh_bits & MACH_MSGH_BITS_COMPLEX)) + return MACH_SEND_MSG_TOO_SMALL; + msg_addr += sizeof(legacy_base.header); #if defined(__LP64__) size += LEGACY_HEADER_SIZE_DELTA; @@ -1325,6 +1698,8 @@ ipc_kmsg_get_from_kernel( (void) memcpy((void *) kmsg->ikm_header, (const void *) msg, size); + ikm_qos_init(kmsg); + kmsg->ikm_header->msgh_size = size; /* @@ -1364,8 +1739,6 @@ ipc_kmsg_get_from_kernel( * MACH_SEND_INTERRUPTED Caller still has message. * MACH_SEND_INVALID_DEST Caller still has message. */ - - mach_msg_return_t ipc_kmsg_send( ipc_kmsg_t kmsg, @@ -1373,23 +1746,36 @@ ipc_kmsg_send( mach_msg_timeout_t send_timeout) { ipc_port_t port; + thread_t th = current_thread(); mach_msg_return_t error = MACH_MSG_SUCCESS; - spl_t s; + boolean_t kernel_reply = FALSE; + + /* Check if honor qlimit flag is set on thread. */ + if ((th->options & TH_OPT_HONOR_QLIMIT) == TH_OPT_HONOR_QLIMIT) { + /* Remove the MACH_SEND_ALWAYS flag to honor queue limit. */ + option &= (~MACH_SEND_ALWAYS); + /* Add the timeout flag since the message queue might be full. */ + option |= MACH_SEND_TIMEOUT; + th->options &= (~TH_OPT_HONOR_QLIMIT); + } #if IMPORTANCE_INHERITANCE - boolean_t did_importance = FALSE; -#if IMPORTANCE_DEBUG + bool did_importance = false; +#if IMPORTANCE_TRACE mach_msg_id_t imp_msgh_id = -1; int sender_pid = -1; -#endif /* IMPORTANCE_DEBUG */ +#endif /* IMPORTANCE_TRACE */ #endif /* IMPORTANCE_INHERITANCE */ /* don't allow the creation of a circular loop */ if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_CIRCULAR) { ipc_kmsg_destroy(kmsg); + KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_MSGH_BITS_CIRCULAR); return MACH_MSG_SUCCESS; } + ipc_voucher_send_preprocessing(kmsg); + port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port; assert(IP_VALID(port)); ip_lock(port); @@ -1404,9 +1790,25 @@ retry: */ if (!ip_active(port)) { ip_unlock(port); +#if MACH_FLIPC + if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) + flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE); +#endif + if (did_importance) { + /* + * We're going to pretend we delivered this message + * successfully, and just eat the kmsg. However, the + * kmsg is actually visible via the importance_task! + * We need to cleanup this linkage before we destroy + * the message, and more importantly before we set the + * msgh_remote_port to NULL. See: 34302571 + */ + ipc_importance_clean(kmsg); + } ip_release(port); /* JMM - Future: release right, not just ref */ kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL; ipc_kmsg_destroy(kmsg); + KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST); return MACH_MSG_SUCCESS; } @@ -1427,14 +1829,19 @@ retry: /* * Call the server routine, and get the reply message to send. */ - kmsg = ipc_kobject_server(kmsg); + kmsg = ipc_kobject_server(kmsg, option); if (kmsg == IKM_NULL) return MACH_MSG_SUCCESS; + /* restart the KMSG_INFO tracing for the reply message */ + KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_START); port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port; assert(IP_VALID(port)); ip_lock(port); /* fall thru with reply - same options */ + kernel_reply = TRUE; + if (!ip_active(port)) + error = MACH_SEND_INVALID_DEST; } #if IMPORTANCE_INHERITANCE @@ -1443,27 +1850,33 @@ retry: * propagation. That routine can drop the port lock temporarily. * If it does we'll have to revalidate the destination. */ - if (did_importance == FALSE) { - did_importance = TRUE; + if (!did_importance) { + did_importance = true; if (ipc_importance_send(kmsg, option)) goto retry; } #endif /* IMPORTANCE_INHERITANCE */ - /* - * We have a valid message and a valid reference on the port. - * we can unlock the port and call mqueue_send() on its message - * queue. Lock message queue while port is locked. - */ - s = splsched(); - imq_lock(&port->ip_messages); - ip_unlock(port); + if (error != MACH_MSG_SUCCESS) { + ip_unlock(port); + } else { + /* + * We have a valid message and a valid reference on the port. + * we can unlock the port and call mqueue_send() on its message + * queue. Lock message queue while port is locked. + */ + imq_lock(&port->ip_messages); + + set_ip_srp_msg_sent(port); - error = ipc_mqueue_send(&port->ip_messages, kmsg, option, - send_timeout, s); + ip_unlock(port); + + error = ipc_mqueue_send(&port->ip_messages, kmsg, option, + send_timeout); + } #if IMPORTANCE_INHERITANCE - if (did_importance == TRUE) { + if (did_importance) { __unused int importance_cleared = 0; switch (error) { case MACH_SEND_TIMED_OUT: @@ -1493,10 +1906,10 @@ retry: default: break; } -#if IMPORTANCE_DEBUG +#if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_END, - audit_token_pid_from_task(current_task()), sender_pid, imp_msgh_id, importance_cleared, 0); -#endif /* IMPORTANCE_DEBUG */ + task_pid(current_task()), sender_pid, imp_msgh_id, importance_cleared, 0); +#endif /* IMPORTANCE_TRACE */ } #endif /* IMPORTANCE_INHERITANCE */ @@ -1505,9 +1918,31 @@ retry: * as a successful delivery (like we do for an inactive port). */ if (error == MACH_SEND_INVALID_DEST) { +#if MACH_FLIPC + if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) + flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE); +#endif ip_release(port); /* JMM - Future: release right, not just ref */ kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL; ipc_kmsg_destroy(kmsg); + KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST); + return MACH_MSG_SUCCESS; + } + + if (error != MACH_MSG_SUCCESS && kernel_reply) { + /* + * Kernel reply messages that fail can't be allowed to + * pseudo-receive on error conditions. We need to just treat + * the message as a successful delivery. + */ +#if MACH_FLIPC + if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) + flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE); +#endif + ip_release(port); /* JMM - Future: release right, not just ref */ + kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL; + ipc_kmsg_destroy(kmsg); + KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, error); return MACH_MSG_SUCCESS; } return error; @@ -1529,10 +1964,14 @@ retry: mach_msg_return_t ipc_kmsg_put( - mach_vm_address_t msg_addr, ipc_kmsg_t kmsg, - mach_msg_size_t size) + mach_msg_option_t option, + mach_vm_address_t rcv_addr, + mach_msg_size_t rcv_size, + mach_msg_size_t trailer_size, + mach_msg_size_t *sizep) { + mach_msg_size_t size = kmsg->ikm_header->msgh_size + trailer_size; mach_msg_return_t mr; DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_put()"); @@ -1553,7 +1992,7 @@ ipc_kmsg_put( kmsg->ikm_header->msgh_id); #if defined(__LP64__) - if (current_task() != kernel_task) { /* don't if receiver expects fully-cooked in-kernel msg; ux_exception */ + if (current_task() != kernel_task) { /* don't if receiver expects fully-cooked in-kernel msg; */ mach_msg_legacy_header_t *legacy_header = (mach_msg_legacy_header_t *)((vm_offset_t)(kmsg->ikm_header) + LEGACY_HEADER_SIZE_DELTA); @@ -1588,12 +2027,29 @@ ipc_kmsg_put( kprintf("type: %d\n", ((mach_msg_type_descriptor_t *)(((mach_msg_base_t *)kmsg->ikm_header)+1))->type); } __unreachable_ok_pop - if (copyoutmsg((const char *) kmsg->ikm_header, msg_addr, size)) + + /* Re-Compute target address if using stack-style delivery */ + if (option & MACH_RCV_STACK) { + rcv_addr += rcv_size - size; + } + + if (copyoutmsg((const char *) kmsg->ikm_header, rcv_addr, size)) { mr = MACH_RCV_INVALID_DATA; - else + size = 0; + } else mr = MACH_MSG_SUCCESS; + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_LINK) | DBG_FUNC_NONE, + (rcv_addr >= VM_MIN_KERNEL_AND_KEXT_ADDRESS || + rcv_addr + size >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) ? (uintptr_t)0 : (uintptr_t)rcv_addr, + VM_KERNEL_ADDRPERM((uintptr_t)kmsg), + 1 /* this is on the receive/copyout path */, + 0, + 0); ipc_kmsg_free(kmsg); + + if (sizep) + *sizep = size; return mr; } @@ -1618,6 +2074,61 @@ ipc_kmsg_put_to_kernel( ipc_kmsg_free(kmsg); } +static mach_msg_priority_t +ipc_get_current_thread_priority(void) +{ + thread_t thread = current_thread(); + thread_qos_t qos; + int relpri; + + qos = thread_get_requested_qos(thread, &relpri); + if (!qos) { + qos = thread_user_promotion_qos_for_pri(thread->base_pri); + relpri = 0; + } + return (mach_msg_priority_t)_pthread_priority_make_from_thread_qos(qos, relpri, 0); +} + +static kern_return_t +ipc_kmsg_set_qos( + ipc_kmsg_t kmsg, + mach_msg_option_t options, + mach_msg_priority_t override) +{ + kern_return_t kr; + ipc_port_t special_reply_port = kmsg->ikm_header->msgh_local_port; + ipc_port_t dest_port = kmsg->ikm_header->msgh_remote_port; + + kr = ipc_get_pthpriority_from_kmsg_voucher(kmsg, &kmsg->ikm_qos); + if (kr != KERN_SUCCESS) { + if (options & MACH_SEND_PROPAGATE_QOS) { + kmsg->ikm_qos = ipc_get_current_thread_priority(); + } else { + kmsg->ikm_qos = MACH_MSG_PRIORITY_UNSPECIFIED; + } + } + kmsg->ikm_qos_override = kmsg->ikm_qos; + + if (options & MACH_SEND_OVERRIDE) { + pthread_priority_t pp = _pthread_priority_normalize_for_ipc(override); + if (pp > kmsg->ikm_qos) + kmsg->ikm_qos_override = (mach_msg_priority_t)pp; + } + + kr = KERN_SUCCESS; + if ((options & MACH_SEND_SYNC_OVERRIDE)) { + if (IP_VALID(special_reply_port) && + MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits) == MACH_MSG_TYPE_PORT_SEND_ONCE) { + /* + * Link the destination port to special reply port and make sure that + * dest port has a send turnstile, else allocate one. + */ + ipc_port_link_special_reply_port(special_reply_port, dest_port); + } + } + return kr; +} + /* * Routine: ipc_kmsg_copyin_header * Purpose: @@ -1646,6 +2157,7 @@ mach_msg_return_t ipc_kmsg_copyin_header( ipc_kmsg_t kmsg, ipc_space_t space, + mach_msg_priority_t override, mach_msg_option_t *optionp) { mach_msg_header_t *msg = kmsg->ikm_header; @@ -1670,17 +2182,11 @@ ipc_kmsg_copyin_header( ipc_entry_t reply_entry = IE_NULL; ipc_entry_t voucher_entry = IE_NULL; -#if IMPORTANCE_INHERITANCE int assertcnt = 0; +#if IMPORTANCE_INHERITANCE boolean_t needboost = FALSE; #endif /* IMPORTANCE_INHERITANCE */ - queue_head_t links_data; - queue_t links = &links_data; - wait_queue_link_t wql; - - queue_init(links); - if ((mbits != msg->msgh_bits) || (!MACH_MSG_TYPE_PORT_ANY_SEND(dest_type)) || ((reply_type == 0) ? @@ -1711,6 +2217,9 @@ ipc_kmsg_copyin_header( (voucher_type != MACH_MSG_TYPE_MOVE_SEND && voucher_type != MACH_MSG_TYPE_COPY_SEND)) { is_write_unlock(space); + if ((*optionp & MACH_SEND_KERNEL) == 0) { + mach_port_guard_exception(voucher_name, 0, 0, kGUARD_EXC_SEND_INVALID_VOUCHER); + } return MACH_SEND_INVALID_VOUCHER; } @@ -1720,6 +2229,9 @@ ipc_kmsg_copyin_header( (voucher_entry->ie_bits & MACH_PORT_TYPE_SEND) == 0 || io_kotype(voucher_entry->ie_object) != IKOT_VOUCHER) { is_write_unlock(space); + if ((*optionp & MACH_SEND_KERNEL) == 0) { + mach_port_guard_exception(voucher_name, 0, 0, kGUARD_EXC_SEND_INVALID_VOUCHER); + } return MACH_SEND_INVALID_VOUCHER; } } else { @@ -1796,21 +2308,11 @@ ipc_kmsg_copyin_header( * Perform the delayed reply right copyin (guaranteed success). */ if (reply_entry != IE_NULL) { -#if IMPORTANCE_INHERITANCE kr = ipc_right_copyin(space, reply_name, reply_entry, reply_type, TRUE, &reply_port, &reply_soright, - &release_port, - &assertcnt, - links); + &release_port, &assertcnt); assert(assertcnt == 0); -#else - kr = ipc_right_copyin(space, reply_name, reply_entry, - reply_type, TRUE, - &reply_port, &reply_soright, - &release_port, - links); -#endif /* IMPORTANCE_INHERITANCE */ assert(kr == KERN_SUCCESS); } @@ -1901,21 +2403,11 @@ ipc_kmsg_copyin_header( /* * copyin the destination. */ -#if IMPORTANCE_INHERITANCE kr = ipc_right_copyin(space, dest_name, dest_entry, dest_type, FALSE, &dest_port, &dest_soright, - &release_port, - &assertcnt, - links); + &release_port, &assertcnt); assert(assertcnt == 0); -#else - kr = ipc_right_copyin(space, dest_name, dest_entry, - dest_type, FALSE, - &dest_port, &dest_soright, - &release_port, - links); -#endif /* IMPORTANCE_INHERITANCE */ if (kr != KERN_SUCCESS) { goto invalid_dest; } @@ -1927,21 +2419,11 @@ ipc_kmsg_copyin_header( * It's OK if the reply right has gone dead in the meantime. */ if (MACH_PORT_VALID(reply_name)) { -#if IMPORTANCE_INHERITANCE kr = ipc_right_copyin(space, reply_name, reply_entry, reply_type, TRUE, &reply_port, &reply_soright, - &release_port, - &assertcnt, - links); + &release_port, &assertcnt); assert(assertcnt == 0); -#else - kr = ipc_right_copyin(space, reply_name, reply_entry, - reply_type, TRUE, - &reply_port, &reply_soright, - &release_port, - links); -#endif /* IMPORTANCE_INHERITANCE */ assert(kr == KERN_SUCCESS); } else { /* convert invalid name to equivalent ipc_object type */ @@ -1954,56 +2436,55 @@ ipc_kmsg_copyin_header( * are fully copied in (guaranteed success). */ if (IE_NULL != voucher_entry) { -#if IMPORTANCE_INHERITANCE kr = ipc_right_copyin(space, voucher_name, voucher_entry, voucher_type, FALSE, (ipc_object_t *)&voucher_port, &voucher_soright, &voucher_release_port, - &assertcnt, - links); + &assertcnt); assert(assertcnt == 0); -#else - kr = ipc_right_copyin(space, voucher_name, voucher_entry, - voucher_type, FALSE, - (ipc_object_t *)&voucher_port, - &voucher_soright, - &voucher_release_port, - links); -#endif /* IMPORTANCE_INHERITANCE */ assert(KERN_SUCCESS == kr); assert(IP_VALID(voucher_port)); assert(ip_active(voucher_port)); } } - /* the entry(s) might need to be deallocated */ + /* + * The entries might need to be deallocated. + * + * Each entry should be deallocated only once, + * even if it was specified in more than one slot in the header. + * Note that dest can be the same entry as reply or voucher, + * but reply and voucher must be distinct entries. + */ assert(IE_NULL != dest_entry); + if (IE_NULL != reply_entry) + assert(reply_entry != voucher_entry); + if (IE_BITS_TYPE(dest_entry->ie_bits) == MACH_PORT_TYPE_NONE) { ipc_entry_dealloc(space, dest_name, dest_entry); + + if (dest_entry == reply_entry) { + reply_entry = IE_NULL; + } + + if (dest_entry == voucher_entry) { + voucher_entry = IE_NULL; + } + dest_entry = IE_NULL; } - if (dest_entry != reply_entry && IE_NULL != reply_entry && + if (IE_NULL != reply_entry && IE_BITS_TYPE(reply_entry->ie_bits) == MACH_PORT_TYPE_NONE) { ipc_entry_dealloc(space, reply_name, reply_entry); reply_entry = IE_NULL; } - if (dest_entry != voucher_entry && IE_NULL != voucher_entry && + if (IE_NULL != voucher_entry && IE_BITS_TYPE(voucher_entry->ie_bits) == MACH_PORT_TYPE_NONE) { ipc_entry_dealloc(space, voucher_name, voucher_entry); voucher_entry = IE_NULL; } - /* - * No room to store voucher port in in-kernel msg header, - * so we store it back in the kmsg itself. - */ - if (IP_VALID(voucher_port)) { - assert(ip_active(voucher_port)); - kmsg->ikm_voucher = voucher_port; - voucher_type = MACH_MSG_TYPE_MOVE_SEND; - } - dest_type = ipc_object_copyin_type(dest_type); reply_type = ipc_object_copyin_type(reply_type); @@ -2024,12 +2505,16 @@ ipc_kmsg_copyin_header( if (ip_full(dport)) { #if IMPORTANCE_INHERITANCE needboost = ipc_port_request_sparm(dport, dest_name, - dest_entry->ie_request, - (*optionp & MACH_SEND_NOIMPORTANCE)); + dest_entry->ie_request, + *optionp, + override); if (needboost == FALSE) ip_unlock(dport); #else - ipc_port_request_sparm(dport, dest_name, dest_entry->ie_request); + ipc_port_request_sparm(dport, dest_name, + dest_entry->ie_request, + *optionp, + override); ip_unlock(dport); #endif /* IMPORTANCE_INHERITANCE */ } else { @@ -2053,7 +2538,7 @@ ipc_kmsg_copyin_header( ipc_port_t dport = (ipc_port_t)dest_port; /* dport still locked from above */ - if (ipc_port_importance_delta(dport, 1) == FALSE) { + if (ipc_port_importance_delta(dport, IPID_OPTION_SENDPOSSIBLE, 1) == FALSE) { ip_unlock(dport); } } @@ -2068,14 +2553,24 @@ ipc_kmsg_copyin_header( if (voucher_soright != IP_NULL) { ipc_notify_port_deleted(voucher_soright, voucher_name); } + + /* + * No room to store voucher port in in-kernel msg header, + * so we store it back in the kmsg itself. Extract the + * qos, and apply any override before we enqueue the kmsg. + */ + if (IP_VALID(voucher_port)) { + + kmsg->ikm_voucher = voucher_port; + voucher_type = MACH_MSG_TYPE_MOVE_SEND; + } + msg->msgh_bits = MACH_MSGH_BITS_SET(dest_type, reply_type, voucher_type, mbits); msg->msgh_remote_port = (ipc_port_t)dest_port; msg->msgh_local_port = (ipc_port_t)reply_port; - while(!queue_empty(links)) { - wql = (wait_queue_link_t) dequeue(links); - wait_queue_link_free(wql); - } + /* capture the qos value(s) for the kmsg */ + ipc_kmsg_set_qos(kmsg, *optionp, override); if (release_port != IP_NULL) ip_release(release_port); @@ -2088,27 +2583,20 @@ ipc_kmsg_copyin_header( invalid_reply: is_write_unlock(space); - while(!queue_empty(links)) { - wql = (wait_queue_link_t) dequeue(links); - wait_queue_link_free(wql); - } - if (release_port != IP_NULL) ip_release(release_port); assert(voucher_port == IP_NULL); assert(voucher_soright == IP_NULL); + if ((*optionp & MACH_SEND_KERNEL) == 0) { + mach_port_guard_exception(reply_name, 0, 0, kGUARD_EXC_SEND_INVALID_REPLY); + } return MACH_SEND_INVALID_REPLY; invalid_dest: is_write_unlock(space); - while(!queue_empty(links)) { - wql = (wait_queue_link_t) dequeue(links); - wait_queue_link_free(wql); - } - if (release_port != IP_NULL) ip_release(release_port); @@ -2122,23 +2610,26 @@ invalid_dest: } mach_msg_descriptor_t *ipc_kmsg_copyin_port_descriptor( - volatile mach_msg_port_descriptor_t *dsc, - mach_msg_legacy_port_descriptor_t *user_dsc, - ipc_space_t space, - ipc_object_t dest, - ipc_kmsg_t kmsg, - mach_msg_return_t *mr); + volatile mach_msg_port_descriptor_t *dsc, + mach_msg_legacy_port_descriptor_t *user_dsc, + ipc_space_t space, + ipc_object_t dest, + ipc_kmsg_t kmsg, + mach_msg_option_t *optionp, + mach_msg_return_t *mr); void ipc_print_type_name( int type_name); + mach_msg_descriptor_t * ipc_kmsg_copyin_port_descriptor( - volatile mach_msg_port_descriptor_t *dsc, - mach_msg_legacy_port_descriptor_t *user_dsc_in, - ipc_space_t space, - ipc_object_t dest, - ipc_kmsg_t kmsg, - mach_msg_return_t *mr) + volatile mach_msg_port_descriptor_t *dsc, + mach_msg_legacy_port_descriptor_t *user_dsc_in, + ipc_space_t space, + ipc_object_t dest, + ipc_kmsg_t kmsg, + mach_msg_option_t *optionp, + mach_msg_return_t *mr) { volatile mach_msg_legacy_port_descriptor_t *user_dsc = user_dsc_in; mach_msg_type_name_t user_disp; @@ -2154,6 +2645,9 @@ ipc_kmsg_copyin_port_descriptor( kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object); if (kr != KERN_SUCCESS) { + if ((*optionp & MACH_SEND_KERNEL) == 0) { + mach_port_guard_exception(name, 0, 0, kGUARD_EXC_SEND_INVALID_RIGHT); + } *mr = MACH_SEND_INVALID_RIGHT; return NULL; } @@ -2176,24 +2670,27 @@ ipc_kmsg_copyin_port_descriptor( } mach_msg_descriptor_t * ipc_kmsg_copyin_ool_descriptor( - mach_msg_ool_descriptor_t *dsc, - mach_msg_descriptor_t *user_dsc, - int is_64bit, - vm_offset_t *paddr, - vm_map_copy_t *copy, - vm_size_t *space_needed, - vm_map_t map, - mach_msg_return_t *mr); + mach_msg_ool_descriptor_t *dsc, + mach_msg_descriptor_t *user_dsc, + int is_64bit, + vm_offset_t *paddr, + vm_map_copy_t *copy, + vm_size_t *space_needed, + vm_map_t map, + mach_msg_option_t *optionp, + mach_msg_return_t *mr); + mach_msg_descriptor_t * ipc_kmsg_copyin_ool_descriptor( - mach_msg_ool_descriptor_t *dsc, - mach_msg_descriptor_t *user_dsc, - int is_64bit, - vm_offset_t *paddr, - vm_map_copy_t *copy, - vm_size_t *space_needed, - vm_map_t map, - mach_msg_return_t *mr) + mach_msg_ool_descriptor_t *dsc, + mach_msg_descriptor_t *user_dsc, + int is_64bit, + vm_offset_t *paddr, + vm_map_copy_t *copy, + vm_size_t *space_needed, + vm_map_t map, + __unused mach_msg_option_t *optionp, + mach_msg_return_t *mr) { vm_size_t length; boolean_t dealloc; @@ -2289,36 +2786,39 @@ ipc_kmsg_copyin_ool_descriptor( } mach_msg_descriptor_t * ipc_kmsg_copyin_ool_ports_descriptor( - mach_msg_ool_ports_descriptor_t *dsc, - mach_msg_descriptor_t *user_dsc, - int is_64bit, - vm_map_t map, - ipc_space_t space, - ipc_object_t dest, - ipc_kmsg_t kmsg, - mach_msg_return_t *mr); + mach_msg_ool_ports_descriptor_t *dsc, + mach_msg_descriptor_t *user_dsc, + int is_64bit, + vm_map_t map, + ipc_space_t space, + ipc_object_t dest, + ipc_kmsg_t kmsg, + mach_msg_option_t *optionp, + mach_msg_return_t *mr); + mach_msg_descriptor_t * ipc_kmsg_copyin_ool_ports_descriptor( - mach_msg_ool_ports_descriptor_t *dsc, - mach_msg_descriptor_t *user_dsc, - int is_64bit, - vm_map_t map, - ipc_space_t space, - ipc_object_t dest, - ipc_kmsg_t kmsg, - mach_msg_return_t *mr) + mach_msg_ool_ports_descriptor_t *dsc, + mach_msg_descriptor_t *user_dsc, + int is_64bit, + vm_map_t map, + ipc_space_t space, + ipc_object_t dest, + ipc_kmsg_t kmsg, + mach_msg_option_t *optionp, + mach_msg_return_t *mr) { - void *data; - ipc_object_t *objects; - unsigned int i; - mach_vm_offset_t addr; - mach_msg_type_name_t user_disp; - mach_msg_type_name_t result_disp; - mach_msg_type_number_t count; - mach_msg_copy_options_t copy_option; - boolean_t deallocate; - mach_msg_descriptor_type_t type; - vm_size_t ports_length, names_length; + void *data; + ipc_object_t *objects; + unsigned int i; + mach_vm_offset_t addr; + mach_msg_type_name_t user_disp; + mach_msg_type_name_t result_disp; + mach_msg_type_number_t count; + mach_msg_copy_options_t copy_option; + boolean_t deallocate; + mach_msg_descriptor_type_t type; + vm_size_t ports_length, names_length; if (is_64bit) { mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; @@ -2353,14 +2853,24 @@ ipc_kmsg_copyin_ool_ports_descriptor( result_disp = ipc_object_copyin_type(user_disp); dsc->disposition = result_disp; - if (count > (INT_MAX / sizeof(mach_port_t))) { - *mr = MACH_SEND_TOO_LARGE; + /* We always do a 'physical copy', but you have to specify something valid */ + if (copy_option != MACH_MSG_PHYSICAL_COPY && + copy_option != MACH_MSG_VIRTUAL_COPY) { + *mr = MACH_SEND_INVALID_TYPE; return NULL; } /* calculate length of data in bytes, rounding up */ - ports_length = count * sizeof(mach_port_t); - names_length = count * sizeof(mach_port_name_t); + + if (os_mul_overflow(count, sizeof(mach_port_t), &ports_length)) { + *mr = MACH_SEND_TOO_LARGE; + return NULL; + } + + if (os_mul_overflow(count, sizeof(mach_port_name_t), &names_length)) { + *mr = MACH_SEND_TOO_LARGE; + return NULL; + } if (ports_length == 0) { return user_dsc; @@ -2413,6 +2923,9 @@ ipc_kmsg_copyin_ool_ports_descriptor( } kfree(data, ports_length); dsc->address = NULL; + if ((*optionp & MACH_SEND_KERNEL) == 0) { + mach_port_guard_exception(name, 0, 0, kGUARD_EXC_SEND_INVALID_RIGHT); + } *mr = MACH_SEND_INVALID_RIGHT; return NULL; } @@ -2455,7 +2968,8 @@ mach_msg_return_t ipc_kmsg_copyin_body( ipc_kmsg_t kmsg, ipc_space_t space, - vm_map_t map) + vm_map_t map, + mach_msg_option_t *optionp) { ipc_object_t dest; mach_msg_body_t *body; @@ -2472,6 +2986,8 @@ ipc_kmsg_copyin_body( vm_size_t descriptor_size = 0; + mach_msg_type_number_t total_ool_port_count = 0; + /* * Determine if the target is a kernel port. */ @@ -2491,6 +3007,7 @@ ipc_kmsg_copyin_body( daddr = NULL; for (i = 0; i < dsc_count; i++) { mach_msg_size_t size; + mach_msg_type_number_t ool_port_count = 0; daddr = naddr; @@ -2515,9 +3032,8 @@ ipc_kmsg_copyin_body( if (naddr > (mach_msg_descriptor_t *) ((vm_offset_t)kmsg->ikm_header + kmsg->ikm_header->msgh_size)) { - ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0); - mr = MACH_SEND_MSG_TOO_SMALL; - goto out; + mr = MACH_SEND_MSG_TOO_SMALL; + goto clean_message; } switch (daddr->type.type) { @@ -2532,11 +3048,10 @@ ipc_kmsg_copyin_body( /* * Invalid copy option */ - ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0); mr = MACH_SEND_INVALID_TYPE; - goto out; + goto clean_message; } - + if ((size >= MSG_OOL_SIZE_SMALL) && (daddr->out_of_line.copy == MACH_MSG_PHYSICAL_COPY) && !(daddr->out_of_line.deallocate)) { @@ -2546,37 +3061,62 @@ ipc_kmsg_copyin_body( * memory requirements */ if (space_needed + round_page(size) <= space_needed) { - /* Overflow dectected */ - ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0); - mr = MACH_MSG_VM_KERNEL; - goto out; - } - + /* Overflow dectected */ + mr = MACH_MSG_VM_KERNEL; + goto clean_message; + } + space_needed += round_page(size); if (space_needed > ipc_kmsg_max_vm_space) { - - /* - * Per message kernel memory limit exceeded - */ - ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0); + /* Per message kernel memory limit exceeded */ mr = MACH_MSG_VM_KERNEL; - goto out; + goto clean_message; } } + break; + case MACH_MSG_PORT_DESCRIPTOR: + if (os_add_overflow(total_ool_port_count, 1, &total_ool_port_count)) { + /* Overflow detected */ + mr = MACH_SEND_TOO_LARGE; + goto clean_message; + } + break; + case MACH_MSG_OOL_PORTS_DESCRIPTOR: + ool_port_count = (is_task_64bit) ? + ((mach_msg_ool_ports_descriptor64_t *)daddr)->count : + daddr->ool_ports.count; + + if (os_add_overflow(total_ool_port_count, ool_port_count, &total_ool_port_count)) { + /* Overflow detected */ + mr = MACH_SEND_TOO_LARGE; + goto clean_message; + } + + if (ool_port_count > (ipc_kmsg_max_vm_space/sizeof(mach_port_t))) { + /* Per message kernel memory limit exceeded */ + mr = MACH_SEND_TOO_LARGE; + goto clean_message; + } + break; } } + /* Sending more than 16383 rights in one message seems crazy */ + if (total_ool_port_count >= (MACH_PORT_UREFS_MAX / 4)) { + mr = MACH_SEND_TOO_LARGE; + goto clean_message; + } + /* * Allocate space in the pageable kernel ipc copy map for all the * ool data that is to be physically copied. Map is marked wait for * space. */ if (space_needed) { - if (vm_allocate(ipc_kernel_copy_map, &paddr, space_needed, - VM_FLAGS_ANYWHERE) != KERN_SUCCESS) { - ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0); + if (vm_allocate_kernel(ipc_kernel_copy_map, &paddr, space_needed, + VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC) != KERN_SUCCESS) { mr = MACH_MSG_VM_KERNEL; - goto out; + goto clean_message; } } @@ -2603,20 +3143,20 @@ ipc_kmsg_copyin_body( switch (user_addr->type.type) { case MACH_MSG_PORT_DESCRIPTOR: user_addr = ipc_kmsg_copyin_port_descriptor((mach_msg_port_descriptor_t *)kern_addr, - (mach_msg_legacy_port_descriptor_t *)user_addr, space, dest, kmsg, &mr); + (mach_msg_legacy_port_descriptor_t *)user_addr, space, dest, kmsg, optionp, &mr); kern_addr++; complex = TRUE; break; case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: case MACH_MSG_OOL_DESCRIPTOR: user_addr = ipc_kmsg_copyin_ool_descriptor((mach_msg_ool_descriptor_t *)kern_addr, - user_addr, is_task_64bit, &paddr, ©, &space_needed, map, &mr); + user_addr, is_task_64bit, &paddr, ©, &space_needed, map, optionp, &mr); kern_addr++; complex = TRUE; break; case MACH_MSG_OOL_PORTS_DESCRIPTOR: user_addr = ipc_kmsg_copyin_ool_ports_descriptor((mach_msg_ool_ports_descriptor_t *)kern_addr, - user_addr, is_task_64bit, map, space, dest, kmsg, &mr); + user_addr, is_task_64bit, map, space, dest, kmsg, optionp, &mr); kern_addr++; complex = TRUE; break; @@ -2640,6 +3180,11 @@ ipc_kmsg_copyin_body( } out: return mr; + +clean_message: + /* no descriptors have been copied in yet */ + ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0); + return mr; } @@ -2672,13 +3217,14 @@ ipc_kmsg_copyin( ipc_kmsg_t kmsg, ipc_space_t space, vm_map_t map, + mach_msg_priority_t override, mach_msg_option_t *optionp) { mach_msg_return_t mr; kmsg->ikm_header->msgh_bits &= MACH_MSGH_BITS_USER; - mr = ipc_kmsg_copyin_header(kmsg, space, optionp); + mr = ipc_kmsg_copyin_header(kmsg, space, override, optionp); if (mr != MACH_MSG_SUCCESS) return mr; @@ -2701,7 +3247,7 @@ ipc_kmsg_copyin( if ((kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) == 0) return MACH_MSG_SUCCESS; - mr = ipc_kmsg_copyin_body( kmsg, space, map); + mr = ipc_kmsg_copyin_body( kmsg, space, map, optionp); /* unreachable if !DEBUG */ __unreachable_ok_push @@ -3451,6 +3997,7 @@ ipc_kmsg_copyout_port_descriptor(mach_msg_descriptor_t *dsc, { mach_msg_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc; user_dsc--; // point to the start of this port descriptor + bzero((void *)user_dsc, sizeof(*user_dsc)); user_dsc->name = CAST_MACH_NAME_TO_PORT(name); user_dsc->disposition = disp; user_dsc->type = MACH_MSG_PORT_DESCRIPTOR; @@ -3458,6 +4005,7 @@ ipc_kmsg_copyout_port_descriptor(mach_msg_descriptor_t *dsc, } else { mach_msg_legacy_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc; user_dsc--; // point to the start of this port descriptor + bzero((void *)user_dsc, sizeof(*user_dsc)); user_dsc->name = CAST_MACH_PORT_TO_NAME(name); user_dsc->disposition = disp; user_dsc->type = MACH_MSG_PORT_DESCRIPTOR; @@ -3475,65 +4023,25 @@ ipc_kmsg_copyout_ool_descriptor(mach_msg_ool_descriptor_t *dsc, mach_msg_descrip vm_map_copy_t copy; vm_map_address_t rcv_addr; mach_msg_copy_options_t copy_options; - mach_msg_size_t size; + vm_map_size_t size; mach_msg_descriptor_type_t dsc_type; //SKIP_PORT_DESCRIPTORS(saddr, sdsc_count); - copy = (vm_map_copy_t) dsc->address; - size = dsc->size; + copy = (vm_map_copy_t)dsc->address; + size = (vm_map_size_t)dsc->size; copy_options = dsc->copy; assert(copy_options != MACH_MSG_KALLOC_COPY_T); dsc_type = dsc->type; - rcv_addr = 0; if (copy != VM_MAP_COPY_NULL) { - /* - * Check to see if there is an overwrite descriptor - * specified in the scatter list for this ool data. - * The descriptor has already been verified. - */ -#if 0 - if (saddr != MACH_MSG_DESCRIPTOR_NULL) { - if (differs) { - OTHER_OOL_DESCRIPTOR *scatter_dsc; - - scatter_dsc = (OTHER_OOL_DESCRIPTOR *)saddr; - if (scatter_dsc->copy == MACH_MSG_OVERWRITE) { - rcv_addr = (mach_vm_offset_t) scatter_dsc->address; - copy_options = MACH_MSG_OVERWRITE; - } else { - copy_options = MACH_MSG_VIRTUAL_COPY; - } - } else { - mach_msg_ool_descriptor_t *scatter_dsc; - - scatter_dsc = &saddr->out_of_line; - if (scatter_dsc->copy == MACH_MSG_OVERWRITE) { - rcv_addr = CAST_USER_ADDR_T(scatter_dsc->address); - copy_options = MACH_MSG_OVERWRITE; - } else { - copy_options = MACH_MSG_VIRTUAL_COPY; - } - } - INCREMENT_SCATTER(saddr, sdsc_count, differs); - } -#endif - + kern_return_t kr; - /* - * Whether the data was virtually or physically - * copied we have a vm_map_copy_t for it. - * If there's an overwrite region specified - * overwrite it, otherwise do a virtual copy out. - */ - kern_return_t kr; - if (copy_options == MACH_MSG_OVERWRITE && rcv_addr != 0) { - kr = vm_map_copy_overwrite(map, rcv_addr, - copy, TRUE); - } else { - kr = vm_map_copyout(map, &rcv_addr, copy); - } + rcv_addr = 0; + if (vm_map_copy_validate_size(map, copy, &size) == FALSE) + panic("Inconsistent OOL/copyout size on %p: expected %d, got %lld @%p", + dsc, dsc->size, (unsigned long long)copy->size, copy); + kr = vm_map_copyout_size(map, &rcv_addr, copy, size); if (kr != KERN_SUCCESS) { if (kr == KERN_RESOURCE_SHORTAGE) *mr |= MACH_MSG_VM_KERNEL; @@ -3558,33 +4066,36 @@ ipc_kmsg_copyout_ool_descriptor(mach_msg_ool_descriptor_t *dsc, mach_msg_descrip { mach_msg_ool_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; user_ool_dsc--; + bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); user_ool_dsc->address = (void *)(uintptr_t)rcv_addr; user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? TRUE : FALSE; user_ool_dsc->copy = copy_options; user_ool_dsc->type = dsc_type; - user_ool_dsc->size = size; + user_ool_dsc->size = (mach_msg_size_t)size; user_dsc = (typeof(user_dsc))user_ool_dsc; } else if (is_64bit) { mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; user_ool_dsc--; + bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); user_ool_dsc->address = rcv_addr; user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? TRUE : FALSE; user_ool_dsc->copy = copy_options; user_ool_dsc->type = dsc_type; - user_ool_dsc->size = size; + user_ool_dsc->size = (mach_msg_size_t)size; user_dsc = (typeof(user_dsc))user_ool_dsc; } else { mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; user_ool_dsc--; + bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr); - user_ool_dsc->size = size; + user_ool_dsc->size = (mach_msg_size_t)size; user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? TRUE : FALSE; user_ool_dsc->copy = copy_options; @@ -3656,13 +4167,14 @@ ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc, /* * Dynamically allocate the region */ - int anywhere = VM_MAKE_TAG(VM_MEMORY_MACH_MSG)| - VM_FLAGS_ANYWHERE; + vm_tag_t tag; + if (vm_kernel_map_is_kernel(map)) tag = VM_KERN_MEMORY_IPC; + else tag = VM_MEMORY_MACH_MSG; kern_return_t kr; - if ((kr = mach_vm_allocate(map, &rcv_addr, + if ((kr = mach_vm_allocate_kernel(map, &rcv_addr, (mach_vm_size_t)names_length, - anywhere)) != KERN_SUCCESS) { + VM_FLAGS_ANYWHERE, tag)) != KERN_SUCCESS) { ipc_kmsg_clean_body(kmsg, 1, (mach_msg_descriptor_t *)dsc); rcv_addr = 0; @@ -3708,6 +4220,7 @@ ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc, if(current_task() == kernel_task) { mach_msg_ool_ports_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; user_ool_dsc--; + bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); user_ool_dsc->address = (void *)(uintptr_t)rcv_addr; user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? @@ -3721,6 +4234,7 @@ ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc, } if (is_64bit) { mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; user_ool_dsc--; + bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); user_ool_dsc->address = rcv_addr; user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? @@ -3734,6 +4248,7 @@ ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc, } else { mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; user_ool_dsc--; + bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr); user_ool_dsc->count = count; @@ -3967,6 +4482,9 @@ ipc_kmsg_copyout_pseudo( mach_port_name_t dest_name, reply_name; mach_msg_return_t mr; + /* Set ith_knote to ITH_KNOTE_PSEUDO */ + current_thread()->ith_knote = ITH_KNOTE_PSEUDO; + assert(IO_VALID(dest)); #if 0 @@ -4078,163 +4596,6 @@ ipc_kmsg_copyout_dest( } } -/* - * Routine: ipc_kmsg_copyin_scatter - * Purpose: - * allocate and copyin a scatter list - * Algorithm: - * The gather (kmsg) is valid since it has been copied in. - * Gather list descriptors are sequentially paired with scatter - * list descriptors, with port descriptors in either list ignored. - * Descriptors are consistent if the type fileds match and size - * of the scatter descriptor is less than or equal to the - * size of the gather descriptor. A MACH_MSG_ALLOCATE copy - * strategy in a scatter descriptor matches any size in the - * corresponding gather descriptor assuming they are the same type. - * Either list may be larger than the other. During the - * subsequent copy out, excess scatter descriptors are ignored - * and excess gather descriptors default to dynamic allocation. - * - * In the case of a size error, the scatter list is released. - * Conditions: - * Nothing locked. - * Returns: - * the allocated message body containing the scatter list. - */ - -mach_msg_body_t * -ipc_kmsg_get_scatter( - mach_vm_address_t msg_addr, - mach_msg_size_t slist_size, - ipc_kmsg_t kmsg) -{ - mach_msg_body_t *slist; - mach_msg_body_t *body; - mach_msg_descriptor_t *gstart, *gend; - mach_msg_descriptor_t *sstart, *send; - -#if defined(__LP64__) - panic("ipc_kmsg_get_scatter called!"); -#endif - - if (slist_size < sizeof(mach_msg_base_t)) - return MACH_MSG_BODY_NULL; - - slist_size -= (mach_msg_size_t)sizeof(mach_msg_header_t); - slist = (mach_msg_body_t *)kalloc(slist_size); - if (slist == MACH_MSG_BODY_NULL) - return slist; - - if (copyin(msg_addr + sizeof(mach_msg_header_t), (char *)slist, slist_size)) { - kfree(slist, slist_size); - return MACH_MSG_BODY_NULL; - } - - if ((slist->msgh_descriptor_count* sizeof(mach_msg_descriptor_t) - + sizeof(mach_msg_size_t)) > slist_size) { - kfree(slist, slist_size); - return MACH_MSG_BODY_NULL; - } - - body = (mach_msg_body_t *) (kmsg->ikm_header + 1); - gstart = (mach_msg_descriptor_t *) (body + 1); - gend = gstart + body->msgh_descriptor_count; - - sstart = (mach_msg_descriptor_t *) (slist + 1); - send = sstart + slist->msgh_descriptor_count; - - while (gstart < gend) { - mach_msg_descriptor_type_t g_type; - - /* - * Skip port descriptors in gather list. - */ - g_type = gstart->type.type; - - if (g_type != MACH_MSG_PORT_DESCRIPTOR) { - - /* - * A scatter list with a 0 descriptor count is treated as an - * automatic size mismatch. - */ - if (slist->msgh_descriptor_count == 0) { - kfree(slist, slist_size); - return MACH_MSG_BODY_NULL; - } - - /* - * Skip port descriptors in scatter list. - */ - while (sstart < send) { - if (sstart->type.type != MACH_MSG_PORT_DESCRIPTOR) - break; - sstart++; - } - - /* - * No more scatter descriptors, we're done - */ - if (sstart >= send) { - break; - } - - /* - * Check type, copy and size fields - */ - if (g_type == MACH_MSG_OOL_DESCRIPTOR || - g_type == MACH_MSG_OOL_VOLATILE_DESCRIPTOR) { - if (sstart->type.type != MACH_MSG_OOL_DESCRIPTOR && - sstart->type.type != MACH_MSG_OOL_VOLATILE_DESCRIPTOR) { - kfree(slist, slist_size); - return MACH_MSG_BODY_NULL; - } - if (sstart->out_of_line.copy == MACH_MSG_OVERWRITE && - gstart->out_of_line.size > sstart->out_of_line.size) { - kfree(slist, slist_size); - return MACH_MSG_BODY_NULL; - } - } - else { - if (sstart->type.type != MACH_MSG_OOL_PORTS_DESCRIPTOR) { - kfree(slist, slist_size); - return MACH_MSG_BODY_NULL; - } - if (sstart->ool_ports.copy == MACH_MSG_OVERWRITE && - gstart->ool_ports.count > sstart->ool_ports.count) { - kfree(slist, slist_size); - return MACH_MSG_BODY_NULL; - } - } - sstart++; - } - gstart++; - } - return slist; -} - - -/* - * Routine: ipc_kmsg_free_scatter - * Purpose: - * Deallocate a scatter list. Since we actually allocated - * a body without a header, and since the header was originally - * accounted for in slist_size, we have to ajust it down - * before freeing the scatter list. - */ -void -ipc_kmsg_free_scatter( - mach_msg_body_t *slist, - mach_msg_size_t slist_size) -{ -#if defined(__LP64__) - panic("%s called; halting!", __func__); -#endif - - slist_size -= (mach_msg_size_t)sizeof(mach_msg_header_t); - kfree(slist, slist_size); -} - - /* * Routine: ipc_kmsg_copyout_to_kernel * Purpose: @@ -4254,13 +4615,13 @@ ipc_kmsg_copyout_to_kernel( ipc_space_t space) { ipc_object_t dest; - ipc_object_t reply; + mach_port_t reply; mach_msg_type_name_t dest_type; mach_msg_type_name_t reply_type; - mach_port_name_t dest_name, reply_name; + mach_port_name_t dest_name; dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; - reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port; + reply = kmsg->ikm_header->msgh_local_port; dest_type = MACH_MSGH_BITS_REMOTE(kmsg->ikm_header->msgh_bits); reply_type = MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits); @@ -4276,13 +4637,11 @@ ipc_kmsg_copyout_to_kernel( dest_name = MACH_PORT_DEAD; } - reply_name = CAST_MACH_PORT_TO_NAME(reply); - kmsg->ikm_header->msgh_bits = (MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) | MACH_MSGH_BITS(reply_type, dest_type)); kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name); - kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name); + kmsg->ikm_header->msgh_remote_port = reply; } #if IKM_SUPPORT_LEGACY @@ -4404,6 +4763,28 @@ ipc_kmsg_copyout_to_kernel_legacy( } #endif /* IKM_SUPPORT_LEGACY */ +#ifdef __arm64__ +/* + * Just sets those parts of the trailer that aren't set up at allocation time. + */ +static void +ipc_kmsg_munge_trailer(mach_msg_max_trailer_t *in, void *_out, boolean_t is64bit) +{ + if (is64bit) { + mach_msg_max_trailer64_t *out = (mach_msg_max_trailer64_t*)_out; + out->msgh_seqno = in->msgh_seqno; + out->msgh_context = in->msgh_context; + out->msgh_trailer_size = in->msgh_trailer_size; + out->msgh_ad = in->msgh_ad; + } else { + mach_msg_max_trailer32_t *out = (mach_msg_max_trailer32_t*)_out; + out->msgh_seqno = in->msgh_seqno; + out->msgh_context = (mach_port_context32_t)in->msgh_context; + out->msgh_trailer_size = in->msgh_trailer_size; + out->msgh_ad = in->msgh_ad; + } +} +#endif /* __arm64__ */ mach_msg_trailer_size_t ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused, @@ -4413,10 +4794,25 @@ ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused, { mach_msg_max_trailer_t *trailer; +#ifdef __arm64__ + mach_msg_max_trailer_t tmp_trailer; /* This accommodates U64, and we'll munge */ + void *real_trailer_out = (void*)(mach_msg_max_trailer_t *) + ((vm_offset_t)kmsg->ikm_header + + round_msg(kmsg->ikm_header->msgh_size)); + + /* + * Populate scratch with initial values set up at message allocation time. + * After, we reinterpret the space in the message as the right type + * of trailer for the address space in question. + */ + bcopy(real_trailer_out, &tmp_trailer, MAX_TRAILER_SIZE); + trailer = &tmp_trailer; +#else /* __arm64__ */ (void)thread; trailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + round_msg(kmsg->ikm_header->msgh_size)); +#endif /* __arm64__ */ if (!(option & MACH_RCV_TRAILER_MASK)) { return trailer->msgh_trailer_size; @@ -4424,7 +4820,7 @@ ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused, trailer->msgh_seqno = seqno; trailer->msgh_context = context; - trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(thread_is_64bit(thread), option); + trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(thread_is_64bit_addr(thread), option); if (minimal_trailer) { goto done; @@ -4446,6 +4842,9 @@ ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused, } done: +#ifdef __arm64__ + ipc_kmsg_munge_trailer(trailer, real_trailer_out, thread_is_64bit_addr(thread)); +#endif /* __arm64__ */ return trailer->msgh_trailer_size; }