X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/a39ff7e25e19b3a8c3020042a3872ca9ec9659f1..4ba76501152d51ccb5647018f3192c6096367d48:/osfmk/ipc/ipc_kmsg.c diff --git a/osfmk/ipc/ipc_kmsg.c b/osfmk/ipc/ipc_kmsg.c index 736fe824c..f1611fc82 100644 --- a/osfmk/ipc/ipc_kmsg.c +++ b/osfmk/ipc/ipc_kmsg.c @@ -2,7 +2,7 @@ * Copyright (c) 2000-2007 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,34 +22,34 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ -/* +/* * Mach Operating System * Copyright (c) 1991,1990,1989 Carnegie Mellon University * All Rights Reserved. - * + * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. - * + * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. - * + * * Carnegie Mellon requests users of this software to return to - * + * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 - * + * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ @@ -94,7 +94,9 @@ #include #include -#include +#include + +#include #include #include @@ -139,38 +141,35 @@ #pragma pack(4) -typedef struct -{ - mach_msg_bits_t msgh_bits; - mach_msg_size_t msgh_size; - mach_port_name_t msgh_remote_port; - mach_port_name_t msgh_local_port; - mach_port_name_t msgh_voucher_port; - mach_msg_id_t msgh_id; +typedef struct{ + mach_msg_bits_t msgh_bits; + mach_msg_size_t msgh_size; + mach_port_name_t msgh_remote_port; + mach_port_name_t msgh_local_port; + mach_port_name_t msgh_voucher_port; + mach_msg_id_t msgh_id; } mach_msg_legacy_header_t; -typedef struct -{ - mach_msg_legacy_header_t header; - mach_msg_body_t body; +typedef struct{ + mach_msg_legacy_header_t header; + mach_msg_body_t body; } mach_msg_legacy_base_t; -typedef struct -{ - mach_port_name_t name; - mach_msg_size_t pad1; - uint32_t pad2 : 16; - mach_msg_type_name_t disposition : 8; - mach_msg_descriptor_type_t type : 8; +typedef struct{ + mach_port_name_t name; + mach_msg_size_t pad1; + uint32_t pad2 : 16; + mach_msg_type_name_t disposition : 8; + mach_msg_descriptor_type_t type : 8; } mach_msg_legacy_port_descriptor_t; -typedef union -{ - mach_msg_legacy_port_descriptor_t port; - mach_msg_ool_descriptor32_t out_of_line32; - mach_msg_ool_ports_descriptor32_t ool_ports32; - mach_msg_type_descriptor_t type; +typedef union{ + mach_msg_legacy_port_descriptor_t port; + mach_msg_ool_descriptor32_t out_of_line32; + mach_msg_ool_ports_descriptor32_t ool_ports32; + mach_msg_guarded_port_descriptor32_t guarded_port32; + mach_msg_type_descriptor_t type; } mach_msg_legacy_descriptor_t; #pragma pack() @@ -182,49 +181,49 @@ typedef union #if DEBUG_MSGS_K64 extern void ipc_pset_print64( - ipc_pset_t pset); + ipc_pset_t pset); -extern void ipc_kmsg_print64( - ipc_kmsg_t kmsg, - const char *str); +extern void ipc_kmsg_print64( + ipc_kmsg_t kmsg, + const char *str); -extern void ipc_msg_print64( - mach_msg_header_t *msgh); +extern void ipc_msg_print64( + mach_msg_header_t *msgh); extern ipc_port_t ipc_name_to_data64( - task_t task, - mach_port_name_t name); + task_t task, + mach_port_name_t name); /* * Forward declarations */ void ipc_msg_print_untyped64( - mach_msg_body_t *body); + mach_msg_body_t *body); const char * ipc_type_name64( - int type_name, - boolean_t received); + int type_name, + boolean_t received); void ipc_print_type_name64( - int type_name); + int type_name); const char * msgh_bit_decode64( - mach_msg_bits_t bit); + mach_msg_bits_t bit); const char * mm_copy_options_string64( - mach_msg_copy_options_t option); + mach_msg_copy_options_t option); void db_print_msg_uid64(mach_msg_header_t *); static void ipc_msg_body_print64(void *body, int size) { - uint32_t *word = (uint32_t *) body; - uint32_t *end = (uint32_t *)(((uintptr_t) body) + size - - sizeof(mach_msg_header_t)); - int i; + uint32_t *word = (uint32_t *) body; + uint32_t *end = (uint32_t *)(((uintptr_t) body) + size + - sizeof(mach_msg_header_t)); + int i; kprintf(" body(%p-%p):\n %p: ", body, end, word); for (;;) { @@ -233,7 +232,7 @@ ipc_msg_body_print64(void *body, int size) kprintf("\n"); return; } - kprintf("%08x ", *word); + kprintf("%08x ", *word); } kprintf("\n %p: ", word); } @@ -242,51 +241,51 @@ ipc_msg_body_print64(void *body, int size) const char * ipc_type_name64( - int type_name, - boolean_t received) + int type_name, + boolean_t received) { switch (type_name) { - case MACH_MSG_TYPE_PORT_NAME: + case MACH_MSG_TYPE_PORT_NAME: return "port_name"; - - case MACH_MSG_TYPE_MOVE_RECEIVE: + + case MACH_MSG_TYPE_MOVE_RECEIVE: if (received) { return "port_receive"; } else { return "move_receive"; } - - case MACH_MSG_TYPE_MOVE_SEND: + + case MACH_MSG_TYPE_MOVE_SEND: if (received) { return "port_send"; } else { return "move_send"; } - - case MACH_MSG_TYPE_MOVE_SEND_ONCE: + + case MACH_MSG_TYPE_MOVE_SEND_ONCE: if (received) { return "port_send_once"; } else { return "move_send_once"; } - - case MACH_MSG_TYPE_COPY_SEND: + + case MACH_MSG_TYPE_COPY_SEND: return "copy_send"; - - case MACH_MSG_TYPE_MAKE_SEND: + + case MACH_MSG_TYPE_MAKE_SEND: return "make_send"; - - case MACH_MSG_TYPE_MAKE_SEND_ONCE: + + case MACH_MSG_TYPE_MAKE_SEND_ONCE: return "make_send_once"; - - default: + + default: return (char *) 0; } } - + void ipc_print_type_name64( - int type_name) + int type_name) { const char *name = ipc_type_name64(type_name, TRUE); if (name) { @@ -301,26 +300,26 @@ ipc_print_type_name64( */ void ipc_kmsg_print64( - ipc_kmsg_t kmsg, - const char *str) + ipc_kmsg_t kmsg, + const char *str) { kprintf("%s kmsg=%p:\n", str, kmsg); kprintf(" next=%p, prev=%p, size=%d", - kmsg->ikm_next, - kmsg->ikm_prev, - kmsg->ikm_size); + kmsg->ikm_next, + kmsg->ikm_prev, + kmsg->ikm_size); kprintf("\n"); ipc_msg_print64(kmsg->ikm_header); } const char * msgh_bit_decode64( - mach_msg_bits_t bit) + mach_msg_bits_t bit) { switch (bit) { - case MACH_MSGH_BITS_COMPLEX: return "complex"; - case MACH_MSGH_BITS_CIRCULAR: return "circular"; - default: return (char *) 0; + case MACH_MSGH_BITS_COMPLEX: return "complex"; + case MACH_MSGH_BITS_CIRCULAR: return "circular"; + default: return (char *) 0; } } @@ -329,35 +328,37 @@ msgh_bit_decode64( */ void ipc_msg_print64( - mach_msg_header_t *msgh) + mach_msg_header_t *msgh) { - mach_msg_bits_t mbits; - unsigned int bit, i; - const char *bit_name; - int needs_comma; + mach_msg_bits_t mbits; + unsigned int bit, i; + const char *bit_name; + int needs_comma; mbits = msgh->msgh_bits; kprintf(" msgh_bits=0x%x: l=0x%x,r=0x%x\n", - mbits, - MACH_MSGH_BITS_LOCAL(msgh->msgh_bits), - MACH_MSGH_BITS_REMOTE(msgh->msgh_bits)); + mbits, + MACH_MSGH_BITS_LOCAL(msgh->msgh_bits), + MACH_MSGH_BITS_REMOTE(msgh->msgh_bits)); mbits = MACH_MSGH_BITS_OTHER(mbits) & MACH_MSGH_BITS_USED; kprintf(" decoded bits: "); needs_comma = 0; for (i = 0, bit = 1; i < sizeof(mbits) * 8; ++i, bit <<= 1) { - if ((mbits & bit) == 0) + if ((mbits & bit) == 0) { continue; + } bit_name = msgh_bit_decode64((mach_msg_bits_t)bit); - if (bit_name) + if (bit_name) { kprintf("%s%s", needs_comma ? "," : "", bit_name); - else + } else { kprintf("%sunknown(0x%x),", needs_comma ? "," : "", bit); + } ++needs_comma; } if (msgh->msgh_bits & ~MACH_MSGH_BITS_USED) { kprintf("%sunused=0x%x,", needs_comma ? "," : "", - msgh->msgh_bits & ~MACH_MSGH_BITS_USED); + msgh->msgh_bits & ~MACH_MSGH_BITS_USED); } kprintf("\n"); @@ -372,7 +373,7 @@ ipc_msg_print64( if (msgh->msgh_local_port) { kprintf("%slocal=%p(", needs_comma ? "," : "", - msgh->msgh_local_port); + msgh->msgh_local_port); ipc_print_type_name64(MACH_MSGH_BITS_LOCAL(msgh->msgh_bits)); kprintf(")\n"); } else { @@ -380,10 +381,10 @@ ipc_msg_print64( } kprintf(" msgh_id=%d, size=%d\n", - msgh->msgh_id, - msgh->msgh_size); + msgh->msgh_id, + msgh->msgh_size); - if (mbits & MACH_MSGH_BITS_COMPLEX) { + if (mbits & MACH_MSGH_BITS_COMPLEX) { ipc_msg_print_untyped64((mach_msg_body_t *) (msgh + 1)); } @@ -393,27 +394,27 @@ ipc_msg_print64( const char * mm_copy_options_string64( - mach_msg_copy_options_t option) + mach_msg_copy_options_t option) { - const char *name; + const char *name; switch (option) { - case MACH_MSG_PHYSICAL_COPY: + case MACH_MSG_PHYSICAL_COPY: name = "PHYSICAL"; break; - case MACH_MSG_VIRTUAL_COPY: + case MACH_MSG_VIRTUAL_COPY: name = "VIRTUAL"; break; - case MACH_MSG_OVERWRITE: + case MACH_MSG_OVERWRITE: name = "OVERWRITE(DEPRECATED)"; break; - case MACH_MSG_ALLOCATE: + case MACH_MSG_ALLOCATE: name = "ALLOCATE"; break; - case MACH_MSG_KALLOC_COPY_T: + case MACH_MSG_KALLOC_COPY_T: name = "KALLOC_COPY_T"; break; - default: + default: name = "unknown"; break; } @@ -422,128 +423,134 @@ mm_copy_options_string64( void ipc_msg_print_untyped64( - mach_msg_body_t *body) + mach_msg_body_t *body) { - mach_msg_descriptor_t *saddr, *send; - mach_msg_descriptor_type_t type; + mach_msg_descriptor_t *saddr, *send; + mach_msg_descriptor_type_t type; - kprintf(" %d descriptors: \n", body->msgh_descriptor_count); + kprintf(" %d descriptors: \n", body->msgh_descriptor_count); - saddr = (mach_msg_descriptor_t *) (body + 1); - send = saddr + body->msgh_descriptor_count; + saddr = (mach_msg_descriptor_t *) (body + 1); + send = saddr + body->msgh_descriptor_count; - for ( ; saddr < send; saddr++ ) { - - type = saddr->type.type; + for (; saddr < send; saddr++) { + type = saddr->type.type; - switch (type) { - - case MACH_MSG_PORT_DESCRIPTOR: { - mach_msg_port_descriptor_t *dsc; + switch (type) { + case MACH_MSG_PORT_DESCRIPTOR: { + mach_msg_port_descriptor_t *dsc; - dsc = &saddr->port; - kprintf(" PORT name = %p disp = ", dsc->name); - ipc_print_type_name64(dsc->disposition); - kprintf("\n"); - break; - } - case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: - case MACH_MSG_OOL_DESCRIPTOR: { - mach_msg_ool_descriptor_t *dsc; - - dsc = (mach_msg_ool_descriptor_t *) &saddr->out_of_line; - kprintf(" OOL%s addr = %p size = 0x%x copy = %s %s\n", - type == MACH_MSG_OOL_DESCRIPTOR ? "" : " VOLATILE", - dsc->address, dsc->size, - mm_copy_options_string64(dsc->copy), - dsc->deallocate ? "DEALLOC" : ""); - break; - } - case MACH_MSG_OOL_PORTS_DESCRIPTOR : { - mach_msg_ool_ports_descriptor_t *dsc; - - dsc = (mach_msg_ool_ports_descriptor_t *) &saddr->ool_ports; - - kprintf(" OOL_PORTS addr = %p count = 0x%x ", - dsc->address, dsc->count); - kprintf("disp = "); - ipc_print_type_name64(dsc->disposition); - kprintf(" copy = %s %s\n", - mm_copy_options_string64(dsc->copy), - dsc->deallocate ? "DEALLOC" : ""); - break; - } + dsc = &saddr->port; + kprintf(" PORT name = %p disp = ", dsc->name); + ipc_print_type_name64(dsc->disposition); + kprintf("\n"); + break; + } + case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: + case MACH_MSG_OOL_DESCRIPTOR: { + mach_msg_ool_descriptor_t *dsc; + + dsc = (mach_msg_ool_descriptor_t *) &saddr->out_of_line; + kprintf(" OOL%s addr = %p size = 0x%x copy = %s %s\n", + type == MACH_MSG_OOL_DESCRIPTOR ? "" : " VOLATILE", + dsc->address, dsc->size, + mm_copy_options_string64(dsc->copy), + dsc->deallocate ? "DEALLOC" : ""); + break; + } + case MACH_MSG_OOL_PORTS_DESCRIPTOR: { + mach_msg_ool_ports_descriptor_t *dsc; + + dsc = (mach_msg_ool_ports_descriptor_t *) &saddr->ool_ports; + + kprintf(" OOL_PORTS addr = %p count = 0x%x ", + dsc->address, dsc->count); + kprintf("disp = "); + ipc_print_type_name64(dsc->disposition); + kprintf(" copy = %s %s\n", + mm_copy_options_string64(dsc->copy), + dsc->deallocate ? "DEALLOC" : ""); + break; + } + case MACH_MSG_GUARDED_PORT_DESCRIPTOR: { + mach_msg_guarded_port_descriptor_t *dsc; - default: { - kprintf(" UNKNOWN DESCRIPTOR 0x%x\n", type); - break; - } + dsc = (mach_msg_guarded_port_descriptor_t *)&saddr->guarded_port; + kprintf(" GUARDED_PORT name = %p flags = 0x%x disp = ", dsc->name, dsc->flags); + ipc_print_type_name64(dsc->disposition); + kprintf("\n"); + break; + } + default: { + kprintf(" UNKNOWN DESCRIPTOR 0x%x\n", type); + break; + } + } } - } } -#define DEBUG_IPC_KMSG_PRINT(kmsg,string) \ - __unreachable_ok_push \ - if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \ - ipc_kmsg_print64(kmsg, string); \ - } \ +#define DEBUG_IPC_KMSG_PRINT(kmsg, string) \ + __unreachable_ok_push \ + if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \ + ipc_kmsg_print64(kmsg, string); \ + } \ __unreachable_ok_pop -#define DEBUG_IPC_MSG_BODY_PRINT(body,size) \ - __unreachable_ok_push \ - if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \ - ipc_msg_body_print64(body,size);\ - } \ +#define DEBUG_IPC_MSG_BODY_PRINT(body, size) \ + __unreachable_ok_push \ + if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \ + ipc_msg_body_print64(body,size);\ + } \ __unreachable_ok_pop #else /* !DEBUG_MSGS_K64 */ -#define DEBUG_IPC_KMSG_PRINT(kmsg,string) -#define DEBUG_IPC_MSG_BODY_PRINT(body,size) +#define DEBUG_IPC_KMSG_PRINT(kmsg, string) +#define DEBUG_IPC_MSG_BODY_PRINT(body, size) #endif /* !DEBUG_MSGS_K64 */ -extern vm_map_t ipc_kernel_copy_map; -extern vm_size_t ipc_kmsg_max_space; -extern vm_size_t ipc_kmsg_max_vm_space; -extern vm_size_t ipc_kmsg_max_body_space; -extern vm_size_t msg_ool_size_small; +extern vm_map_t ipc_kernel_copy_map; +extern vm_size_t ipc_kmsg_max_space; +extern vm_size_t ipc_kmsg_max_vm_space; +extern vm_size_t ipc_kmsg_max_body_space; +extern vm_size_t msg_ool_size_small; -#define MSG_OOL_SIZE_SMALL msg_ool_size_small +#define MSG_OOL_SIZE_SMALL msg_ool_size_small #if defined(__LP64__) -#define MAP_SIZE_DIFFERS(map) (map->max_offset < MACH_VM_MAX_ADDRESS) -#define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor32_t -#define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor32_t +#define MAP_SIZE_DIFFERS(map) (map->max_offset < MACH_VM_MAX_ADDRESS) +#define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor32_t +#define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor32_t #else -#define MAP_SIZE_DIFFERS(map) (map->max_offset > VM_MAX_ADDRESS) -#define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor64_t -#define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor64_t +#define MAP_SIZE_DIFFERS(map) (map->max_offset > VM_MAX_ADDRESS) +#define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor64_t +#define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor64_t #endif -#define DESC_SIZE_ADJUSTMENT ((mach_msg_size_t)(sizeof(mach_msg_ool_descriptor64_t) - \ - sizeof(mach_msg_ool_descriptor32_t))) +#define DESC_SIZE_ADJUSTMENT ((mach_msg_size_t)(sizeof(mach_msg_ool_descriptor64_t) - \ + sizeof(mach_msg_ool_descriptor32_t))) /* scatter list macros */ -#define SKIP_PORT_DESCRIPTORS(s, c) \ -MACRO_BEGIN \ - if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \ - while ((c) > 0) { \ - if ((s)->type.type != MACH_MSG_PORT_DESCRIPTOR) \ - break; \ - (s)++; (c)--; \ - } \ - if (c == 0) \ - (s) = MACH_MSG_DESCRIPTOR_NULL; \ - } \ +#define SKIP_PORT_DESCRIPTORS(s, c) \ +MACRO_BEGIN \ + if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \ + while ((c) > 0) { \ + if ((s)->type.type != MACH_MSG_PORT_DESCRIPTOR) \ + break; \ + (s)++; (c)--; \ + } \ + if (c == 0) \ + (s) = MACH_MSG_DESCRIPTOR_NULL; \ + } \ MACRO_END -#define INCREMENT_SCATTER(s, c, d) \ -MACRO_BEGIN \ - if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \ - s = (d) ? (mach_msg_descriptor_t *) \ - ((OTHER_OOL_DESCRIPTOR *)(s) + 1) : \ - (s + 1); \ - (c)--; \ - } \ +#define INCREMENT_SCATTER(s, c, d) \ +MACRO_BEGIN \ + if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \ + s = (d) ? (mach_msg_descriptor_t *) \ + ((OTHER_OOL_DESCRIPTOR *)(s) + 1) : \ + (s + 1); \ + (c)--; \ + } \ MACRO_END #define KMSG_TRACE_FLAG_TRACED 0x000001 @@ -570,8 +577,9 @@ MACRO_END #define KMSG_TRACE_FLAG_TIMER 0x200000 #define KMSG_TRACE_FLAG_SEMA 0x400000 #define KMSG_TRACE_FLAG_DTMPOWNER 0x800000 +#define KMSG_TRACE_FLAG_GUARDED_DESC 0x1000000 -#define KMSG_TRACE_FLAGS_MASK 0xffffff +#define KMSG_TRACE_FLAGS_MASK 0x1ffffff #define KMSG_TRACE_FLAGS_SHIFT 8 #define KMSG_TRACE_PORTS_MASK 0xff @@ -579,10 +587,10 @@ MACRO_END #if (KDEBUG_LEVEL >= KDEBUG_LEVEL_STANDARD) #include -extern boolean_t kdebug_debugid_enabled(uint32_t debugid); -void ipc_kmsg_trace_send(ipc_kmsg_t kmsg, - mach_msg_option_t option) +void +ipc_kmsg_trace_send(ipc_kmsg_t kmsg, + mach_msg_option_t option) { task_t send_task = TASK_NULL; ipc_port_t dst_port, src_port; @@ -592,7 +600,7 @@ void ipc_kmsg_trace_send(ipc_kmsg_t kmsg, int kotype = 0; uint32_t msg_size = 0; - uint32_t msg_flags = KMSG_TRACE_FLAG_TRACED; + uint64_t msg_flags = KMSG_TRACE_FLAG_TRACED; uint32_t num_ports = 0; uint32_t send_pid, dst_pid; @@ -602,35 +610,43 @@ void ipc_kmsg_trace_send(ipc_kmsg_t kmsg, * significant amount of processing (and a port lock hold) in * the non-tracing case. */ - if (__probable((kdebug_enable & KDEBUG_TRACE) == 0)) + if (__probable((kdebug_enable & KDEBUG_TRACE) == 0)) { return; - if (!kdebug_debugid_enabled(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO))) + } + if (!kdebug_debugid_enabled(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO))) { return; + } msg = kmsg->ikm_header; - dst_port = (ipc_port_t)(msg->msgh_remote_port); - if (!IPC_PORT_VALID(dst_port)) + dst_port = msg->msgh_remote_port; + if (!IPC_PORT_VALID(dst_port)) { return; + } /* * Message properties / options */ - if ((option & (MACH_SEND_MSG|MACH_RCV_MSG)) == (MACH_SEND_MSG|MACH_RCV_MSG)) + if ((option & (MACH_SEND_MSG | MACH_RCV_MSG)) == (MACH_SEND_MSG | MACH_RCV_MSG)) { msg_flags |= KMSG_TRACE_FLAG_SNDRCV; + } if (msg->msgh_id >= is_iokit_subsystem.start && - msg->msgh_id < is_iokit_subsystem.end + 100) + msg->msgh_id < is_iokit_subsystem.end + 100) { msg_flags |= KMSG_TRACE_FLAG_IOKIT; + } /* magic XPC checkin message id (XPC_MESSAGE_ID_CHECKIN) from libxpc */ - else if (msg->msgh_id == 0x77303074u /* w00t */) + else if (msg->msgh_id == 0x77303074u /* w00t */) { msg_flags |= KMSG_TRACE_FLAG_CHECKIN; + } - if (msg->msgh_bits & MACH_MSGH_BITS_RAISEIMP) + if (msg->msgh_bits & MACH_MSGH_BITS_RAISEIMP) { msg_flags |= KMSG_TRACE_FLAG_RAISEIMP; + } - if (unsafe_convert_port_to_voucher(kmsg->ikm_voucher)) + if (unsafe_convert_port_to_voucher(kmsg->ikm_voucher)) { msg_flags |= KMSG_TRACE_FLAG_VOUCHER; + } /* * Sending task / port @@ -639,20 +655,23 @@ void ipc_kmsg_trace_send(ipc_kmsg_t kmsg, send_pid = task_pid(send_task); if (send_pid != 0) { - if (task_is_daemon(send_task)) + if (task_is_daemon(send_task)) { msg_flags |= KMSG_TRACE_FLAG_DAEMON_SRC; - else if (task_is_app(send_task)) + } else if (task_is_app(send_task)) { msg_flags |= KMSG_TRACE_FLAG_APP_SRC; + } } is_task_64bit = (send_task->map->max_offset > VM_MAX_ADDRESS); - if (is_task_64bit) + if (is_task_64bit) { msg_flags |= KMSG_TRACE_FLAG_SND64; + } - src_port = (ipc_port_t)(msg->msgh_local_port); + src_port = msg->msgh_local_port; if (src_port) { - if (src_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT) + if (src_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT) { msg_flags |= KMSG_TRACE_FLAG_SRC_NDFLTQ; + } switch (MACH_MSGH_BITS_LOCAL(msg->msgh_bits)) { case MACH_MSG_TYPE_MOVE_SEND_ONCE: msg_flags |= KMSG_TRACE_FLAG_SRC_SONCE; @@ -674,10 +693,11 @@ void ipc_kmsg_trace_send(ipc_kmsg_t kmsg, dst_pid = (uint32_t)0xfffffff0; } else if (dst_port->ip_tempowner) { msg_flags |= KMSG_TRACE_FLAG_DTMPOWNER; - if (IIT_NULL != dst_port->ip_imp_task) + if (IIT_NULL != dst_port->ip_imp_task) { dst_pid = task_pid(dst_port->ip_imp_task->iit_task); - else + } else { dst_pid = (uint32_t)0xfffffff1; + } } else if (dst_port->ip_receiver_name == MACH_PORT_NULL) { /* dst_port is otherwise in-transit */ dst_pid = (uint32_t)0xfffffff2; @@ -689,10 +709,11 @@ void ipc_kmsg_trace_send(ipc_kmsg_t kmsg, dst_space = dst_port->ip_receiver; if (dst_space && is_active(dst_space)) { dst_pid = task_pid(dst_space->is_task); - if (task_is_daemon(dst_space->is_task)) + if (task_is_daemon(dst_space->is_task)) { msg_flags |= KMSG_TRACE_FLAG_DAEMON_DST; - else if (task_is_app(dst_space->is_task)) + } else if (task_is_app(dst_space->is_task)) { msg_flags |= KMSG_TRACE_FLAG_APP_DST; + } } else { /* receiving task is being torn down */ dst_pid = (uint32_t)0xfffffff3; @@ -700,10 +721,12 @@ void ipc_kmsg_trace_send(ipc_kmsg_t kmsg, } } - if (dst_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT) + if (dst_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT) { msg_flags |= KMSG_TRACE_FLAG_DST_NDFLTQ; - if (imq_full(&dst_port->ip_messages)) + } + if (imq_full(&dst_port->ip_messages)) { msg_flags |= KMSG_TRACE_FLAG_DSTQFULL; + } kotype = ip_kotype(dst_port); @@ -721,13 +744,14 @@ void ipc_kmsg_trace_send(ipc_kmsg_t kmsg, case IKOT_IOKIT_CONNECT: case IKOT_IOKIT_OBJECT: case IKOT_IOKIT_IDENT: + case IKOT_UEXT_OBJECT: msg_flags |= KMSG_TRACE_FLAG_IOKIT; break; default: break; } - switch(MACH_MSGH_BITS_REMOTE(msg->msgh_bits)) { + switch (MACH_MSGH_BITS_REMOTE(msg->msgh_bits)) { case MACH_MSG_TYPE_PORT_SEND_ONCE: msg_flags |= KMSG_TRACE_FLAG_DST_SONCE; break; @@ -753,15 +777,17 @@ void ipc_kmsg_trace_send(ipc_kmsg_t kmsg, kern_dsc = (mach_msg_descriptor_t *)(msg_body + 1); /* this is gross: see ipc_kmsg_copyin_body()... */ - if (!is_task_64bit) + if (!is_task_64bit) { msg_size -= (dsc_count * 12); + } for (int i = 0; i < dsc_count; i++) { switch (kern_dsc[i].type.type) { case MACH_MSG_PORT_DESCRIPTOR: num_ports++; - if (is_task_64bit) + if (is_task_64bit) { msg_size -= 12; + } break; case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: case MACH_MSG_OOL_DESCRIPTOR: { @@ -771,22 +797,31 @@ void ipc_kmsg_trace_send(ipc_kmsg_t kmsg, msg_size += dsc->size; if ((dsc->size >= MSG_OOL_SIZE_SMALL) && (dsc->copy == MACH_MSG_PHYSICAL_COPY) && - !dsc->deallocate) + !dsc->deallocate) { msg_flags |= KMSG_TRACE_FLAG_PCPY; - else if (dsc->size <= MSG_OOL_SIZE_SMALL) + } else if (dsc->size <= MSG_OOL_SIZE_SMALL) { msg_flags |= KMSG_TRACE_FLAG_PCPY; - else + } else { msg_flags |= KMSG_TRACE_FLAG_VCPY; - if (is_task_64bit) + } + if (is_task_64bit) { msg_size -= 16; - } break; + } + } break; case MACH_MSG_OOL_PORTS_DESCRIPTOR: { - mach_msg_ool_ports_descriptor_t *dsc; + mach_msg_ool_ports_descriptor_t *dsc; dsc = (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i]; num_ports += dsc->count; - if (is_task_64bit) + if (is_task_64bit) { + msg_size -= 16; + } + } break; + case MACH_MSG_GUARDED_PORT_DESCRIPTOR: + num_ports++; + msg_flags |= KMSG_TRACE_FLAG_GUARDED_DESC; + if (is_task_64bit) { msg_size -= 16; - } break; + } default: break; } @@ -797,9 +832,9 @@ void ipc_kmsg_trace_send(ipc_kmsg_t kmsg, * Trailer contents */ trailer = (mach_msg_trailer_t *)((vm_offset_t)msg + - (vm_offset_t)msg->msgh_size); + round_msg((vm_offset_t)msg->msgh_size)); if (trailer->msgh_trailer_size <= sizeof(mach_msg_security_trailer_t)) { - extern security_token_t KERNEL_SECURITY_TOKEN; + extern const security_token_t KERNEL_SECURITY_TOKEN; mach_msg_security_trailer_t *strailer; strailer = (mach_msg_security_trailer_t *)trailer; /* @@ -807,53 +842,77 @@ void ipc_kmsg_trace_send(ipc_kmsg_t kmsg, * like self-talk because the sending port is not reset. */ if (memcmp(&strailer->msgh_sender, - &KERNEL_SECURITY_TOKEN, - sizeof(KERNEL_SECURITY_TOKEN)) == 0) { + &KERNEL_SECURITY_TOKEN, + sizeof(KERNEL_SECURITY_TOKEN)) == 0) { send_pid = 0; msg_flags &= ~(KMSG_TRACE_FLAG_APP_SRC | KMSG_TRACE_FLAG_DAEMON_SRC); } } - KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, - (uintptr_t)send_pid, - (uintptr_t)dst_pid, - (uintptr_t)msg_size, - (uintptr_t)( - ((msg_flags & KMSG_TRACE_FLAGS_MASK) << KMSG_TRACE_FLAGS_SHIFT) | - ((num_ports & KMSG_TRACE_PORTS_MASK) << KMSG_TRACE_PORTS_SHIFT) - ) - ); + KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, + (uintptr_t)send_pid, + (uintptr_t)dst_pid, + (uintptr_t)msg_size, + (uintptr_t)( + ((msg_flags & KMSG_TRACE_FLAGS_MASK) << KMSG_TRACE_FLAGS_SHIFT) | + ((num_ports & KMSG_TRACE_PORTS_MASK) << KMSG_TRACE_PORTS_SHIFT) + ) + ); } #endif /* zone for cached ipc_kmsg_t structures */ -zone_t ipc_kmsg_zone; +zone_t ipc_kmsg_zone; /* * Forward declarations */ void ipc_kmsg_clean( - ipc_kmsg_t kmsg); + ipc_kmsg_t kmsg); void ipc_kmsg_clean_body( - ipc_kmsg_t kmsg, - mach_msg_type_number_t number, - mach_msg_descriptor_t *desc); + ipc_kmsg_t kmsg, + mach_msg_type_number_t number, + mach_msg_descriptor_t *desc); void ipc_kmsg_clean_partial( - ipc_kmsg_t kmsg, - mach_msg_type_number_t number, - mach_msg_descriptor_t *desc, - vm_offset_t paddr, - vm_size_t length); + ipc_kmsg_t kmsg, + mach_msg_type_number_t number, + mach_msg_descriptor_t *desc, + vm_offset_t paddr, + vm_size_t length); mach_msg_return_t ipc_kmsg_copyin_body( - ipc_kmsg_t kmsg, - ipc_space_t space, - vm_map_t map); + ipc_kmsg_t kmsg, + ipc_space_t space, + vm_map_t map, + mach_msg_option_t *optionp); + + +extern int enforce_strict_reply; + +static void +ipc_kmsg_link_reply_context_locked( + ipc_port_t reply_port, + ipc_port_t voucher_port); -extern int thread_qos_from_pthread_priority(unsigned long, unsigned long *); +static kern_return_t +ipc_kmsg_validate_reply_port_locked( + ipc_port_t reply_port, + mach_msg_option_t options); + +static mach_msg_return_t +ipc_kmsg_validate_reply_context_locked( + mach_msg_option_t option, + ipc_port_t dest_port, + ipc_voucher_t voucher, + mach_port_name_t voucher_name); + +/* we can't include the BSD header here... */ +#ifndef PERSONA_ID_NONE +#define PERSONA_ID_NONE ((uint32_t)-1) +#endif /* * We keep a per-processor cache of kernel message buffers. @@ -881,7 +940,7 @@ ipc_kmsg_alloc( /* * LP64support - * Pad the allocation in case we need to expand the - * message descrptors for user spaces with pointers larger than + * message descriptors for user spaces with pointers larger than * the kernel's own, or vice versa. We don't know how many descriptors * there are yet, so just assume the whole body could be * descriptors (if there could be any at all). @@ -894,41 +953,29 @@ ipc_kmsg_alloc( mach_msg_size_t size = msg_and_trailer_size - MAX_TRAILER_SIZE; /* compare against implementation upper limit for the body */ - if (size > ipc_kmsg_max_body_space) + if (size > ipc_kmsg_max_body_space) { return IKM_NULL; + } if (size > sizeof(mach_msg_base_t)) { mach_msg_size_t max_desc = (mach_msg_size_t)(((size - sizeof(mach_msg_base_t)) / - sizeof(mach_msg_ool_descriptor32_t)) * - DESC_SIZE_ADJUSTMENT); + sizeof(mach_msg_ool_descriptor32_t)) * + DESC_SIZE_ADJUSTMENT); /* make sure expansion won't cause wrap */ - if (msg_and_trailer_size > MACH_MSG_SIZE_MAX - max_desc) + if (msg_and_trailer_size > MACH_MSG_SIZE_MAX - max_desc) { return IKM_NULL; + } max_expanded_size = msg_and_trailer_size + max_desc; - } else - max_expanded_size = msg_and_trailer_size; - - if (max_expanded_size < IKM_SAVED_MSG_SIZE) - max_expanded_size = IKM_SAVED_MSG_SIZE; /* round up for ikm_cache */ + } else { + max_expanded_size = msg_and_trailer_size; + } + if (max_expanded_size < IKM_SAVED_MSG_SIZE) { + max_expanded_size = IKM_SAVED_MSG_SIZE; /* round up for ikm_cache */ + } if (max_expanded_size == IKM_SAVED_MSG_SIZE) { - struct ikm_cache *cache; - unsigned int i; - - disable_preemption(); - cache = &PROCESSOR_DATA(current_processor(), ikm_cache); - if ((i = cache->avail) > 0) { - assert(i <= IKM_STASH); - kmsg = cache->entries[--i]; - cache->avail = i; - enable_preemption(); - ikm_check_init(kmsg, max_expanded_size); - ikm_set_header(kmsg, msg_and_trailer_size); - return (kmsg); - } - enable_preemption(); kmsg = (ipc_kmsg_t)zalloc(ipc_kmsg_zone); } else { kmsg = (ipc_kmsg_t)kalloc(ikm_plus_overhead(max_expanded_size)); @@ -939,7 +986,7 @@ ipc_kmsg_alloc( ikm_set_header(kmsg, msg_and_trailer_size); } - return(kmsg); + return kmsg; } /* @@ -956,16 +1003,16 @@ ipc_kmsg_alloc( void ipc_kmsg_free( - ipc_kmsg_t kmsg) + ipc_kmsg_t kmsg) { mach_msg_size_t size = kmsg->ikm_size; ipc_port_t port; assert(!IP_VALID(kmsg->ikm_voucher)); - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_FREE) | DBG_FUNC_NONE, - VM_KERNEL_ADDRPERM((uintptr_t)kmsg), - 0, 0, 0, 0); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_FREE) | DBG_FUNC_NONE, + VM_KERNEL_ADDRPERM((uintptr_t)kmsg), + 0, 0, 0, 0); /* * Check to see if the message is bound to the port. If so, @@ -982,26 +1029,11 @@ ipc_kmsg_free( ip_release(port); return; } - ip_unlock(port); + ip_unlock(port); ip_release(port); /* May be last reference */ } - /* - * Peek and see if it has to go back in the cache. - */ if (kmsg->ikm_size == IKM_SAVED_MSG_SIZE) { - struct ikm_cache *cache; - unsigned int i; - - disable_preemption(); - cache = &PROCESSOR_DATA(current_processor(), ikm_cache); - if ((i = cache->avail) < IKM_STASH) { - cache->entries[i] = kmsg; - cache->avail = i + 1; - enable_preemption(); - return; - } - enable_preemption(); zfree(ipc_kmsg_zone, kmsg); return; } @@ -1017,8 +1049,8 @@ ipc_kmsg_free( void ipc_kmsg_enqueue( - ipc_kmsg_queue_t queue, - ipc_kmsg_t kmsg) + ipc_kmsg_queue_t queue, + ipc_kmsg_t kmsg) { ipc_kmsg_t first = queue->ikmq_base; ipc_kmsg_t last; @@ -1050,8 +1082,8 @@ ipc_kmsg_enqueue( boolean_t ipc_kmsg_enqueue_qos( - ipc_kmsg_queue_t queue, - ipc_kmsg_t kmsg) + ipc_kmsg_queue_t queue, + ipc_kmsg_t kmsg) { ipc_kmsg_t first = queue->ikmq_base; ipc_kmsg_t prev; @@ -1075,13 +1107,13 @@ ipc_kmsg_enqueue_qos( /* apply QoS overrides towards the head */ override = kmsg->ikm_qos_override; while (prev != kmsg && - override > prev->ikm_qos_override) { + override > prev->ikm_qos_override) { prev->ikm_qos_override = override; prev = prev->ikm_prev; } /* did we adjust everything? */ - return (prev == kmsg); + return prev == kmsg; } /* @@ -1099,7 +1131,7 @@ ipc_kmsg_enqueue_qos( boolean_t ipc_kmsg_override_qos( - ipc_kmsg_queue_t queue, + ipc_kmsg_queue_t queue, ipc_kmsg_t kmsg, mach_msg_priority_t override) { @@ -1109,9 +1141,10 @@ ipc_kmsg_override_qos( /* apply QoS overrides towards the head */ while (override > cur->ikm_qos_override) { cur->ikm_qos_override = override; - if (cur == first) + if (cur == first) { return TRUE; - cur = cur->ikm_prev; + } + cur = cur->ikm_prev; } return FALSE; } @@ -1124,14 +1157,15 @@ ipc_kmsg_override_qos( ipc_kmsg_t ipc_kmsg_dequeue( - ipc_kmsg_queue_t queue) + ipc_kmsg_queue_t queue) { ipc_kmsg_t first; first = ipc_kmsg_queue_first(queue); - if (first != IKM_NULL) + if (first != IKM_NULL) { ipc_kmsg_rmqueue(queue, first); + } return first; } @@ -1144,8 +1178,8 @@ ipc_kmsg_dequeue( void ipc_kmsg_rmqueue( - ipc_kmsg_queue_t queue, - ipc_kmsg_t kmsg) + ipc_kmsg_queue_t queue, + ipc_kmsg_t kmsg) { ipc_kmsg_t next, prev; @@ -1162,12 +1196,13 @@ ipc_kmsg_rmqueue( } else { if (__improbable(next->ikm_prev != kmsg || prev->ikm_next != kmsg)) { panic("ipc_kmsg_rmqueue: inconsistent prev/next pointers. " - "(prev->next: %p, next->prev: %p, kmsg: %p)", - prev->ikm_next, next->ikm_prev, kmsg); + "(prev->next: %p, next->prev: %p, kmsg: %p)", + prev->ikm_next, next->ikm_prev, kmsg); } - if (queue->ikmq_base == kmsg) + if (queue->ikmq_base == kmsg) { queue->ikmq_base = next; + } next->ikm_prev = prev; prev->ikm_next = next; @@ -1186,16 +1221,17 @@ ipc_kmsg_rmqueue( ipc_kmsg_t ipc_kmsg_queue_next( - ipc_kmsg_queue_t queue, - ipc_kmsg_t kmsg) + ipc_kmsg_queue_t queue, + ipc_kmsg_t kmsg) { ipc_kmsg_t next; assert(queue->ikmq_base != IKM_NULL); next = kmsg->ikm_next; - if (queue->ikmq_base == next) + if (queue->ikmq_base == next) { next = IKM_NULL; + } return next; } @@ -1212,7 +1248,7 @@ ipc_kmsg_queue_next( void ipc_kmsg_destroy( - ipc_kmsg_t kmsg) + ipc_kmsg_t kmsg) { /* * Destroying a message can cause more messages to be destroyed. @@ -1220,8 +1256,9 @@ ipc_kmsg_destroy( * destruction queue. If this was the first message on the * queue, this instance must process the full queue. */ - if (ipc_kmsg_delayed_destroy(kmsg)) + if (ipc_kmsg_delayed_destroy(kmsg)) { ipc_kmsg_reap_delayed(); + } } /* @@ -1233,7 +1270,8 @@ ipc_kmsg_destroy( * deferred messages. */ -boolean_t ipc_kmsg_delayed_destroy( +boolean_t +ipc_kmsg_delayed_destroy( ipc_kmsg_t kmsg) { ipc_kmsg_queue_t queue = &(current_thread()->ith_messages); @@ -1281,86 +1319,99 @@ ipc_kmsg_reap_delayed(void) static unsigned int _ipc_kmsg_clean_invalid_desc = 0; void ipc_kmsg_clean_body( - __unused ipc_kmsg_t kmsg, - mach_msg_type_number_t number, - mach_msg_descriptor_t *saddr) + __unused ipc_kmsg_t kmsg, + mach_msg_type_number_t number, + mach_msg_descriptor_t *saddr) { - mach_msg_type_number_t i; + mach_msg_type_number_t i; - if ( number == 0 ) - return; + if (number == 0) { + return; + } - for (i = 0 ; i < number; i++, saddr++ ) { - - switch (saddr->type.type) { - - case MACH_MSG_PORT_DESCRIPTOR: { - mach_msg_port_descriptor_t *dsc; + for (i = 0; i < number; i++, saddr++) { + switch (saddr->type.type) { + case MACH_MSG_PORT_DESCRIPTOR: { + mach_msg_port_descriptor_t *dsc; - dsc = &saddr->port; + dsc = &saddr->port; - /* - * Destroy port rights carried in the message - */ - if (!IO_VALID((ipc_object_t) dsc->name)) - continue; - ipc_object_destroy((ipc_object_t) dsc->name, dsc->disposition); - break; - } - case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: - case MACH_MSG_OOL_DESCRIPTOR : { - mach_msg_ool_descriptor_t *dsc; - - dsc = (mach_msg_ool_descriptor_t *)&saddr->out_of_line; - - /* - * Destroy memory carried in the message - */ - if (dsc->size == 0) { - assert(dsc->address == (void *) 0); - } else { - vm_map_copy_discard((vm_map_copy_t) dsc->address); + /* + * Destroy port rights carried in the message + */ + if (!IP_VALID(dsc->name)) { + continue; + } + ipc_object_destroy(ip_to_object(dsc->name), dsc->disposition); + break; } - break; - } - case MACH_MSG_OOL_PORTS_DESCRIPTOR : { - ipc_object_t *objects; - mach_msg_type_number_t j; - mach_msg_ool_ports_descriptor_t *dsc; + case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: + case MACH_MSG_OOL_DESCRIPTOR: { + mach_msg_ool_descriptor_t *dsc; - dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports; - objects = (ipc_object_t *) dsc->address; + dsc = (mach_msg_ool_descriptor_t *)&saddr->out_of_line; - if (dsc->count == 0) { + /* + * Destroy memory carried in the message + */ + if (dsc->size == 0) { + assert(dsc->address == (void *) 0); + } else { + vm_map_copy_discard((vm_map_copy_t) dsc->address); + } break; } + case MACH_MSG_OOL_PORTS_DESCRIPTOR: { + ipc_object_t *objects; + mach_msg_type_number_t j; + mach_msg_ool_ports_descriptor_t *dsc; - assert(objects != (ipc_object_t *) 0); - - /* destroy port rights carried in the message */ - - for (j = 0; j < dsc->count; j++) { - ipc_object_t object = objects[j]; - - if (!IO_VALID(object)) - continue; - - ipc_object_destroy(object, dsc->disposition); - } + dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports; + objects = (ipc_object_t *) dsc->address; - /* destroy memory carried in the message */ + if (dsc->count == 0) { + break; + } - assert(dsc->count != 0); + assert(objects != (ipc_object_t *) 0); - kfree(dsc->address, - (vm_size_t) dsc->count * sizeof(mach_port_t)); - break; - } - default : { - _ipc_kmsg_clean_invalid_desc++; /* don't understand this type of descriptor */ - } + /* destroy port rights carried in the message */ + + for (j = 0; j < dsc->count; j++) { + ipc_object_t object = objects[j]; + + if (!IO_VALID(object)) { + continue; + } + + ipc_object_destroy(object, dsc->disposition); + } + + /* destroy memory carried in the message */ + + assert(dsc->count != 0); + + kfree(dsc->address, + (vm_size_t) dsc->count * sizeof(mach_port_t)); + break; + } + case MACH_MSG_GUARDED_PORT_DESCRIPTOR: { + mach_msg_guarded_port_descriptor_t *dsc = (typeof(dsc)) & saddr->guarded_port; + + /* + * Destroy port rights carried in the message + */ + if (!IP_VALID(dsc->name)) { + continue; + } + ipc_object_destroy(ip_to_object(dsc->name), dsc->disposition); + break; + } + default: { + _ipc_kmsg_clean_invalid_desc++; /* don't understand this type of descriptor */ + } + } } - } } /* @@ -1378,11 +1429,11 @@ ipc_kmsg_clean_body( void ipc_kmsg_clean_partial( - ipc_kmsg_t kmsg, - mach_msg_type_number_t number, - mach_msg_descriptor_t *desc, - vm_offset_t paddr, - vm_size_t length) + ipc_kmsg_t kmsg, + mach_msg_type_number_t number, + mach_msg_descriptor_t *desc, + vm_offset_t paddr, + vm_size_t length) { ipc_object_t object; mach_msg_bits_t mbits = kmsg->ikm_header->msgh_bits; @@ -1390,15 +1441,16 @@ ipc_kmsg_clean_partial( /* deal with importance chain while we still have dest and voucher references */ ipc_importance_clean(kmsg); - object = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; + object = ip_to_object(kmsg->ikm_header->msgh_remote_port); assert(IO_VALID(object)); ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits)); - object = (ipc_object_t) kmsg->ikm_header->msgh_local_port; - if (IO_VALID(object)) + object = ip_to_object(kmsg->ikm_header->msgh_local_port); + if (IO_VALID(object)) { ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits)); + } - object = (ipc_object_t) kmsg->ikm_voucher; + object = ip_to_object(kmsg->ikm_voucher); if (IO_VALID(object)) { assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND); ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND); @@ -1423,7 +1475,7 @@ ipc_kmsg_clean_partial( void ipc_kmsg_clean( - ipc_kmsg_t kmsg) + ipc_kmsg_t kmsg) { ipc_object_t object; mach_msg_bits_t mbits; @@ -1432,15 +1484,17 @@ ipc_kmsg_clean( ipc_importance_clean(kmsg); mbits = kmsg->ikm_header->msgh_bits; - object = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; - if (IO_VALID(object)) + object = ip_to_object(kmsg->ikm_header->msgh_remote_port); + if (IO_VALID(object)) { ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits)); + } - object = (ipc_object_t) kmsg->ikm_header->msgh_local_port; - if (IO_VALID(object)) + object = ip_to_object(kmsg->ikm_header->msgh_local_port); + if (IO_VALID(object)) { ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits)); + } - object = (ipc_object_t) kmsg->ikm_voucher; + object = ip_to_object(kmsg->ikm_voucher); if (IO_VALID(object)) { assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND); ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND); @@ -1452,7 +1506,7 @@ ipc_kmsg_clean( body = (mach_msg_body_t *) (kmsg->ikm_header + 1); ipc_kmsg_clean_body(kmsg, body->msgh_descriptor_count, - (mach_msg_descriptor_t *)(body + 1)); + (mach_msg_descriptor_t *)(body + 1)); } } @@ -1466,22 +1520,16 @@ ipc_kmsg_clean( void ipc_kmsg_set_prealloc( - ipc_kmsg_t kmsg, - ipc_port_t port) + ipc_kmsg_t kmsg, + ipc_port_t port) { assert(kmsg->ikm_prealloc == IP_NULL); - + kmsg->ikm_prealloc = IP_NULL; - /* take the mqueue lock since the sync qos is protected under it */ - imq_lock(&port->ip_messages); - /* copy the sync qos values to kmsg */ - for (int i = 0; i < THREAD_QOS_LAST; i++) { - kmsg->sync_qos[i] = port_sync_qos(port, i); - } - kmsg->special_port_qos = port_special_qos(port); + assert(port_send_turnstile(port) == TURNSTILE_NULL); + kmsg->ikm_turnstile = TURNSTILE_NULL; IP_SET_PREALLOC(port, kmsg); - imq_unlock(&port->ip_messages); } /* @@ -1493,23 +1541,14 @@ ipc_kmsg_set_prealloc( */ void ipc_kmsg_clear_prealloc( - ipc_kmsg_t kmsg, - ipc_port_t port) + ipc_kmsg_t kmsg, + ipc_port_t port) { - assert(kmsg->ikm_prealloc == port); - - kmsg->ikm_prealloc = IP_NULL; - - /* take the mqueue lock since the sync qos is protected under it */ + /* take the mqueue lock since the turnstile is protected under it */ imq_lock(&port->ip_messages); IP_CLEAR_PREALLOC(port, kmsg); - - /* copy the sync qos values from kmsg to port */ - for (int i = 0; i < THREAD_QOS_LAST; i++) { - set_port_sync_qos(port, i, kmsg->sync_qos[i]); - } - set_port_special_qos(port, kmsg->special_port_qos); + set_port_send_turnstile(port, kmsg->ikm_turnstile); imq_unlock(&port->ip_messages); } @@ -1523,8 +1562,9 @@ ipc_kmsg_t ipc_kmsg_prealloc(mach_msg_size_t size) { #if defined(__LP64__) - if (size > MACH_MSG_SIZE_MAX - LEGACY_HEADER_SIZE_DELTA) + if (size > MACH_MSG_SIZE_MAX - LEGACY_HEADER_SIZE_DELTA) { return IKM_NULL; + } size += LEGACY_HEADER_SIZE_DELTA; #endif @@ -1550,38 +1590,43 @@ ipc_kmsg_prealloc(mach_msg_size_t size) mach_msg_return_t ipc_kmsg_get( - mach_vm_address_t msg_addr, - mach_msg_size_t size, - ipc_kmsg_t *kmsgp) + mach_vm_address_t msg_addr, + mach_msg_size_t size, + ipc_kmsg_t *kmsgp) { - mach_msg_size_t msg_and_trailer_size; - ipc_kmsg_t kmsg; - mach_msg_max_trailer_t *trailer; - mach_msg_legacy_base_t legacy_base; + mach_msg_size_t msg_and_trailer_size; + ipc_kmsg_t kmsg; + mach_msg_max_trailer_t *trailer; + mach_msg_legacy_base_t legacy_base; mach_msg_size_t len_copied; legacy_base.body.msgh_descriptor_count = 0; - if ((size < sizeof(mach_msg_legacy_header_t)) || (size & 3)) + if ((size < sizeof(mach_msg_legacy_header_t)) || (size & 3)) { return MACH_SEND_MSG_TOO_SMALL; + } - if (size > ipc_kmsg_max_body_space) + if (size > ipc_kmsg_max_body_space) { return MACH_SEND_TOO_LARGE; + } - if(size == sizeof(mach_msg_legacy_header_t)) + if (size == sizeof(mach_msg_legacy_header_t)) { len_copied = sizeof(mach_msg_legacy_header_t); - else + } else { len_copied = sizeof(mach_msg_legacy_base_t); + } - if (copyinmsg(msg_addr, (char *)&legacy_base, len_copied)) + if (copyinmsg(msg_addr, (char *)&legacy_base, len_copied)) { return MACH_SEND_INVALID_DATA; + } /* * If the message claims to be complex, it must at least * have the length of a "base" message (header + dsc_count). */ if (len_copied < sizeof(mach_msg_legacy_base_t) && - (legacy_base.header.msgh_bits & MACH_MSGH_BITS_COMPLEX)) + (legacy_base.header.msgh_bits & MACH_MSGH_BITS_COMPLEX)) { return MACH_SEND_MSG_TOO_SMALL; + } msg_addr += sizeof(legacy_base.header); #if defined(__LP64__) @@ -1591,37 +1636,38 @@ ipc_kmsg_get( __unreachable_ok_push if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { unsigned int j; - for (j=0; jikm_header->msgh_size = size; - kmsg->ikm_header->msgh_bits = legacy_base.header.msgh_bits; - kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(legacy_base.header.msgh_remote_port); - kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(legacy_base.header.msgh_local_port); - kmsg->ikm_header->msgh_voucher_port = legacy_base.header.msgh_voucher_port; - kmsg->ikm_header->msgh_id = legacy_base.header.msgh_id; + kmsg->ikm_header->msgh_size = size; + kmsg->ikm_header->msgh_bits = legacy_base.header.msgh_bits; + kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(legacy_base.header.msgh_remote_port); + kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(legacy_base.header.msgh_local_port); + kmsg->ikm_header->msgh_voucher_port = legacy_base.header.msgh_voucher_port; + kmsg->ikm_header->msgh_id = legacy_base.header.msgh_id; DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_get header:\n" - " size: 0x%.8x\n" - " bits: 0x%.8x\n" - " remote_port: %p\n" - " local_port: %p\n" - " voucher_port: 0x%.8x\n" - " id: %.8d\n", - kmsg->ikm_header->msgh_size, - kmsg->ikm_header->msgh_bits, - kmsg->ikm_header->msgh_remote_port, - kmsg->ikm_header->msgh_local_port, - kmsg->ikm_header->msgh_voucher_port, - kmsg->ikm_header->msgh_id); + " size: 0x%.8x\n" + " bits: 0x%.8x\n" + " remote_port: %p\n" + " local_port: %p\n" + " voucher_port: 0x%.8x\n" + " id: %.8d\n", + kmsg->ikm_header->msgh_size, + kmsg->ikm_header->msgh_bits, + kmsg->ikm_header->msgh_remote_port, + kmsg->ikm_header->msgh_local_port, + kmsg->ikm_header->msgh_voucher_port, + kmsg->ikm_header->msgh_id); if (copyinmsg(msg_addr, (char *)(kmsg->ikm_header + 1), size - (mach_msg_size_t)sizeof(mach_msg_header_t))) { ipc_kmsg_free(kmsg); @@ -1630,19 +1676,17 @@ ipc_kmsg_get( /* unreachable if !DEBUG */ __unreachable_ok_push - if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) - { + if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { kprintf("body: size: %lu\n", (size - sizeof(mach_msg_header_t))); uint32_t i; - for(i=0;i*4 < (size - sizeof(mach_msg_header_t));i++) - { - kprintf("%.4x\n",((uint32_t *)(kmsg->ikm_header + 1))[i]); + for (i = 0; i * 4 < (size - sizeof(mach_msg_header_t)); i++) { + kprintf("%.4x\n", ((uint32_t *)(kmsg->ikm_header + 1))[i]); } } __unreachable_ok_pop DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_get()"); - /* + /* * I reserve for the trailer the largest space (MAX_TRAILER_SIZE) * However, the internal size field of the trailer (msgh_trailer_size) * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to optimize @@ -1655,9 +1699,11 @@ ipc_kmsg_get( trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE; #ifdef ppc - if(trcWork.traceMask) dbgTrace(0x1100, (unsigned int)kmsg->ikm_header->msgh_id, - (unsigned int)kmsg->ikm_header->msgh_remote_port, - (unsigned int)kmsg->ikm_header->msgh_local_port, 0); + if (trcWork.traceMask) { + dbgTrace(0x1100, (unsigned int)kmsg->ikm_header->msgh_id, + (unsigned int)kmsg->ikm_header->msgh_remote_port, + (unsigned int)kmsg->ikm_header->msgh_local_port, 0); + } #endif trailer->msgh_labels.sender = 0; @@ -1683,19 +1729,19 @@ ipc_kmsg_get( mach_msg_return_t ipc_kmsg_get_from_kernel( - mach_msg_header_t *msg, - mach_msg_size_t size, - ipc_kmsg_t *kmsgp) + mach_msg_header_t *msg, + mach_msg_size_t size, + ipc_kmsg_t *kmsgp) { - ipc_kmsg_t kmsg; - mach_msg_size_t msg_and_trailer_size; + ipc_kmsg_t kmsg; + mach_msg_size_t msg_and_trailer_size; mach_msg_max_trailer_t *trailer; - ipc_port_t dest_port; + ipc_port_t dest_port; assert(size >= sizeof(mach_msg_header_t)); assert((size & 3) == 0); - dest_port = (ipc_port_t)msg->msgh_remote_port; + dest_port = msg->msgh_remote_port; msg_and_trailer_size = size + MAX_TRAILER_SIZE; @@ -1722,7 +1768,7 @@ ipc_kmsg_get_from_kernel( if (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) { assert(size > sizeof(mach_msg_base_t)); max_desc = ((mach_msg_base_t *)msg)->body.msgh_descriptor_count * - DESC_SIZE_ADJUSTMENT; + DESC_SIZE_ADJUSTMENT; } #endif if (msg_and_trailer_size > kmsg->ikm_size - max_desc) { @@ -1732,12 +1778,11 @@ ipc_kmsg_get_from_kernel( ikm_prealloc_set_inuse(kmsg, dest_port); ikm_set_header(kmsg, msg_and_trailer_size); ip_unlock(dest_port); - } - else - { + } else { kmsg = ipc_kmsg_alloc(msg_and_trailer_size); - if (kmsg == IKM_NULL) + if (kmsg == IKM_NULL) { return MACH_SEND_NO_BUFFER; + } } (void) memcpy((void *) kmsg->ikm_header, (const void *) msg, size); @@ -1746,14 +1791,14 @@ ipc_kmsg_get_from_kernel( kmsg->ikm_header->msgh_size = size; - /* + /* * I reserve for the trailer the largest space (MAX_TRAILER_SIZE) * However, the internal size field of the trailer (msgh_trailer_size) * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to * optimize the cases where no implicit data is requested. */ - trailer = (mach_msg_max_trailer_t *) - ((vm_offset_t)kmsg->ikm_header + size); + trailer = (mach_msg_max_trailer_t *) + ((vm_offset_t)kmsg->ikm_header + size); trailer->msgh_sender = KERNEL_SECURITY_TOKEN; trailer->msgh_audit = KERNEL_AUDIT_TOKEN; trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0; @@ -1783,13 +1828,11 @@ ipc_kmsg_get_from_kernel( * MACH_SEND_INTERRUPTED Caller still has message. * MACH_SEND_INVALID_DEST Caller still has message. */ - - mach_msg_return_t ipc_kmsg_send( - ipc_kmsg_t kmsg, - mach_msg_option_t option, - mach_msg_timeout_t send_timeout) + ipc_kmsg_t kmsg, + mach_msg_option_t option, + mach_msg_timeout_t send_timeout) { ipc_port_t port; thread_t th = current_thread(); @@ -1806,7 +1849,7 @@ ipc_kmsg_send( } #if IMPORTANCE_INHERITANCE - boolean_t did_importance = FALSE; + bool did_importance = false; #if IMPORTANCE_TRACE mach_msg_id_t imp_msgh_id = -1; int sender_pid = -1; @@ -1816,16 +1859,32 @@ ipc_kmsg_send( /* don't allow the creation of a circular loop */ if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_CIRCULAR) { ipc_kmsg_destroy(kmsg); - KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_MSGH_BITS_CIRCULAR); + KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_MSGH_BITS_CIRCULAR); return MACH_MSG_SUCCESS; } ipc_voucher_send_preprocessing(kmsg); - port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port; + port = kmsg->ikm_header->msgh_remote_port; assert(IP_VALID(port)); ip_lock(port); + /* + * If the destination has been guarded with a reply context, and the + * sender is consuming a send-once right, then assume this is a reply + * to an RPC and we need to validate that this sender is currently in + * the correct context. + */ + if (enforce_strict_reply && port->ip_reply_context != 0 && + ((option & MACH_SEND_KERNEL) == 0) && + MACH_MSGH_BITS_REMOTE(kmsg->ikm_header->msgh_bits) == MACH_MSG_TYPE_PORT_SEND_ONCE) { + error = ipc_kmsg_validate_reply_context_locked(option, port, th->ith_voucher, th->ith_voucher_name); + if (error != MACH_MSG_SUCCESS) { + ip_unlock(port); + return error; + } + } + #if IMPORTANCE_INHERITANCE retry: #endif /* IMPORTANCE_INHERITANCE */ @@ -1837,25 +1896,36 @@ retry: if (!ip_active(port)) { ip_unlock(port); #if MACH_FLIPC - if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) - flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE); + if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) { + flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE); + } #endif + if (did_importance) { + /* + * We're going to pretend we delivered this message + * successfully, and just eat the kmsg. However, the + * kmsg is actually visible via the importance_task! + * We need to cleanup this linkage before we destroy + * the message, and more importantly before we set the + * msgh_remote_port to NULL. See: 34302571 + */ + ipc_importance_clean(kmsg); + } ip_release(port); /* JMM - Future: release right, not just ref */ kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL; ipc_kmsg_destroy(kmsg); - KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST); + KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST); return MACH_MSG_SUCCESS; } if (port->ip_receiver == ipc_space_kernel) { - /* * We can check ip_receiver == ipc_space_kernel * before checking that the port is active because * ipc_port_dealloc_kernel clears ip_receiver * before destroying a kernel port. */ - assert(ip_active(port)); + require_ip_active(port); port->ip_messages.imq_seqno++; ip_unlock(port); @@ -1865,18 +1935,20 @@ retry: * Call the server routine, and get the reply message to send. */ kmsg = ipc_kobject_server(kmsg, option); - if (kmsg == IKM_NULL) + if (kmsg == IKM_NULL) { return MACH_MSG_SUCCESS; + } /* restart the KMSG_INFO tracing for the reply message */ - KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_START); - port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port; + KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_START); + port = kmsg->ikm_header->msgh_remote_port; assert(IP_VALID(port)); ip_lock(port); /* fall thru with reply - same options */ kernel_reply = TRUE; - if (!ip_active(port)) + if (!ip_active(port)) { error = MACH_SEND_INVALID_DEST; + } } #if IMPORTANCE_INHERITANCE @@ -1885,10 +1957,11 @@ retry: * propagation. That routine can drop the port lock temporarily. * If it does we'll have to revalidate the destination. */ - if (did_importance == FALSE) { - did_importance = TRUE; - if (ipc_importance_send(kmsg, option)) - goto retry; + if (!did_importance) { + did_importance = true; + if (ipc_importance_send(kmsg, option)) { + goto retry; + } } #endif /* IMPORTANCE_INHERITANCE */ @@ -1901,46 +1974,49 @@ retry: * queue. Lock message queue while port is locked. */ imq_lock(&port->ip_messages); + + ipc_special_reply_port_msg_sent(port); + ip_unlock(port); error = ipc_mqueue_send(&port->ip_messages, kmsg, option, - send_timeout); + send_timeout); } #if IMPORTANCE_INHERITANCE - if (did_importance == TRUE) { + if (did_importance) { __unused int importance_cleared = 0; switch (error) { - case MACH_SEND_TIMED_OUT: - case MACH_SEND_NO_BUFFER: - case MACH_SEND_INTERRUPTED: - case MACH_SEND_INVALID_DEST: - /* - * We still have the kmsg and its - * reference on the port. But we - * have to back out the importance - * boost. - * - * The port could have changed hands, - * be inflight to another destination, - * etc... But in those cases our - * back-out will find the new owner - * (and all the operations that - * transferred the right should have - * applied their own boost adjustments - * to the old owner(s)). - */ - importance_cleared = 1; - ipc_importance_clean(kmsg); - break; + case MACH_SEND_TIMED_OUT: + case MACH_SEND_NO_BUFFER: + case MACH_SEND_INTERRUPTED: + case MACH_SEND_INVALID_DEST: + /* + * We still have the kmsg and its + * reference on the port. But we + * have to back out the importance + * boost. + * + * The port could have changed hands, + * be inflight to another destination, + * etc... But in those cases our + * back-out will find the new owner + * (and all the operations that + * transferred the right should have + * applied their own boost adjustments + * to the old owner(s)). + */ + importance_cleared = 1; + ipc_importance_clean(kmsg); + break; - case MACH_MSG_SUCCESS: - default: - break; + case MACH_MSG_SUCCESS: + default: + break; } #if IMPORTANCE_TRACE KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_END, - task_pid(current_task()), sender_pid, imp_msgh_id, importance_cleared, 0); + task_pid(current_task()), sender_pid, imp_msgh_id, importance_cleared, 0); #endif /* IMPORTANCE_TRACE */ } #endif /* IMPORTANCE_INHERITANCE */ @@ -1951,13 +2027,14 @@ retry: */ if (error == MACH_SEND_INVALID_DEST) { #if MACH_FLIPC - if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) - flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE); + if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) { + flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE); + } #endif ip_release(port); /* JMM - Future: release right, not just ref */ kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL; ipc_kmsg_destroy(kmsg); - KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST); + KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST); return MACH_MSG_SUCCESS; } @@ -1968,13 +2045,14 @@ retry: * the message as a successful delivery. */ #if MACH_FLIPC - if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) - flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE); + if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) { + flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE); + } #endif ip_release(port); /* JMM - Future: release right, not just ref */ kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL; ipc_kmsg_destroy(kmsg); - KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, error); + KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, error); return MACH_MSG_SUCCESS; } return error; @@ -1996,12 +2074,12 @@ retry: mach_msg_return_t ipc_kmsg_put( - ipc_kmsg_t kmsg, - mach_msg_option_t option, - mach_vm_address_t rcv_addr, - mach_msg_size_t rcv_size, - mach_msg_size_t trailer_size, - mach_msg_size_t *sizep) + ipc_kmsg_t kmsg, + mach_msg_option_t option, + mach_vm_address_t rcv_addr, + mach_msg_size_t rcv_size, + mach_msg_size_t trailer_size, + mach_msg_size_t *sizep) { mach_msg_size_t size = kmsg->ikm_header->msgh_size + trailer_size; mach_msg_return_t mr; @@ -2010,37 +2088,37 @@ ipc_kmsg_put( DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_put header:\n" - " size: 0x%.8x\n" - " bits: 0x%.8x\n" - " remote_port: %p\n" - " local_port: %p\n" - " voucher_port: 0x%.8x\n" - " id: %.8d\n", - kmsg->ikm_header->msgh_size, - kmsg->ikm_header->msgh_bits, - kmsg->ikm_header->msgh_remote_port, - kmsg->ikm_header->msgh_local_port, - kmsg->ikm_header->msgh_voucher_port, - kmsg->ikm_header->msgh_id); + " size: 0x%.8x\n" + " bits: 0x%.8x\n" + " remote_port: %p\n" + " local_port: %p\n" + " voucher_port: 0x%.8x\n" + " id: %.8d\n", + kmsg->ikm_header->msgh_size, + kmsg->ikm_header->msgh_bits, + kmsg->ikm_header->msgh_remote_port, + kmsg->ikm_header->msgh_local_port, + kmsg->ikm_header->msgh_voucher_port, + kmsg->ikm_header->msgh_id); #if defined(__LP64__) - if (current_task() != kernel_task) { /* don't if receiver expects fully-cooked in-kernel msg; ux_exception */ - mach_msg_legacy_header_t *legacy_header = - (mach_msg_legacy_header_t *)((vm_offset_t)(kmsg->ikm_header) + LEGACY_HEADER_SIZE_DELTA); - - mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits; - mach_msg_size_t msg_size = kmsg->ikm_header->msgh_size; - mach_port_name_t remote_port = CAST_MACH_PORT_TO_NAME(kmsg->ikm_header->msgh_remote_port); - mach_port_name_t local_port = CAST_MACH_PORT_TO_NAME(kmsg->ikm_header->msgh_local_port); - mach_port_name_t voucher_port = kmsg->ikm_header->msgh_voucher_port; - mach_msg_id_t id = kmsg->ikm_header->msgh_id; - - legacy_header->msgh_id = id; + if (current_task() != kernel_task) { /* don't if receiver expects fully-cooked in-kernel msg; */ + mach_msg_legacy_header_t *legacy_header = + (mach_msg_legacy_header_t *)((vm_offset_t)(kmsg->ikm_header) + LEGACY_HEADER_SIZE_DELTA); + + mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits; + mach_msg_size_t msg_size = kmsg->ikm_header->msgh_size; + mach_port_name_t remote_port = CAST_MACH_PORT_TO_NAME(kmsg->ikm_header->msgh_remote_port); + mach_port_name_t local_port = CAST_MACH_PORT_TO_NAME(kmsg->ikm_header->msgh_local_port); + mach_port_name_t voucher_port = kmsg->ikm_header->msgh_voucher_port; + mach_msg_id_t id = kmsg->ikm_header->msgh_id; + + legacy_header->msgh_id = id; legacy_header->msgh_local_port = local_port; legacy_header->msgh_remote_port = remote_port; legacy_header->msgh_voucher_port = voucher_port; - legacy_header->msgh_size = msg_size - LEGACY_HEADER_SIZE_DELTA; - legacy_header->msgh_bits = bits; + legacy_header->msgh_size = msg_size - LEGACY_HEADER_SIZE_DELTA; + legacy_header->msgh_bits = bits; size -= LEGACY_HEADER_SIZE_DELTA; kmsg->ikm_header = (mach_msg_header_t *)legacy_header; @@ -2052,15 +2130,14 @@ ipc_kmsg_put( if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { kprintf("ipc_kmsg_put header+body: %d\n", (size)); uint32_t i; - for(i=0;i*4 < size;i++) - { - kprintf("%.4x\n",((uint32_t *)kmsg->ikm_header)[i]); + for (i = 0; i * 4 < size; i++) { + kprintf("%.4x\n", ((uint32_t *)kmsg->ikm_header)[i]); } - kprintf("type: %d\n", ((mach_msg_type_descriptor_t *)(((mach_msg_base_t *)kmsg->ikm_header)+1))->type); + kprintf("type: %d\n", ((mach_msg_type_descriptor_t *)(((mach_msg_base_t *)kmsg->ikm_header) + 1))->type); } __unreachable_ok_pop - /* Re-Compute target address if using stack-style delivery */ + /* Re-Compute target address if using stack-style delivery */ if (option & MACH_RCV_STACK) { rcv_addr += rcv_size - size; } @@ -2068,20 +2145,22 @@ ipc_kmsg_put( if (copyoutmsg((const char *) kmsg->ikm_header, rcv_addr, size)) { mr = MACH_RCV_INVALID_DATA; size = 0; - } else + } else { mr = MACH_MSG_SUCCESS; + } - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_LINK) | DBG_FUNC_NONE, - (rcv_addr >= VM_MIN_KERNEL_AND_KEXT_ADDRESS || - rcv_addr + size >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) ? (uintptr_t)0 : (uintptr_t)rcv_addr, - VM_KERNEL_ADDRPERM((uintptr_t)kmsg), - 1 /* this is on the receive/copyout path */, - 0, - 0); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_LINK) | DBG_FUNC_NONE, + (rcv_addr >= VM_MIN_KERNEL_AND_KEXT_ADDRESS || + rcv_addr + size >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) ? (uintptr_t)0 : (uintptr_t)rcv_addr, + VM_KERNEL_ADDRPERM((uintptr_t)kmsg), + 1 /* this is on the receive/copyout path */, + 0, + 0); ipc_kmsg_free(kmsg); - if (sizep) + if (sizep) { *sizep = size; + } return mr; } @@ -2097,16 +2176,29 @@ ipc_kmsg_put( void ipc_kmsg_put_to_kernel( - mach_msg_header_t *msg, - ipc_kmsg_t kmsg, - mach_msg_size_t size) + mach_msg_header_t *msg, + ipc_kmsg_t kmsg, + mach_msg_size_t size) { (void) memcpy((void *) msg, (const void *) kmsg->ikm_header, size); ipc_kmsg_free(kmsg); } -unsigned long pthread_priority_canonicalize(unsigned long priority, boolean_t propagation); +static mach_msg_priority_t +ipc_get_current_thread_priority(void) +{ + thread_t thread = current_thread(); + thread_qos_t qos; + int relpri; + + qos = thread_get_requested_qos(thread, &relpri); + if (!qos) { + qos = thread_user_promotion_qos_for_pri(thread->base_pri); + relpri = 0; + } + return (mach_msg_priority_t)_pthread_priority_make_from_thread_qos(qos, relpri, 0); +} static kern_return_t ipc_kmsg_set_qos( @@ -2115,78 +2207,260 @@ ipc_kmsg_set_qos( mach_msg_priority_t override) { kern_return_t kr; - unsigned long flags = 0; ipc_port_t special_reply_port = kmsg->ikm_header->msgh_local_port; ipc_port_t dest_port = kmsg->ikm_header->msgh_remote_port; kr = ipc_get_pthpriority_from_kmsg_voucher(kmsg, &kmsg->ikm_qos); if (kr != KERN_SUCCESS) { - kmsg->ikm_qos = MACH_MSG_PRIORITY_UNSPECIFIED; + if (options & MACH_SEND_PROPAGATE_QOS) { + kmsg->ikm_qos = ipc_get_current_thread_priority(); + } else { + kmsg->ikm_qos = MACH_MSG_PRIORITY_UNSPECIFIED; + } } kmsg->ikm_qos_override = kmsg->ikm_qos; if (options & MACH_SEND_OVERRIDE) { - unsigned long canonical; - mach_msg_priority_t canon; - - canonical = pthread_priority_canonicalize(override, TRUE); - canon = (mach_msg_priority_t)canonical; - if (canon > kmsg->ikm_qos) - kmsg->ikm_qos_override = canon; + pthread_priority_t pp = _pthread_priority_normalize_for_ipc(override); + if (pp > kmsg->ikm_qos) { + kmsg->ikm_qos_override = (mach_msg_priority_t)pp; + } } kr = KERN_SUCCESS; - if ((options & MACH_SEND_SYNC_OVERRIDE)) { - if (IP_VALID(special_reply_port) && - MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits) == MACH_MSG_TYPE_PORT_SEND_ONCE) { + + if (IP_VALID(special_reply_port) && + MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits) == MACH_MSG_TYPE_PORT_SEND_ONCE) { + if ((options & MACH_SEND_SYNC_OVERRIDE)) { + boolean_t sync_bootstrap_checkin = !!(options & MACH_SEND_SYNC_BOOTSTRAP_CHECKIN); /* - * Update the sync override count if the reply port is a special reply port, - * link the destination port to special reply port and update the qos count - * of destination port. - * - * Use the qos value passed by voucher and not the one passed by notify field. + * Link the destination port to special reply port and make sure that + * dest port has a send turnstile, else allocate one. */ - kr = ipc_port_link_special_reply_port_with_qos(special_reply_port, dest_port, - thread_qos_from_pthread_priority(kmsg->ikm_qos, &flags)); - } else { - kr = KERN_FAILURE; + ipc_port_link_special_reply_port(special_reply_port, dest_port, sync_bootstrap_checkin); } } return kr; } +static inline void +ipc_kmsg_allow_immovable_send( + ipc_kmsg_t kmsg, + ipc_entry_t dest_entry) +{ + ipc_object_t object = dest_entry->ie_object; + /* + * If the dest port is a kobject, allow copyin of immovable send + * rights in the message body to succeed + */ + if (IO_VALID(object) && io_is_kobject(object)) { + kmsg->ikm_flags |= IPC_KMSG_FLAGS_ALLOW_IMMOVABLE_SEND; + } +} + /* - * Routine: ipc_kmsg_copyin_header + * Routine: ipc_kmsg_link_reply_context_locked * Purpose: - * "Copy-in" port rights in the header of a message. - * Operates atomically; if it doesn't succeed the - * message header and the space are left untouched. - * If it does succeed the remote/local port fields - * contain object pointers instead of port names, - * and the bits field is updated. The destination port - * will be a valid port pointer. - * + * Link any required context from the sending voucher + * to the reply port. The ipc_kmsg_copyin function will + * enforce that the sender calls mach_msg in this context. * Conditions: - * Nothing locked. - * Returns: - * MACH_MSG_SUCCESS Successful copyin. - * MACH_SEND_INVALID_HEADER - * Illegal value in the message header bits. - * MACH_SEND_INVALID_DEST The space is dead. - * MACH_SEND_INVALID_DEST Can't copyin destination port. - * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.) - * MACH_SEND_INVALID_REPLY Can't copyin reply port. - * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.) + * reply port is locked */ - -mach_msg_return_t -ipc_kmsg_copyin_header( - ipc_kmsg_t kmsg, - ipc_space_t space, - mach_msg_priority_t override, - mach_msg_option_t *optionp) +static void +ipc_kmsg_link_reply_context_locked( + ipc_port_t reply_port, + ipc_port_t voucher_port) { - mach_msg_header_t *msg = kmsg->ikm_header; + kern_return_t __assert_only kr; + uint32_t persona_id = 0; + ipc_voucher_t voucher; + + ip_lock_held(reply_port); + + if (!ip_active(reply_port)) { + return; + } + + voucher = convert_port_to_voucher(voucher_port); + + kr = bank_get_bank_ledger_thread_group_and_persona(voucher, NULL, NULL, &persona_id); + assert(kr == KERN_SUCCESS); + ipc_voucher_release(voucher); + + if (persona_id == 0 || persona_id == PERSONA_ID_NONE) { + /* there was no persona context to record */ + return; + } + + /* + * Set the persona_id as the context on the reply port. + * This will force the thread that replies to have adopted a voucher + * with a matching persona. + */ + reply_port->ip_reply_context = persona_id; + + return; +} + +static kern_return_t +ipc_kmsg_validate_reply_port_locked(ipc_port_t reply_port, mach_msg_option_t options) +{ + ip_lock_held(reply_port); + + if (!ip_active(reply_port)) { + /* + * Ideally, we would enforce that the reply receive right is + * active, but asynchronous XPC cancellation destroys the + * receive right, so we just have to return success here. + */ + return KERN_SUCCESS; + } + + if (options & MACH_SEND_MSG) { + /* + * If the rely port is active, then it should not be + * in-transit, and the receive right should be in the caller's + * IPC space. + */ + if (!reply_port->ip_receiver_name || reply_port->ip_receiver != current_task()->itk_space) { + return KERN_INVALID_CAPABILITY; + } + + /* + * A port used as a reply port in an RPC should have exactly 1 + * extant send-once right which we either just made or are + * moving as part of the IPC. + */ + if (reply_port->ip_sorights != 1) { + return KERN_INVALID_CAPABILITY; + } + /* + * XPC uses an extra send-right to keep the name of the reply + * right around through cancellation. That makes it harder to + * enforce a particular semantic kere, so for now, we say that + * you can have a maximum of 1 send right (in addition to your + * send once right). In the future, it would be great to lock + * this down even further. + */ + if (reply_port->ip_srights > 1) { + return KERN_INVALID_CAPABILITY; + } + + /* + * The sender can also specify that the receive right should + * be immovable. Note that this check only applies to + * send-only operations. Combined send/receive or rcv-only + * operations can specify an immovable receive right by + * opt-ing into guarded descriptors (MACH_RCV_GUARDED_DESC) + * and using the MACH_MSG_STRICT_REPLY options flag. + */ + if (MACH_SEND_REPLY_IS_IMMOVABLE(options)) { + if (!reply_port->ip_immovable_receive) { + return KERN_INVALID_CAPABILITY; + } + } + } + + /* + * don't enforce this yet: need a better way of indicating the + * receiver wants this... + */ +#if 0 + if (MACH_RCV_WITH_IMMOVABLE_REPLY(options)) { + if (!reply_port->ip_immovable_receive) { + return KERN_INVALID_CAPABILITY; + } + } +#endif /* 0 */ + + return KERN_SUCCESS; +} + +/* + * Routine: ipc_kmsg_validate_reply_context_locked + * Purpose: + * Validate that the current thread is running in the context + * required by the destination port. + * Conditions: + * dest_port is locked + * Returns: + * MACH_MSG_SUCCESS on success. + * On error, an EXC_GUARD exception is also raised. + * This function *always* resets the port reply context. + */ +static mach_msg_return_t +ipc_kmsg_validate_reply_context_locked( + mach_msg_option_t option, + ipc_port_t dest_port, + ipc_voucher_t voucher, + mach_port_name_t voucher_name) +{ + uint32_t dest_ctx = dest_port->ip_reply_context; + dest_port->ip_reply_context = 0; + + if (!ip_active(dest_port)) { + return MACH_MSG_SUCCESS; + } + + if (voucher == IPC_VOUCHER_NULL || !MACH_PORT_VALID(voucher_name)) { + if ((option & MACH_SEND_KERNEL) == 0) { + mach_port_guard_exception(voucher_name, 0, + (MPG_FLAGS_STRICT_REPLY_INVALID_VOUCHER | dest_ctx), + kGUARD_EXC_STRICT_REPLY); + } + return MACH_SEND_INVALID_CONTEXT; + } + + kern_return_t __assert_only kr; + uint32_t persona_id = 0; + kr = bank_get_bank_ledger_thread_group_and_persona(voucher, NULL, NULL, &persona_id); + assert(kr == KERN_SUCCESS); + + if (dest_ctx != persona_id) { + if ((option & MACH_SEND_KERNEL) == 0) { + mach_port_guard_exception(voucher_name, 0, + (MPG_FLAGS_STRICT_REPLY_MISMATCHED_PERSONA | ((((uint64_t)persona_id << 32) & MPG_FLAGS_STRICT_REPLY_MASK) | dest_ctx)), + kGUARD_EXC_STRICT_REPLY); + } + return MACH_SEND_INVALID_CONTEXT; + } + + return MACH_MSG_SUCCESS; +} + +/* + * Routine: ipc_kmsg_copyin_header + * Purpose: + * "Copy-in" port rights in the header of a message. + * Operates atomically; if it doesn't succeed the + * message header and the space are left untouched. + * If it does succeed the remote/local port fields + * contain object pointers instead of port names, + * and the bits field is updated. The destination port + * will be a valid port pointer. + * + * Conditions: + * Nothing locked. + * Returns: + * MACH_MSG_SUCCESS Successful copyin. + * MACH_SEND_INVALID_HEADER + * Illegal value in the message header bits. + * MACH_SEND_INVALID_DEST The space is dead. + * MACH_SEND_INVALID_DEST Can't copyin destination port. + * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.) + * MACH_SEND_INVALID_REPLY Can't copyin reply port. + * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.) + */ + +mach_msg_return_t +ipc_kmsg_copyin_header( + ipc_kmsg_t kmsg, + ipc_space_t space, + mach_msg_priority_t override, + mach_msg_option_t *optionp) +{ + mach_msg_header_t *msg = kmsg->ikm_header; mach_msg_bits_t mbits = msg->msgh_bits & MACH_MSGH_BITS_USER; mach_port_name_t dest_name = CAST_MACH_PORT_TO_NAME(msg->msgh_remote_port); mach_port_name_t reply_name = CAST_MACH_PORT_TO_NAME(msg->msgh_local_port); @@ -2216,12 +2490,14 @@ ipc_kmsg_copyin_header( if ((mbits != msg->msgh_bits) || (!MACH_MSG_TYPE_PORT_ANY_SEND(dest_type)) || ((reply_type == 0) ? - (reply_name != MACH_PORT_NULL) : - !MACH_MSG_TYPE_PORT_ANY_SEND(reply_type))) + (reply_name != MACH_PORT_NULL) : + !MACH_MSG_TYPE_PORT_ANY_SEND(reply_type))) { return MACH_SEND_INVALID_HEADER; + } - if (!MACH_PORT_VALID(dest_name)) + if (!MACH_PORT_VALID(dest_name)) { return MACH_SEND_INVALID_DEST; + } is_write_lock(space); if (!is_active(space)) { @@ -2236,22 +2512,27 @@ ipc_kmsg_copyin_header( * actually copy in until we validate destination and reply. */ if (voucher_type != MACH_MSGH_BITS_ZERO) { - voucher_name = msg->msgh_voucher_port; if (voucher_name == MACH_PORT_DEAD || (voucher_type != MACH_MSG_TYPE_MOVE_SEND && - voucher_type != MACH_MSG_TYPE_COPY_SEND)) { + voucher_type != MACH_MSG_TYPE_COPY_SEND)) { is_write_unlock(space); + if ((*optionp & MACH_SEND_KERNEL) == 0) { + mach_port_guard_exception(voucher_name, 0, 0, kGUARD_EXC_SEND_INVALID_VOUCHER); + } return MACH_SEND_INVALID_VOUCHER; } if (voucher_name != MACH_PORT_NULL) { voucher_entry = ipc_entry_lookup(space, voucher_name); - if (voucher_entry == IE_NULL || + if (voucher_entry == IE_NULL || (voucher_entry->ie_bits & MACH_PORT_TYPE_SEND) == 0 || io_kotype(voucher_entry->ie_object) != IKOT_VOUCHER) { is_write_unlock(space); + if ((*optionp & MACH_SEND_KERNEL) == 0) { + mach_port_guard_exception(voucher_name, 0, 0, kGUARD_EXC_SEND_INVALID_VOUCHER); + } return MACH_SEND_INVALID_VOUCHER; } } else { @@ -2259,13 +2540,29 @@ ipc_kmsg_copyin_header( } } + if (enforce_strict_reply && MACH_SEND_WITH_STRICT_REPLY(*optionp) && + (!MACH_PORT_VALID(reply_name) || + ((reply_type != MACH_MSG_TYPE_MAKE_SEND_ONCE) && (reply_type != MACH_MSG_TYPE_MOVE_SEND_ONCE)) + )) { + /* + * The caller cannot enforce a reply context with an invalid + * reply port name, or a non-send_once reply disposition. + */ + is_write_unlock(space); + if ((*optionp & MACH_SEND_KERNEL) == 0) { + mach_port_guard_exception(reply_name, 0, + (MPG_FLAGS_STRICT_REPLY_INVALID_REPLY_DISP | reply_type), + kGUARD_EXC_STRICT_REPLY); + } + return MACH_SEND_INVALID_REPLY; + } + /* * Handle combinations of validating destination and reply; along * with copying in destination, reply, and voucher in an atomic way. */ if (dest_name == voucher_name) { - /* * If the destination name is the same as the voucher name, * the voucher_entry must already be known. Either that or @@ -2275,6 +2572,8 @@ ipc_kmsg_copyin_header( if (dest_entry == IE_NULL) { goto invalid_dest; } + /* Check if dest port allows immovable send rights to be sent in the kmsg body */ + ipc_kmsg_allow_immovable_send(kmsg, dest_entry); /* * Make sure a future copyin of the reply port will succeed. @@ -2293,49 +2592,47 @@ ipc_kmsg_copyin_header( goto invalid_reply; } assert(dest_entry != reply_entry); /* names are not equal */ - if (!ipc_right_copyin_check(space, reply_name, reply_entry, reply_type)) { + if (!ipc_right_copyin_check_reply(space, reply_name, reply_entry, reply_type)) { goto invalid_reply; } } - /* - * Do the joint copyin of the dest disposition and + /* + * Do the joint copyin of the dest disposition and * voucher disposition from the one entry/port. We * already validated that the voucher copyin would * succeed (above). So, any failure in combining * the copyins can be blamed on the destination. */ kr = ipc_right_copyin_two(space, dest_name, dest_entry, - dest_type, voucher_type, - &dest_port, &dest_soright, - &release_port); + dest_type, voucher_type, &dest_port, &dest_soright, + &release_port); if (kr != KERN_SUCCESS) { assert(kr != KERN_INVALID_CAPABILITY); goto invalid_dest; } - voucher_port = (ipc_port_t)dest_port; + voucher_port = ip_object_to_port(dest_port); - /* - * could not have been one of these dispositions, + /* + * could not have been one of these dispositions, * validated the port was a true kernel voucher port above, * AND was successfully able to copyin both dest and voucher. */ assert(dest_type != MACH_MSG_TYPE_MAKE_SEND); assert(dest_type != MACH_MSG_TYPE_MAKE_SEND_ONCE); assert(dest_type != MACH_MSG_TYPE_MOVE_SEND_ONCE); - + /* * Perform the delayed reply right copyin (guaranteed success). */ if (reply_entry != IE_NULL) { kr = ipc_right_copyin(space, reply_name, reply_entry, - reply_type, TRUE, - &reply_port, &reply_soright, - &release_port, &assertcnt); + reply_type, IPC_RIGHT_COPYIN_FLAGS_DEADOK, + &reply_port, &reply_soright, + &release_port, &assertcnt, 0, NULL); assert(assertcnt == 0); assert(kr == KERN_SUCCESS); } - } else { if (dest_name == reply_name) { /* @@ -2349,25 +2646,31 @@ ipc_kmsg_copyin_header( if (dest_entry == IE_NULL) { goto invalid_dest; } + ipc_kmsg_allow_immovable_send(kmsg, dest_entry); + reply_entry = dest_entry; assert(reply_type != 0); /* because name not null */ - /* - * Do the joint copyin of the dest disposition and + /* + * Pre-validate that the reply right can be copied in by itself + */ + if (!ipc_right_copyin_check_reply(space, reply_name, reply_entry, reply_type)) { + goto invalid_reply; + } + + /* + * Do the joint copyin of the dest disposition and * reply disposition from the one entry/port. */ kr = ipc_right_copyin_two(space, dest_name, dest_entry, - dest_type, reply_type, - &dest_port, &dest_soright, - &release_port); + dest_type, reply_type, &dest_port, &dest_soright, + &release_port); if (kr == KERN_INVALID_CAPABILITY) { goto invalid_reply; } else if (kr != KERN_SUCCESS) { goto invalid_dest; } reply_port = dest_port; - - } else { /* * Handle destination and reply independently, as @@ -2400,6 +2703,7 @@ ipc_kmsg_copyin_header( goto invalid_dest; } assert(dest_entry != voucher_entry); + ipc_kmsg_allow_immovable_send(kmsg, dest_entry); /* * Make sure reply port entry is valid before dest copyin. @@ -2415,7 +2719,7 @@ ipc_kmsg_copyin_header( assert(dest_entry != reply_entry); /* names are not equal */ assert(reply_type != 0); /* because reply_name not null */ - if (!ipc_right_copyin_check(space, reply_name, reply_entry, reply_type)) { + if (!ipc_right_copyin_check_reply(space, reply_name, reply_entry, reply_type)) { goto invalid_reply; } } @@ -2424,9 +2728,10 @@ ipc_kmsg_copyin_header( * copyin the destination. */ kr = ipc_right_copyin(space, dest_name, dest_entry, - dest_type, FALSE, - &dest_port, &dest_soright, - &release_port, &assertcnt); + dest_type, (IPC_RIGHT_COPYIN_FLAGS_ALLOW_IMMOVABLE_SEND | + IPC_RIGHT_COPYIN_FLAGS_ALLOW_DEAD_SEND_ONCE), + &dest_port, &dest_soright, + &release_port, &assertcnt, 0, NULL); assert(assertcnt == 0); if (kr != KERN_SUCCESS) { goto invalid_dest; @@ -2440,14 +2745,14 @@ ipc_kmsg_copyin_header( */ if (MACH_PORT_VALID(reply_name)) { kr = ipc_right_copyin(space, reply_name, reply_entry, - reply_type, TRUE, - &reply_port, &reply_soright, - &release_port, &assertcnt); + reply_type, IPC_RIGHT_COPYIN_FLAGS_DEADOK, + &reply_port, &reply_soright, + &release_port, &assertcnt, 0, NULL); assert(assertcnt == 0); assert(kr == KERN_SUCCESS); } else { /* convert invalid name to equivalent ipc_object type */ - reply_port = (ipc_object_t)CAST_MACH_NAME_TO_PORT(reply_name); + reply_port = ip_to_object(CAST_MACH_NAME_TO_PORT(reply_name)); } } @@ -2457,15 +2762,15 @@ ipc_kmsg_copyin_header( */ if (IE_NULL != voucher_entry) { kr = ipc_right_copyin(space, voucher_name, voucher_entry, - voucher_type, FALSE, - (ipc_object_t *)&voucher_port, - &voucher_soright, - &voucher_release_port, - &assertcnt); + voucher_type, IPC_RIGHT_COPYIN_FLAGS_NONE, + (ipc_object_t *)&voucher_port, + &voucher_soright, + &voucher_release_port, + &assertcnt, 0, NULL); assert(assertcnt == 0); assert(KERN_SUCCESS == kr); assert(IP_VALID(voucher_port)); - assert(ip_active(voucher_port)); + require_ip_active(voucher_port); } } @@ -2478,8 +2783,9 @@ ipc_kmsg_copyin_header( * but reply and voucher must be distinct entries. */ assert(IE_NULL != dest_entry); - if (IE_NULL != reply_entry) + if (IE_NULL != reply_entry) { assert(reply_entry != voucher_entry); + } if (IE_BITS_TYPE(dest_entry->ie_bits) == MACH_PORT_TYPE_NONE) { ipc_entry_dealloc(space, dest_name, dest_entry); @@ -2514,27 +2820,28 @@ ipc_kmsg_copyin_header( * early (send may NOT have failed), but better than missing. We assure * we won't miss by forcing MACH_SEND_ALWAYS if we got past arming. */ - if (((*optionp & MACH_SEND_NOTIFY) != 0) && + if (((*optionp & MACH_SEND_NOTIFY) != 0) && dest_type != MACH_MSG_TYPE_PORT_SEND_ONCE && dest_entry != IE_NULL && dest_entry->ie_request != IE_REQ_NONE) { - ipc_port_t dport = (ipc_port_t)dest_port; + ipc_port_t dport = ip_object_to_port(dest_port); assert(dport != IP_NULL); ip_lock(dport); if (ip_active(dport) && dport->ip_receiver != ipc_space_kernel) { if (ip_full(dport)) { #if IMPORTANCE_INHERITANCE - needboost = ipc_port_request_sparm(dport, dest_name, - dest_entry->ie_request, - *optionp, - override); - if (needboost == FALSE) + needboost = ipc_port_request_sparm(dport, dest_name, + dest_entry->ie_request, + *optionp, + override); + if (needboost == FALSE) { ip_unlock(dport); + } #else ipc_port_request_sparm(dport, dest_name, - dest_entry->ie_request, - *optionp, - override); + dest_entry->ie_request, + *optionp, + override); ip_unlock(dport); #endif /* IMPORTANCE_INHERITANCE */ } else { @@ -2549,13 +2856,13 @@ ipc_kmsg_copyin_header( is_write_unlock(space); #if IMPORTANCE_INHERITANCE - /* + /* * If our request is the first boosting send-possible * notification this cycle, push the boost down the * destination port. */ if (needboost == TRUE) { - ipc_port_t dport = (ipc_port_t)dest_port; + ipc_port_t dport = ip_object_to_port(dest_port); /* dport still locked from above */ if (ipc_port_importance_delta(dport, IPID_OPTION_SENDPOSSIBLE, 1) == FALSE) { @@ -2580,45 +2887,83 @@ ipc_kmsg_copyin_header( * qos, and apply any override before we enqueue the kmsg. */ if (IP_VALID(voucher_port)) { - kmsg->ikm_voucher = voucher_port; voucher_type = MACH_MSG_TYPE_MOVE_SEND; } msg->msgh_bits = MACH_MSGH_BITS_SET(dest_type, reply_type, voucher_type, mbits); - msg->msgh_remote_port = (ipc_port_t)dest_port; - msg->msgh_local_port = (ipc_port_t)reply_port; + msg->msgh_remote_port = ip_object_to_port(dest_port); + msg->msgh_local_port = ip_object_to_port(reply_port); /* capture the qos value(s) for the kmsg */ ipc_kmsg_set_qos(kmsg, *optionp, override); - if (release_port != IP_NULL) + if (release_port != IP_NULL) { ip_release(release_port); + } - if (voucher_release_port != IP_NULL) + if (voucher_release_port != IP_NULL) { ip_release(voucher_release_port); + } + + if (enforce_strict_reply && MACH_SEND_WITH_STRICT_REPLY(*optionp) && IP_VALID(msg->msgh_local_port)) { + /* + * We've already validated that the reply disposition is a + * [make/move] send-once. Ideally, we should enforce that the + * reply port is also not dead, but XPC asynchronous + * cancellation can make the reply port dead before we + * actually make it to the mach_msg send. + * + * Here, we ensure that if we have a non-dead reply port, then + * the reply port's receive right should not be in-transit, + * and should live in the caller's IPC space. + */ + ipc_port_t rport = msg->msgh_local_port; + ip_lock(rport); + kr = ipc_kmsg_validate_reply_port_locked(rport, *optionp); + ip_unlock(rport); + if (kr != KERN_SUCCESS) { + /* + * no descriptors have been copied in yet, but the + * full header has been copied in: clean it up + */ + ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0); + if ((*optionp & MACH_SEND_KERNEL) == 0) { + mach_port_guard_exception(reply_name, 0, + (MPG_FLAGS_STRICT_REPLY_INVALID_REPLY_PORT | kr), + kGUARD_EXC_STRICT_REPLY); + } + return MACH_SEND_INVALID_REPLY; + } + } return MACH_MSG_SUCCESS; invalid_reply: is_write_unlock(space); - if (release_port != IP_NULL) + if (release_port != IP_NULL) { ip_release(release_port); + } assert(voucher_port == IP_NULL); assert(voucher_soright == IP_NULL); + if ((*optionp & MACH_SEND_KERNEL) == 0) { + mach_port_guard_exception(reply_name, 0, 0, kGUARD_EXC_SEND_INVALID_REPLY); + } return MACH_SEND_INVALID_REPLY; invalid_dest: is_write_unlock(space); - if (release_port != IP_NULL) + if (release_port != IP_NULL) { ip_release(release_port); + } - if (reply_soright != IP_NULL) + if (reply_soright != IP_NULL) { ipc_notify_port_deleted(reply_soright, reply_name); + } assert(voucher_port == IP_NULL); assert(voucher_soright == IP_NULL); @@ -2626,324 +2971,371 @@ invalid_dest: return MACH_SEND_INVALID_DEST; } -mach_msg_descriptor_t *ipc_kmsg_copyin_port_descriptor( - volatile mach_msg_port_descriptor_t *dsc, - mach_msg_legacy_port_descriptor_t *user_dsc, - ipc_space_t space, - ipc_object_t dest, - ipc_kmsg_t kmsg, - mach_msg_return_t *mr); - -void ipc_print_type_name( - int type_name); -mach_msg_descriptor_t * +static mach_msg_descriptor_t * ipc_kmsg_copyin_port_descriptor( - volatile mach_msg_port_descriptor_t *dsc, - mach_msg_legacy_port_descriptor_t *user_dsc_in, - ipc_space_t space, - ipc_object_t dest, - ipc_kmsg_t kmsg, - mach_msg_return_t *mr) + volatile mach_msg_port_descriptor_t *dsc, + mach_msg_legacy_port_descriptor_t *user_dsc_in, + ipc_space_t space, + ipc_object_t dest, + ipc_kmsg_t kmsg, + mach_msg_option_t *optionp, + mach_msg_return_t *mr) { - volatile mach_msg_legacy_port_descriptor_t *user_dsc = user_dsc_in; - mach_msg_type_name_t user_disp; - mach_msg_type_name_t result_disp; - mach_port_name_t name; - ipc_object_t object; - - user_disp = user_dsc->disposition; - result_disp = ipc_object_copyin_type(user_disp); - - name = (mach_port_name_t)user_dsc->name; - if (MACH_PORT_VALID(name)) { - - kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object); - if (kr != KERN_SUCCESS) { - *mr = MACH_SEND_INVALID_RIGHT; - return NULL; - } - - if ((result_disp == MACH_MSG_TYPE_PORT_RECEIVE) && - ipc_port_check_circularity((ipc_port_t) object, - (ipc_port_t) dest)) { - kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; - } - dsc->name = (ipc_port_t) object; - } else { - dsc->name = CAST_MACH_NAME_TO_PORT(name); - } - dsc->disposition = result_disp; - dsc->type = MACH_MSG_PORT_DESCRIPTOR; - - dsc->pad_end = 0; // debug, unnecessary - - return (mach_msg_descriptor_t *)(user_dsc_in+1); + volatile mach_msg_legacy_port_descriptor_t *user_dsc = user_dsc_in; + mach_msg_type_name_t user_disp; + mach_msg_type_name_t result_disp; + mach_port_name_t name; + ipc_object_t object; + + user_disp = user_dsc->disposition; + result_disp = ipc_object_copyin_type(user_disp); + + name = (mach_port_name_t)user_dsc->name; + if (MACH_PORT_VALID(name)) { + kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object, 0, NULL, kmsg->ikm_flags); + if (kr != KERN_SUCCESS) { + if (((*optionp & MACH_SEND_KERNEL) == 0) && (kr == KERN_INVALID_RIGHT)) { + mach_port_guard_exception(name, 0, 0, kGUARD_EXC_SEND_INVALID_RIGHT); + } + *mr = MACH_SEND_INVALID_RIGHT; + return NULL; + } + + if ((result_disp == MACH_MSG_TYPE_PORT_RECEIVE) && + ipc_port_check_circularity(ip_object_to_port(object), + ip_object_to_port(dest))) { + kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; + } + dsc->name = ip_object_to_port(object); + } else { + dsc->name = CAST_MACH_NAME_TO_PORT(name); + } + dsc->disposition = result_disp; + dsc->type = MACH_MSG_PORT_DESCRIPTOR; + + dsc->pad_end = 0; // debug, unnecessary + + return (mach_msg_descriptor_t *)(user_dsc_in + 1); } -mach_msg_descriptor_t * ipc_kmsg_copyin_ool_descriptor( - mach_msg_ool_descriptor_t *dsc, - mach_msg_descriptor_t *user_dsc, - int is_64bit, - vm_offset_t *paddr, - vm_map_copy_t *copy, - vm_size_t *space_needed, - vm_map_t map, - mach_msg_return_t *mr); -mach_msg_descriptor_t * +static mach_msg_descriptor_t * ipc_kmsg_copyin_ool_descriptor( - mach_msg_ool_descriptor_t *dsc, - mach_msg_descriptor_t *user_dsc, - int is_64bit, - vm_offset_t *paddr, - vm_map_copy_t *copy, - vm_size_t *space_needed, - vm_map_t map, - mach_msg_return_t *mr) + mach_msg_ool_descriptor_t *dsc, + mach_msg_descriptor_t *user_dsc, + int is_64bit, + vm_offset_t *paddr, + vm_map_copy_t *copy, + vm_size_t *space_needed, + vm_map_t map, + __unused mach_msg_option_t *optionp, + mach_msg_return_t *mr) { - vm_size_t length; - boolean_t dealloc; - mach_msg_copy_options_t copy_options; - mach_vm_offset_t addr; - mach_msg_descriptor_type_t dsc_type; - - if (is_64bit) { - mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; - - addr = (mach_vm_offset_t) user_ool_dsc->address; - length = user_ool_dsc->size; - dealloc = user_ool_dsc->deallocate; - copy_options = user_ool_dsc->copy; - dsc_type = user_ool_dsc->type; - - user_dsc = (typeof(user_dsc))(user_ool_dsc+1); - } else { - mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; - - addr = CAST_USER_ADDR_T(user_ool_dsc->address); - dealloc = user_ool_dsc->deallocate; - copy_options = user_ool_dsc->copy; - dsc_type = user_ool_dsc->type; - length = user_ool_dsc->size; - - user_dsc = (typeof(user_dsc))(user_ool_dsc+1); - } - - dsc->size = (mach_msg_size_t)length; - dsc->deallocate = dealloc; - dsc->copy = copy_options; - dsc->type = dsc_type; - - if (length == 0) { - dsc->address = NULL; - } else if ((length >= MSG_OOL_SIZE_SMALL) && - (copy_options == MACH_MSG_PHYSICAL_COPY) && !dealloc) { - - /* - * If the request is a physical copy and the source - * is not being deallocated, then allocate space - * in the kernel's pageable ipc copy map and copy - * the data in. The semantics guarantee that the - * data will have been physically copied before - * the send operation terminates. Thus if the data - * is not being deallocated, we must be prepared - * to page if the region is sufficiently large. - */ - if (copyin(addr, (char *)*paddr, length)) { - *mr = MACH_SEND_INVALID_MEMORY; - return NULL; - } - - /* - * The kernel ipc copy map is marked no_zero_fill. - * If the transfer is not a page multiple, we need - * to zero fill the balance. - */ - if (!page_aligned(length)) { - (void) memset((void *) (*paddr + length), 0, - round_page(length) - length); - } - if (vm_map_copyin(ipc_kernel_copy_map, (vm_map_address_t)*paddr, - (vm_map_size_t)length, TRUE, copy) != KERN_SUCCESS) { - *mr = MACH_MSG_VM_KERNEL; - return NULL; - } - dsc->address = (void *)*copy; - *paddr += round_page(length); - *space_needed -= round_page(length); - } else { - - /* - * Make a vm_map_copy_t of the of the data. If the - * data is small, this will do an optimized physical - * copy. Otherwise, it will do a virtual copy. - * - * NOTE: A virtual copy is OK if the original is being - * deallocted, even if a physical copy was requested. - */ - kern_return_t kr = vm_map_copyin(map, addr, - (vm_map_size_t)length, dealloc, copy); - if (kr != KERN_SUCCESS) { - *mr = (kr == KERN_RESOURCE_SHORTAGE) ? - MACH_MSG_VM_KERNEL : - MACH_SEND_INVALID_MEMORY; - return NULL; - } - dsc->address = (void *)*copy; - } - return user_dsc; + vm_size_t length; + boolean_t dealloc; + mach_msg_copy_options_t copy_options; + mach_vm_offset_t addr; + mach_msg_descriptor_type_t dsc_type; + + if (is_64bit) { + mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; + + addr = (mach_vm_offset_t) user_ool_dsc->address; + length = user_ool_dsc->size; + dealloc = user_ool_dsc->deallocate; + copy_options = user_ool_dsc->copy; + dsc_type = user_ool_dsc->type; + + user_dsc = (typeof(user_dsc))(user_ool_dsc + 1); + } else { + mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; + + addr = CAST_USER_ADDR_T(user_ool_dsc->address); + dealloc = user_ool_dsc->deallocate; + copy_options = user_ool_dsc->copy; + dsc_type = user_ool_dsc->type; + length = user_ool_dsc->size; + + user_dsc = (typeof(user_dsc))(user_ool_dsc + 1); + } + + dsc->size = (mach_msg_size_t)length; + dsc->deallocate = dealloc; + dsc->copy = copy_options; + dsc->type = dsc_type; + + if (length == 0) { + dsc->address = NULL; + } else if ((length >= MSG_OOL_SIZE_SMALL) && + (copy_options == MACH_MSG_PHYSICAL_COPY) && !dealloc) { + /* + * If the request is a physical copy and the source + * is not being deallocated, then allocate space + * in the kernel's pageable ipc copy map and copy + * the data in. The semantics guarantee that the + * data will have been physically copied before + * the send operation terminates. Thus if the data + * is not being deallocated, we must be prepared + * to page if the region is sufficiently large. + */ + if (copyin(addr, (char *)*paddr, length)) { + *mr = MACH_SEND_INVALID_MEMORY; + return NULL; + } + + /* + * The kernel ipc copy map is marked no_zero_fill. + * If the transfer is not a page multiple, we need + * to zero fill the balance. + */ + if (!page_aligned(length)) { + (void) memset((void *) (*paddr + length), 0, + round_page(length) - length); + } + if (vm_map_copyin(ipc_kernel_copy_map, (vm_map_address_t)*paddr, + (vm_map_size_t)length, TRUE, copy) != KERN_SUCCESS) { + *mr = MACH_MSG_VM_KERNEL; + return NULL; + } + dsc->address = (void *)*copy; + *paddr += round_page(length); + *space_needed -= round_page(length); + } else { + /* + * Make a vm_map_copy_t of the of the data. If the + * data is small, this will do an optimized physical + * copy. Otherwise, it will do a virtual copy. + * + * NOTE: A virtual copy is OK if the original is being + * deallocted, even if a physical copy was requested. + */ + kern_return_t kr = vm_map_copyin(map, addr, + (vm_map_size_t)length, dealloc, copy); + if (kr != KERN_SUCCESS) { + *mr = (kr == KERN_RESOURCE_SHORTAGE) ? + MACH_MSG_VM_KERNEL : + MACH_SEND_INVALID_MEMORY; + return NULL; + } + dsc->address = (void *)*copy; + } + return user_dsc; } -mach_msg_descriptor_t * ipc_kmsg_copyin_ool_ports_descriptor( - mach_msg_ool_ports_descriptor_t *dsc, - mach_msg_descriptor_t *user_dsc, - int is_64bit, - vm_map_t map, - ipc_space_t space, - ipc_object_t dest, - ipc_kmsg_t kmsg, - mach_msg_return_t *mr); -mach_msg_descriptor_t * +static mach_msg_descriptor_t * ipc_kmsg_copyin_ool_ports_descriptor( - mach_msg_ool_ports_descriptor_t *dsc, - mach_msg_descriptor_t *user_dsc, - int is_64bit, - vm_map_t map, - ipc_space_t space, - ipc_object_t dest, - ipc_kmsg_t kmsg, - mach_msg_return_t *mr) + mach_msg_ool_ports_descriptor_t *dsc, + mach_msg_descriptor_t *user_dsc, + int is_64bit, + vm_map_t map, + ipc_space_t space, + ipc_object_t dest, + ipc_kmsg_t kmsg, + mach_msg_option_t *optionp, + mach_msg_return_t *mr) { - void *data; - ipc_object_t *objects; - unsigned int i; - mach_vm_offset_t addr; - mach_msg_type_name_t user_disp; - mach_msg_type_name_t result_disp; - mach_msg_type_number_t count; - mach_msg_copy_options_t copy_option; - boolean_t deallocate; - mach_msg_descriptor_type_t type; - vm_size_t ports_length, names_length; - - if (is_64bit) { - mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; - - addr = (mach_vm_offset_t)user_ool_dsc->address; - count = user_ool_dsc->count; - deallocate = user_ool_dsc->deallocate; - copy_option = user_ool_dsc->copy; - user_disp = user_ool_dsc->disposition; - type = user_ool_dsc->type; - - user_dsc = (typeof(user_dsc))(user_ool_dsc+1); - } else { - mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; - - addr = CAST_USER_ADDR_T(user_ool_dsc->address); - count = user_ool_dsc->count; - deallocate = user_ool_dsc->deallocate; - copy_option = user_ool_dsc->copy; - user_disp = user_ool_dsc->disposition; - type = user_ool_dsc->type; - - user_dsc = (typeof(user_dsc))(user_ool_dsc+1); - } - - dsc->deallocate = deallocate; - dsc->copy = copy_option; - dsc->type = type; - dsc->count = count; - dsc->address = NULL; /* for now */ - - result_disp = ipc_object_copyin_type(user_disp); - dsc->disposition = result_disp; - - /* We always do a 'physical copy', but you have to specify something valid */ - if (copy_option != MACH_MSG_PHYSICAL_COPY && - copy_option != MACH_MSG_VIRTUAL_COPY) { - *mr = MACH_SEND_INVALID_TYPE; - return NULL; - } - - /* calculate length of data in bytes, rounding up */ - - if (os_mul_overflow(count, sizeof(mach_port_t), &ports_length)) { - *mr = MACH_SEND_TOO_LARGE; - return NULL; - } - - if (os_mul_overflow(count, sizeof(mach_port_name_t), &names_length)) { - *mr = MACH_SEND_TOO_LARGE; - return NULL; - } - - if (ports_length == 0) { - return user_dsc; - } - - data = kalloc(ports_length); - - if (data == NULL) { - *mr = MACH_SEND_NO_BUFFER; - return NULL; - } - + void *data; + ipc_object_t *objects; + unsigned int i; + mach_vm_offset_t addr; + mach_msg_type_name_t user_disp; + mach_msg_type_name_t result_disp; + mach_msg_type_number_t count; + mach_msg_copy_options_t copy_option; + boolean_t deallocate; + mach_msg_descriptor_type_t type; + vm_size_t ports_length, names_length; + + if (is_64bit) { + mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; + + addr = (mach_vm_offset_t)user_ool_dsc->address; + count = user_ool_dsc->count; + deallocate = user_ool_dsc->deallocate; + copy_option = user_ool_dsc->copy; + user_disp = user_ool_dsc->disposition; + type = user_ool_dsc->type; + + user_dsc = (typeof(user_dsc))(user_ool_dsc + 1); + } else { + mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; + + addr = CAST_USER_ADDR_T(user_ool_dsc->address); + count = user_ool_dsc->count; + deallocate = user_ool_dsc->deallocate; + copy_option = user_ool_dsc->copy; + user_disp = user_ool_dsc->disposition; + type = user_ool_dsc->type; + + user_dsc = (typeof(user_dsc))(user_ool_dsc + 1); + } + + dsc->deallocate = deallocate; + dsc->copy = copy_option; + dsc->type = type; + dsc->count = count; + dsc->address = NULL; /* for now */ + + result_disp = ipc_object_copyin_type(user_disp); + dsc->disposition = result_disp; + + /* We always do a 'physical copy', but you have to specify something valid */ + if (copy_option != MACH_MSG_PHYSICAL_COPY && + copy_option != MACH_MSG_VIRTUAL_COPY) { + *mr = MACH_SEND_INVALID_TYPE; + return NULL; + } + + /* calculate length of data in bytes, rounding up */ + + if (os_mul_overflow(count, sizeof(mach_port_t), &ports_length)) { + *mr = MACH_SEND_TOO_LARGE; + return NULL; + } + + if (os_mul_overflow(count, sizeof(mach_port_name_t), &names_length)) { + *mr = MACH_SEND_TOO_LARGE; + return NULL; + } + + if (ports_length == 0) { + return user_dsc; + } + + data = kalloc(ports_length); + + if (data == NULL) { + *mr = MACH_SEND_NO_BUFFER; + return NULL; + } + #ifdef __LP64__ - mach_port_name_t *names = &((mach_port_name_t *)data)[count]; + mach_port_name_t *names = &((mach_port_name_t *)data)[count]; #else - mach_port_name_t *names = ((mach_port_name_t *)data); + mach_port_name_t *names = ((mach_port_name_t *)data); #endif - if (copyinmap(map, addr, names, names_length) != KERN_SUCCESS) { - kfree(data, ports_length); - *mr = MACH_SEND_INVALID_MEMORY; - return NULL; - } - - if (deallocate) { - (void) mach_vm_deallocate(map, addr, (mach_vm_size_t)ports_length); - } - - objects = (ipc_object_t *) data; - dsc->address = data; - - for ( i = 0; i < count; i++) { - mach_port_name_t name = names[i]; - ipc_object_t object; - - if (!MACH_PORT_VALID(name)) { - objects[i] = (ipc_object_t)CAST_MACH_NAME_TO_PORT(name); - continue; - } - - kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object); - - if (kr != KERN_SUCCESS) { - unsigned int j; - - for(j = 0; j < i; j++) { - object = objects[j]; - if (IPC_OBJECT_VALID(object)) - ipc_object_destroy(object, result_disp); - } - kfree(data, ports_length); - dsc->address = NULL; - *mr = MACH_SEND_INVALID_RIGHT; - return NULL; - } - - if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && - ipc_port_check_circularity( - (ipc_port_t) object, - (ipc_port_t) dest)) - kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; - - objects[i] = object; - } - - return user_dsc; + if (copyinmap(map, addr, names, names_length) != KERN_SUCCESS) { + kfree(data, ports_length); + *mr = MACH_SEND_INVALID_MEMORY; + return NULL; + } + + if (deallocate) { + (void) mach_vm_deallocate(map, addr, (mach_vm_size_t)ports_length); + } + + objects = (ipc_object_t *) data; + dsc->address = data; + + for (i = 0; i < count; i++) { + mach_port_name_t name = names[i]; + ipc_object_t object; + + if (!MACH_PORT_VALID(name)) { + objects[i] = ip_to_object(CAST_MACH_NAME_TO_PORT(name)); + continue; + } + + kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object, 0, NULL, kmsg->ikm_flags); + + if (kr != KERN_SUCCESS) { + unsigned int j; + + for (j = 0; j < i; j++) { + object = objects[j]; + if (IPC_OBJECT_VALID(object)) { + ipc_object_destroy(object, result_disp); + } + } + kfree(data, ports_length); + dsc->address = NULL; + if (((*optionp & MACH_SEND_KERNEL) == 0) && (kr == KERN_INVALID_RIGHT)) { + mach_port_guard_exception(name, 0, 0, kGUARD_EXC_SEND_INVALID_RIGHT); + } + *mr = MACH_SEND_INVALID_RIGHT; + return NULL; + } + + if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && + ipc_port_check_circularity(ip_object_to_port(object), + ip_object_to_port(dest))) { + kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; + } + + objects[i] = object; + } + + return user_dsc; +} + +static mach_msg_descriptor_t * +ipc_kmsg_copyin_guarded_port_descriptor( + mach_msg_guarded_port_descriptor_t *dsc, + mach_msg_descriptor_t *user_addr, + int is_64bit, + ipc_space_t space, + ipc_object_t dest, + ipc_kmsg_t kmsg, + mach_msg_option_t *optionp, + mach_msg_return_t *mr) +{ + mach_msg_descriptor_t *user_dsc; + mach_msg_type_name_t disp; + mach_msg_type_name_t result_disp; + mach_port_name_t name; + mach_msg_guard_flags_t guard_flags; + ipc_object_t object; + mach_port_context_t context; + + if (!is_64bit) { + mach_msg_guarded_port_descriptor32_t *user_gp_dsc = (typeof(user_gp_dsc))user_addr; + name = user_gp_dsc->name; + guard_flags = user_gp_dsc->flags; + disp = user_gp_dsc->disposition; + context = user_gp_dsc->context; + user_dsc = (mach_msg_descriptor_t *)(user_gp_dsc + 1); + } else { + mach_msg_guarded_port_descriptor64_t *user_gp_dsc = (typeof(user_gp_dsc))user_addr; + name = user_gp_dsc->name; + guard_flags = user_gp_dsc->flags; + disp = user_gp_dsc->disposition; + context = user_gp_dsc->context; + user_dsc = (mach_msg_descriptor_t *)(user_gp_dsc + 1); + } + + guard_flags &= MACH_MSG_GUARD_FLAGS_MASK; + result_disp = ipc_object_copyin_type(disp); + + if (MACH_PORT_VALID(name)) { + kern_return_t kr = ipc_object_copyin(space, name, disp, &object, context, &guard_flags, kmsg->ikm_flags); + if (kr != KERN_SUCCESS) { + if (((*optionp & MACH_SEND_KERNEL) == 0) && (kr == KERN_INVALID_RIGHT)) { + mach_port_guard_exception(name, 0, 0, kGUARD_EXC_SEND_INVALID_RIGHT); + } + *mr = MACH_SEND_INVALID_RIGHT; + return NULL; + } + + if ((result_disp == MACH_MSG_TYPE_PORT_RECEIVE) && + ipc_port_check_circularity(ip_object_to_port(object), + ip_object_to_port(dest))) { + kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; + } + dsc->name = ip_object_to_port(object); + } else { + dsc->name = CAST_MACH_NAME_TO_PORT(name); + } + dsc->flags = guard_flags; + dsc->disposition = result_disp; + dsc->type = MACH_MSG_GUARDED_PORT_DESCRIPTOR; + +#if __LP64__ + dsc->pad_end = 0; // debug, unnecessary +#endif + return user_dsc; } + /* * Routine: ipc_kmsg_copyin_body * Purpose: @@ -2964,143 +3356,186 @@ ipc_kmsg_copyin_ool_ports_descriptor( * MACH_SEND_MSG_TOO_SMALL Body is too small for types/data. * MACH_SEND_INVALID_RT_OOL_SIZE OOL Buffer too large for RT * MACH_MSG_INVALID_RT_DESCRIPTOR Dealloc and RT are incompatible + * MACH_SEND_NO_GRANT_DEST Dest port doesn't accept ports in body */ mach_msg_return_t ipc_kmsg_copyin_body( - ipc_kmsg_t kmsg, - ipc_space_t space, - vm_map_t map) + ipc_kmsg_t kmsg, + ipc_space_t space, + vm_map_t map, + mach_msg_option_t *optionp) { - ipc_object_t dest; - mach_msg_body_t *body; - mach_msg_descriptor_t *daddr, *naddr; - mach_msg_descriptor_t *user_addr, *kern_addr; - mach_msg_type_number_t dsc_count; - boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS); - boolean_t complex = FALSE; - vm_size_t space_needed = 0; - vm_offset_t paddr = 0; - vm_map_copy_t copy = VM_MAP_COPY_NULL; - mach_msg_type_number_t i; - mach_msg_return_t mr = MACH_MSG_SUCCESS; - - vm_size_t descriptor_size = 0; - - mach_msg_type_number_t total_ool_port_count = 0; - - /* - * Determine if the target is a kernel port. - */ - dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; - body = (mach_msg_body_t *) (kmsg->ikm_header + 1); - naddr = (mach_msg_descriptor_t *) (body + 1); - - dsc_count = body->msgh_descriptor_count; - if (dsc_count == 0) - return MACH_MSG_SUCCESS; - - /* - * Make an initial pass to determine kernal VM space requirements for - * physical copies and possible contraction of the descriptors from - * processes with pointers larger than the kernel's. - */ - daddr = NULL; - for (i = 0; i < dsc_count; i++) { - mach_msg_size_t size; - mach_msg_type_number_t ool_port_count = 0; + ipc_object_t dest; + mach_msg_body_t *body; + mach_msg_descriptor_t *daddr, *naddr, *end; + mach_msg_descriptor_t *user_addr, *kern_addr; + mach_msg_type_number_t dsc_count; + boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS); + boolean_t complex = FALSE; + boolean_t contains_port_desc = FALSE; + vm_size_t space_needed = 0; + vm_offset_t paddr = 0; + vm_map_copy_t copy = VM_MAP_COPY_NULL; + mach_msg_type_number_t i; + mach_msg_return_t mr = MACH_MSG_SUCCESS; + ipc_port_t remote_port = kmsg->ikm_header->msgh_remote_port; + + vm_size_t descriptor_size = 0; + + mach_msg_type_number_t total_ool_port_count = 0; + mach_msg_guard_flags_t guard_flags = 0; + mach_port_context_t context; + mach_msg_type_name_t disp; - daddr = naddr; - - /* make sure the descriptor fits in the message */ - if (is_task_64bit) { - switch (daddr->type.type) { - case MACH_MSG_OOL_DESCRIPTOR: - case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: - case MACH_MSG_OOL_PORTS_DESCRIPTOR: - descriptor_size += 16; - naddr = (typeof(naddr))((vm_offset_t)daddr + 16); - break; - default: - descriptor_size += 12; - naddr = (typeof(naddr))((vm_offset_t)daddr + 12); - break; - } - } else { - descriptor_size += 12; - naddr = (typeof(naddr))((vm_offset_t)daddr + 12); - } + /* + * Determine if the target is a kernel port. + */ + dest = ip_to_object(remote_port); + body = (mach_msg_body_t *) (kmsg->ikm_header + 1); + naddr = (mach_msg_descriptor_t *) (body + 1); + end = (mach_msg_descriptor_t *) ((vm_offset_t)kmsg->ikm_header + kmsg->ikm_header->msgh_size); - if (naddr > (mach_msg_descriptor_t *) - ((vm_offset_t)kmsg->ikm_header + kmsg->ikm_header->msgh_size)) { - mr = MACH_SEND_MSG_TOO_SMALL; - goto clean_message; + dsc_count = body->msgh_descriptor_count; + if (dsc_count == 0) { + return MACH_MSG_SUCCESS; } - switch (daddr->type.type) { - case MACH_MSG_OOL_DESCRIPTOR: - case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: - size = (is_task_64bit) ? - ((mach_msg_ool_descriptor64_t *)daddr)->size : - daddr->out_of_line.size; - - if (daddr->out_of_line.copy != MACH_MSG_PHYSICAL_COPY && - daddr->out_of_line.copy != MACH_MSG_VIRTUAL_COPY) { - /* - * Invalid copy option - */ - mr = MACH_SEND_INVALID_TYPE; - goto clean_message; - } - - if ((size >= MSG_OOL_SIZE_SMALL) && - (daddr->out_of_line.copy == MACH_MSG_PHYSICAL_COPY) && - !(daddr->out_of_line.deallocate)) { + /* + * Make an initial pass to determine kernal VM space requirements for + * physical copies and possible contraction of the descriptors from + * processes with pointers larger than the kernel's. + */ + daddr = NULL; + for (i = 0; i < dsc_count; i++) { + mach_msg_size_t size; + mach_msg_type_number_t ool_port_count = 0; + + daddr = naddr; + + /* make sure the descriptor fits in the message */ + if (is_task_64bit) { + if ((mach_msg_descriptor_t*)((vm_offset_t)daddr + 12) > end) { + mr = MACH_SEND_MSG_TOO_SMALL; + goto clean_message; + } - /* - * Out-of-line memory descriptor, accumulate kernel - * memory requirements - */ - if (space_needed + round_page(size) <= space_needed) { - /* Overflow dectected */ - mr = MACH_MSG_VM_KERNEL; - goto clean_message; + switch (daddr->type.type) { + case MACH_MSG_OOL_DESCRIPTOR: + case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: + case MACH_MSG_OOL_PORTS_DESCRIPTOR: + case MACH_MSG_GUARDED_PORT_DESCRIPTOR: + descriptor_size += 16; + naddr = (typeof(naddr))((vm_offset_t)daddr + 16); + break; + default: + descriptor_size += 12; + naddr = (typeof(naddr))((vm_offset_t)daddr + 12); + break; + } + } else { + descriptor_size += 12; + naddr = (typeof(naddr))((vm_offset_t)daddr + 12); } - space_needed += round_page(size); - if (space_needed > ipc_kmsg_max_vm_space) { - /* Per message kernel memory limit exceeded */ - mr = MACH_MSG_VM_KERNEL; - goto clean_message; - } - } - break; - case MACH_MSG_PORT_DESCRIPTOR: - if (os_add_overflow(total_ool_port_count, 1, &total_ool_port_count)) { - /* Overflow detected */ - mr = MACH_SEND_TOO_LARGE; - goto clean_message; - } - break; - case MACH_MSG_OOL_PORTS_DESCRIPTOR: - ool_port_count = (is_task_64bit) ? - ((mach_msg_ool_ports_descriptor64_t *)daddr)->count : - daddr->ool_ports.count; - - if (os_add_overflow(total_ool_port_count, ool_port_count, &total_ool_port_count)) { - /* Overflow detected */ - mr = MACH_SEND_TOO_LARGE; + if (naddr > end) { + mr = MACH_SEND_MSG_TOO_SMALL; goto clean_message; } - if (ool_port_count > (ipc_kmsg_max_vm_space/sizeof(mach_port_t))) { - /* Per message kernel memory limit exceeded */ - mr = MACH_SEND_TOO_LARGE; - goto clean_message; + switch (daddr->type.type) { + case MACH_MSG_OOL_DESCRIPTOR: + case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: + size = (is_task_64bit) ? + ((mach_msg_ool_descriptor64_t *)daddr)->size : + daddr->out_of_line.size; + + if (daddr->out_of_line.copy != MACH_MSG_PHYSICAL_COPY && + daddr->out_of_line.copy != MACH_MSG_VIRTUAL_COPY) { + /* + * Invalid copy option + */ + mr = MACH_SEND_INVALID_TYPE; + goto clean_message; + } + + if ((size >= MSG_OOL_SIZE_SMALL) && + (daddr->out_of_line.copy == MACH_MSG_PHYSICAL_COPY) && + !(daddr->out_of_line.deallocate)) { + /* + * Out-of-line memory descriptor, accumulate kernel + * memory requirements + */ + if (space_needed + round_page(size) <= space_needed) { + /* Overflow dectected */ + mr = MACH_MSG_VM_KERNEL; + goto clean_message; + } + + space_needed += round_page(size); + if (space_needed > ipc_kmsg_max_vm_space) { + /* Per message kernel memory limit exceeded */ + mr = MACH_MSG_VM_KERNEL; + goto clean_message; + } + } + break; + case MACH_MSG_PORT_DESCRIPTOR: + if (os_add_overflow(total_ool_port_count, 1, &total_ool_port_count)) { + /* Overflow detected */ + mr = MACH_SEND_TOO_LARGE; + goto clean_message; + } + contains_port_desc = TRUE; + break; + case MACH_MSG_OOL_PORTS_DESCRIPTOR: + ool_port_count = (is_task_64bit) ? + ((mach_msg_ool_ports_descriptor64_t *)daddr)->count : + daddr->ool_ports.count; + + if (os_add_overflow(total_ool_port_count, ool_port_count, &total_ool_port_count)) { + /* Overflow detected */ + mr = MACH_SEND_TOO_LARGE; + goto clean_message; + } + + if (ool_port_count > (ipc_kmsg_max_vm_space / sizeof(mach_port_t))) { + /* Per message kernel memory limit exceeded */ + mr = MACH_SEND_TOO_LARGE; + goto clean_message; + } + contains_port_desc = TRUE; + break; + case MACH_MSG_GUARDED_PORT_DESCRIPTOR: + guard_flags = (is_task_64bit) ? + ((mach_msg_guarded_port_descriptor64_t *)daddr)->flags : + ((mach_msg_guarded_port_descriptor32_t *)daddr)->flags; + context = (is_task_64bit) ? + ((mach_msg_guarded_port_descriptor64_t *)daddr)->context : + ((mach_msg_guarded_port_descriptor32_t *)daddr)->context; + disp = (is_task_64bit) ? + ((mach_msg_guarded_port_descriptor64_t *)daddr)->disposition : + ((mach_msg_guarded_port_descriptor32_t *)daddr)->disposition; + + /* Only MACH_MSG_TYPE_MOVE_RECEIVE is supported for now */ + if (!guard_flags || ((guard_flags & ~MACH_MSG_GUARD_FLAGS_MASK) != 0) || + ((guard_flags & MACH_MSG_GUARD_FLAGS_UNGUARDED_ON_SEND) && (context != 0)) || + (disp != MACH_MSG_TYPE_MOVE_RECEIVE)) { + /* + * Invalid guard flags, context or disposition + */ + mr = MACH_SEND_INVALID_TYPE; + goto clean_message; + } + if (os_add_overflow(total_ool_port_count, 1, &total_ool_port_count)) { + /* Overflow detected */ + mr = MACH_SEND_TOO_LARGE; + goto clean_message; + } + contains_port_desc = TRUE; + break; } - break; } - } /* Sending more than 16383 rights in one message seems crazy */ if (total_ool_port_count >= (MACH_PORT_UREFS_MAX / 4)) { @@ -3108,79 +3543,96 @@ ipc_kmsg_copyin_body( goto clean_message; } - /* - * Allocate space in the pageable kernel ipc copy map for all the - * ool data that is to be physically copied. Map is marked wait for - * space. - */ - if (space_needed) { - if (vm_allocate_kernel(ipc_kernel_copy_map, &paddr, space_needed, - VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC) != KERN_SUCCESS) { - mr = MACH_MSG_VM_KERNEL; - goto clean_message; - } - } - - /* user_addr = just after base as it was copied in */ - user_addr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t)); - - /* Shift the mach_msg_base_t down to make room for dsc_count*16bytes of descriptors */ - if(descriptor_size != 16*dsc_count) { - vm_offset_t dsc_adjust = 16*dsc_count - descriptor_size; - - memmove((char *)(((vm_offset_t)kmsg->ikm_header) - dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t)); - kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header - dsc_adjust); - - /* Update the message size for the larger in-kernel representation */ - kmsg->ikm_header->msgh_size += (mach_msg_size_t)dsc_adjust; - } - - - /* kern_addr = just after base after it has been (conditionally) moved */ - kern_addr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t)); - - /* handle the OOL regions and port descriptors. */ - for(i=0;itype.type) { - case MACH_MSG_PORT_DESCRIPTOR: - user_addr = ipc_kmsg_copyin_port_descriptor((mach_msg_port_descriptor_t *)kern_addr, - (mach_msg_legacy_port_descriptor_t *)user_addr, space, dest, kmsg, &mr); - kern_addr++; - complex = TRUE; - break; - case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: - case MACH_MSG_OOL_DESCRIPTOR: - user_addr = ipc_kmsg_copyin_ool_descriptor((mach_msg_ool_descriptor_t *)kern_addr, - user_addr, is_task_64bit, &paddr, ©, &space_needed, map, &mr); - kern_addr++; - complex = TRUE; - break; - case MACH_MSG_OOL_PORTS_DESCRIPTOR: - user_addr = ipc_kmsg_copyin_ool_ports_descriptor((mach_msg_ool_ports_descriptor_t *)kern_addr, - user_addr, is_task_64bit, map, space, dest, kmsg, &mr); - kern_addr++; - complex = TRUE; - break; - default: - /* Invalid descriptor */ - mr = MACH_SEND_INVALID_TYPE; - break; - } - - if (MACH_MSG_SUCCESS != mr) { - /* clean from start of message descriptors to i */ - ipc_kmsg_clean_partial(kmsg, i, - (mach_msg_descriptor_t *)((mach_msg_base_t *)kmsg->ikm_header + 1), - paddr, space_needed); - goto out; - } - } /* End of loop */ - - if (!complex) { - kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_COMPLEX; - } - out: - return mr; + /* + * Check if dest is a no-grant port; Since this bit is set only on + * port construction and cannot be unset later, we can peek at the + * bit without paying the cost of locking the port. + */ + if (contains_port_desc && remote_port->ip_no_grant) { + mr = MACH_SEND_NO_GRANT_DEST; + goto clean_message; + } + + /* + * Allocate space in the pageable kernel ipc copy map for all the + * ool data that is to be physically copied. Map is marked wait for + * space. + */ + if (space_needed) { + if (vm_allocate_kernel(ipc_kernel_copy_map, &paddr, space_needed, + VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC) != KERN_SUCCESS) { + mr = MACH_MSG_VM_KERNEL; + goto clean_message; + } + } + + /* user_addr = just after base as it was copied in */ + user_addr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t)); + + /* Shift the mach_msg_base_t down to make room for dsc_count*16bytes of descriptors on 64 bit kernels + */ + if (descriptor_size != 16 * dsc_count) { + vm_offset_t dsc_adjust = 16 * dsc_count - descriptor_size; + + memmove((char *)(((vm_offset_t)kmsg->ikm_header) - dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t)); + kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header - dsc_adjust); + + /* Update the message size for the larger in-kernel representation */ + kmsg->ikm_header->msgh_size += (mach_msg_size_t)dsc_adjust; + } + + + /* kern_addr = just after base after it has been (conditionally) moved */ + kern_addr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t)); + + /* handle the OOL regions and port descriptors. */ + for (i = 0; i < dsc_count; i++) { + switch (user_addr->type.type) { + case MACH_MSG_PORT_DESCRIPTOR: + user_addr = ipc_kmsg_copyin_port_descriptor((mach_msg_port_descriptor_t *)kern_addr, + (mach_msg_legacy_port_descriptor_t *)user_addr, space, dest, kmsg, optionp, &mr); + kern_addr++; + complex = TRUE; + break; + case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: + case MACH_MSG_OOL_DESCRIPTOR: + user_addr = ipc_kmsg_copyin_ool_descriptor((mach_msg_ool_descriptor_t *)kern_addr, + user_addr, is_task_64bit, &paddr, ©, &space_needed, map, optionp, &mr); + kern_addr++; + complex = TRUE; + break; + case MACH_MSG_OOL_PORTS_DESCRIPTOR: + user_addr = ipc_kmsg_copyin_ool_ports_descriptor((mach_msg_ool_ports_descriptor_t *)kern_addr, + user_addr, is_task_64bit, map, space, dest, kmsg, optionp, &mr); + kern_addr++; + complex = TRUE; + break; + case MACH_MSG_GUARDED_PORT_DESCRIPTOR: + user_addr = ipc_kmsg_copyin_guarded_port_descriptor((mach_msg_guarded_port_descriptor_t *)kern_addr, + user_addr, is_task_64bit, space, dest, kmsg, optionp, &mr); + kern_addr++; + complex = TRUE; + break; + default: + /* Invalid descriptor */ + mr = MACH_SEND_INVALID_TYPE; + break; + } + + if (MACH_MSG_SUCCESS != mr) { + /* clean from start of message descriptors to i */ + ipc_kmsg_clean_partial(kmsg, i, + (mach_msg_descriptor_t *)((mach_msg_base_t *)kmsg->ikm_header + 1), + paddr, space_needed); + goto out; + } + } /* End of loop */ + + if (!complex) { + kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_COMPLEX; + } +out: + return mr; clean_message: /* no descriptors have been copied in yet */ @@ -3215,50 +3667,49 @@ clean_message: mach_msg_return_t ipc_kmsg_copyin( - ipc_kmsg_t kmsg, - ipc_space_t space, - vm_map_t map, + ipc_kmsg_t kmsg, + ipc_space_t space, + vm_map_t map, mach_msg_priority_t override, - mach_msg_option_t *optionp) + mach_msg_option_t *optionp) { - mach_msg_return_t mr; + mach_msg_return_t mr; - kmsg->ikm_header->msgh_bits &= MACH_MSGH_BITS_USER; + kmsg->ikm_header->msgh_bits &= MACH_MSGH_BITS_USER; - mr = ipc_kmsg_copyin_header(kmsg, space, override, optionp); + mr = ipc_kmsg_copyin_header(kmsg, space, override, optionp); - if (mr != MACH_MSG_SUCCESS) - return mr; + if (mr != MACH_MSG_SUCCESS) { + return mr; + } - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_MSG_SEND) | DBG_FUNC_NONE, - VM_KERNEL_ADDRPERM((uintptr_t)kmsg), - (uintptr_t)kmsg->ikm_header->msgh_bits, - (uintptr_t)kmsg->ikm_header->msgh_id, - VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(kmsg->ikm_voucher)), - 0); - - DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_copyin header:\n%.8x\n%.8x\n%p\n%p\n%p\n%.8x\n", - kmsg->ikm_header->msgh_size, - kmsg->ikm_header->msgh_bits, - kmsg->ikm_header->msgh_remote_port, - kmsg->ikm_header->msgh_local_port, - kmsg->ikm_voucher, - kmsg->ikm_header->msgh_id); - - if ((kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) == 0) - return MACH_MSG_SUCCESS; - - mr = ipc_kmsg_copyin_body( kmsg, space, map); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_SEND) | DBG_FUNC_NONE, + VM_KERNEL_ADDRPERM((uintptr_t)kmsg), + (uintptr_t)kmsg->ikm_header->msgh_bits, + (uintptr_t)kmsg->ikm_header->msgh_id, + VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(kmsg->ikm_voucher)), + 0); + + DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_copyin header:\n%.8x\n%.8x\n%p\n%p\n%p\n%.8x\n", + kmsg->ikm_header->msgh_size, + kmsg->ikm_header->msgh_bits, + kmsg->ikm_header->msgh_remote_port, + kmsg->ikm_header->msgh_local_port, + kmsg->ikm_voucher, + kmsg->ikm_header->msgh_id); + + if ((kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) == 0) { + return MACH_MSG_SUCCESS; + } + mr = ipc_kmsg_copyin_body( kmsg, space, map, optionp); /* unreachable if !DEBUG */ __unreachable_ok_push - if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) - { + if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { kprintf("body:\n"); uint32_t i; - for(i=0;i*4 < (kmsg->ikm_header->msgh_size - sizeof(mach_msg_header_t));i++) - { - kprintf("%.4x\n",((uint32_t *)(kmsg->ikm_header + 1))[i]); + for (i = 0; i * 4 < (kmsg->ikm_header->msgh_size - sizeof(mach_msg_header_t)); i++) { + kprintf("%.4x\n", ((uint32_t *)(kmsg->ikm_header + 1))[i]); } } __unreachable_ok_pop @@ -3281,21 +3732,24 @@ ipc_kmsg_copyin( mach_msg_return_t ipc_kmsg_copyin_from_kernel( - ipc_kmsg_t kmsg) + ipc_kmsg_t kmsg) { mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits; mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits); mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits); - ipc_object_t remote = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; - ipc_object_t local = (ipc_object_t) kmsg->ikm_header->msgh_local_port; + ipc_object_t remote = ip_to_object(kmsg->ikm_header->msgh_remote_port); + ipc_object_t local = ip_to_object(kmsg->ikm_header->msgh_local_port); + ipc_port_t dest = kmsg->ikm_header->msgh_remote_port; /* translate the destination and reply ports */ - if (!IO_VALID(remote)) + if (!IO_VALID(remote)) { return MACH_SEND_INVALID_DEST; + } ipc_object_copyin_from_kernel(remote, rname); - if (IO_VALID(local)) + if (IO_VALID(local)) { ipc_object_copyin_from_kernel(local, lname); + } /* * The common case is a complex message with no reply port, @@ -3303,133 +3757,188 @@ ipc_kmsg_copyin_from_kernel( */ if (bits == (MACH_MSGH_BITS_COMPLEX | - MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) { + MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) { bits = (MACH_MSGH_BITS_COMPLEX | - MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0)); + MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0)); kmsg->ikm_header->msgh_bits = bits; } else { bits = (MACH_MSGH_BITS_OTHER(bits) | - MACH_MSGH_BITS(ipc_object_copyin_type(rname), - ipc_object_copyin_type(lname))); + MACH_MSGH_BITS(ipc_object_copyin_type(rname), + ipc_object_copyin_type(lname))); kmsg->ikm_header->msgh_bits = bits; - if ((bits & MACH_MSGH_BITS_COMPLEX) == 0) + if ((bits & MACH_MSGH_BITS_COMPLEX) == 0) { return MACH_MSG_SUCCESS; + } } - { - mach_msg_descriptor_t *saddr; - mach_msg_body_t *body; - mach_msg_type_number_t i, count; - - body = (mach_msg_body_t *) (kmsg->ikm_header + 1); - saddr = (mach_msg_descriptor_t *) (body + 1); - count = body->msgh_descriptor_count; - - for (i = 0; i < count; i++, saddr++) { - - switch (saddr->type.type) { - - case MACH_MSG_PORT_DESCRIPTOR: { - mach_msg_type_name_t name; - ipc_object_t object; - mach_msg_port_descriptor_t *dsc; - - dsc = &saddr->port; - - /* this is really the type SEND, SEND_ONCE, etc. */ - name = dsc->disposition; - object = (ipc_object_t) dsc->name; - dsc->disposition = ipc_object_copyin_type(name); - - if (!IO_VALID(object)) { - break; - } - - ipc_object_copyin_from_kernel(object, name); - - /* CDY avoid circularity when the destination is also */ - /* the kernel. This check should be changed into an */ - /* assert when the new kobject model is in place since*/ - /* ports will not be used in kernel to kernel chats */ - - if (((ipc_port_t)remote)->ip_receiver != ipc_space_kernel) { - if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && - ipc_port_check_circularity((ipc_port_t) object, - (ipc_port_t) remote)) { - kmsg->ikm_header->msgh_bits |= - MACH_MSGH_BITS_CIRCULAR; - } - } - break; - } - case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: - case MACH_MSG_OOL_DESCRIPTOR: { - /* - * The sender should supply ready-made memory, i.e. - * a vm_map_copy_t, so we don't need to do anything. - */ - break; - } - case MACH_MSG_OOL_PORTS_DESCRIPTOR: { - ipc_object_t *objects; - unsigned int j; - mach_msg_type_name_t name; - mach_msg_ool_ports_descriptor_t *dsc; - - dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports; - - /* this is really the type SEND, SEND_ONCE, etc. */ - name = dsc->disposition; - dsc->disposition = ipc_object_copyin_type(name); - - objects = (ipc_object_t *) dsc->address; - - for ( j = 0; j < dsc->count; j++) { - ipc_object_t object = objects[j]; - - if (!IO_VALID(object)) - continue; - - ipc_object_copyin_from_kernel(object, name); - - if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && - ipc_port_check_circularity( - (ipc_port_t) object, - (ipc_port_t) remote)) - kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; - } - break; - } - default: { -#if MACH_ASSERT - panic("ipc_kmsg_copyin_from_kernel: bad descriptor"); -#endif /* MACH_ASSERT */ + + /* + * Check if the remote port accepts ports in the body. + */ + if (dest->ip_no_grant) { + mach_msg_descriptor_t *saddr; + mach_msg_body_t *body; + mach_msg_type_number_t i, count; + + body = (mach_msg_body_t *) (kmsg->ikm_header + 1); + saddr = (mach_msg_descriptor_t *) (body + 1); + count = body->msgh_descriptor_count; + + for (i = 0; i < count; i++, saddr++) { + switch (saddr->type.type) { + case MACH_MSG_PORT_DESCRIPTOR: + case MACH_MSG_OOL_PORTS_DESCRIPTOR: + case MACH_MSG_GUARDED_PORT_DESCRIPTOR: + /* no descriptors have been copied in yet */ + ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0); + return MACH_SEND_NO_GRANT_DEST; + } + } + } + { + mach_msg_descriptor_t *saddr; + mach_msg_body_t *body; + mach_msg_type_number_t i, count; + + body = (mach_msg_body_t *) (kmsg->ikm_header + 1); + saddr = (mach_msg_descriptor_t *) (body + 1); + count = body->msgh_descriptor_count; + + for (i = 0; i < count; i++, saddr++) { + switch (saddr->type.type) { + case MACH_MSG_PORT_DESCRIPTOR: { + mach_msg_type_name_t name; + ipc_object_t object; + mach_msg_port_descriptor_t *dsc; + + dsc = &saddr->port; + + /* this is really the type SEND, SEND_ONCE, etc. */ + name = dsc->disposition; + object = ip_to_object(dsc->name); + dsc->disposition = ipc_object_copyin_type(name); + + if (!IO_VALID(object)) { + break; + } + + ipc_object_copyin_from_kernel(object, name); + + /* CDY avoid circularity when the destination is also */ + /* the kernel. This check should be changed into an */ + /* assert when the new kobject model is in place since*/ + /* ports will not be used in kernel to kernel chats */ + + if (ip_object_to_port(remote)->ip_receiver != ipc_space_kernel) { + if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && + ipc_port_check_circularity(ip_object_to_port(object), + ip_object_to_port(remote))) { + kmsg->ikm_header->msgh_bits |= + MACH_MSGH_BITS_CIRCULAR; + } + } + break; + } + case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: + case MACH_MSG_OOL_DESCRIPTOR: { + /* + * The sender should supply ready-made memory, i.e. + * a vm_map_copy_t, so we don't need to do anything. + */ + break; + } + case MACH_MSG_OOL_PORTS_DESCRIPTOR: { + ipc_object_t *objects; + unsigned int j; + mach_msg_type_name_t name; + mach_msg_ool_ports_descriptor_t *dsc; + + dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports; + + /* this is really the type SEND, SEND_ONCE, etc. */ + name = dsc->disposition; + dsc->disposition = ipc_object_copyin_type(name); + + objects = (ipc_object_t *) dsc->address; + + for (j = 0; j < dsc->count; j++) { + ipc_object_t object = objects[j]; + + if (!IO_VALID(object)) { + continue; + } + + ipc_object_copyin_from_kernel(object, name); + + if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && + ipc_port_check_circularity(ip_object_to_port(object), + ip_object_to_port(remote))) { + kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; + } + } + break; + } + case MACH_MSG_GUARDED_PORT_DESCRIPTOR: { + mach_msg_guarded_port_descriptor_t *dsc = (typeof(dsc)) & saddr->guarded_port; + mach_msg_type_name_t disp = dsc->disposition; + ipc_object_t object = ip_to_object(dsc->name); + dsc->disposition = ipc_object_copyin_type(disp); + assert(dsc->flags == 0); + + if (!IO_VALID(object)) { + break; + } + + ipc_object_copyin_from_kernel(object, disp); + /* + * avoid circularity when the destination is also + * the kernel. This check should be changed into an + * assert when the new kobject model is in place since + * ports will not be used in kernel to kernel chats + */ + + if (ip_object_to_port(remote)->ip_receiver != ipc_space_kernel) { + if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && + ipc_port_check_circularity(ip_object_to_port(object), + ip_object_to_port(remote))) { + kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; + } + } + break; + } + default: { +#if MACH_ASSERT + panic("ipc_kmsg_copyin_from_kernel: bad descriptor"); +#endif /* MACH_ASSERT */ + } + } } - } } - } - return MACH_MSG_SUCCESS; + return MACH_MSG_SUCCESS; } #if IKM_SUPPORT_LEGACY mach_msg_return_t ipc_kmsg_copyin_from_kernel_legacy( - ipc_kmsg_t kmsg) + ipc_kmsg_t kmsg) { mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits; mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits); mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits); - ipc_object_t remote = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; - ipc_object_t local = (ipc_object_t) kmsg->ikm_header->msgh_local_port; + ipc_object_t remote = ip_to_object(kmsg->ikm_header->msgh_remote_port); + ipc_object_t local = ip_to_object(kmsg->ikm_header->msgh_local_port); + ipc_port_t dest = kmsg->ikm_header->msgh_remote_port; /* translate the destination and reply ports */ - if (!IO_VALID(remote)) + if (!IO_VALID(remote)) { return MACH_SEND_INVALID_DEST; + } ipc_object_copyin_from_kernel(remote, rname); - if (IO_VALID(local)) + if (IO_VALID(local)) { ipc_object_copyin_from_kernel(local, lname); + } /* * The common case is a complex message with no reply port, @@ -3437,152 +3946,215 @@ ipc_kmsg_copyin_from_kernel_legacy( */ if (bits == (MACH_MSGH_BITS_COMPLEX | - MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) { + MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) { bits = (MACH_MSGH_BITS_COMPLEX | - MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0)); + MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0)); kmsg->ikm_header->msgh_bits = bits; } else { bits = (MACH_MSGH_BITS_OTHER(bits) | - MACH_MSGH_BITS(ipc_object_copyin_type(rname), - ipc_object_copyin_type(lname))); + MACH_MSGH_BITS(ipc_object_copyin_type(rname), + ipc_object_copyin_type(lname))); kmsg->ikm_header->msgh_bits = bits; - if ((bits & MACH_MSGH_BITS_COMPLEX) == 0) + if ((bits & MACH_MSGH_BITS_COMPLEX) == 0) { return MACH_MSG_SUCCESS; + } } - { - mach_msg_legacy_descriptor_t *saddr; - mach_msg_descriptor_t *daddr; - mach_msg_body_t *body; - mach_msg_type_number_t i, count; - - body = (mach_msg_body_t *) (kmsg->ikm_header + 1); - saddr = (typeof(saddr)) (body + 1); - count = body->msgh_descriptor_count; - - if(count) { - vm_offset_t dsc_adjust = 4*count; - memmove((char *)(((vm_offset_t)kmsg->ikm_header) - dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t)); - kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header - dsc_adjust); - /* Update the message size for the larger in-kernel representation */ - kmsg->ikm_header->msgh_size += dsc_adjust; - } - daddr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t)); - - for (i = 0; i < count; i++, saddr++, daddr++) { - switch (saddr->type.type) { - - case MACH_MSG_PORT_DESCRIPTOR: { - mach_msg_type_name_t name; - ipc_object_t object; - mach_msg_legacy_port_descriptor_t *dsc; - mach_msg_port_descriptor_t *dest_dsc; - - dsc = (typeof(dsc))&saddr->port; - dest_dsc = &daddr->port; - - /* this is really the type SEND, SEND_ONCE, etc. */ - name = dsc->disposition; - object = (ipc_object_t) CAST_MACH_NAME_TO_PORT(dsc->name); - dest_dsc->disposition = ipc_object_copyin_type(name); - dest_dsc->name = (mach_port_t)object; - dest_dsc->type = MACH_MSG_PORT_DESCRIPTOR; - - if (!IO_VALID(object)) { - break; - } - - ipc_object_copyin_from_kernel(object, name); - - /* CDY avoid circularity when the destination is also */ - /* the kernel. This check should be changed into an */ - /* assert when the new kobject model is in place since*/ - /* ports will not be used in kernel to kernel chats */ - - if (((ipc_port_t)remote)->ip_receiver != ipc_space_kernel) { - if ((dest_dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && - ipc_port_check_circularity((ipc_port_t) object, - (ipc_port_t) remote)) { - kmsg->ikm_header->msgh_bits |= - MACH_MSGH_BITS_CIRCULAR; - } - } - break; - } - case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: - case MACH_MSG_OOL_DESCRIPTOR: { - /* The sender should supply ready-made memory, i.e. a vm_map_copy_t - * so we don't need to do anything special. */ - - mach_msg_ool_descriptor32_t *source_dsc = &saddr->out_of_line32; - mach_msg_ool_descriptor_t *dest_dsc = (typeof(dest_dsc))&daddr->out_of_line; - - vm_offset_t address = source_dsc->address; - vm_size_t size = source_dsc->size; - boolean_t deallocate = source_dsc->deallocate; - mach_msg_copy_options_t copy = source_dsc->copy; - mach_msg_descriptor_type_t type = source_dsc->type; - - dest_dsc->address = (void *)address; - dest_dsc->size = size; - dest_dsc->deallocate = deallocate; - dest_dsc->copy = copy; - dest_dsc->type = type; - break; - } - case MACH_MSG_OOL_PORTS_DESCRIPTOR: { - ipc_object_t *objects; - unsigned int j; - mach_msg_type_name_t name; - mach_msg_ool_ports_descriptor_t *dest_dsc; - - mach_msg_ool_ports_descriptor32_t *source_dsc = &saddr->ool_ports32; - dest_dsc = (typeof(dest_dsc))&daddr->ool_ports; - - boolean_t deallocate = source_dsc->deallocate; - mach_msg_copy_options_t copy = source_dsc->copy; - mach_msg_size_t port_count = source_dsc->count; - mach_msg_type_name_t disposition = source_dsc->disposition; - - /* this is really the type SEND, SEND_ONCE, etc. */ - name = disposition; - disposition = ipc_object_copyin_type(name); - - objects = (ipc_object_t *) (uintptr_t)source_dsc->address; - - for ( j = 0; j < port_count; j++) { - ipc_object_t object = objects[j]; - - if (!IO_VALID(object)) - continue; - - ipc_object_copyin_from_kernel(object, name); - - if ((disposition == MACH_MSG_TYPE_PORT_RECEIVE) && - ipc_port_check_circularity( - (ipc_port_t) object, - (ipc_port_t) remote)) - kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; - } - - dest_dsc->address = objects; - dest_dsc->deallocate = deallocate; - dest_dsc->copy = copy; - dest_dsc->disposition = disposition; - dest_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR; - dest_dsc->count = port_count; - break; - } - default: { -#if MACH_ASSERT - panic("ipc_kmsg_copyin_from_kernel: bad descriptor"); -#endif /* MACH_ASSERT */ + + if (dest->ip_no_grant) { + mach_msg_descriptor_t *saddr; + mach_msg_body_t *body; + mach_msg_type_number_t i, count; + + body = (mach_msg_body_t *) (kmsg->ikm_header + 1); + saddr = (mach_msg_descriptor_t *) (body + 1); + count = body->msgh_descriptor_count; + + for (i = 0; i < count; i++, saddr++) { + switch (saddr->type.type) { + case MACH_MSG_PORT_DESCRIPTOR: + case MACH_MSG_OOL_PORTS_DESCRIPTOR: + case MACH_MSG_GUARDED_PORT_DESCRIPTOR: + /* no descriptors have been copied in yet */ + ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0); + return MACH_SEND_NO_GRANT_DEST; + } } - } } - } - return MACH_MSG_SUCCESS; + + { + mach_msg_legacy_descriptor_t *saddr; + mach_msg_descriptor_t *daddr; + mach_msg_body_t *body; + mach_msg_type_number_t i, count; + + body = (mach_msg_body_t *) (kmsg->ikm_header + 1); + saddr = (typeof(saddr))(body + 1); + count = body->msgh_descriptor_count; + + if (count) { + vm_offset_t dsc_adjust = 4 * count; + memmove((char *)(((vm_offset_t)kmsg->ikm_header) - dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t)); + kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header - dsc_adjust); + /* Update the message size for the larger in-kernel representation */ + kmsg->ikm_header->msgh_size += dsc_adjust; + } + daddr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t)); + + for (i = 0; i < count; i++, saddr++, daddr++) { + switch (saddr->type.type) { + case MACH_MSG_PORT_DESCRIPTOR: { + mach_msg_type_name_t name; + ipc_object_t object; + mach_msg_legacy_port_descriptor_t *dsc; + mach_msg_port_descriptor_t *dest_dsc; + + dsc = (typeof(dsc)) & saddr->port; + dest_dsc = &daddr->port; + + /* this is really the type SEND, SEND_ONCE, etc. */ + name = dsc->disposition; + object = ip_to_object(CAST_MACH_NAME_TO_PORT(dsc->name)); + dest_dsc->disposition = ipc_object_copyin_type(name); + dest_dsc->name = ip_object_to_port(object); + dest_dsc->type = MACH_MSG_PORT_DESCRIPTOR; + + if (!IO_VALID(object)) { + break; + } + + ipc_object_copyin_from_kernel(object, name); + + /* CDY avoid circularity when the destination is also */ + /* the kernel. This check should be changed into an */ + /* assert when the new kobject model is in place since*/ + /* ports will not be used in kernel to kernel chats */ + + if (ip_object_to_port(remote)->ip_receiver != ipc_space_kernel) { + if ((dest_dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && + ipc_port_check_circularity(ip_object_to_port(object), + ip_object_to_port(remote))) { + kmsg->ikm_header->msgh_bits |= + MACH_MSGH_BITS_CIRCULAR; + } + } + break; + } + case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: + case MACH_MSG_OOL_DESCRIPTOR: { + /* The sender should supply ready-made memory, i.e. a vm_map_copy_t + * so we don't need to do anything special. */ + + mach_msg_ool_descriptor32_t *source_dsc = &saddr->out_of_line32; + mach_msg_ool_descriptor_t *dest_dsc = (typeof(dest_dsc)) & daddr->out_of_line; + + vm_offset_t address = source_dsc->address; + vm_size_t size = source_dsc->size; + boolean_t deallocate = source_dsc->deallocate; + mach_msg_copy_options_t copy = source_dsc->copy; + mach_msg_descriptor_type_t type = source_dsc->type; + + dest_dsc->address = (void *)address; + dest_dsc->size = size; + dest_dsc->deallocate = deallocate; + dest_dsc->copy = copy; + dest_dsc->type = type; + break; + } + case MACH_MSG_OOL_PORTS_DESCRIPTOR: { + ipc_object_t *objects; + unsigned int j; + mach_msg_type_name_t name; + mach_msg_ool_ports_descriptor_t *dest_dsc; + + mach_msg_ool_ports_descriptor32_t *source_dsc = &saddr->ool_ports32; + dest_dsc = (typeof(dest_dsc)) & daddr->ool_ports; + + boolean_t deallocate = source_dsc->deallocate; + mach_msg_copy_options_t copy = source_dsc->copy; + mach_msg_size_t port_count = source_dsc->count; + mach_msg_type_name_t disposition = source_dsc->disposition; + + /* this is really the type SEND, SEND_ONCE, etc. */ + name = disposition; + disposition = ipc_object_copyin_type(name); + + objects = (ipc_object_t *) (uintptr_t)source_dsc->address; + + for (j = 0; j < port_count; j++) { + ipc_object_t object = objects[j]; + + if (!IO_VALID(object)) { + continue; + } + + ipc_object_copyin_from_kernel(object, name); + + if ((disposition == MACH_MSG_TYPE_PORT_RECEIVE) && + ipc_port_check_circularity(ip_object_to_port(object), + ip_object_to_port(remote))) { + kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; + } + } + + dest_dsc->address = objects; + dest_dsc->deallocate = deallocate; + dest_dsc->copy = copy; + dest_dsc->disposition = disposition; + dest_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR; + dest_dsc->count = port_count; + break; + } + case MACH_MSG_GUARDED_PORT_DESCRIPTOR: { + mach_msg_type_name_t disp; + ipc_object_t object; + mach_msg_guarded_port_descriptor32_t *dsc; + mach_msg_guarded_port_descriptor_t *dest_dsc; + + dsc = (typeof(dsc)) & saddr->guarded_port32; + dest_dsc = &daddr->guarded_port; + + disp = dsc->disposition; + object = ip_to_object(CAST_MACH_NAME_TO_PORT(dsc->name)); + assert(dsc->flags == 0); + assert(dsc->context == 0); + + dest_dsc->disposition = ipc_object_copyin_type(disp); + dest_dsc->name = ip_object_to_port(object); + dest_dsc->type = MACH_MSG_GUARDED_PORT_DESCRIPTOR; + dest_dsc->flags = 0; + + if (!IO_VALID(object)) { + break; + } + + ipc_object_copyin_from_kernel(object, disp); + + /* CDY avoid circularity when the destination is also */ + /* the kernel. This check should be changed into an */ + /* assert when the new kobject model is in place since*/ + /* ports will not be used in kernel to kernel chats */ + + if (ip_object_to_port(remote)->ip_receiver != ipc_space_kernel) { + if ((dest_dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && + ipc_port_check_circularity(ip_object_to_port(object), + ip_object_to_port(remote))) { + kmsg->ikm_header->msgh_bits |= + MACH_MSGH_BITS_CIRCULAR; + } + } + break; + } + default: { +#if MACH_ASSERT + panic("ipc_kmsg_copyin_from_kernel: bad descriptor"); +#endif /* MACH_ASSERT */ + } + } + } + } + return MACH_MSG_SUCCESS; } #endif /* IKM_SUPPORT_LEGACY */ @@ -3599,7 +4171,7 @@ ipc_kmsg_copyin_from_kernel_legacy( * Nothing locked. * Returns: * MACH_MSG_SUCCESS Copied out port rights. - * MACH_RCV_INVALID_NOTIFY + * MACH_RCV_INVALID_NOTIFY * Notify is non-null and doesn't name a receive right. * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.) * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE @@ -3615,12 +4187,12 @@ ipc_kmsg_copyin_from_kernel_legacy( mach_msg_return_t ipc_kmsg_copyout_header( ipc_kmsg_t kmsg, - ipc_space_t space, - mach_msg_option_t option) + ipc_space_t space, + mach_msg_option_t option) { mach_msg_header_t *msg = kmsg->ikm_header; mach_msg_bits_t mbits = msg->msgh_bits; - ipc_port_t dest = (ipc_port_t) msg->msgh_remote_port; + ipc_port_t dest = msg->msgh_remote_port; assert(IP_VALID(dest)); @@ -3629,295 +4201,329 @@ ipc_kmsg_copyout_header( * process all send-possible notfications we received along with * the message. */ - ipc_port_spnotify(dest); - - { - mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits); - mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits); - mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits); - ipc_port_t reply = msg->msgh_local_port; - ipc_port_t release_reply_port = IP_NULL; - mach_port_name_t dest_name, reply_name; - - ipc_port_t voucher = kmsg->ikm_voucher; - ipc_port_t release_voucher_port = IP_NULL; - mach_port_name_t voucher_name; - - uint32_t entries_held = 0; - boolean_t need_write_lock = FALSE; - kern_return_t kr; - - /* - * Reserve any potentially needed entries in the target space. - * We'll free any unused before unlocking the space. - */ - if (IP_VALID(reply)) { - entries_held++; - need_write_lock = TRUE; - } - if (IP_VALID(voucher)) { - assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); - - if ((option & MACH_RCV_VOUCHER) != 0) - entries_held++; - need_write_lock = TRUE; - } - - if (need_write_lock) { + ipc_port_spnotify(dest); - is_write_lock(space); + { + mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits); + mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits); + mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits); + ipc_port_t reply = msg->msgh_local_port; + ipc_port_t release_reply_port = IP_NULL; + mach_port_name_t dest_name, reply_name; - while(entries_held) { - if (!is_active(space)) { - is_write_unlock(space); - return (MACH_RCV_HEADER_ERROR| - MACH_MSG_IPC_SPACE); - } - - kr = ipc_entries_hold(space, entries_held); - if (KERN_SUCCESS == kr) - break; + ipc_port_t voucher = kmsg->ikm_voucher; + ipc_port_t release_voucher_port = IP_NULL; + mach_port_name_t voucher_name; - kr = ipc_entry_grow_table(space, ITS_SIZE_NONE); - if (KERN_SUCCESS != kr) - return(MACH_RCV_HEADER_ERROR| - MACH_MSG_IPC_SPACE); - /* space was unlocked and relocked - retry */ - } + uint32_t entries_held = 0; + boolean_t need_write_lock = FALSE; + kern_return_t kr; - /* Handle reply port. */ + /* + * Reserve any potentially needed entries in the target space. + * We'll free any unused before unlocking the space. + */ if (IP_VALID(reply)) { - ipc_entry_t entry; + entries_held++; + need_write_lock = TRUE; + } + if (IP_VALID(voucher)) { + assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); - /* Is there already an entry we can use? */ - if ((reply_type != MACH_MSG_TYPE_PORT_SEND_ONCE) && - ipc_right_reverse(space, (ipc_object_t) reply, &reply_name, &entry)) { - /* reply port is locked and active */ - assert(entry->ie_bits & MACH_PORT_TYPE_SEND_RECEIVE); - } else { - ip_lock(reply); - if (!ip_active(reply)) { - ip_unlock(reply); - - release_reply_port = reply; - reply = IP_DEAD; - reply_name = MACH_PORT_DEAD; - goto done_with_reply; - } - - /* claim a held entry for the reply port */ - assert(entries_held > 0); - entries_held--; - ipc_entry_claim(space, &reply_name, &entry); - assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE); - assert(entry->ie_object == IO_NULL); - entry->ie_object = (ipc_object_t) reply; + if ((option & MACH_RCV_VOUCHER) != 0) { + entries_held++; } + need_write_lock = TRUE; + } - /* space and reply port are locked and active */ - ip_reference(reply); /* hold onto the reply port */ - - kr = ipc_right_copyout(space, reply_name, entry, - reply_type, TRUE, (ipc_object_t) reply); - assert(kr == KERN_SUCCESS); - /* reply port is unlocked */ - } else - reply_name = CAST_MACH_PORT_TO_NAME(reply); + if (need_write_lock) { + is_write_lock(space); - done_with_reply: + while (entries_held) { + if (!is_active(space)) { + is_write_unlock(space); + return MACH_RCV_HEADER_ERROR | + MACH_MSG_IPC_SPACE; + } - /* Handle voucher port. */ - if (voucher_type != MACH_MSGH_BITS_ZERO) { - assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); + kr = ipc_entries_hold(space, entries_held); + if (KERN_SUCCESS == kr) { + break; + } - if (!IP_VALID(voucher)) { - if ((option & MACH_RCV_VOUCHER) == 0) { - voucher_type = MACH_MSGH_BITS_ZERO; + kr = ipc_entry_grow_table(space, ITS_SIZE_NONE); + if (KERN_SUCCESS != kr) { + return MACH_RCV_HEADER_ERROR | + MACH_MSG_IPC_SPACE; } - voucher_name = MACH_PORT_NULL; - goto done_with_voucher; + /* space was unlocked and relocked - retry */ } - - /* clear voucher from its hiding place back in the kmsg */ - kmsg->ikm_voucher = IP_NULL; - if ((option & MACH_RCV_VOUCHER) != 0) { + /* Handle reply port. */ + if (IP_VALID(reply)) { ipc_entry_t entry; - if (ipc_right_reverse(space, (ipc_object_t) voucher, - &voucher_name, &entry)) { - /* voucher port locked */ - assert(entry->ie_bits & MACH_PORT_TYPE_SEND); + /* Is there already an entry we can use? */ + if ((reply_type != MACH_MSG_TYPE_PORT_SEND_ONCE) && + ipc_right_reverse(space, ip_to_object(reply), &reply_name, &entry)) { + /* reply port is locked and active */ + assert(entry->ie_bits & MACH_PORT_TYPE_SEND_RECEIVE); } else { + ip_lock(reply); + if (!ip_active(reply)) { + /* clear the context value */ + reply->ip_reply_context = 0; + ip_unlock(reply); + + release_reply_port = reply; + reply = IP_DEAD; + reply_name = MACH_PORT_DEAD; + goto done_with_reply; + } + + /* claim a held entry for the reply port */ assert(entries_held > 0); entries_held--; - ipc_entry_claim(space, &voucher_name, &entry); + ipc_entry_claim(space, &reply_name, &entry); assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE); - assert(entry->ie_object == IO_NULL); - entry->ie_object = (ipc_object_t) voucher; - ip_lock(voucher); + assert(entry->ie_object == IO_NULL); + entry->ie_object = ip_to_object(reply); + } + + /* space and reply port are locked and active */ + ip_reference(reply); /* hold onto the reply port */ + + /* + * If the receiver would like to enforce strict reply + * semantics, and the message looks like it expects a reply, + * and contains a voucher, then link the context in the + * voucher with the reply port so that the next message sent + * to the reply port must come from a thread that has a + * matching context (voucher). + */ + if (enforce_strict_reply && MACH_RCV_WITH_STRICT_REPLY(option) && IP_VALID(voucher)) { + if (ipc_kmsg_validate_reply_port_locked(reply, option) != KERN_SUCCESS) { + /* if the receiver isn't happy with the reply port: fail the receive. */ + ip_unlock(reply); + ipc_entry_dealloc(space, reply_name, entry); + is_write_unlock(space); + ip_release(reply); + return MACH_RCV_INVALID_REPLY; + } + ipc_kmsg_link_reply_context_locked(reply, voucher); + } else { + /* + * if the receive did not choose to participate + * in the strict reply/RPC, then don't enforce + * anything (as this could lead to booby-trapped + * messages that kill the server). + */ + reply->ip_reply_context = 0; } - /* space is locked and active */ - - assert(ip_active(voucher)); - assert(ip_kotype(voucher) == IKOT_VOUCHER); - kr = ipc_right_copyout(space, voucher_name, entry, - MACH_MSG_TYPE_MOVE_SEND, TRUE, - (ipc_object_t) voucher); - /* voucher port is unlocked */ + + kr = ipc_right_copyout(space, reply_name, entry, + reply_type, NULL, NULL, ip_to_object(reply)); + assert(kr == KERN_SUCCESS); + /* reply port is unlocked */ } else { - voucher_type = MACH_MSGH_BITS_ZERO; - release_voucher_port = voucher; - voucher_name = MACH_PORT_NULL; + reply_name = CAST_MACH_PORT_TO_NAME(reply); } - } else { - voucher_name = msg->msgh_voucher_port; - } - done_with_voucher: +done_with_reply: - ip_lock(dest); - is_write_unlock(space); + /* Handle voucher port. */ + if (voucher_type != MACH_MSGH_BITS_ZERO) { + assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); - } else { - /* - * No reply or voucher port! This is an easy case. - * We only need to have the space locked - * when locking the destination. - */ + if (!IP_VALID(voucher)) { + if ((option & MACH_RCV_VOUCHER) == 0) { + voucher_type = MACH_MSGH_BITS_ZERO; + } + voucher_name = MACH_PORT_NULL; + goto done_with_voucher; + } - is_read_lock(space); - if (!is_active(space)) { - is_read_unlock(space); - return MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE; - } + /* clear voucher from its hiding place back in the kmsg */ + kmsg->ikm_voucher = IP_NULL; + + if ((option & MACH_RCV_VOUCHER) != 0) { + ipc_entry_t entry; + + if (ipc_right_reverse(space, ip_to_object(voucher), + &voucher_name, &entry)) { + /* voucher port locked */ + assert(entry->ie_bits & MACH_PORT_TYPE_SEND); + } else { + assert(entries_held > 0); + entries_held--; + ipc_entry_claim(space, &voucher_name, &entry); + assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE); + assert(entry->ie_object == IO_NULL); + entry->ie_object = ip_to_object(voucher); + ip_lock(voucher); + } + /* space is locked and active */ + require_ip_active(voucher); + assert(ip_kotype(voucher) == IKOT_VOUCHER); + kr = ipc_right_copyout(space, voucher_name, entry, + MACH_MSG_TYPE_MOVE_SEND, NULL, NULL, + ip_to_object(voucher)); + /* voucher port is unlocked */ + } else { + voucher_type = MACH_MSGH_BITS_ZERO; + release_voucher_port = voucher; + voucher_name = MACH_PORT_NULL; + } + } else { + voucher_name = msg->msgh_voucher_port; + } - ip_lock(dest); - is_read_unlock(space); +done_with_voucher: - reply_name = CAST_MACH_PORT_TO_NAME(reply); + ip_lock(dest); + is_write_unlock(space); + } else { + /* + * No reply or voucher port! This is an easy case. + * We only need to have the space locked + * when locking the destination. + */ - if (voucher_type != MACH_MSGH_BITS_ZERO) { - assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); - if ((option & MACH_RCV_VOUCHER) == 0) { - voucher_type = MACH_MSGH_BITS_ZERO; + is_read_lock(space); + if (!is_active(space)) { + is_read_unlock(space); + return MACH_RCV_HEADER_ERROR | MACH_MSG_IPC_SPACE; + } + + ip_lock(dest); + is_read_unlock(space); + + reply_name = CAST_MACH_PORT_TO_NAME(reply); + + if (voucher_type != MACH_MSGH_BITS_ZERO) { + assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); + if ((option & MACH_RCV_VOUCHER) == 0) { + voucher_type = MACH_MSGH_BITS_ZERO; + } + voucher_name = MACH_PORT_NULL; + } else { + voucher_name = msg->msgh_voucher_port; } - voucher_name = MACH_PORT_NULL; - } else { - voucher_name = msg->msgh_voucher_port; } - } - /* - * At this point, the space is unlocked and the destination - * port is locked. (Lock taken while space was locked.) - * reply_name is taken care of; we still need dest_name. - * We still hold a ref for reply (if it is valid). - * - * If the space holds receive rights for the destination, - * we return its name for the right. Otherwise the task - * managed to destroy or give away the receive right between - * receiving the message and this copyout. If the destination - * is dead, return MACH_PORT_DEAD, and if the receive right - * exists somewhere else (another space, in transit) - * return MACH_PORT_NULL. - * - * Making this copyout operation atomic with the previous - * copyout of the reply port is a bit tricky. If there was - * no real reply port (it wasn't IP_VALID) then this isn't - * an issue. If the reply port was dead at copyout time, - * then we are OK, because if dest is dead we serialize - * after the death of both ports and if dest is alive - * we serialize after reply died but before dest's (later) death. - * So assume reply was alive when we copied it out. If dest - * is alive, then we are OK because we serialize before - * the ports' deaths. So assume dest is dead when we look at it. - * If reply dies/died after dest, then we are OK because - * we serialize after dest died but before reply dies. - * So the hard case is when reply is alive at copyout, - * dest is dead at copyout, and reply died before dest died. - * In this case pretend that dest is still alive, so - * we serialize while both ports are alive. - * - * Because the space lock is held across the copyout of reply - * and locking dest, the receive right for dest can't move - * in or out of the space while the copyouts happen, so - * that isn't an atomicity problem. In the last hard case - * above, this implies that when dest is dead that the - * space couldn't have had receive rights for dest at - * the time reply was copied-out, so when we pretend - * that dest is still alive, we can return MACH_PORT_NULL. - * - * If dest == reply, then we have to make it look like - * either both copyouts happened before the port died, - * or both happened after the port died. This special - * case works naturally if the timestamp comparison - * is done correctly. - */ + /* + * At this point, the space is unlocked and the destination + * port is locked. (Lock taken while space was locked.) + * reply_name is taken care of; we still need dest_name. + * We still hold a ref for reply (if it is valid). + * + * If the space holds receive rights for the destination, + * we return its name for the right. Otherwise the task + * managed to destroy or give away the receive right between + * receiving the message and this copyout. If the destination + * is dead, return MACH_PORT_DEAD, and if the receive right + * exists somewhere else (another space, in transit) + * return MACH_PORT_NULL. + * + * Making this copyout operation atomic with the previous + * copyout of the reply port is a bit tricky. If there was + * no real reply port (it wasn't IP_VALID) then this isn't + * an issue. If the reply port was dead at copyout time, + * then we are OK, because if dest is dead we serialize + * after the death of both ports and if dest is alive + * we serialize after reply died but before dest's (later) death. + * So assume reply was alive when we copied it out. If dest + * is alive, then we are OK because we serialize before + * the ports' deaths. So assume dest is dead when we look at it. + * If reply dies/died after dest, then we are OK because + * we serialize after dest died but before reply dies. + * So the hard case is when reply is alive at copyout, + * dest is dead at copyout, and reply died before dest died. + * In this case pretend that dest is still alive, so + * we serialize while both ports are alive. + * + * Because the space lock is held across the copyout of reply + * and locking dest, the receive right for dest can't move + * in or out of the space while the copyouts happen, so + * that isn't an atomicity problem. In the last hard case + * above, this implies that when dest is dead that the + * space couldn't have had receive rights for dest at + * the time reply was copied-out, so when we pretend + * that dest is still alive, we can return MACH_PORT_NULL. + * + * If dest == reply, then we have to make it look like + * either both copyouts happened before the port died, + * or both happened after the port died. This special + * case works naturally if the timestamp comparison + * is done correctly. + */ - if (ip_active(dest)) { - ipc_object_copyout_dest(space, (ipc_object_t) dest, - dest_type, &dest_name); - /* dest is unlocked */ + if (ip_active(dest)) { + ipc_object_copyout_dest(space, ip_to_object(dest), + dest_type, &dest_name); + /* dest is unlocked */ + } else { + ipc_port_timestamp_t timestamp; - } else { - ipc_port_timestamp_t timestamp; + timestamp = dest->ip_timestamp; + ip_unlock(dest); + ip_release(dest); - timestamp = dest->ip_timestamp; - ip_unlock(dest); - ip_release(dest); + if (IP_VALID(reply)) { + ip_lock(reply); + if (ip_active(reply) || + IP_TIMESTAMP_ORDER(timestamp, + reply->ip_timestamp)) { + dest_name = MACH_PORT_DEAD; + } else { + dest_name = MACH_PORT_NULL; + } + ip_unlock(reply); + } else { + dest_name = MACH_PORT_DEAD; + } + } if (IP_VALID(reply)) { - ip_lock(reply); - if (ip_active(reply) || - IP_TIMESTAMP_ORDER(timestamp, - reply->ip_timestamp)) - dest_name = MACH_PORT_DEAD; - else - dest_name = MACH_PORT_NULL; - ip_unlock(reply); - } else - dest_name = MACH_PORT_DEAD; - } + ip_release(reply); + } - if (IP_VALID(reply)) - ip_release(reply); + if (IP_VALID(release_reply_port)) { + if (reply_type == MACH_MSG_TYPE_PORT_SEND_ONCE) { + ipc_port_release_sonce(release_reply_port); + } else { + ipc_port_release_send(release_reply_port); + } + } - if (IP_VALID(release_reply_port)) { - if (reply_type == MACH_MSG_TYPE_PORT_SEND_ONCE) - ipc_port_release_sonce(release_reply_port); - else - ipc_port_release_send(release_reply_port); - } + if ((option & MACH_RCV_VOUCHER) != 0) { + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV) | DBG_FUNC_NONE, + VM_KERNEL_ADDRPERM((uintptr_t)kmsg), + (uintptr_t)kmsg->ikm_header->msgh_bits, + (uintptr_t)kmsg->ikm_header->msgh_id, + VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(voucher)), + 0); + } else { + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV_VOUCHER_REFUSED) | DBG_FUNC_NONE, + VM_KERNEL_ADDRPERM((uintptr_t)kmsg), + (uintptr_t)kmsg->ikm_header->msgh_bits, + (uintptr_t)kmsg->ikm_header->msgh_id, + VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(voucher)), + 0); + } - if (IP_VALID(release_voucher_port)) - ipc_port_release_send(release_voucher_port); + if (IP_VALID(release_voucher_port)) { + ipc_port_release_send(release_voucher_port); + } + msg->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type, + voucher_type, mbits); + msg->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name); + msg->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name); + msg->msgh_voucher_port = voucher_name; + } - if ((option & MACH_RCV_VOUCHER) != 0) { - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV) | DBG_FUNC_NONE, - VM_KERNEL_ADDRPERM((uintptr_t)kmsg), - (uintptr_t)kmsg->ikm_header->msgh_bits, - (uintptr_t)kmsg->ikm_header->msgh_id, - VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(voucher)), - 0); - } else { - KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV_VOUCHER_REFUSED) | DBG_FUNC_NONE, - VM_KERNEL_ADDRPERM((uintptr_t)kmsg), - (uintptr_t)kmsg->ikm_header->msgh_bits, - (uintptr_t)kmsg->ikm_header->msgh_id, - VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(voucher)), - 0); - } - - msg->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type, - voucher_type, mbits); - msg->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name); - msg->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name); - msg->msgh_voucher_port = voucher_name; - } - - return MACH_MSG_SUCCESS; + return MACH_MSG_SUCCESS; } /* @@ -3939,10 +4545,12 @@ ipc_kmsg_copyout_header( mach_msg_return_t ipc_kmsg_copyout_object( - ipc_space_t space, - ipc_object_t object, - mach_msg_type_name_t msgt_name, - mach_port_name_t *namep) + ipc_space_t space, + ipc_object_t object, + mach_msg_type_name_t msgt_name, + mach_port_context_t *context, + mach_msg_guard_flags_t *guard_flags, + mach_port_name_t *namep) { kern_return_t kr; @@ -3951,69 +4559,61 @@ ipc_kmsg_copyout_object( return MACH_MSG_SUCCESS; } - kr = ipc_object_copyout(space, object, msgt_name, TRUE, namep); + kr = ipc_object_copyout(space, object, msgt_name, context, guard_flags, namep); if (kr != KERN_SUCCESS) { ipc_object_destroy(object, msgt_name); - if (kr == KERN_INVALID_CAPABILITY) + if (kr == KERN_INVALID_CAPABILITY) { *namep = MACH_PORT_DEAD; - else { + } else { *namep = MACH_PORT_NULL; - if (kr == KERN_RESOURCE_SHORTAGE) + if (kr == KERN_RESOURCE_SHORTAGE) { return MACH_MSG_IPC_KERNEL; - else + } else { return MACH_MSG_IPC_SPACE; + } } } return MACH_MSG_SUCCESS; } -mach_msg_descriptor_t * +static mach_msg_descriptor_t * ipc_kmsg_copyout_port_descriptor(mach_msg_descriptor_t *dsc, - mach_msg_descriptor_t *user_dsc, - ipc_space_t space, - kern_return_t *mr); -mach_msg_descriptor_t * -ipc_kmsg_copyout_port_descriptor(mach_msg_descriptor_t *dsc, - mach_msg_descriptor_t *dest_dsc, - ipc_space_t space, - kern_return_t *mr) + mach_msg_descriptor_t *dest_dsc, + ipc_space_t space, + kern_return_t *mr) { - mach_port_t port; - mach_port_name_t name; - mach_msg_type_name_t disp; - - - /* Copyout port right carried in the message */ - port = dsc->port.name; - disp = dsc->port.disposition; - *mr |= ipc_kmsg_copyout_object(space, - (ipc_object_t)port, - disp, - &name); - - if(current_task() == kernel_task) - { - mach_msg_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc; - user_dsc--; // point to the start of this port descriptor - bzero((void *)user_dsc, sizeof(*user_dsc)); - user_dsc->name = CAST_MACH_NAME_TO_PORT(name); - user_dsc->disposition = disp; - user_dsc->type = MACH_MSG_PORT_DESCRIPTOR; - dest_dsc = (typeof(dest_dsc))user_dsc; - } else { - mach_msg_legacy_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc; - user_dsc--; // point to the start of this port descriptor - bzero((void *)user_dsc, sizeof(*user_dsc)); - user_dsc->name = CAST_MACH_PORT_TO_NAME(name); - user_dsc->disposition = disp; - user_dsc->type = MACH_MSG_PORT_DESCRIPTOR; - dest_dsc = (typeof(dest_dsc))user_dsc; - } - - return (mach_msg_descriptor_t *)dest_dsc; + mach_port_t port; + mach_port_name_t name; + mach_msg_type_name_t disp; + + /* Copyout port right carried in the message */ + port = dsc->port.name; + disp = dsc->port.disposition; + *mr |= ipc_kmsg_copyout_object(space, + ip_to_object(port), disp, NULL, NULL, &name); + + if (current_task() == kernel_task) { + mach_msg_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc; + user_dsc--; // point to the start of this port descriptor + bzero((void *)user_dsc, sizeof(*user_dsc)); + user_dsc->name = CAST_MACH_NAME_TO_PORT(name); + user_dsc->disposition = disp; + user_dsc->type = MACH_MSG_PORT_DESCRIPTOR; + dest_dsc = (typeof(dest_dsc))user_dsc; + } else { + mach_msg_legacy_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc; + user_dsc--; // point to the start of this port descriptor + bzero((void *)user_dsc, sizeof(*user_dsc)); + user_dsc->name = CAST_MACH_PORT_TO_NAME(name); + user_dsc->disposition = disp; + user_dsc->type = MACH_MSG_PORT_DESCRIPTOR; + dest_dsc = (typeof(dest_dsc))user_dsc; + } + + return (mach_msg_descriptor_t *)dest_dsc; } mach_msg_descriptor_t * @@ -4021,247 +4621,326 @@ ipc_kmsg_copyout_ool_descriptor(mach_msg_ool_descriptor_t *dsc, mach_msg_descrip mach_msg_descriptor_t * ipc_kmsg_copyout_ool_descriptor(mach_msg_ool_descriptor_t *dsc, mach_msg_descriptor_t *user_dsc, int is_64bit, vm_map_t map, mach_msg_return_t *mr) { - vm_map_copy_t copy; - vm_map_address_t rcv_addr; - mach_msg_copy_options_t copy_options; - vm_map_size_t size; - mach_msg_descriptor_type_t dsc_type; - - //SKIP_PORT_DESCRIPTORS(saddr, sdsc_count); - - copy = (vm_map_copy_t)dsc->address; - size = (vm_map_size_t)dsc->size; - copy_options = dsc->copy; - assert(copy_options != MACH_MSG_KALLOC_COPY_T); - dsc_type = dsc->type; - - if (copy != VM_MAP_COPY_NULL) { - kern_return_t kr; + vm_map_copy_t copy; + vm_map_address_t rcv_addr; + mach_msg_copy_options_t copy_options; + vm_map_size_t size; + mach_msg_descriptor_type_t dsc_type; + + //SKIP_PORT_DESCRIPTORS(saddr, sdsc_count); + + copy = (vm_map_copy_t)dsc->address; + size = (vm_map_size_t)dsc->size; + copy_options = dsc->copy; + assert(copy_options != MACH_MSG_KALLOC_COPY_T); + dsc_type = dsc->type; + + if (copy != VM_MAP_COPY_NULL) { + kern_return_t kr; + + rcv_addr = 0; + if (vm_map_copy_validate_size(map, copy, &size) == FALSE) { + panic("Inconsistent OOL/copyout size on %p: expected %d, got %lld @%p", + dsc, dsc->size, (unsigned long long)copy->size, copy); + } + kr = vm_map_copyout_size(map, &rcv_addr, copy, size); + if (kr != KERN_SUCCESS) { + if (kr == KERN_RESOURCE_SHORTAGE) { + *mr |= MACH_MSG_VM_KERNEL; + } else { + *mr |= MACH_MSG_VM_SPACE; + } + vm_map_copy_discard(copy); + rcv_addr = 0; + size = 0; + } + } else { + rcv_addr = 0; + size = 0; + } - rcv_addr = 0; - if (vm_map_copy_validate_size(map, copy, &size) == FALSE) - panic("Inconsistent OOL/copyout size on %p: expected %d, got %lld @%p", - dsc, dsc->size, (unsigned long long)copy->size, copy); - kr = vm_map_copyout_size(map, &rcv_addr, copy, size); - if (kr != KERN_SUCCESS) { - if (kr == KERN_RESOURCE_SHORTAGE) - *mr |= MACH_MSG_VM_KERNEL; - else - *mr |= MACH_MSG_VM_SPACE; - vm_map_copy_discard(copy); - rcv_addr = 0; - size = 0; - } - } else { - rcv_addr = 0; - size = 0; - } - - /* - * Now update the descriptor as the user would see it. - * This may require expanding the descriptor to the user - * visible size. There is already space allocated for - * this in what naddr points to. - */ - if(current_task() == kernel_task) - { - mach_msg_ool_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; - user_ool_dsc--; - bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); - - user_ool_dsc->address = (void *)(uintptr_t)rcv_addr; - user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? - TRUE : FALSE; - user_ool_dsc->copy = copy_options; - user_ool_dsc->type = dsc_type; - user_ool_dsc->size = (mach_msg_size_t)size; - - user_dsc = (typeof(user_dsc))user_ool_dsc; - } else if (is_64bit) { - mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; - user_ool_dsc--; - bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); - - user_ool_dsc->address = rcv_addr; - user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? - TRUE : FALSE; - user_ool_dsc->copy = copy_options; - user_ool_dsc->type = dsc_type; - user_ool_dsc->size = (mach_msg_size_t)size; - - user_dsc = (typeof(user_dsc))user_ool_dsc; - } else { - mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; - user_ool_dsc--; - bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); - - user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr); - user_ool_dsc->size = (mach_msg_size_t)size; - user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? - TRUE : FALSE; - user_ool_dsc->copy = copy_options; - user_ool_dsc->type = dsc_type; - - user_dsc = (typeof(user_dsc))user_ool_dsc; - } - return user_dsc; + /* + * Now update the descriptor as the user would see it. + * This may require expanding the descriptor to the user + * visible size. There is already space allocated for + * this in what naddr points to. + */ + if (current_task() == kernel_task) { + mach_msg_ool_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; + user_ool_dsc--; + bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); + + user_ool_dsc->address = (void *)(uintptr_t)rcv_addr; + user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? + TRUE : FALSE; + user_ool_dsc->copy = copy_options; + user_ool_dsc->type = dsc_type; + user_ool_dsc->size = (mach_msg_size_t)size; + + user_dsc = (typeof(user_dsc))user_ool_dsc; + } else if (is_64bit) { + mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; + user_ool_dsc--; + bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); + + user_ool_dsc->address = rcv_addr; + user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? + TRUE : FALSE; + user_ool_dsc->copy = copy_options; + user_ool_dsc->type = dsc_type; + user_ool_dsc->size = (mach_msg_size_t)size; + + user_dsc = (typeof(user_dsc))user_ool_dsc; + } else { + mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; + user_ool_dsc--; + bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); + + user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr); + user_ool_dsc->size = (mach_msg_size_t)size; + user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? + TRUE : FALSE; + user_ool_dsc->copy = copy_options; + user_ool_dsc->type = dsc_type; + + user_dsc = (typeof(user_dsc))user_ool_dsc; + } + return user_dsc; } mach_msg_descriptor_t * ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc, - mach_msg_descriptor_t *user_dsc, - int is_64bit, - vm_map_t map, - ipc_space_t space, - ipc_kmsg_t kmsg, - mach_msg_return_t *mr); + mach_msg_descriptor_t *user_dsc, + int is_64bit, + vm_map_t map, + ipc_space_t space, + ipc_kmsg_t kmsg, + mach_msg_return_t *mr); mach_msg_descriptor_t * ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc, - mach_msg_descriptor_t *user_dsc, - int is_64bit, - vm_map_t map, - ipc_space_t space, - ipc_kmsg_t kmsg, - mach_msg_return_t *mr) + mach_msg_descriptor_t *user_dsc, + int is_64bit, + vm_map_t map, + ipc_space_t space, + ipc_kmsg_t kmsg, + mach_msg_return_t *mr) { - mach_vm_offset_t rcv_addr = 0; - mach_msg_type_name_t disp; - mach_msg_type_number_t count, i; - vm_size_t ports_length, names_length; - - mach_msg_copy_options_t copy_options = MACH_MSG_VIRTUAL_COPY; + mach_vm_offset_t rcv_addr = 0; + mach_msg_type_name_t disp; + mach_msg_type_number_t count, i; + vm_size_t ports_length, names_length; - //SKIP_PORT_DESCRIPTORS(saddr, sdsc_count); + mach_msg_copy_options_t copy_options = MACH_MSG_VIRTUAL_COPY; - count = dsc->count; - disp = dsc->disposition; - ports_length = count * sizeof(mach_port_t); - names_length = count * sizeof(mach_port_name_t); + //SKIP_PORT_DESCRIPTORS(saddr, sdsc_count); - if (ports_length != 0 && dsc->address != 0) { + count = dsc->count; + disp = dsc->disposition; + ports_length = count * sizeof(mach_port_t); + names_length = count * sizeof(mach_port_name_t); - /* - * Check to see if there is an overwrite descriptor - * specified in the scatter list for this ool data. - * The descriptor has already been verified. - */ + if (ports_length != 0 && dsc->address != 0) { + /* + * Check to see if there is an overwrite descriptor + * specified in the scatter list for this ool data. + * The descriptor has already been verified. + */ #if 0 - if (saddr != MACH_MSG_DESCRIPTOR_NULL) { - if (differs) { - OTHER_OOL_DESCRIPTOR *scatter_dsc; - - scatter_dsc = (OTHER_OOL_DESCRIPTOR *)saddr; - rcv_addr = (mach_vm_offset_t) scatter_dsc->address; - copy_options = scatter_dsc->copy; - } else { - mach_msg_ool_descriptor_t *scatter_dsc; - - scatter_dsc = &saddr->out_of_line; - rcv_addr = CAST_USER_ADDR_T(scatter_dsc->address); - copy_options = scatter_dsc->copy; - } - INCREMENT_SCATTER(saddr, sdsc_count, differs); - } + if (saddr != MACH_MSG_DESCRIPTOR_NULL) { + if (differs) { + OTHER_OOL_DESCRIPTOR *scatter_dsc; + + scatter_dsc = (OTHER_OOL_DESCRIPTOR *)saddr; + rcv_addr = (mach_vm_offset_t) scatter_dsc->address; + copy_options = scatter_dsc->copy; + } else { + mach_msg_ool_descriptor_t *scatter_dsc; + + scatter_dsc = &saddr->out_of_line; + rcv_addr = CAST_USER_ADDR_T(scatter_dsc->address); + copy_options = scatter_dsc->copy; + } + INCREMENT_SCATTER(saddr, sdsc_count, differs); + } +#endif + + if (copy_options == MACH_MSG_VIRTUAL_COPY) { + /* + * Dynamically allocate the region + */ + vm_tag_t tag; + if (vm_kernel_map_is_kernel(map)) { + tag = VM_KERN_MEMORY_IPC; + } else { + tag = VM_MEMORY_MACH_MSG; + } + + kern_return_t kr; + if ((kr = mach_vm_allocate_kernel(map, &rcv_addr, + (mach_vm_size_t)names_length, + VM_FLAGS_ANYWHERE, tag)) != KERN_SUCCESS) { + ipc_kmsg_clean_body(kmsg, 1, (mach_msg_descriptor_t *)dsc); + rcv_addr = 0; + + if (kr == KERN_RESOURCE_SHORTAGE) { + *mr |= MACH_MSG_VM_KERNEL; + } else { + *mr |= MACH_MSG_VM_SPACE; + } + } + } + + /* + * Handle the port rights and copy out the names + * for those rights out to user-space. + */ + if (rcv_addr != 0) { + ipc_object_t *objects = (ipc_object_t *) dsc->address; + mach_port_name_t *names = (mach_port_name_t *) dsc->address; + + /* copyout port rights carried in the message */ + + for (i = 0; i < count; i++) { + ipc_object_t object = objects[i]; + + *mr |= ipc_kmsg_copyout_object(space, object, + disp, NULL, NULL, &names[i]); + } + + /* copyout to memory allocated above */ + void *data = dsc->address; + if (copyoutmap(map, data, rcv_addr, names_length) != KERN_SUCCESS) { + *mr |= MACH_MSG_VM_SPACE; + } + kfree(data, ports_length); + } + } else { + rcv_addr = 0; + } + + /* + * Now update the descriptor based on the information + * calculated above. + */ + if (current_task() == kernel_task) { + mach_msg_ool_ports_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; + user_ool_dsc--; + bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); + + user_ool_dsc->address = (void *)(uintptr_t)rcv_addr; + user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? + TRUE : FALSE; + user_ool_dsc->copy = copy_options; + user_ool_dsc->disposition = disp; + user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR; + user_ool_dsc->count = count; + + user_dsc = (typeof(user_dsc))user_ool_dsc; + } else if (is_64bit) { + mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; + user_ool_dsc--; + bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); + + user_ool_dsc->address = rcv_addr; + user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? + TRUE : FALSE; + user_ool_dsc->copy = copy_options; + user_ool_dsc->disposition = disp; + user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR; + user_ool_dsc->count = count; + + user_dsc = (typeof(user_dsc))user_ool_dsc; + } else { + mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; + user_ool_dsc--; + bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); + + user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr); + user_ool_dsc->count = count; + user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? + TRUE : FALSE; + user_ool_dsc->copy = copy_options; + user_ool_dsc->disposition = disp; + user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR; + + user_dsc = (typeof(user_dsc))user_ool_dsc; + } + return user_dsc; +} + +static mach_msg_descriptor_t * +ipc_kmsg_copyout_guarded_port_descriptor( + mach_msg_guarded_port_descriptor_t *dsc, + mach_msg_descriptor_t *dest_dsc, + int is_64bit, + __unused ipc_kmsg_t kmsg, + ipc_space_t space, + mach_msg_option_t option, + kern_return_t *mr) +{ + mach_port_t port; + mach_port_name_t name = MACH_PORT_NULL; + mach_msg_type_name_t disp; + mach_msg_guard_flags_t guard_flags; + mach_port_context_t context; + + /* Copyout port right carried in the message */ + port = dsc->name; + disp = dsc->disposition; + guard_flags = dsc->flags; + context = 0; + + /* Currently kernel_task doesnt support receiving guarded port descriptors */ + struct knote *kn = current_thread()->ith_knote; + if ((kn != ITH_KNOTE_PSEUDO) && (((option & MACH_RCV_GUARDED_DESC) == 0) || + (current_task() == kernel_task))) { +#if DEVELOPMENT || DEBUG + if (current_task() != kernel_task) { + /* + * Simulated crash needed for debugging, notifies the receiver to opt into receiving + * guarded descriptors. + */ + mach_port_guard_exception(current_thread()->ith_receiver_name, 0, 0, kGUARD_EXC_RCV_GUARDED_DESC); + } #endif + KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_DESTROY_GUARDED_DESC), current_thread()->ith_receiver_name, + VM_KERNEL_ADDRPERM(port), disp, guard_flags); + ipc_object_destroy(ip_to_object(port), disp); + mach_msg_legacy_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc; + user_dsc--; // point to the start of this port descriptor + bzero((void *)user_dsc, sizeof(*user_dsc)); + user_dsc->name = name; + user_dsc->disposition = disp; + user_dsc->type = MACH_MSG_PORT_DESCRIPTOR; + dest_dsc = (typeof(dest_dsc))user_dsc; + } else { + *mr |= ipc_kmsg_copyout_object(space, + ip_to_object(port), disp, &context, &guard_flags, &name); + + if (!is_64bit) { + mach_msg_guarded_port_descriptor32_t *user_dsc = (typeof(user_dsc))dest_dsc; + user_dsc--; // point to the start of this port descriptor + bzero((void *)user_dsc, sizeof(*user_dsc)); + user_dsc->name = name; + user_dsc->flags = guard_flags; + user_dsc->disposition = disp; + user_dsc->type = MACH_MSG_GUARDED_PORT_DESCRIPTOR; + user_dsc->context = CAST_DOWN_EXPLICIT(uint32_t, context); + dest_dsc = (typeof(dest_dsc))user_dsc; + } else { + mach_msg_guarded_port_descriptor64_t *user_dsc = (typeof(user_dsc))dest_dsc; + user_dsc--; // point to the start of this port descriptor + bzero((void *)user_dsc, sizeof(*user_dsc)); + user_dsc->name = name; + user_dsc->flags = guard_flags; + user_dsc->disposition = disp; + user_dsc->type = MACH_MSG_GUARDED_PORT_DESCRIPTOR; + user_dsc->context = context; + dest_dsc = (typeof(dest_dsc))user_dsc; + } + } - if (copy_options == MACH_MSG_VIRTUAL_COPY) { - /* - * Dynamically allocate the region - */ - vm_tag_t tag; - if (vm_kernel_map_is_kernel(map)) tag = VM_KERN_MEMORY_IPC; - else tag = VM_MEMORY_MACH_MSG; - - kern_return_t kr; - if ((kr = mach_vm_allocate_kernel(map, &rcv_addr, - (mach_vm_size_t)names_length, - VM_FLAGS_ANYWHERE, tag)) != KERN_SUCCESS) { - ipc_kmsg_clean_body(kmsg, 1, (mach_msg_descriptor_t *)dsc); - rcv_addr = 0; - - if (kr == KERN_RESOURCE_SHORTAGE){ - *mr |= MACH_MSG_VM_KERNEL; - } else { - *mr |= MACH_MSG_VM_SPACE; - } - } - } - - /* - * Handle the port rights and copy out the names - * for those rights out to user-space. - */ - if (rcv_addr != 0) { - mach_port_t *objects = (mach_port_t *) dsc->address; - mach_port_name_t *names = (mach_port_name_t *) dsc->address; - - /* copyout port rights carried in the message */ - - for ( i = 0; i < count ; i++) { - ipc_object_t object = (ipc_object_t)objects[i]; - - *mr |= ipc_kmsg_copyout_object(space, object, - disp, &names[i]); - } - - /* copyout to memory allocated above */ - void *data = dsc->address; - if (copyoutmap(map, data, rcv_addr, names_length) != KERN_SUCCESS) - *mr |= MACH_MSG_VM_SPACE; - kfree(data, ports_length); - } - } else { - rcv_addr = 0; - } - - /* - * Now update the descriptor based on the information - * calculated above. - */ - if(current_task() == kernel_task) { - mach_msg_ool_ports_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; - user_ool_dsc--; - bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); - - user_ool_dsc->address = (void *)(uintptr_t)rcv_addr; - user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? - TRUE : FALSE; - user_ool_dsc->copy = copy_options; - user_ool_dsc->disposition = disp; - user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR; - user_ool_dsc->count = count; - - user_dsc = (typeof(user_dsc))user_ool_dsc; - } if (is_64bit) { - mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; - user_ool_dsc--; - bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); - - user_ool_dsc->address = rcv_addr; - user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? - TRUE : FALSE; - user_ool_dsc->copy = copy_options; - user_ool_dsc->disposition = disp; - user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR; - user_ool_dsc->count = count; - - user_dsc = (typeof(user_dsc))user_ool_dsc; - } else { - mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; - user_ool_dsc--; - bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc)); - - user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr); - user_ool_dsc->count = count; - user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? - TRUE : FALSE; - user_ool_dsc->copy = copy_options; - user_ool_dsc->disposition = disp; - user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR; - - user_dsc = (typeof(user_dsc))user_ool_dsc; - } - return user_dsc; + return (mach_msg_descriptor_t *)dest_dsc; } /* @@ -4285,67 +4964,70 @@ ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc, mach_msg_return_t ipc_kmsg_copyout_body( - ipc_kmsg_t kmsg, - ipc_space_t space, - vm_map_t map, - mach_msg_body_t *slist) + ipc_kmsg_t kmsg, + ipc_space_t space, + vm_map_t map, + mach_msg_option_t option, + mach_msg_body_t *slist) { - mach_msg_body_t *body; - mach_msg_descriptor_t *kern_dsc, *user_dsc; - mach_msg_descriptor_t *saddr; - mach_msg_type_number_t dsc_count, sdsc_count; - int i; - mach_msg_return_t mr = MACH_MSG_SUCCESS; - boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS); - - body = (mach_msg_body_t *) (kmsg->ikm_header + 1); - dsc_count = body->msgh_descriptor_count; - kern_dsc = (mach_msg_descriptor_t *) (body + 1); - /* Point user_dsc just after the end of all the descriptors */ - user_dsc = &kern_dsc[dsc_count]; - - /* Do scatter list setup */ - if (slist != MACH_MSG_BODY_NULL) { - panic("Scatter lists disabled"); - saddr = (mach_msg_descriptor_t *) (slist + 1); - sdsc_count = slist->msgh_descriptor_count; - } - else { - saddr = MACH_MSG_DESCRIPTOR_NULL; - sdsc_count = 0; - } - - /* Now process the descriptors */ - for (i = dsc_count-1; i >= 0; i--) { - switch (kern_dsc[i].type.type) { - - case MACH_MSG_PORT_DESCRIPTOR: - user_dsc = ipc_kmsg_copyout_port_descriptor(&kern_dsc[i], user_dsc, space, &mr); - break; - case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: - case MACH_MSG_OOL_DESCRIPTOR : - user_dsc = ipc_kmsg_copyout_ool_descriptor( - (mach_msg_ool_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, map, &mr); - break; - case MACH_MSG_OOL_PORTS_DESCRIPTOR : - user_dsc = ipc_kmsg_copyout_ool_ports_descriptor( - (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, map, space, kmsg, &mr); - break; - default : { - panic("untyped IPC copyout body: invalid message descriptor"); - } - } - } - - if(user_dsc != kern_dsc) { - vm_offset_t dsc_adjust = (vm_offset_t)user_dsc - (vm_offset_t)kern_dsc; - memmove((char *)((vm_offset_t)kmsg->ikm_header + dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t)); - kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header + dsc_adjust); - /* Update the message size for the smaller user representation */ - kmsg->ikm_header->msgh_size -= (mach_msg_size_t)dsc_adjust; - } - - return mr; + mach_msg_body_t *body; + mach_msg_descriptor_t *kern_dsc, *user_dsc; + mach_msg_descriptor_t *saddr; + mach_msg_type_number_t dsc_count, sdsc_count; + int i; + mach_msg_return_t mr = MACH_MSG_SUCCESS; + boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS); + + body = (mach_msg_body_t *) (kmsg->ikm_header + 1); + dsc_count = body->msgh_descriptor_count; + kern_dsc = (mach_msg_descriptor_t *) (body + 1); + /* Point user_dsc just after the end of all the descriptors */ + user_dsc = &kern_dsc[dsc_count]; + + /* Do scatter list setup */ + if (slist != MACH_MSG_BODY_NULL) { + panic("Scatter lists disabled"); + saddr = (mach_msg_descriptor_t *) (slist + 1); + sdsc_count = slist->msgh_descriptor_count; + } else { + saddr = MACH_MSG_DESCRIPTOR_NULL; + sdsc_count = 0; + } + + /* Now process the descriptors */ + for (i = dsc_count - 1; i >= 0; i--) { + switch (kern_dsc[i].type.type) { + case MACH_MSG_PORT_DESCRIPTOR: + user_dsc = ipc_kmsg_copyout_port_descriptor(&kern_dsc[i], user_dsc, space, &mr); + break; + case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: + case MACH_MSG_OOL_DESCRIPTOR: + user_dsc = ipc_kmsg_copyout_ool_descriptor( + (mach_msg_ool_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, map, &mr); + break; + case MACH_MSG_OOL_PORTS_DESCRIPTOR: + user_dsc = ipc_kmsg_copyout_ool_ports_descriptor( + (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, map, space, kmsg, &mr); + break; + case MACH_MSG_GUARDED_PORT_DESCRIPTOR: + user_dsc = ipc_kmsg_copyout_guarded_port_descriptor( + (mach_msg_guarded_port_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, kmsg, space, option, &mr); + break; + default: { + panic("untyped IPC copyout body: invalid message descriptor"); + } + } + } + + if (user_dsc != kern_dsc) { + vm_offset_t dsc_adjust = (vm_offset_t)user_dsc - (vm_offset_t)kern_dsc; + memmove((char *)((vm_offset_t)kmsg->ikm_header + dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t)); + kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header + dsc_adjust); + /* Update the message size for the smaller user representation */ + kmsg->ikm_header->msgh_size -= (mach_msg_size_t)dsc_adjust; + } + + return mr; } /* @@ -4363,45 +5045,46 @@ ipc_kmsg_copyout_body( mach_msg_size_t ipc_kmsg_copyout_size( - ipc_kmsg_t kmsg, - vm_map_t map) + ipc_kmsg_t kmsg, + vm_map_t map) { - mach_msg_size_t send_size; + mach_msg_size_t send_size; - send_size = kmsg->ikm_header->msgh_size; + send_size = kmsg->ikm_header->msgh_size; - boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS); + boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS); #if defined(__LP64__) send_size -= LEGACY_HEADER_SIZE_DELTA; #endif - if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) { - - mach_msg_body_t *body; - mach_msg_descriptor_t *saddr, *eaddr; - - body = (mach_msg_body_t *) (kmsg->ikm_header + 1); - saddr = (mach_msg_descriptor_t *) (body + 1); - eaddr = saddr + body->msgh_descriptor_count; - - for ( ; saddr < eaddr; saddr++ ) { - switch (saddr->type.type) { - case MACH_MSG_OOL_DESCRIPTOR: - case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: - case MACH_MSG_OOL_PORTS_DESCRIPTOR: - if(!is_task_64bit) - send_size -= DESC_SIZE_ADJUSTMENT; - break; - case MACH_MSG_PORT_DESCRIPTOR: - send_size -= DESC_SIZE_ADJUSTMENT; - break; - default: - break; - } - } - } - return send_size; + if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) { + mach_msg_body_t *body; + mach_msg_descriptor_t *saddr, *eaddr; + + body = (mach_msg_body_t *) (kmsg->ikm_header + 1); + saddr = (mach_msg_descriptor_t *) (body + 1); + eaddr = saddr + body->msgh_descriptor_count; + + for (; saddr < eaddr; saddr++) { + switch (saddr->type.type) { + case MACH_MSG_OOL_DESCRIPTOR: + case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: + case MACH_MSG_OOL_PORTS_DESCRIPTOR: + case MACH_MSG_GUARDED_PORT_DESCRIPTOR: + if (!is_task_64bit) { + send_size -= DESC_SIZE_ADJUSTMENT; + } + break; + case MACH_MSG_PORT_DESCRIPTOR: + send_size -= DESC_SIZE_ADJUSTMENT; + break; + default: + break; + } + } + } + return send_size; } /* @@ -4422,11 +5105,11 @@ ipc_kmsg_copyout_size( mach_msg_return_t ipc_kmsg_copyout( - ipc_kmsg_t kmsg, - ipc_space_t space, - vm_map_t map, - mach_msg_body_t *slist, - mach_msg_option_t option) + ipc_kmsg_t kmsg, + ipc_space_t space, + vm_map_t map, + mach_msg_body_t *slist, + mach_msg_option_t option) { mach_msg_return_t mr; @@ -4436,10 +5119,11 @@ ipc_kmsg_copyout( } if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) { - mr = ipc_kmsg_copyout_body(kmsg, space, map, slist); + mr = ipc_kmsg_copyout_body(kmsg, space, map, option, slist); - if (mr != MACH_MSG_SUCCESS) + if (mr != MACH_MSG_SUCCESS) { mr |= MACH_RCV_BODY_ERROR; + } } return mr; @@ -4467,15 +5151,15 @@ ipc_kmsg_copyout( mach_msg_return_t ipc_kmsg_copyout_pseudo( - ipc_kmsg_t kmsg, - ipc_space_t space, - vm_map_t map, - mach_msg_body_t *slist) + ipc_kmsg_t kmsg, + ipc_space_t space, + vm_map_t map, + mach_msg_body_t *slist) { mach_msg_bits_t mbits = kmsg->ikm_header->msgh_bits; - ipc_object_t dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; - ipc_object_t reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port; - ipc_object_t voucher = (ipc_object_t) kmsg->ikm_voucher; + ipc_object_t dest = ip_to_object(kmsg->ikm_header->msgh_remote_port); + ipc_object_t reply = ip_to_object(kmsg->ikm_header->msgh_local_port); + ipc_object_t voucher = ip_to_object(kmsg->ikm_voucher); mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits); mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits); mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits); @@ -4500,8 +5184,8 @@ ipc_kmsg_copyout_pseudo( ipc_importance_assert_clean(kmsg); #endif - mr = (ipc_kmsg_copyout_object(space, dest, dest_type, &dest_name) | - ipc_kmsg_copyout_object(space, reply, reply_type, &reply_name)); + mr = (ipc_kmsg_copyout_object(space, dest, dest_type, NULL, NULL, &dest_name) | + ipc_kmsg_copyout_object(space, reply, reply_type, NULL, NULL, &reply_name)); kmsg->ikm_header->msgh_bits = mbits & MACH_MSGH_BITS_USER; kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(dest_name); @@ -4511,12 +5195,12 @@ ipc_kmsg_copyout_pseudo( assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); kmsg->ikm_voucher = IP_NULL; - mr |= ipc_kmsg_copyout_object(space, voucher, voucher_type, &voucher_name); + mr |= ipc_kmsg_copyout_object(space, voucher, voucher_type, NULL, NULL, &voucher_name); kmsg->ikm_header->msgh_voucher_port = voucher_name; } - + if (mbits & MACH_MSGH_BITS_COMPLEX) { - mr |= ipc_kmsg_copyout_body(kmsg, space, map, slist); + mr |= ipc_kmsg_copyout_body(kmsg, space, map, 0, slist); } return mr; @@ -4533,8 +5217,8 @@ ipc_kmsg_copyout_pseudo( void ipc_kmsg_copyout_dest( - ipc_kmsg_t kmsg, - ipc_space_t space) + ipc_kmsg_t kmsg, + ipc_space_t space) { mach_msg_bits_t mbits; ipc_object_t dest; @@ -4546,9 +5230,9 @@ ipc_kmsg_copyout_dest( mach_port_name_t dest_name, reply_name, voucher_name; mbits = kmsg->ikm_header->msgh_bits; - dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; - reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port; - voucher = (ipc_object_t) kmsg->ikm_voucher; + dest = ip_to_object(kmsg->ikm_header->msgh_remote_port); + reply = ip_to_object(kmsg->ikm_header->msgh_local_port); + voucher = ip_to_object(kmsg->ikm_voucher); voucher_name = kmsg->ikm_header->msgh_voucher_port; dest_type = MACH_MSGH_BITS_REMOTE(mbits); reply_type = MACH_MSGH_BITS_LOCAL(mbits); @@ -4571,19 +5255,20 @@ ipc_kmsg_copyout_dest( if (IO_VALID(reply)) { ipc_object_destroy(reply, reply_type); reply_name = MACH_PORT_NULL; - } else + } else { reply_name = CAST_MACH_PORT_TO_NAME(reply); + } if (IO_VALID(voucher)) { assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); kmsg->ikm_voucher = IP_NULL; - ipc_object_destroy((ipc_object_t)voucher, voucher_type); + ipc_object_destroy(voucher, voucher_type); voucher_name = MACH_PORT_NULL; } kmsg->ikm_header->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type, - voucher_type, mbits); + voucher_type, mbits); kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name); kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name); kmsg->ikm_header->msgh_voucher_port = voucher_name; @@ -4592,8 +5277,8 @@ ipc_kmsg_copyout_dest( mach_msg_body_t *body; body = (mach_msg_body_t *) (kmsg->ikm_header + 1); - ipc_kmsg_clean_body(kmsg, body->msgh_descriptor_count, - (mach_msg_descriptor_t *)(body + 1)); + ipc_kmsg_clean_body(kmsg, body->msgh_descriptor_count, + (mach_msg_descriptor_t *)(body + 1)); } } @@ -4612,17 +5297,17 @@ ipc_kmsg_copyout_dest( void ipc_kmsg_copyout_to_kernel( - ipc_kmsg_t kmsg, - ipc_space_t space) + ipc_kmsg_t kmsg, + ipc_space_t space) { ipc_object_t dest; - ipc_object_t reply; + mach_port_t reply; mach_msg_type_name_t dest_type; mach_msg_type_name_t reply_type; - mach_port_name_t dest_name, reply_name; + mach_port_name_t dest_name; - dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; - reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port; + dest = ip_to_object(kmsg->ikm_header->msgh_remote_port); + reply = kmsg->ikm_header->msgh_local_port; dest_type = MACH_MSGH_BITS_REMOTE(kmsg->ikm_header->msgh_bits); reply_type = MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits); @@ -4638,29 +5323,50 @@ ipc_kmsg_copyout_to_kernel( dest_name = MACH_PORT_DEAD; } - reply_name = CAST_MACH_PORT_TO_NAME(reply); + /* + * While MIG kernel users don't receive vouchers, the + * msgh_voucher_port field is intended to be round-tripped through the + * kernel if there is no voucher disposition set. Here we check for a + * non-zero voucher disposition, and consume the voucher send right as + * there is no possible way to specify MACH_RCV_VOUCHER semantics. + */ + mach_msg_type_name_t voucher_type; + voucher_type = MACH_MSGH_BITS_VOUCHER(kmsg->ikm_header->msgh_bits); + if (voucher_type != MACH_MSGH_BITS_ZERO) { + assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); + /* + * someone managed to send this kernel routine a message with + * a voucher in it. Cleanup the reference in + * kmsg->ikm_voucher. + */ + if (IP_VALID(kmsg->ikm_voucher)) { + ipc_port_release_send(kmsg->ikm_voucher); + } + kmsg->ikm_voucher = IP_NULL; + kmsg->ikm_header->msgh_voucher_port = 0; + } kmsg->ikm_header->msgh_bits = - (MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) | - MACH_MSGH_BITS(reply_type, dest_type)); + (MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) | + MACH_MSGH_BITS(reply_type, dest_type)); kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name); - kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name); + kmsg->ikm_header->msgh_remote_port = reply; } #if IKM_SUPPORT_LEGACY void ipc_kmsg_copyout_to_kernel_legacy( - ipc_kmsg_t kmsg, - ipc_space_t space) + ipc_kmsg_t kmsg, + ipc_space_t space) { ipc_object_t dest; - ipc_object_t reply; + mach_port_t reply; mach_msg_type_name_t dest_type; mach_msg_type_name_t reply_type; - mach_port_name_t dest_name, reply_name; + mach_port_name_t dest_name; - dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; - reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port; + dest = ip_to_object(kmsg->ikm_header->msgh_remote_port); + reply = kmsg->ikm_header->msgh_local_port; dest_type = MACH_MSGH_BITS_REMOTE(kmsg->ikm_header->msgh_bits); reply_type = MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits); @@ -4676,93 +5382,117 @@ ipc_kmsg_copyout_to_kernel_legacy( dest_name = MACH_PORT_DEAD; } - reply_name = CAST_MACH_PORT_TO_NAME(reply); + mach_msg_type_name_t voucher_type; + voucher_type = MACH_MSGH_BITS_VOUCHER(kmsg->ikm_header->msgh_bits); + if (voucher_type != MACH_MSGH_BITS_ZERO) { + assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); + assert(IP_VALID(kmsg->ikm_voucher)); + /* + * someone managed to send this kernel routine a message with + * a voucher in it. Cleanup the reference in + * kmsg->ikm_voucher. + */ + ipc_port_release_send(kmsg->ikm_voucher); + kmsg->ikm_voucher = IP_NULL; + kmsg->ikm_header->msgh_voucher_port = 0; + } kmsg->ikm_header->msgh_bits = - (MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) | - MACH_MSGH_BITS(reply_type, dest_type)); + (MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) | + MACH_MSGH_BITS(reply_type, dest_type)); kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name); - kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name); + kmsg->ikm_header->msgh_remote_port = reply; + + mach_msg_descriptor_t *saddr; + mach_msg_legacy_descriptor_t *daddr; + mach_msg_type_number_t i, count = ((mach_msg_base_t *)kmsg->ikm_header)->body.msgh_descriptor_count; + saddr = (mach_msg_descriptor_t *) (((mach_msg_base_t *)kmsg->ikm_header) + 1); + saddr = &saddr[count - 1]; + daddr = (mach_msg_legacy_descriptor_t *)&saddr[count]; + daddr--; + + vm_offset_t dsc_adjust = 0; + + for (i = 0; i < count; i++, saddr--, daddr--) { + switch (saddr->type.type) { + case MACH_MSG_PORT_DESCRIPTOR: { + mach_msg_port_descriptor_t *dsc = &saddr->port; + mach_msg_legacy_port_descriptor_t *dest_dsc = &daddr->port; + + mach_port_t name = dsc->name; + mach_msg_type_name_t disposition = dsc->disposition; + + dest_dsc->name = CAST_MACH_PORT_TO_NAME(name); + dest_dsc->disposition = disposition; + dest_dsc->type = MACH_MSG_PORT_DESCRIPTOR; + break; + } + case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: + case MACH_MSG_OOL_DESCRIPTOR: { + /* The sender should supply ready-made memory, i.e. a vm_map_copy_t + * so we don't need to do anything special. */ + + mach_msg_ool_descriptor_t *source_dsc = (typeof(source_dsc)) & saddr->out_of_line; + + mach_msg_ool_descriptor32_t *dest_dsc = &daddr->out_of_line32; + + vm_offset_t address = (vm_offset_t)source_dsc->address; + vm_size_t size = source_dsc->size; + boolean_t deallocate = source_dsc->deallocate; + mach_msg_copy_options_t copy = source_dsc->copy; + mach_msg_descriptor_type_t type = source_dsc->type; - mach_msg_descriptor_t *saddr; - mach_msg_legacy_descriptor_t *daddr; - mach_msg_type_number_t i, count = ((mach_msg_base_t *)kmsg->ikm_header)->body.msgh_descriptor_count; - saddr = (mach_msg_descriptor_t *) (((mach_msg_base_t *)kmsg->ikm_header) + 1); - saddr = &saddr[count-1]; - daddr = (mach_msg_legacy_descriptor_t *)&saddr[count]; - daddr--; - - vm_offset_t dsc_adjust = 0; - - for (i = 0; i < count; i++, saddr--, daddr--) { - switch (saddr->type.type) { - case MACH_MSG_PORT_DESCRIPTOR: { - mach_msg_port_descriptor_t *dsc = &saddr->port; - mach_msg_legacy_port_descriptor_t *dest_dsc = &daddr->port; - - mach_port_t name = dsc->name; - mach_msg_type_name_t disposition = dsc->disposition; - - dest_dsc->name = CAST_MACH_PORT_TO_NAME(name); - dest_dsc->disposition = disposition; - dest_dsc->type = MACH_MSG_PORT_DESCRIPTOR; - break; - } - case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: - case MACH_MSG_OOL_DESCRIPTOR: { - /* The sender should supply ready-made memory, i.e. a vm_map_copy_t - * so we don't need to do anything special. */ - - mach_msg_ool_descriptor_t *source_dsc = (typeof(source_dsc))&saddr->out_of_line; - - mach_msg_ool_descriptor32_t *dest_dsc = &daddr->out_of_line32; - - vm_offset_t address = (vm_offset_t)source_dsc->address; - vm_size_t size = source_dsc->size; - boolean_t deallocate = source_dsc->deallocate; - mach_msg_copy_options_t copy = source_dsc->copy; - mach_msg_descriptor_type_t type = source_dsc->type; - - dest_dsc->address = address; - dest_dsc->size = size; - dest_dsc->deallocate = deallocate; - dest_dsc->copy = copy; - dest_dsc->type = type; - break; - } - case MACH_MSG_OOL_PORTS_DESCRIPTOR: { - mach_msg_ool_ports_descriptor_t *source_dsc = (typeof(source_dsc))&saddr->ool_ports; - - mach_msg_ool_ports_descriptor32_t *dest_dsc = &daddr->ool_ports32; - - vm_offset_t address = (vm_offset_t)source_dsc->address; - vm_size_t port_count = source_dsc->count; - boolean_t deallocate = source_dsc->deallocate; - mach_msg_copy_options_t copy = source_dsc->copy; - mach_msg_descriptor_type_t type = source_dsc->type; - - dest_dsc->address = address; - dest_dsc->count = port_count; - dest_dsc->deallocate = deallocate; - dest_dsc->copy = copy; - dest_dsc->type = type; - break; - } - default: { -#if MACH_ASSERT - panic("ipc_kmsg_copyin_from_kernel: bad descriptor"); -#endif /* MACH_ASSERT */ - } - } - } - - if(count) { - dsc_adjust = 4*count; - memmove((char *)((vm_offset_t)kmsg->ikm_header + dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t)); - kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header + dsc_adjust); - /* Update the message size for the smaller user representation */ - kmsg->ikm_header->msgh_size -= dsc_adjust; - } + dest_dsc->address = address; + dest_dsc->size = size; + dest_dsc->deallocate = deallocate; + dest_dsc->copy = copy; + dest_dsc->type = type; + break; + } + case MACH_MSG_OOL_PORTS_DESCRIPTOR: { + mach_msg_ool_ports_descriptor_t *source_dsc = (typeof(source_dsc)) & saddr->ool_ports; + + mach_msg_ool_ports_descriptor32_t *dest_dsc = &daddr->ool_ports32; + + vm_offset_t address = (vm_offset_t)source_dsc->address; + vm_size_t port_count = source_dsc->count; + boolean_t deallocate = source_dsc->deallocate; + mach_msg_copy_options_t copy = source_dsc->copy; + mach_msg_descriptor_type_t type = source_dsc->type; + + dest_dsc->address = address; + dest_dsc->count = port_count; + dest_dsc->deallocate = deallocate; + dest_dsc->copy = copy; + dest_dsc->type = type; + break; + } + case MACH_MSG_GUARDED_PORT_DESCRIPTOR: { + mach_msg_guarded_port_descriptor_t *source_dsc = (typeof(source_dsc)) & saddr->guarded_port; + mach_msg_guarded_port_descriptor32_t *dest_dsc = &daddr->guarded_port32; + + dest_dsc->name = CAST_MACH_PORT_TO_NAME(source_dsc->name); + dest_dsc->disposition = source_dsc->disposition; + dest_dsc->flags = 0; + dest_dsc->type = MACH_MSG_GUARDED_PORT_DESCRIPTOR; + dest_dsc->context = 0; + break; + } + default: { +#if MACH_ASSERT + panic("ipc_kmsg_copyout_to_kernel_legacy: bad descriptor"); +#endif /* MACH_ASSERT */ + } + } + } + + if (count) { + dsc_adjust = 4 * count; + memmove((char *)((vm_offset_t)kmsg->ikm_header + dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t)); + kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header + dsc_adjust); + /* Update the message size for the smaller user representation */ + kmsg->ikm_header->msgh_size -= dsc_adjust; + } } #endif /* IKM_SUPPORT_LEGACY */ @@ -4771,7 +5501,7 @@ ipc_kmsg_copyout_to_kernel_legacy( * Just sets those parts of the trailer that aren't set up at allocation time. */ static void -ipc_kmsg_munge_trailer(mach_msg_max_trailer_t *in, void *_out, boolean_t is64bit) +ipc_kmsg_munge_trailer(mach_msg_max_trailer_t *in, void *_out, boolean_t is64bit) { if (is64bit) { mach_msg_max_trailer64_t *out = (mach_msg_max_trailer64_t*)_out; @@ -4790,22 +5520,22 @@ ipc_kmsg_munge_trailer(mach_msg_max_trailer_t *in, void *_out, boolean_t is64bit #endif /* __arm64__ */ mach_msg_trailer_size_t -ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused, - mach_msg_option_t option, thread_t thread, - mach_port_seqno_t seqno, boolean_t minimal_trailer, - mach_vm_offset_t context) +ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused, + mach_msg_option_t option, thread_t thread, + mach_port_seqno_t seqno, boolean_t minimal_trailer, + mach_vm_offset_t context) { mach_msg_max_trailer_t *trailer; #ifdef __arm64__ mach_msg_max_trailer_t tmp_trailer; /* This accommodates U64, and we'll munge */ void *real_trailer_out = (void*)(mach_msg_max_trailer_t *) - ((vm_offset_t)kmsg->ikm_header + - round_msg(kmsg->ikm_header->msgh_size)); + ((vm_offset_t)kmsg->ikm_header + + round_msg(kmsg->ikm_header->msgh_size)); - /* + /* * Populate scratch with initial values set up at message allocation time. - * After, we reinterpret the space in the message as the right type + * After, we reinterpret the space in the message as the right type * of trailer for the address space in question. */ bcopy(real_trailer_out, &tmp_trailer, MAX_TRAILER_SIZE); @@ -4813,8 +5543,8 @@ ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused, #else /* __arm64__ */ (void)thread; trailer = (mach_msg_max_trailer_t *) - ((vm_offset_t)kmsg->ikm_header + - round_msg(kmsg->ikm_header->msgh_size)); + ((vm_offset_t)kmsg->ikm_header + + round_msg(kmsg->ikm_header->msgh_size)); #endif /* __arm64__ */ if (!(option & MACH_RCV_TRAILER_MASK)) { @@ -4823,14 +5553,13 @@ ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused, trailer->msgh_seqno = seqno; trailer->msgh_context = context; - trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(thread_is_64bit(thread), option); + trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(thread_is_64bit_addr(thread), option); - if (minimal_trailer) { + if (minimal_trailer) { goto done; } - if (MACH_RCV_TRAILER_ELEMENTS(option) >= - MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_AV)){ + if (GET_RCV_ELEMENTS(option) >= MACH_RCV_TRAILER_AV) { trailer->msgh_ad = 0; } @@ -4840,14 +5569,23 @@ ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused, * and a send right to copyout to the receiver. */ - if (option & MACH_RCV_TRAILER_ELEMENTS (MACH_RCV_TRAILER_LABELS)) { + if (option & MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_LABELS)) { trailer->msgh_labels.sender = 0; } done: #ifdef __arm64__ - ipc_kmsg_munge_trailer(trailer, real_trailer_out, thread_is_64bit(thread)); + ipc_kmsg_munge_trailer(trailer, real_trailer_out, thread_is_64bit_addr(thread)); #endif /* __arm64__ */ return trailer->msgh_trailer_size; } + +mach_msg_header_t * +ipc_kmsg_msg_header(ipc_kmsg_t kmsg) +{ + if (NULL == kmsg) { + return NULL; + } + return kmsg->ikm_header; +}