+ case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
+ case MACH_MSG_OOL_DESCRIPTOR: {
+ mach_msg_ool_descriptor_t *dsc;
+
+ dsc = (mach_msg_ool_descriptor_t *) &saddr->out_of_line;
+ kprintf(" OOL%s addr = %p size = 0x%x copy = %s %s\n",
+ type == MACH_MSG_OOL_DESCRIPTOR ? "" : " VOLATILE",
+ dsc->address, dsc->size,
+ mm_copy_options_string64(dsc->copy),
+ dsc->deallocate ? "DEALLOC" : "");
+ break;
+ }
+ case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
+ mach_msg_ool_ports_descriptor_t *dsc;
+
+ dsc = (mach_msg_ool_ports_descriptor_t *) &saddr->ool_ports;
+
+ kprintf(" OOL_PORTS addr = %p count = 0x%x ",
+ dsc->address, dsc->count);
+ kprintf("disp = ");
+ ipc_print_type_name64(dsc->disposition);
+ kprintf(" copy = %s %s\n",
+ mm_copy_options_string64(dsc->copy),
+ dsc->deallocate ? "DEALLOC" : "");
+ break;
+ }
+ case MACH_MSG_GUARDED_PORT_DESCRIPTOR: {
+ mach_msg_guarded_port_descriptor_t *dsc;
+
+ dsc = (mach_msg_guarded_port_descriptor_t *)&saddr->guarded_port;
+ kprintf(" GUARDED_PORT name = %p flags = 0x%x disp = ", dsc->name, dsc->flags);
+ ipc_print_type_name64(dsc->disposition);
+ kprintf("\n");
+ break;
+ }
+ default: {
+ kprintf(" UNKNOWN DESCRIPTOR 0x%x\n", type);
+ break;
+ }
+ }
+ }
+}
+
+#define DEBUG_IPC_KMSG_PRINT(kmsg, string) \
+ __unreachable_ok_push \
+ if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \
+ ipc_kmsg_print64(kmsg, string); \
+ } \
+ __unreachable_ok_pop
+
+#define DEBUG_IPC_MSG_BODY_PRINT(body, size) \
+ __unreachable_ok_push \
+ if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \
+ ipc_msg_body_print64(body,size);\
+ } \
+ __unreachable_ok_pop
+#else /* !DEBUG_MSGS_K64 */
+#define DEBUG_IPC_KMSG_PRINT(kmsg, string)
+#define DEBUG_IPC_MSG_BODY_PRINT(body, size)
+#endif /* !DEBUG_MSGS_K64 */
+
+extern vm_map_t ipc_kernel_copy_map;
+extern vm_size_t ipc_kmsg_max_space;
+extern vm_size_t ipc_kmsg_max_vm_space;
+extern vm_size_t ipc_kmsg_max_body_space;
+extern vm_size_t msg_ool_size_small;
+
+#define MSG_OOL_SIZE_SMALL msg_ool_size_small
+
+#if defined(__LP64__)
+#define MAP_SIZE_DIFFERS(map) (map->max_offset < MACH_VM_MAX_ADDRESS)
+#define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor32_t
+#define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor32_t
+#else
+#define MAP_SIZE_DIFFERS(map) (map->max_offset > VM_MAX_ADDRESS)
+#define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor64_t
+#define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor64_t
+#endif
+
+#define DESC_SIZE_ADJUSTMENT ((mach_msg_size_t)(sizeof(mach_msg_ool_descriptor64_t) - \
+ sizeof(mach_msg_ool_descriptor32_t)))
+
+/* scatter list macros */
+
+#define SKIP_PORT_DESCRIPTORS(s, c) \
+MACRO_BEGIN \
+ if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \
+ while ((c) > 0) { \
+ if ((s)->type.type != MACH_MSG_PORT_DESCRIPTOR) \
+ break; \
+ (s)++; (c)--; \
+ } \
+ if (c == 0) \
+ (s) = MACH_MSG_DESCRIPTOR_NULL; \
+ } \
+MACRO_END
+
+#define INCREMENT_SCATTER(s, c, d) \
+MACRO_BEGIN \
+ if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \
+ s = (d) ? (mach_msg_descriptor_t *) \
+ ((OTHER_OOL_DESCRIPTOR *)(s) + 1) : \
+ (s + 1); \
+ (c)--; \
+ } \
+MACRO_END
+
+#define KMSG_TRACE_FLAG_TRACED 0x000001
+#define KMSG_TRACE_FLAG_COMPLEX 0x000002
+#define KMSG_TRACE_FLAG_OOLMEM 0x000004
+#define KMSG_TRACE_FLAG_VCPY 0x000008
+#define KMSG_TRACE_FLAG_PCPY 0x000010
+#define KMSG_TRACE_FLAG_SND64 0x000020
+#define KMSG_TRACE_FLAG_RAISEIMP 0x000040
+#define KMSG_TRACE_FLAG_APP_SRC 0x000080
+#define KMSG_TRACE_FLAG_APP_DST 0x000100
+#define KMSG_TRACE_FLAG_DAEMON_SRC 0x000200
+#define KMSG_TRACE_FLAG_DAEMON_DST 0x000400
+#define KMSG_TRACE_FLAG_DST_NDFLTQ 0x000800
+#define KMSG_TRACE_FLAG_SRC_NDFLTQ 0x001000
+#define KMSG_TRACE_FLAG_DST_SONCE 0x002000
+#define KMSG_TRACE_FLAG_SRC_SONCE 0x004000
+#define KMSG_TRACE_FLAG_CHECKIN 0x008000
+#define KMSG_TRACE_FLAG_ONEWAY 0x010000
+#define KMSG_TRACE_FLAG_IOKIT 0x020000
+#define KMSG_TRACE_FLAG_SNDRCV 0x040000
+#define KMSG_TRACE_FLAG_DSTQFULL 0x080000
+#define KMSG_TRACE_FLAG_VOUCHER 0x100000
+#define KMSG_TRACE_FLAG_TIMER 0x200000
+#define KMSG_TRACE_FLAG_SEMA 0x400000
+#define KMSG_TRACE_FLAG_DTMPOWNER 0x800000
+#define KMSG_TRACE_FLAG_GUARDED_DESC 0x1000000
+
+#define KMSG_TRACE_FLAGS_MASK 0x1ffffff
+#define KMSG_TRACE_FLAGS_SHIFT 8
+
+#define KMSG_TRACE_PORTS_MASK 0xff
+#define KMSG_TRACE_PORTS_SHIFT 0
+
+#if (KDEBUG_LEVEL >= KDEBUG_LEVEL_STANDARD)
+#include <stdint.h>
+
+void
+ipc_kmsg_trace_send(ipc_kmsg_t kmsg,
+ mach_msg_option_t option)
+{
+ task_t send_task = TASK_NULL;
+ ipc_port_t dst_port, src_port;
+ boolean_t is_task_64bit;
+ mach_msg_header_t *msg;
+ mach_msg_trailer_t *trailer;
+
+ int kotype = 0;
+ uint32_t msg_size = 0;
+ uint64_t msg_flags = KMSG_TRACE_FLAG_TRACED;
+ uint32_t num_ports = 0;
+ uint32_t send_pid, dst_pid;
+
+ /*
+ * check to see not only if ktracing is enabled, but if we will
+ * _actually_ emit the KMSG_INFO tracepoint. This saves us a
+ * significant amount of processing (and a port lock hold) in
+ * the non-tracing case.
+ */
+ if (__probable((kdebug_enable & KDEBUG_TRACE) == 0)) {
+ return;
+ }
+ if (!kdebug_debugid_enabled(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO))) {
+ return;
+ }
+
+ msg = kmsg->ikm_header;
+
+ dst_port = msg->msgh_remote_port;
+ if (!IPC_PORT_VALID(dst_port)) {
+ return;
+ }
+
+ /*
+ * Message properties / options
+ */
+ if ((option & (MACH_SEND_MSG | MACH_RCV_MSG)) == (MACH_SEND_MSG | MACH_RCV_MSG)) {
+ msg_flags |= KMSG_TRACE_FLAG_SNDRCV;
+ }
+
+ if (msg->msgh_id >= is_iokit_subsystem.start &&
+ msg->msgh_id < is_iokit_subsystem.end + 100) {
+ msg_flags |= KMSG_TRACE_FLAG_IOKIT;
+ }
+ /* magic XPC checkin message id (XPC_MESSAGE_ID_CHECKIN) from libxpc */
+ else if (msg->msgh_id == 0x77303074u /* w00t */) {
+ msg_flags |= KMSG_TRACE_FLAG_CHECKIN;
+ }
+
+ if (msg->msgh_bits & MACH_MSGH_BITS_RAISEIMP) {
+ msg_flags |= KMSG_TRACE_FLAG_RAISEIMP;
+ }
+
+ if (unsafe_convert_port_to_voucher(kmsg->ikm_voucher)) {
+ msg_flags |= KMSG_TRACE_FLAG_VOUCHER;
+ }
+
+ /*
+ * Sending task / port
+ */
+ send_task = current_task();
+ send_pid = task_pid(send_task);
+
+ if (send_pid != 0) {
+ if (task_is_daemon(send_task)) {
+ msg_flags |= KMSG_TRACE_FLAG_DAEMON_SRC;
+ } else if (task_is_app(send_task)) {
+ msg_flags |= KMSG_TRACE_FLAG_APP_SRC;
+ }
+ }
+
+ is_task_64bit = (send_task->map->max_offset > VM_MAX_ADDRESS);
+ if (is_task_64bit) {
+ msg_flags |= KMSG_TRACE_FLAG_SND64;
+ }
+
+ src_port = msg->msgh_local_port;
+ if (src_port) {
+ if (src_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT) {
+ msg_flags |= KMSG_TRACE_FLAG_SRC_NDFLTQ;
+ }
+ switch (MACH_MSGH_BITS_LOCAL(msg->msgh_bits)) {
+ case MACH_MSG_TYPE_MOVE_SEND_ONCE:
+ msg_flags |= KMSG_TRACE_FLAG_SRC_SONCE;
+ break;
+ default:
+ break;
+ }
+ } else {
+ msg_flags |= KMSG_TRACE_FLAG_ONEWAY;
+ }
+
+
+ /*
+ * Destination task / port
+ */
+ ip_lock(dst_port);
+ if (!ip_active(dst_port)) {
+ /* dst port is being torn down */
+ dst_pid = (uint32_t)0xfffffff0;
+ } else if (dst_port->ip_tempowner) {
+ msg_flags |= KMSG_TRACE_FLAG_DTMPOWNER;
+ if (IIT_NULL != dst_port->ip_imp_task) {
+ dst_pid = task_pid(dst_port->ip_imp_task->iit_task);
+ } else {
+ dst_pid = (uint32_t)0xfffffff1;
+ }
+ } else if (dst_port->ip_receiver_name == MACH_PORT_NULL) {
+ /* dst_port is otherwise in-transit */
+ dst_pid = (uint32_t)0xfffffff2;
+ } else {
+ if (dst_port->ip_receiver == ipc_space_kernel) {
+ dst_pid = 0;
+ } else {
+ ipc_space_t dst_space;
+ dst_space = dst_port->ip_receiver;
+ if (dst_space && is_active(dst_space)) {
+ dst_pid = task_pid(dst_space->is_task);
+ if (task_is_daemon(dst_space->is_task)) {
+ msg_flags |= KMSG_TRACE_FLAG_DAEMON_DST;
+ } else if (task_is_app(dst_space->is_task)) {
+ msg_flags |= KMSG_TRACE_FLAG_APP_DST;
+ }
+ } else {
+ /* receiving task is being torn down */
+ dst_pid = (uint32_t)0xfffffff3;
+ }
+ }
+ }
+
+ if (dst_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT) {
+ msg_flags |= KMSG_TRACE_FLAG_DST_NDFLTQ;
+ }
+ if (imq_full(&dst_port->ip_messages)) {
+ msg_flags |= KMSG_TRACE_FLAG_DSTQFULL;
+ }
+
+ kotype = ip_kotype(dst_port);
+
+ ip_unlock(dst_port);
+
+ switch (kotype) {
+ case IKOT_SEMAPHORE:
+ msg_flags |= KMSG_TRACE_FLAG_SEMA;
+ break;
+ case IKOT_TIMER:
+ case IKOT_CLOCK:
+ msg_flags |= KMSG_TRACE_FLAG_TIMER;
+ break;
+ case IKOT_MASTER_DEVICE:
+ case IKOT_IOKIT_CONNECT:
+ case IKOT_IOKIT_OBJECT:
+ case IKOT_IOKIT_IDENT:
+ case IKOT_UEXT_OBJECT:
+ msg_flags |= KMSG_TRACE_FLAG_IOKIT;
+ break;
+ default:
+ break;
+ }
+
+ switch (MACH_MSGH_BITS_REMOTE(msg->msgh_bits)) {
+ case MACH_MSG_TYPE_PORT_SEND_ONCE:
+ msg_flags |= KMSG_TRACE_FLAG_DST_SONCE;
+ break;
+ default:
+ break;
+ }
+
+
+ /*
+ * Message size / content
+ */
+ msg_size = msg->msgh_size - sizeof(mach_msg_header_t);
+
+ if (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
+ mach_msg_body_t *msg_body;
+ mach_msg_descriptor_t *kern_dsc;
+ int dsc_count;
+
+ msg_flags |= KMSG_TRACE_FLAG_COMPLEX;
+
+ msg_body = (mach_msg_body_t *)(kmsg->ikm_header + 1);
+ dsc_count = (int)msg_body->msgh_descriptor_count;
+ kern_dsc = (mach_msg_descriptor_t *)(msg_body + 1);
+
+ /* this is gross: see ipc_kmsg_copyin_body()... */
+ if (!is_task_64bit) {
+ msg_size -= (dsc_count * 12);
+ }
+
+ for (int i = 0; i < dsc_count; i++) {
+ switch (kern_dsc[i].type.type) {
+ case MACH_MSG_PORT_DESCRIPTOR:
+ num_ports++;
+ if (is_task_64bit) {
+ msg_size -= 12;
+ }
+ break;
+ case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
+ case MACH_MSG_OOL_DESCRIPTOR: {
+ mach_msg_ool_descriptor_t *dsc;
+ dsc = (mach_msg_ool_descriptor_t *)&kern_dsc[i];
+ msg_flags |= KMSG_TRACE_FLAG_OOLMEM;
+ msg_size += dsc->size;
+ if ((dsc->size >= MSG_OOL_SIZE_SMALL) &&
+ (dsc->copy == MACH_MSG_PHYSICAL_COPY) &&
+ !dsc->deallocate) {
+ msg_flags |= KMSG_TRACE_FLAG_PCPY;
+ } else if (dsc->size <= MSG_OOL_SIZE_SMALL) {
+ msg_flags |= KMSG_TRACE_FLAG_PCPY;
+ } else {
+ msg_flags |= KMSG_TRACE_FLAG_VCPY;
+ }
+ if (is_task_64bit) {
+ msg_size -= 16;
+ }
+ } break;
+ case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
+ mach_msg_ool_ports_descriptor_t *dsc;
+ dsc = (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i];
+ num_ports += dsc->count;
+ if (is_task_64bit) {
+ msg_size -= 16;
+ }
+ } break;
+ case MACH_MSG_GUARDED_PORT_DESCRIPTOR:
+ num_ports++;
+ msg_flags |= KMSG_TRACE_FLAG_GUARDED_DESC;
+ if (is_task_64bit) {
+ msg_size -= 16;
+ }
+ default:
+ break;
+ }
+ }
+ }
+
+ /*
+ * Trailer contents
+ */
+ trailer = (mach_msg_trailer_t *)((vm_offset_t)msg +
+ round_msg((vm_offset_t)msg->msgh_size));
+ if (trailer->msgh_trailer_size <= sizeof(mach_msg_security_trailer_t)) {
+ extern const security_token_t KERNEL_SECURITY_TOKEN;
+ mach_msg_security_trailer_t *strailer;
+ strailer = (mach_msg_security_trailer_t *)trailer;
+ /*
+ * verify the sender PID: replies from the kernel often look
+ * like self-talk because the sending port is not reset.
+ */
+ if (memcmp(&strailer->msgh_sender,
+ &KERNEL_SECURITY_TOKEN,
+ sizeof(KERNEL_SECURITY_TOKEN)) == 0) {
+ send_pid = 0;
+ msg_flags &= ~(KMSG_TRACE_FLAG_APP_SRC | KMSG_TRACE_FLAG_DAEMON_SRC);
+ }
+ }
+
+ KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END,
+ (uintptr_t)send_pid,
+ (uintptr_t)dst_pid,
+ (uintptr_t)msg_size,
+ (uintptr_t)(
+ ((msg_flags & KMSG_TRACE_FLAGS_MASK) << KMSG_TRACE_FLAGS_SHIFT) |
+ ((num_ports & KMSG_TRACE_PORTS_MASK) << KMSG_TRACE_PORTS_SHIFT)
+ )
+ );
+}
+#endif
+
+/* zone for cached ipc_kmsg_t structures */
+zone_t ipc_kmsg_zone;
+
+/*
+ * Forward declarations
+ */
+
+void ipc_kmsg_clean(
+ ipc_kmsg_t kmsg);
+
+void ipc_kmsg_clean_body(
+ ipc_kmsg_t kmsg,
+ mach_msg_type_number_t number,
+ mach_msg_descriptor_t *desc);
+
+void ipc_kmsg_clean_partial(
+ ipc_kmsg_t kmsg,
+ mach_msg_type_number_t number,
+ mach_msg_descriptor_t *desc,
+ vm_offset_t paddr,
+ vm_size_t length);
+
+mach_msg_return_t ipc_kmsg_copyin_body(
+ ipc_kmsg_t kmsg,
+ ipc_space_t space,
+ vm_map_t map,
+ mach_msg_option_t *optionp);
+
+
+extern int enforce_strict_reply;
+
+static void
+ipc_kmsg_link_reply_context_locked(
+ ipc_port_t reply_port,
+ ipc_port_t voucher_port);
+
+static kern_return_t
+ipc_kmsg_validate_reply_port_locked(
+ ipc_port_t reply_port,
+ mach_msg_option_t options);
+
+static mach_msg_return_t
+ipc_kmsg_validate_reply_context_locked(
+ mach_msg_option_t option,
+ ipc_port_t dest_port,
+ ipc_voucher_t voucher,
+ mach_port_name_t voucher_name);
+
+/* we can't include the BSD <sys/persona.h> header here... */
+#ifndef PERSONA_ID_NONE
+#define PERSONA_ID_NONE ((uint32_t)-1)
+#endif
+
+/*
+ * We keep a per-processor cache of kernel message buffers.
+ * The cache saves the overhead/locking of using kalloc/kfree.
+ * The per-processor cache seems to miss less than a per-thread cache,
+ * and it also uses less memory. Access to the cache doesn't
+ * require locking.
+ */
+
+/*
+ * Routine: ipc_kmsg_alloc
+ * Purpose:
+ * Allocate a kernel message structure. If we can get one from
+ * the cache, that is best. Otherwise, allocate a new one.
+ * Conditions:
+ * Nothing locked.
+ */
+ipc_kmsg_t
+ipc_kmsg_alloc(
+ mach_msg_size_t msg_and_trailer_size)
+{
+ mach_msg_size_t max_expanded_size;
+ ipc_kmsg_t kmsg;
+
+ /*
+ * LP64support -
+ * Pad the allocation in case we need to expand the
+ * message descriptors for user spaces with pointers larger than
+ * the kernel's own, or vice versa. We don't know how many descriptors
+ * there are yet, so just assume the whole body could be
+ * descriptors (if there could be any at all).
+ *
+ * The expansion space is left in front of the header,
+ * because it is easier to pull the header and descriptors
+ * forward as we process them than it is to push all the
+ * data backwards.
+ */
+ mach_msg_size_t size = msg_and_trailer_size - MAX_TRAILER_SIZE;
+
+ /* compare against implementation upper limit for the body */
+ if (size > ipc_kmsg_max_body_space) {
+ return IKM_NULL;
+ }
+
+ if (size > sizeof(mach_msg_base_t)) {
+ mach_msg_size_t max_desc = (mach_msg_size_t)(((size - sizeof(mach_msg_base_t)) /
+ sizeof(mach_msg_ool_descriptor32_t)) *
+ DESC_SIZE_ADJUSTMENT);
+
+ /* make sure expansion won't cause wrap */
+ if (msg_and_trailer_size > MACH_MSG_SIZE_MAX - max_desc) {
+ return IKM_NULL;
+ }
+
+ max_expanded_size = msg_and_trailer_size + max_desc;
+ } else {
+ max_expanded_size = msg_and_trailer_size;
+ }
+
+ if (max_expanded_size < IKM_SAVED_MSG_SIZE) {
+ max_expanded_size = IKM_SAVED_MSG_SIZE; /* round up for ikm_cache */
+ }
+ if (max_expanded_size == IKM_SAVED_MSG_SIZE) {
+ kmsg = (ipc_kmsg_t)zalloc(ipc_kmsg_zone);
+ } else {
+ kmsg = (ipc_kmsg_t)kalloc(ikm_plus_overhead(max_expanded_size));
+ }
+
+ if (kmsg != IKM_NULL) {
+ ikm_init(kmsg, max_expanded_size);
+ ikm_set_header(kmsg, msg_and_trailer_size);
projects / apple / xnu.git / blobdiff