#define KMSG_TRACE_PORTS_SHIFT 0
#if (KDEBUG_LEVEL >= KDEBUG_LEVEL_STANDARD)
+#include <stdint.h>
extern boolean_t kdebug_debugid_enabled(uint32_t debugid);
void ipc_kmsg_trace_send(ipc_kmsg_t kmsg,
case IKOT_MASTER_DEVICE:
case IKOT_IOKIT_CONNECT:
case IKOT_IOKIT_OBJECT:
- case IKOT_IOKIT_SPARE:
+ case IKOT_IOKIT_IDENT:
msg_flags |= KMSG_TRACE_FLAG_IOKIT;
break;
default:
ipc_space_t space,
vm_map_t map);
+extern int thread_qos_from_pthread_priority(unsigned long, unsigned long *);
+
/*
* We keep a per-processor cache of kernel message buffers.
* The cache saves the overhead/locking of using kalloc/kfree.
cur->ikm_qos_override = override;
if (cur == first)
return TRUE;
- cur = cur->ikm_next;
+ cur = cur->ikm_prev;
}
return FALSE;
}
queue->ikmq_base = IKM_NULL;
} else {
+ if (__improbable(next->ikm_prev != kmsg || prev->ikm_next != kmsg)) {
+ panic("ipc_kmsg_rmqueue: inconsistent prev/next pointers. "
+ "(prev->next: %p, next->prev: %p, kmsg: %p)",
+ prev->ikm_next, next->ikm_prev, kmsg);
+ }
+
if (queue->ikmq_base == kmsg)
queue->ikmq_base = next;
assert(kmsg->ikm_prealloc == IP_NULL);
kmsg->ikm_prealloc = IP_NULL;
+ /* take the mqueue lock since the sync qos is protected under it */
+ imq_lock(&port->ip_messages);
+
+ /* copy the sync qos values to kmsg */
+ for (int i = 0; i < THREAD_QOS_LAST; i++) {
+ kmsg->sync_qos[i] = port_sync_qos(port, i);
+ }
+ kmsg->special_port_qos = port_special_qos(port);
IP_SET_PREALLOC(port, kmsg);
+ imq_unlock(&port->ip_messages);
}
/*
assert(kmsg->ikm_prealloc == port);
kmsg->ikm_prealloc = IP_NULL;
+
+ /* take the mqueue lock since the sync qos is protected under it */
+ imq_lock(&port->ip_messages);
+
IP_CLEAR_PREALLOC(port, kmsg);
+
+ /* copy the sync qos values from kmsg to port */
+ for (int i = 0; i < THREAD_QOS_LAST; i++) {
+ set_port_sync_qos(port, i, kmsg->sync_qos[i]);
+ }
+ set_port_special_qos(port, kmsg->special_port_qos);
+ imq_unlock(&port->ip_messages);
}
/*
if (copyinmsg(msg_addr, (char *)&legacy_base, len_copied))
return MACH_SEND_INVALID_DATA;
+ /*
+ * If the message claims to be complex, it must at least
+ * have the length of a "base" message (header + dsc_count).
+ */
+ if (len_copied < sizeof(mach_msg_legacy_base_t) &&
+ (legacy_base.header.msgh_bits & MACH_MSGH_BITS_COMPLEX))
+ return MACH_SEND_MSG_TOO_SMALL;
+
msg_addr += sizeof(legacy_base.header);
#if defined(__LP64__)
size += LEGACY_HEADER_SIZE_DELTA;
(void) memcpy((void *) kmsg->ikm_header, (const void *) msg, size);
+ ikm_qos_init(kmsg);
+
kmsg->ikm_header->msgh_size = size;
/*
#if IMPORTANCE_INHERITANCE
boolean_t did_importance = FALSE;
-#if IMPORTANCE_DEBUG
+#if IMPORTANCE_TRACE
mach_msg_id_t imp_msgh_id = -1;
int sender_pid = -1;
-#endif /* IMPORTANCE_DEBUG */
+#endif /* IMPORTANCE_TRACE */
#endif /* IMPORTANCE_INHERITANCE */
/* don't allow the creation of a circular loop */
default:
break;
}
-#if IMPORTANCE_DEBUG
+#if IMPORTANCE_TRACE
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_END,
task_pid(current_task()), sender_pid, imp_msgh_id, importance_cleared, 0);
-#endif /* IMPORTANCE_DEBUG */
+#endif /* IMPORTANCE_TRACE */
}
#endif /* IMPORTANCE_INHERITANCE */
unsigned long pthread_priority_canonicalize(unsigned long priority, boolean_t propagation);
-static void
+static kern_return_t
ipc_kmsg_set_qos(
ipc_kmsg_t kmsg,
mach_msg_option_t options,
mach_msg_priority_t override)
{
kern_return_t kr;
+ unsigned long flags = 0;
+ ipc_port_t special_reply_port = kmsg->ikm_header->msgh_local_port;
+ ipc_port_t dest_port = kmsg->ikm_header->msgh_remote_port;
kr = ipc_get_pthpriority_from_kmsg_voucher(kmsg, &kmsg->ikm_qos);
if (kr != KERN_SUCCESS) {
if (canon > kmsg->ikm_qos)
kmsg->ikm_qos_override = canon;
}
+
+ kr = KERN_SUCCESS;
+ if ((options & MACH_SEND_SYNC_OVERRIDE)) {
+ if (IP_VALID(special_reply_port) &&
+ MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits) == MACH_MSG_TYPE_PORT_SEND_ONCE) {
+ /*
+ * Update the sync override count if the reply port is a special reply port,
+ * link the destination port to special reply port and update the qos count
+ * of destination port.
+ *
+ * Use the qos value passed by voucher and not the one passed by notify field.
+ */
+ kr = ipc_port_link_special_reply_port_with_qos(special_reply_port, dest_port,
+ thread_qos_from_pthread_priority(kmsg->ikm_qos, &flags));
+ } else {
+ kr = KERN_FAILURE;
+ }
+ }
+ return kr;
}
/*
voucher_type = MACH_MSG_TYPE_MOVE_SEND;
}
- /* capture the qos value(s) for the kmsg */
- ipc_kmsg_set_qos(kmsg, *optionp, override);
-
msg->msgh_bits = MACH_MSGH_BITS_SET(dest_type, reply_type, voucher_type, mbits);
msg->msgh_remote_port = (ipc_port_t)dest_port;
msg->msgh_local_port = (ipc_port_t)reply_port;
+ /* capture the qos value(s) for the kmsg */
+ ipc_kmsg_set_qos(kmsg, *optionp, override);
+
if (release_port != IP_NULL)
ip_release(release_port);
* space.
*/
if (space_needed) {
- if (vm_allocate(ipc_kernel_copy_map, &paddr, space_needed,
- VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IPC)) != KERN_SUCCESS) {
+ if (vm_allocate_kernel(ipc_kernel_copy_map, &paddr, space_needed,
+ VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC) != KERN_SUCCESS) {
mr = MACH_MSG_VM_KERNEL;
goto clean_message;
}
{
mach_msg_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc;
user_dsc--; // point to the start of this port descriptor
+ bzero((void *)user_dsc, sizeof(*user_dsc));
user_dsc->name = CAST_MACH_NAME_TO_PORT(name);
user_dsc->disposition = disp;
user_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
} else {
mach_msg_legacy_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc;
user_dsc--; // point to the start of this port descriptor
+ bzero((void *)user_dsc, sizeof(*user_dsc));
user_dsc->name = CAST_MACH_PORT_TO_NAME(name);
user_dsc->disposition = disp;
user_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
{
mach_msg_ool_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
user_ool_dsc--;
+ bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
user_ool_dsc->address = (void *)(uintptr_t)rcv_addr;
user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
} else if (is_64bit) {
mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
user_ool_dsc--;
+ bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
user_ool_dsc->address = rcv_addr;
user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
} else {
mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
user_ool_dsc--;
+ bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr);
user_ool_dsc->size = (mach_msg_size_t)size;
/*
* Dynamically allocate the region
*/
- int anywhere = VM_FLAGS_ANYWHERE;
- if (vm_kernel_map_is_kernel(map)) anywhere |= VM_MAKE_TAG(VM_KERN_MEMORY_IPC);
- else anywhere |= VM_MAKE_TAG(VM_MEMORY_MACH_MSG);
+ vm_tag_t tag;
+ if (vm_kernel_map_is_kernel(map)) tag = VM_KERN_MEMORY_IPC;
+ else tag = VM_MEMORY_MACH_MSG;
kern_return_t kr;
- if ((kr = mach_vm_allocate(map, &rcv_addr,
+ if ((kr = mach_vm_allocate_kernel(map, &rcv_addr,
(mach_vm_size_t)names_length,
- anywhere)) != KERN_SUCCESS) {
+ VM_FLAGS_ANYWHERE, tag)) != KERN_SUCCESS) {
ipc_kmsg_clean_body(kmsg, 1, (mach_msg_descriptor_t *)dsc);
rcv_addr = 0;
if(current_task() == kernel_task) {
mach_msg_ool_ports_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
user_ool_dsc--;
+ bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
user_ool_dsc->address = (void *)(uintptr_t)rcv_addr;
user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
} if (is_64bit) {
mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
user_ool_dsc--;
+ bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
user_ool_dsc->address = rcv_addr;
user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
} else {
mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
user_ool_dsc--;
+ bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr);
user_ool_dsc->count = count;
mach_port_name_t dest_name, reply_name;
mach_msg_return_t mr;
+ /* Set ith_knote to ITH_KNOTE_PSEUDO */
+ current_thread()->ith_knote = ITH_KNOTE_PSEUDO;
+
assert(IO_VALID(dest));
#if 0
}
#endif /* IKM_SUPPORT_LEGACY */
+#ifdef __arm64__
+/*
+ * Just sets those parts of the trailer that aren't set up at allocation time.
+ */
+static void
+ipc_kmsg_munge_trailer(mach_msg_max_trailer_t *in, void *_out, boolean_t is64bit)
+{
+ if (is64bit) {
+ mach_msg_max_trailer64_t *out = (mach_msg_max_trailer64_t*)_out;
+ out->msgh_seqno = in->msgh_seqno;
+ out->msgh_context = in->msgh_context;
+ out->msgh_trailer_size = in->msgh_trailer_size;
+ out->msgh_ad = in->msgh_ad;
+ } else {
+ mach_msg_max_trailer32_t *out = (mach_msg_max_trailer32_t*)_out;
+ out->msgh_seqno = in->msgh_seqno;
+ out->msgh_context = (mach_port_context32_t)in->msgh_context;
+ out->msgh_trailer_size = in->msgh_trailer_size;
+ out->msgh_ad = in->msgh_ad;
+ }
+}
+#endif /* __arm64__ */
mach_msg_trailer_size_t
ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused,
{
mach_msg_max_trailer_t *trailer;
+#ifdef __arm64__
+ mach_msg_max_trailer_t tmp_trailer; /* This accommodates U64, and we'll munge */
+ void *real_trailer_out = (void*)(mach_msg_max_trailer_t *)
+ ((vm_offset_t)kmsg->ikm_header +
+ round_msg(kmsg->ikm_header->msgh_size));
+
+ /*
+ * Populate scratch with initial values set up at message allocation time.
+ * After, we reinterpret the space in the message as the right type
+ * of trailer for the address space in question.
+ */
+ bcopy(real_trailer_out, &tmp_trailer, MAX_TRAILER_SIZE);
+ trailer = &tmp_trailer;
+#else /* __arm64__ */
(void)thread;
trailer = (mach_msg_max_trailer_t *)
((vm_offset_t)kmsg->ikm_header +
round_msg(kmsg->ikm_header->msgh_size));
+#endif /* __arm64__ */
if (!(option & MACH_RCV_TRAILER_MASK)) {
return trailer->msgh_trailer_size;
}
done:
+#ifdef __arm64__
+ ipc_kmsg_munge_trailer(trailer, real_trailer_out, thread_is_64bit(thread));
+#endif /* __arm64__ */
return trailer->msgh_trailer_size;
}
projects / apple / xnu.git / blobdiff