]> git.saurik.com Git - apple/xnu.git/blob - osfmk/ipc/ipc_kmsg.c
projects / apple / xnu.git / blob
? search:


803b25bc291d24e79b3901a2e2e1d4ba4712dc0f
[apple/xnu.git] / osfmk / ipc / ipc_kmsg.c
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 * NOTICE: This file was modified by McAfee Research in 2004 to introduce
58 * support for mandatory and extensible security protections. This notice
59 * is included in support of clause 2.2 (b) of the Apple Public License,
60 * Version 2.0.
61 * Copyright (c) 2005 SPARTA, Inc.
62 */
63 /*
64 */
65 /*
66 * File: ipc/ipc_kmsg.c
67 * Author: Rich Draves
68 * Date: 1989
69 *
70 * Operations on kernel messages.
71 */
72
73
74 #include <mach/mach_types.h>
75 #include <mach/boolean.h>
76 #include <mach/kern_return.h>
77 #include <mach/message.h>
78 #include <mach/port.h>
79 #include <mach/vm_map.h>
80 #include <mach/mach_vm.h>
81 #include <mach/vm_statistics.h>
82
83 #include <kern/kern_types.h>
84 #include <kern/assert.h>
85 #include <kern/debug.h>
86 #include <kern/ipc_kobject.h>
87 #include <kern/kalloc.h>
88 #include <kern/zalloc.h>
89 #include <kern/processor.h>
90 #include <kern/thread.h>
91 #include <kern/sched_prim.h>
92 #include <kern/misc_protos.h>
93 #include <kern/counters.h>
94 #include <kern/cpu_data.h>
95 #include <kern/policy_internal.h>
96
97 #include <pthread/priority_private.h>
98
99 #include <machine/machlimits.h>
100
101 #include <vm/vm_map.h>
102 #include <vm/vm_object.h>
103 #include <vm/vm_kern.h>
104
105 #include <ipc/port.h>
106 #include <ipc/ipc_types.h>
107 #include <ipc/ipc_entry.h>
108 #include <ipc/ipc_kmsg.h>
109 #include <ipc/ipc_notify.h>
110 #include <ipc/ipc_object.h>
111 #include <ipc/ipc_space.h>
112 #include <ipc/ipc_port.h>
113 #include <ipc/ipc_right.h>
114 #include <ipc/ipc_hash.h>
115 #include <ipc/ipc_table.h>
116 #include <ipc/ipc_importance.h>
117 #if MACH_FLIPC
118 #include <kern/mach_node.h>
119 #include <ipc/flipc.h>
120 #endif
121
122 #include <os/overflow.h>
123
124 #include <security/mac_mach_internal.h>
125
126 #include <device/device_server.h>
127
128 #include <string.h>
129
130 #ifdef ppc
131 #include <ppc/Firmware.h>
132 #include <ppc/low_trace.h>
133 #endif
134
135 #if DEBUG
136 #define DEBUG_MSGS_K64 1
137 #endif
138
139 #include <sys/kdebug.h>
140 #include <libkern/OSAtomic.h>
141
142 #pragma pack(4)
143
144 typedef struct{
145 mach_msg_bits_t msgh_bits;
146 mach_msg_size_t msgh_size;
147 mach_port_name_t msgh_remote_port;
148 mach_port_name_t msgh_local_port;
149 mach_port_name_t msgh_voucher_port;
150 mach_msg_id_t msgh_id;
151 } mach_msg_legacy_header_t;
152
153 typedef struct{
154 mach_msg_legacy_header_t header;
155 mach_msg_body_t body;
156 } mach_msg_legacy_base_t;
157
158 typedef struct{
159 mach_port_name_t name;
160 mach_msg_size_t pad1;
161 uint32_t pad2 : 16;
162 mach_msg_type_name_t disposition : 8;
163 mach_msg_descriptor_type_t type : 8;
164 } mach_msg_legacy_port_descriptor_t;
165
166
167 typedef union{
168 mach_msg_legacy_port_descriptor_t port;
169 mach_msg_ool_descriptor32_t out_of_line32;
170 mach_msg_ool_ports_descriptor32_t ool_ports32;
171 mach_msg_type_descriptor_t type;
172 } mach_msg_legacy_descriptor_t;
173
174 #pragma pack()
175
176 #define LEGACY_HEADER_SIZE_DELTA ((mach_msg_size_t)(sizeof(mach_msg_header_t) - sizeof(mach_msg_legacy_header_t)))
177
178 // END LP64 fixes
179
180
181 #if DEBUG_MSGS_K64
182 extern void ipc_pset_print64(
183 ipc_pset_t pset);
184
185 extern void ipc_kmsg_print64(
186 ipc_kmsg_t kmsg,
187 const char *str);
188
189 extern void ipc_msg_print64(
190 mach_msg_header_t *msgh);
191
192 extern ipc_port_t ipc_name_to_data64(
193 task_t task,
194 mach_port_name_t name);
195
196 /*
197 * Forward declarations
198 */
199 void ipc_msg_print_untyped64(
200 mach_msg_body_t *body);
201
202 const char * ipc_type_name64(
203 int type_name,
204 boolean_t received);
205
206 void ipc_print_type_name64(
207 int type_name);
208
209 const char *
210 msgh_bit_decode64(
211 mach_msg_bits_t bit);
212
213 const char *
214 mm_copy_options_string64(
215 mach_msg_copy_options_t option);
216
217 void db_print_msg_uid64(mach_msg_header_t *);
218
219 static void
220 ipc_msg_body_print64(void *body, int size)
221 {
222 uint32_t *word = (uint32_t *) body;
223 uint32_t *end = (uint32_t *)(((uintptr_t) body) + size
224 - sizeof(mach_msg_header_t));
225 int i;
226
227 kprintf(" body(%p-%p):\n %p: ", body, end, word);
228 for (;;) {
229 for (i = 0; i < 8; i++, word++) {
230 if (word >= end) {
231 kprintf("\n");
232 return;
233 }
234 kprintf("%08x ", *word);
235 }
236 kprintf("\n %p: ", word);
237 }
238 }
239
240
241 const char *
242 ipc_type_name64(
243 int type_name,
244 boolean_t received)
245 {
246 switch (type_name) {
247 case MACH_MSG_TYPE_PORT_NAME:
248 return "port_name";
249
250 case MACH_MSG_TYPE_MOVE_RECEIVE:
251 if (received) {
252 return "port_receive";
253 } else {
254 return "move_receive";
255 }
256
257 case MACH_MSG_TYPE_MOVE_SEND:
258 if (received) {
259 return "port_send";
260 } else {
261 return "move_send";
262 }
263
264 case MACH_MSG_TYPE_MOVE_SEND_ONCE:
265 if (received) {
266 return "port_send_once";
267 } else {
268 return "move_send_once";
269 }
270
271 case MACH_MSG_TYPE_COPY_SEND:
272 return "copy_send";
273
274 case MACH_MSG_TYPE_MAKE_SEND:
275 return "make_send";
276
277 case MACH_MSG_TYPE_MAKE_SEND_ONCE:
278 return "make_send_once";
279
280 default:
281 return (char *) 0;
282 }
283 }
284
285 void
286 ipc_print_type_name64(
287 int type_name)
288 {
289 const char *name = ipc_type_name64(type_name, TRUE);
290 if (name) {
291 kprintf("%s", name);
292 } else {
293 kprintf("type%d", type_name);
294 }
295 }
296
297 /*
298 * ipc_kmsg_print64 [ debug ]
299 */
300 void
301 ipc_kmsg_print64(
302 ipc_kmsg_t kmsg,
303 const char *str)
304 {
305 kprintf("%s kmsg=%p:\n", str, kmsg);
306 kprintf(" next=%p, prev=%p, size=%d",
307 kmsg->ikm_next,
308 kmsg->ikm_prev,
309 kmsg->ikm_size);
310 kprintf("\n");
311 ipc_msg_print64(kmsg->ikm_header);
312 }
313
314 const char *
315 msgh_bit_decode64(
316 mach_msg_bits_t bit)
317 {
318 switch (bit) {
319 case MACH_MSGH_BITS_COMPLEX: return "complex";
320 case MACH_MSGH_BITS_CIRCULAR: return "circular";
321 default: return (char *) 0;
322 }
323 }
324
325 /*
326 * ipc_msg_print64 [ debug ]
327 */
328 void
329 ipc_msg_print64(
330 mach_msg_header_t *msgh)
331 {
332 mach_msg_bits_t mbits;
333 unsigned int bit, i;
334 const char *bit_name;
335 int needs_comma;
336
337 mbits = msgh->msgh_bits;
338 kprintf(" msgh_bits=0x%x: l=0x%x,r=0x%x\n",
339 mbits,
340 MACH_MSGH_BITS_LOCAL(msgh->msgh_bits),
341 MACH_MSGH_BITS_REMOTE(msgh->msgh_bits));
342
343 mbits = MACH_MSGH_BITS_OTHER(mbits) & MACH_MSGH_BITS_USED;
344 kprintf(" decoded bits: ");
345 needs_comma = 0;
346 for (i = 0, bit = 1; i < sizeof(mbits) * 8; ++i, bit <<= 1) {
347 if ((mbits & bit) == 0) {
348 continue;
349 }
350 bit_name = msgh_bit_decode64((mach_msg_bits_t)bit);
351 if (bit_name) {
352 kprintf("%s%s", needs_comma ? "," : "", bit_name);
353 } else {
354 kprintf("%sunknown(0x%x),", needs_comma ? "," : "", bit);
355 }
356 ++needs_comma;
357 }
358 if (msgh->msgh_bits & ~MACH_MSGH_BITS_USED) {
359 kprintf("%sunused=0x%x,", needs_comma ? "," : "",
360 msgh->msgh_bits & ~MACH_MSGH_BITS_USED);
361 }
362 kprintf("\n");
363
364 needs_comma = 1;
365 if (msgh->msgh_remote_port) {
366 kprintf(" remote=%p(", msgh->msgh_remote_port);
367 ipc_print_type_name64(MACH_MSGH_BITS_REMOTE(msgh->msgh_bits));
368 kprintf(")");
369 } else {
370 kprintf(" remote=null");
371 }
372
373 if (msgh->msgh_local_port) {
374 kprintf("%slocal=%p(", needs_comma ? "," : "",
375 msgh->msgh_local_port);
376 ipc_print_type_name64(MACH_MSGH_BITS_LOCAL(msgh->msgh_bits));
377 kprintf(")\n");
378 } else {
379 kprintf("local=null\n");
380 }
381
382 kprintf(" msgh_id=%d, size=%d\n",
383 msgh->msgh_id,
384 msgh->msgh_size);
385
386 if (mbits & MACH_MSGH_BITS_COMPLEX) {
387 ipc_msg_print_untyped64((mach_msg_body_t *) (msgh + 1));
388 }
389
390 ipc_msg_body_print64((void *)(msgh + 1), msgh->msgh_size);
391 }
392
393
394 const char *
395 mm_copy_options_string64(
396 mach_msg_copy_options_t option)
397 {
398 const char *name;
399
400 switch (option) {
401 case MACH_MSG_PHYSICAL_COPY:
402 name = "PHYSICAL";
403 break;
404 case MACH_MSG_VIRTUAL_COPY:
405 name = "VIRTUAL";
406 break;
407 case MACH_MSG_OVERWRITE:
408 name = "OVERWRITE(DEPRECATED)";
409 break;
410 case MACH_MSG_ALLOCATE:
411 name = "ALLOCATE";
412 break;
413 case MACH_MSG_KALLOC_COPY_T:
414 name = "KALLOC_COPY_T";
415 break;
416 default:
417 name = "unknown";
418 break;
419 }
420 return name;
421 }
422
423 void
424 ipc_msg_print_untyped64(
425 mach_msg_body_t *body)
426 {
427 mach_msg_descriptor_t *saddr, *send;
428 mach_msg_descriptor_type_t type;
429
430 kprintf(" %d descriptors: \n", body->msgh_descriptor_count);
431
432 saddr = (mach_msg_descriptor_t *) (body + 1);
433 send = saddr + body->msgh_descriptor_count;
434
435 for (; saddr < send; saddr++) {
436 type = saddr->type.type;
437
438 switch (type) {
439 case MACH_MSG_PORT_DESCRIPTOR: {
440 mach_msg_port_descriptor_t *dsc;
441
442 dsc = &saddr->port;
443 kprintf(" PORT name = %p disp = ", dsc->name);
444 ipc_print_type_name64(dsc->disposition);
445 kprintf("\n");
446 break;
447 }
448 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
449 case MACH_MSG_OOL_DESCRIPTOR: {
450 mach_msg_ool_descriptor_t *dsc;
451
452 dsc = (mach_msg_ool_descriptor_t *) &saddr->out_of_line;
453 kprintf(" OOL%s addr = %p size = 0x%x copy = %s %s\n",
454 type == MACH_MSG_OOL_DESCRIPTOR ? "" : " VOLATILE",
455 dsc->address, dsc->size,
456 mm_copy_options_string64(dsc->copy),
457 dsc->deallocate ? "DEALLOC" : "");
458 break;
459 }
460 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
461 mach_msg_ool_ports_descriptor_t *dsc;
462
463 dsc = (mach_msg_ool_ports_descriptor_t *) &saddr->ool_ports;
464
465 kprintf(" OOL_PORTS addr = %p count = 0x%x ",
466 dsc->address, dsc->count);
467 kprintf("disp = ");
468 ipc_print_type_name64(dsc->disposition);
469 kprintf(" copy = %s %s\n",
470 mm_copy_options_string64(dsc->copy),
471 dsc->deallocate ? "DEALLOC" : "");
472 break;
473 }
474
475 default: {
476 kprintf(" UNKNOWN DESCRIPTOR 0x%x\n", type);
477 break;
478 }
479 }
480 }
481 }
482
483 #define DEBUG_IPC_KMSG_PRINT(kmsg, string) \
484 __unreachable_ok_push \
485 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \
486 ipc_kmsg_print64(kmsg, string); \
487 } \
488 __unreachable_ok_pop
489
490 #define DEBUG_IPC_MSG_BODY_PRINT(body, size) \
491 __unreachable_ok_push \
492 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \
493 ipc_msg_body_print64(body,size);\
494 } \
495 __unreachable_ok_pop
496 #else /* !DEBUG_MSGS_K64 */
497 #define DEBUG_IPC_KMSG_PRINT(kmsg, string)
498 #define DEBUG_IPC_MSG_BODY_PRINT(body, size)
499 #endif /* !DEBUG_MSGS_K64 */
500
501 extern vm_map_t ipc_kernel_copy_map;
502 extern vm_size_t ipc_kmsg_max_space;
503 extern vm_size_t ipc_kmsg_max_vm_space;
504 extern vm_size_t ipc_kmsg_max_body_space;
505 extern vm_size_t msg_ool_size_small;
506
507 #define MSG_OOL_SIZE_SMALL msg_ool_size_small
508
509 #if defined(__LP64__)
510 #define MAP_SIZE_DIFFERS(map) (map->max_offset < MACH_VM_MAX_ADDRESS)
511 #define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor32_t
512 #define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor32_t
513 #else
514 #define MAP_SIZE_DIFFERS(map) (map->max_offset > VM_MAX_ADDRESS)
515 #define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor64_t
516 #define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor64_t
517 #endif
518
519 #define DESC_SIZE_ADJUSTMENT ((mach_msg_size_t)(sizeof(mach_msg_ool_descriptor64_t) - \
520 sizeof(mach_msg_ool_descriptor32_t)))
521
522 /* scatter list macros */
523
524 #define SKIP_PORT_DESCRIPTORS(s, c) \
525 MACRO_BEGIN \
526 if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \
527 while ((c) > 0) { \
528 if ((s)->type.type != MACH_MSG_PORT_DESCRIPTOR) \
529 break; \
530 (s)++; (c)--; \
531 } \
532 if (c == 0) \
533 (s) = MACH_MSG_DESCRIPTOR_NULL; \
534 } \
535 MACRO_END
536
537 #define INCREMENT_SCATTER(s, c, d) \
538 MACRO_BEGIN \
539 if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \
540 s = (d) ? (mach_msg_descriptor_t *) \
541 ((OTHER_OOL_DESCRIPTOR *)(s) + 1) : \
542 (s + 1); \
543 (c)--; \
544 } \
545 MACRO_END
546
547 #define KMSG_TRACE_FLAG_TRACED 0x000001
548 #define KMSG_TRACE_FLAG_COMPLEX 0x000002
549 #define KMSG_TRACE_FLAG_OOLMEM 0x000004
550 #define KMSG_TRACE_FLAG_VCPY 0x000008
551 #define KMSG_TRACE_FLAG_PCPY 0x000010
552 #define KMSG_TRACE_FLAG_SND64 0x000020
553 #define KMSG_TRACE_FLAG_RAISEIMP 0x000040
554 #define KMSG_TRACE_FLAG_APP_SRC 0x000080
555 #define KMSG_TRACE_FLAG_APP_DST 0x000100
556 #define KMSG_TRACE_FLAG_DAEMON_SRC 0x000200
557 #define KMSG_TRACE_FLAG_DAEMON_DST 0x000400
558 #define KMSG_TRACE_FLAG_DST_NDFLTQ 0x000800
559 #define KMSG_TRACE_FLAG_SRC_NDFLTQ 0x001000
560 #define KMSG_TRACE_FLAG_DST_SONCE 0x002000
561 #define KMSG_TRACE_FLAG_SRC_SONCE 0x004000
562 #define KMSG_TRACE_FLAG_CHECKIN 0x008000
563 #define KMSG_TRACE_FLAG_ONEWAY 0x010000
564 #define KMSG_TRACE_FLAG_IOKIT 0x020000
565 #define KMSG_TRACE_FLAG_SNDRCV 0x040000
566 #define KMSG_TRACE_FLAG_DSTQFULL 0x080000
567 #define KMSG_TRACE_FLAG_VOUCHER 0x100000
568 #define KMSG_TRACE_FLAG_TIMER 0x200000
569 #define KMSG_TRACE_FLAG_SEMA 0x400000
570 #define KMSG_TRACE_FLAG_DTMPOWNER 0x800000
571
572 #define KMSG_TRACE_FLAGS_MASK 0xffffff
573 #define KMSG_TRACE_FLAGS_SHIFT 8
574
575 #define KMSG_TRACE_PORTS_MASK 0xff
576 #define KMSG_TRACE_PORTS_SHIFT 0
577
578 #if (KDEBUG_LEVEL >= KDEBUG_LEVEL_STANDARD)
579 #include <stdint.h>
580 extern boolean_t kdebug_debugid_enabled(uint32_t debugid);
581
582 void
583 ipc_kmsg_trace_send(ipc_kmsg_t kmsg,
584 mach_msg_option_t option)
585 {
586 task_t send_task = TASK_NULL;
587 ipc_port_t dst_port, src_port;
588 boolean_t is_task_64bit;
589 mach_msg_header_t *msg;
590 mach_msg_trailer_t *trailer;
591
592 int kotype = 0;
593 uint32_t msg_size = 0;
594 uint32_t msg_flags = KMSG_TRACE_FLAG_TRACED;
595 uint32_t num_ports = 0;
596 uint32_t send_pid, dst_pid;
597
598 /*
599 * check to see not only if ktracing is enabled, but if we will
600 * _actually_ emit the KMSG_INFO tracepoint. This saves us a
601 * significant amount of processing (and a port lock hold) in
602 * the non-tracing case.
603 */
604 if (__probable((kdebug_enable & KDEBUG_TRACE) == 0)) {
605 return;
606 }
607 if (!kdebug_debugid_enabled(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO))) {
608 return;
609 }
610
611 msg = kmsg->ikm_header;
612
613 dst_port = (ipc_port_t)(msg->msgh_remote_port);
614 if (!IPC_PORT_VALID(dst_port)) {
615 return;
616 }
617
618 /*
619 * Message properties / options
620 */
621 if ((option & (MACH_SEND_MSG | MACH_RCV_MSG)) == (MACH_SEND_MSG | MACH_RCV_MSG)) {
622 msg_flags |= KMSG_TRACE_FLAG_SNDRCV;
623 }
624
625 if (msg->msgh_id >= is_iokit_subsystem.start &&
626 msg->msgh_id < is_iokit_subsystem.end + 100) {
627 msg_flags |= KMSG_TRACE_FLAG_IOKIT;
628 }
629 /* magic XPC checkin message id (XPC_MESSAGE_ID_CHECKIN) from libxpc */
630 else if (msg->msgh_id == 0x77303074u /* w00t */) {
631 msg_flags |= KMSG_TRACE_FLAG_CHECKIN;
632 }
633
634 if (msg->msgh_bits & MACH_MSGH_BITS_RAISEIMP) {
635 msg_flags |= KMSG_TRACE_FLAG_RAISEIMP;
636 }
637
638 if (unsafe_convert_port_to_voucher(kmsg->ikm_voucher)) {
639 msg_flags |= KMSG_TRACE_FLAG_VOUCHER;
640 }
641
642 /*
643 * Sending task / port
644 */
645 send_task = current_task();
646 send_pid = task_pid(send_task);
647
648 if (send_pid != 0) {
649 if (task_is_daemon(send_task)) {
650 msg_flags |= KMSG_TRACE_FLAG_DAEMON_SRC;
651 } else if (task_is_app(send_task)) {
652 msg_flags |= KMSG_TRACE_FLAG_APP_SRC;
653 }
654 }
655
656 is_task_64bit = (send_task->map->max_offset > VM_MAX_ADDRESS);
657 if (is_task_64bit) {
658 msg_flags |= KMSG_TRACE_FLAG_SND64;
659 }
660
661 src_port = (ipc_port_t)(msg->msgh_local_port);
662 if (src_port) {
663 if (src_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT) {
664 msg_flags |= KMSG_TRACE_FLAG_SRC_NDFLTQ;
665 }
666 switch (MACH_MSGH_BITS_LOCAL(msg->msgh_bits)) {
667 case MACH_MSG_TYPE_MOVE_SEND_ONCE:
668 msg_flags |= KMSG_TRACE_FLAG_SRC_SONCE;
669 break;
670 default:
671 break;
672 }
673 } else {
674 msg_flags |= KMSG_TRACE_FLAG_ONEWAY;
675 }
676
677
678 /*
679 * Destination task / port
680 */
681 ip_lock(dst_port);
682 if (!ip_active(dst_port)) {
683 /* dst port is being torn down */
684 dst_pid = (uint32_t)0xfffffff0;
685 } else if (dst_port->ip_tempowner) {
686 msg_flags |= KMSG_TRACE_FLAG_DTMPOWNER;
687 if (IIT_NULL != dst_port->ip_imp_task) {
688 dst_pid = task_pid(dst_port->ip_imp_task->iit_task);
689 } else {
690 dst_pid = (uint32_t)0xfffffff1;
691 }
692 } else if (dst_port->ip_receiver_name == MACH_PORT_NULL) {
693 /* dst_port is otherwise in-transit */
694 dst_pid = (uint32_t)0xfffffff2;
695 } else {
696 if (dst_port->ip_receiver == ipc_space_kernel) {
697 dst_pid = 0;
698 } else {
699 ipc_space_t dst_space;
700 dst_space = dst_port->ip_receiver;
701 if (dst_space && is_active(dst_space)) {
702 dst_pid = task_pid(dst_space->is_task);
703 if (task_is_daemon(dst_space->is_task)) {
704 msg_flags |= KMSG_TRACE_FLAG_DAEMON_DST;
705 } else if (task_is_app(dst_space->is_task)) {
706 msg_flags |= KMSG_TRACE_FLAG_APP_DST;
707 }
708 } else {
709 /* receiving task is being torn down */
710 dst_pid = (uint32_t)0xfffffff3;
711 }
712 }
713 }
714
715 if (dst_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT) {
716 msg_flags |= KMSG_TRACE_FLAG_DST_NDFLTQ;
717 }
718 if (imq_full(&dst_port->ip_messages)) {
719 msg_flags |= KMSG_TRACE_FLAG_DSTQFULL;
720 }
721
722 kotype = ip_kotype(dst_port);
723
724 ip_unlock(dst_port);
725
726 switch (kotype) {
727 case IKOT_SEMAPHORE:
728 msg_flags |= KMSG_TRACE_FLAG_SEMA;
729 break;
730 case IKOT_TIMER:
731 case IKOT_CLOCK:
732 msg_flags |= KMSG_TRACE_FLAG_TIMER;
733 break;
734 case IKOT_MASTER_DEVICE:
735 case IKOT_IOKIT_CONNECT:
736 case IKOT_IOKIT_OBJECT:
737 case IKOT_IOKIT_IDENT:
738 msg_flags |= KMSG_TRACE_FLAG_IOKIT;
739 break;
740 default:
741 break;
742 }
743
744 switch (MACH_MSGH_BITS_REMOTE(msg->msgh_bits)) {
745 case MACH_MSG_TYPE_PORT_SEND_ONCE:
746 msg_flags |= KMSG_TRACE_FLAG_DST_SONCE;
747 break;
748 default:
749 break;
750 }
751
752
753 /*
754 * Message size / content
755 */
756 msg_size = msg->msgh_size - sizeof(mach_msg_header_t);
757
758 if (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
759 mach_msg_body_t *msg_body;
760 mach_msg_descriptor_t *kern_dsc;
761 int dsc_count;
762
763 msg_flags |= KMSG_TRACE_FLAG_COMPLEX;
764
765 msg_body = (mach_msg_body_t *)(kmsg->ikm_header + 1);
766 dsc_count = (int)msg_body->msgh_descriptor_count;
767 kern_dsc = (mach_msg_descriptor_t *)(msg_body + 1);
768
769 /* this is gross: see ipc_kmsg_copyin_body()... */
770 if (!is_task_64bit) {
771 msg_size -= (dsc_count * 12);
772 }
773
774 for (int i = 0; i < dsc_count; i++) {
775 switch (kern_dsc[i].type.type) {
776 case MACH_MSG_PORT_DESCRIPTOR:
777 num_ports++;
778 if (is_task_64bit) {
779 msg_size -= 12;
780 }
781 break;
782 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
783 case MACH_MSG_OOL_DESCRIPTOR: {
784 mach_msg_ool_descriptor_t *dsc;
785 dsc = (mach_msg_ool_descriptor_t *)&kern_dsc[i];
786 msg_flags |= KMSG_TRACE_FLAG_OOLMEM;
787 msg_size += dsc->size;
788 if ((dsc->size >= MSG_OOL_SIZE_SMALL) &&
789 (dsc->copy == MACH_MSG_PHYSICAL_COPY) &&
790 !dsc->deallocate) {
791 msg_flags |= KMSG_TRACE_FLAG_PCPY;
792 } else if (dsc->size <= MSG_OOL_SIZE_SMALL) {
793 msg_flags |= KMSG_TRACE_FLAG_PCPY;
794 } else {
795 msg_flags |= KMSG_TRACE_FLAG_VCPY;
796 }
797 if (is_task_64bit) {
798 msg_size -= 16;
799 }
800 } break;
801 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
802 mach_msg_ool_ports_descriptor_t *dsc;
803 dsc = (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i];
804 num_ports += dsc->count;
805 if (is_task_64bit) {
806 msg_size -= 16;
807 }
808 } break;
809 default:
810 break;
811 }
812 }
813 }
814
815 /*
816 * Trailer contents
817 */
818 trailer = (mach_msg_trailer_t *)((vm_offset_t)msg +
819 round_msg((vm_offset_t)msg->msgh_size));
820 if (trailer->msgh_trailer_size <= sizeof(mach_msg_security_trailer_t)) {
821 extern security_token_t KERNEL_SECURITY_TOKEN;
822 mach_msg_security_trailer_t *strailer;
823 strailer = (mach_msg_security_trailer_t *)trailer;
824 /*
825 * verify the sender PID: replies from the kernel often look
826 * like self-talk because the sending port is not reset.
827 */
828 if (memcmp(&strailer->msgh_sender,
829 &KERNEL_SECURITY_TOKEN,
830 sizeof(KERNEL_SECURITY_TOKEN)) == 0) {
831 send_pid = 0;
832 msg_flags &= ~(KMSG_TRACE_FLAG_APP_SRC | KMSG_TRACE_FLAG_DAEMON_SRC);
833 }
834 }
835
836 KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END,
837 (uintptr_t)send_pid,
838 (uintptr_t)dst_pid,
839 (uintptr_t)msg_size,
840 (uintptr_t)(
841 ((msg_flags & KMSG_TRACE_FLAGS_MASK) << KMSG_TRACE_FLAGS_SHIFT) |
842 ((num_ports & KMSG_TRACE_PORTS_MASK) << KMSG_TRACE_PORTS_SHIFT)
843 )
844 );
845 }
846 #endif
847
848 /* zone for cached ipc_kmsg_t structures */
849 zone_t ipc_kmsg_zone;
850
851 /*
852 * Forward declarations
853 */
854
855 void ipc_kmsg_clean(
856 ipc_kmsg_t kmsg);
857
858 void ipc_kmsg_clean_body(
859 ipc_kmsg_t kmsg,
860 mach_msg_type_number_t number,
861 mach_msg_descriptor_t *desc);
862
863 void ipc_kmsg_clean_partial(
864 ipc_kmsg_t kmsg,
865 mach_msg_type_number_t number,
866 mach_msg_descriptor_t *desc,
867 vm_offset_t paddr,
868 vm_size_t length);
869
870 mach_msg_return_t ipc_kmsg_copyin_body(
871 ipc_kmsg_t kmsg,
872 ipc_space_t space,
873 vm_map_t map,
874 mach_msg_option_t *optionp);
875
876 /*
877 * We keep a per-processor cache of kernel message buffers.
878 * The cache saves the overhead/locking of using kalloc/kfree.
879 * The per-processor cache seems to miss less than a per-thread cache,
880 * and it also uses less memory. Access to the cache doesn't
881 * require locking.
882 */
883
884 /*
885 * Routine: ipc_kmsg_alloc
886 * Purpose:
887 * Allocate a kernel message structure. If we can get one from
888 * the cache, that is best. Otherwise, allocate a new one.
889 * Conditions:
890 * Nothing locked.
891 */
892 ipc_kmsg_t
893 ipc_kmsg_alloc(
894 mach_msg_size_t msg_and_trailer_size)
895 {
896 mach_msg_size_t max_expanded_size;
897 ipc_kmsg_t kmsg;
898
899 /*
900 * LP64support -
901 * Pad the allocation in case we need to expand the
902 * message descrptors for user spaces with pointers larger than
903 * the kernel's own, or vice versa. We don't know how many descriptors
904 * there are yet, so just assume the whole body could be
905 * descriptors (if there could be any at all).
906 *
907 * The expansion space is left in front of the header,
908 * because it is easier to pull the header and descriptors
909 * forward as we process them than it is to push all the
910 * data backwards.
911 */
912 mach_msg_size_t size = msg_and_trailer_size - MAX_TRAILER_SIZE;
913
914 /* compare against implementation upper limit for the body */
915 if (size > ipc_kmsg_max_body_space) {
916 return IKM_NULL;
917 }
918
919 if (size > sizeof(mach_msg_base_t)) {
920 mach_msg_size_t max_desc = (mach_msg_size_t)(((size - sizeof(mach_msg_base_t)) /
921 sizeof(mach_msg_ool_descriptor32_t)) *
922 DESC_SIZE_ADJUSTMENT);
923
924 /* make sure expansion won't cause wrap */
925 if (msg_and_trailer_size > MACH_MSG_SIZE_MAX - max_desc) {
926 return IKM_NULL;
927 }
928
929 max_expanded_size = msg_and_trailer_size + max_desc;
930 } else {
931 max_expanded_size = msg_and_trailer_size;
932 }
933
934 if (max_expanded_size < IKM_SAVED_MSG_SIZE) {
935 max_expanded_size = IKM_SAVED_MSG_SIZE; /* round up for ikm_cache */
936 }
937 if (max_expanded_size == IKM_SAVED_MSG_SIZE) {
938 kmsg = (ipc_kmsg_t)zalloc(ipc_kmsg_zone);
939 } else {
940 kmsg = (ipc_kmsg_t)kalloc(ikm_plus_overhead(max_expanded_size));
941 }
942
943 if (kmsg != IKM_NULL) {
944 ikm_init(kmsg, max_expanded_size);
945 ikm_set_header(kmsg, msg_and_trailer_size);
946 }
947
948 return kmsg;
949 }
950
951 /*
952 * Routine: ipc_kmsg_free
953 * Purpose:
954 * Free a kernel message buffer. If the kms is preallocated
955 * to a port, just "put it back (marked unused)." We have to
956 * do this with the port locked. The port may have its hold
957 * on our message released. In that case, we have to just
958 * revert the message to a traditional one and free it normally.
959 * Conditions:
960 * Nothing locked.
961 */
962
963 void
964 ipc_kmsg_free(
965 ipc_kmsg_t kmsg)
966 {
967 mach_msg_size_t size = kmsg->ikm_size;
968 ipc_port_t port;
969
970 assert(!IP_VALID(kmsg->ikm_voucher));
971
972 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_FREE) | DBG_FUNC_NONE,
973 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
974 0, 0, 0, 0);
975
976 /*
977 * Check to see if the message is bound to the port. If so,
978 * mark it not in use. If the port isn't already dead, then
979 * leave the message associated with it. Otherwise, free it.
980 */
981 port = ikm_prealloc_inuse_port(kmsg);
982 if (port != IP_NULL) {
983 ip_lock(port);
984 ikm_prealloc_clear_inuse(kmsg, port);
985 if (ip_active(port) && (port->ip_premsg == kmsg)) {
986 assert(IP_PREALLOC(port));
987 ip_unlock(port);
988 ip_release(port);
989 return;
990 }
991 ip_unlock(port);
992 ip_release(port); /* May be last reference */
993 }
994
995 if (kmsg->ikm_size == IKM_SAVED_MSG_SIZE) {
996 zfree(ipc_kmsg_zone, kmsg);
997 return;
998 }
999 kfree(kmsg, ikm_plus_overhead(size));
1000 }
1001
1002
1003 /*
1004 * Routine: ipc_kmsg_enqueue
1005 * Purpose:
1006 * Enqueue a kmsg.
1007 */
1008
1009 void
1010 ipc_kmsg_enqueue(
1011 ipc_kmsg_queue_t queue,
1012 ipc_kmsg_t kmsg)
1013 {
1014 ipc_kmsg_t first = queue->ikmq_base;
1015 ipc_kmsg_t last;
1016
1017 if (first == IKM_NULL) {
1018 queue->ikmq_base = kmsg;
1019 kmsg->ikm_next = kmsg;
1020 kmsg->ikm_prev = kmsg;
1021 } else {
1022 last = first->ikm_prev;
1023 kmsg->ikm_next = first;
1024 kmsg->ikm_prev = last;
1025 first->ikm_prev = kmsg;
1026 last->ikm_next = kmsg;
1027 }
1028 }
1029
1030 /*
1031 * Routine: ipc_kmsg_enqueue_qos
1032 * Purpose:
1033 * Enqueue a kmsg, propagating qos
1034 * overrides towards the head of the queue.
1035 *
1036 * Returns:
1037 * whether the head of the queue had
1038 * it's override-qos adjusted because
1039 * of this insertion.
1040 */
1041
1042 boolean_t
1043 ipc_kmsg_enqueue_qos(
1044 ipc_kmsg_queue_t queue,
1045 ipc_kmsg_t kmsg)
1046 {
1047 ipc_kmsg_t first = queue->ikmq_base;
1048 ipc_kmsg_t prev;
1049 mach_msg_priority_t override;
1050
1051 if (first == IKM_NULL) {
1052 /* insert a first message */
1053 queue->ikmq_base = kmsg;
1054 kmsg->ikm_next = kmsg;
1055 kmsg->ikm_prev = kmsg;
1056 return TRUE;
1057 }
1058
1059 /* insert at the tail */
1060 prev = first->ikm_prev;
1061 kmsg->ikm_next = first;
1062 kmsg->ikm_prev = prev;
1063 first->ikm_prev = kmsg;
1064 prev->ikm_next = kmsg;
1065
1066 /* apply QoS overrides towards the head */
1067 override = kmsg->ikm_qos_override;
1068 while (prev != kmsg &&
1069 override > prev->ikm_qos_override) {
1070 prev->ikm_qos_override = override;
1071 prev = prev->ikm_prev;
1072 }
1073
1074 /* did we adjust everything? */
1075 return prev == kmsg;
1076 }
1077
1078 /*
1079 * Routine: ipc_kmsg_override_qos
1080 * Purpose:
1081 * Update the override for a given kmsg already
1082 * enqueued, propagating qos override adjustments
1083 * towards the head of the queue.
1084 *
1085 * Returns:
1086 * whether the head of the queue had
1087 * it's override-qos adjusted because
1088 * of this insertion.
1089 */
1090
1091 boolean_t
1092 ipc_kmsg_override_qos(
1093 ipc_kmsg_queue_t queue,
1094 ipc_kmsg_t kmsg,
1095 mach_msg_priority_t override)
1096 {
1097 ipc_kmsg_t first = queue->ikmq_base;
1098 ipc_kmsg_t cur = kmsg;
1099
1100 /* apply QoS overrides towards the head */
1101 while (override > cur->ikm_qos_override) {
1102 cur->ikm_qos_override = override;
1103 if (cur == first) {
1104 return TRUE;
1105 }
1106 cur = cur->ikm_prev;
1107 }
1108 return FALSE;
1109 }
1110
1111 /*
1112 * Routine: ipc_kmsg_dequeue
1113 * Purpose:
1114 * Dequeue and return a kmsg.
1115 */
1116
1117 ipc_kmsg_t
1118 ipc_kmsg_dequeue(
1119 ipc_kmsg_queue_t queue)
1120 {
1121 ipc_kmsg_t first;
1122
1123 first = ipc_kmsg_queue_first(queue);
1124
1125 if (first != IKM_NULL) {
1126 ipc_kmsg_rmqueue(queue, first);
1127 }
1128
1129 return first;
1130 }
1131
1132 /*
1133 * Routine: ipc_kmsg_rmqueue
1134 * Purpose:
1135 * Pull a kmsg out of a queue.
1136 */
1137
1138 void
1139 ipc_kmsg_rmqueue(
1140 ipc_kmsg_queue_t queue,
1141 ipc_kmsg_t kmsg)
1142 {
1143 ipc_kmsg_t next, prev;
1144
1145 assert(queue->ikmq_base != IKM_NULL);
1146
1147 next = kmsg->ikm_next;
1148 prev = kmsg->ikm_prev;
1149
1150 if (next == kmsg) {
1151 assert(prev == kmsg);
1152 assert(queue->ikmq_base == kmsg);
1153
1154 queue->ikmq_base = IKM_NULL;
1155 } else {
1156 if (__improbable(next->ikm_prev != kmsg || prev->ikm_next != kmsg)) {
1157 panic("ipc_kmsg_rmqueue: inconsistent prev/next pointers. "
1158 "(prev->next: %p, next->prev: %p, kmsg: %p)",
1159 prev->ikm_next, next->ikm_prev, kmsg);
1160 }
1161
1162 if (queue->ikmq_base == kmsg) {
1163 queue->ikmq_base = next;
1164 }
1165
1166 next->ikm_prev = prev;
1167 prev->ikm_next = next;
1168 }
1169 /* XXX Temporary debug logic */
1170 assert((kmsg->ikm_next = IKM_BOGUS) == IKM_BOGUS);
1171 assert((kmsg->ikm_prev = IKM_BOGUS) == IKM_BOGUS);
1172 }
1173
1174 /*
1175 * Routine: ipc_kmsg_queue_next
1176 * Purpose:
1177 * Return the kmsg following the given kmsg.
1178 * (Or IKM_NULL if it is the last one in the queue.)
1179 */
1180
1181 ipc_kmsg_t
1182 ipc_kmsg_queue_next(
1183 ipc_kmsg_queue_t queue,
1184 ipc_kmsg_t kmsg)
1185 {
1186 ipc_kmsg_t next;
1187
1188 assert(queue->ikmq_base != IKM_NULL);
1189
1190 next = kmsg->ikm_next;
1191 if (queue->ikmq_base == next) {
1192 next = IKM_NULL;
1193 }
1194
1195 return next;
1196 }
1197
1198 /*
1199 * Routine: ipc_kmsg_destroy
1200 * Purpose:
1201 * Destroys a kernel message. Releases all rights,
1202 * references, and memory held by the message.
1203 * Frees the message.
1204 * Conditions:
1205 * No locks held.
1206 */
1207
1208 void
1209 ipc_kmsg_destroy(
1210 ipc_kmsg_t kmsg)
1211 {
1212 /*
1213 * Destroying a message can cause more messages to be destroyed.
1214 * Curtail recursion by putting messages on the deferred
1215 * destruction queue. If this was the first message on the
1216 * queue, this instance must process the full queue.
1217 */
1218 if (ipc_kmsg_delayed_destroy(kmsg)) {
1219 ipc_kmsg_reap_delayed();
1220 }
1221 }
1222
1223 /*
1224 * Routine: ipc_kmsg_delayed_destroy
1225 * Purpose:
1226 * Enqueues a kernel message for deferred destruction.
1227 * Returns:
1228 * Boolean indicator that the caller is responsible to reap
1229 * deferred messages.
1230 */
1231
1232 boolean_t
1233 ipc_kmsg_delayed_destroy(
1234 ipc_kmsg_t kmsg)
1235 {
1236 ipc_kmsg_queue_t queue = &(current_thread()->ith_messages);
1237 boolean_t first = ipc_kmsg_queue_empty(queue);
1238
1239 ipc_kmsg_enqueue(queue, kmsg);
1240 return first;
1241 }
1242
1243 /*
1244 * Routine: ipc_kmsg_destroy_queue
1245 * Purpose:
1246 * Destroys messages from the per-thread
1247 * deferred reaping queue.
1248 * Conditions:
1249 * No locks held.
1250 */
1251
1252 void
1253 ipc_kmsg_reap_delayed(void)
1254 {
1255 ipc_kmsg_queue_t queue = &(current_thread()->ith_messages);
1256 ipc_kmsg_t kmsg;
1257
1258 /*
1259 * must leave kmsg in queue while cleaning it to assure
1260 * no nested calls recurse into here.
1261 */
1262 while ((kmsg = ipc_kmsg_queue_first(queue)) != IKM_NULL) {
1263 ipc_kmsg_clean(kmsg);
1264 ipc_kmsg_rmqueue(queue, kmsg);
1265 ipc_kmsg_free(kmsg);
1266 }
1267 }
1268
1269 /*
1270 * Routine: ipc_kmsg_clean_body
1271 * Purpose:
1272 * Cleans the body of a kernel message.
1273 * Releases all rights, references, and memory.
1274 *
1275 * Conditions:
1276 * No locks held.
1277 */
1278 static unsigned int _ipc_kmsg_clean_invalid_desc = 0;
1279 void
1280 ipc_kmsg_clean_body(
1281 __unused ipc_kmsg_t kmsg,
1282 mach_msg_type_number_t number,
1283 mach_msg_descriptor_t *saddr)
1284 {
1285 mach_msg_type_number_t i;
1286
1287 if (number == 0) {
1288 return;
1289 }
1290
1291 for (i = 0; i < number; i++, saddr++) {
1292 switch (saddr->type.type) {
1293 case MACH_MSG_PORT_DESCRIPTOR: {
1294 mach_msg_port_descriptor_t *dsc;
1295
1296 dsc = &saddr->port;
1297
1298 /*
1299 * Destroy port rights carried in the message
1300 */
1301 if (!IO_VALID((ipc_object_t) dsc->name)) {
1302 continue;
1303 }
1304 ipc_object_destroy((ipc_object_t) dsc->name, dsc->disposition);
1305 break;
1306 }
1307 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
1308 case MACH_MSG_OOL_DESCRIPTOR: {
1309 mach_msg_ool_descriptor_t *dsc;
1310
1311 dsc = (mach_msg_ool_descriptor_t *)&saddr->out_of_line;
1312
1313 /*
1314 * Destroy memory carried in the message
1315 */
1316 if (dsc->size == 0) {
1317 assert(dsc->address == (void *) 0);
1318 } else {
1319 vm_map_copy_discard((vm_map_copy_t) dsc->address);
1320 }
1321 break;
1322 }
1323 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
1324 ipc_object_t *objects;
1325 mach_msg_type_number_t j;
1326 mach_msg_ool_ports_descriptor_t *dsc;
1327
1328 dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports;
1329 objects = (ipc_object_t *) dsc->address;
1330
1331 if (dsc->count == 0) {
1332 break;
1333 }
1334
1335 assert(objects != (ipc_object_t *) 0);
1336
1337 /* destroy port rights carried in the message */
1338
1339 for (j = 0; j < dsc->count; j++) {
1340 ipc_object_t object = objects[j];
1341
1342 if (!IO_VALID(object)) {
1343 continue;
1344 }
1345
1346 ipc_object_destroy(object, dsc->disposition);
1347 }
1348
1349 /* destroy memory carried in the message */
1350
1351 assert(dsc->count != 0);
1352
1353 kfree(dsc->address,
1354 (vm_size_t) dsc->count * sizeof(mach_port_t));
1355 break;
1356 }
1357 default: {
1358 _ipc_kmsg_clean_invalid_desc++; /* don't understand this type of descriptor */
1359 }
1360 }
1361 }
1362 }
1363
1364 /*
1365 * Routine: ipc_kmsg_clean_partial
1366 * Purpose:
1367 * Cleans a partially-acquired kernel message.
1368 * number is the index of the type descriptor
1369 * in the body of the message that contained the error.
1370 * If dolast, the memory and port rights in this last
1371 * type spec are also cleaned. In that case, number
1372 * specifies the number of port rights to clean.
1373 * Conditions:
1374 * Nothing locked.
1375 */
1376
1377 void
1378 ipc_kmsg_clean_partial(
1379 ipc_kmsg_t kmsg,
1380 mach_msg_type_number_t number,
1381 mach_msg_descriptor_t *desc,
1382 vm_offset_t paddr,
1383 vm_size_t length)
1384 {
1385 ipc_object_t object;
1386 mach_msg_bits_t mbits = kmsg->ikm_header->msgh_bits;
1387
1388 /* deal with importance chain while we still have dest and voucher references */
1389 ipc_importance_clean(kmsg);
1390
1391 object = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
1392 assert(IO_VALID(object));
1393 ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits));
1394
1395 object = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
1396 if (IO_VALID(object)) {
1397 ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits));
1398 }
1399
1400 object = (ipc_object_t) kmsg->ikm_voucher;
1401 if (IO_VALID(object)) {
1402 assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND);
1403 ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND);
1404 kmsg->ikm_voucher = IP_NULL;
1405 }
1406
1407 if (paddr) {
1408 (void) vm_deallocate(ipc_kernel_copy_map, paddr, length);
1409 }
1410
1411 ipc_kmsg_clean_body(kmsg, number, desc);
1412 }
1413
1414 /*
1415 * Routine: ipc_kmsg_clean
1416 * Purpose:
1417 * Cleans a kernel message. Releases all rights,
1418 * references, and memory held by the message.
1419 * Conditions:
1420 * No locks held.
1421 */
1422
1423 void
1424 ipc_kmsg_clean(
1425 ipc_kmsg_t kmsg)
1426 {
1427 ipc_object_t object;
1428 mach_msg_bits_t mbits;
1429
1430 /* deal with importance chain while we still have dest and voucher references */
1431 ipc_importance_clean(kmsg);
1432
1433 mbits = kmsg->ikm_header->msgh_bits;
1434 object = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
1435 if (IO_VALID(object)) {
1436 ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits));
1437 }
1438
1439 object = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
1440 if (IO_VALID(object)) {
1441 ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits));
1442 }
1443
1444 object = (ipc_object_t) kmsg->ikm_voucher;
1445 if (IO_VALID(object)) {
1446 assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND);
1447 ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND);
1448 kmsg->ikm_voucher = IP_NULL;
1449 }
1450
1451 if (mbits & MACH_MSGH_BITS_COMPLEX) {
1452 mach_msg_body_t *body;
1453
1454 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
1455 ipc_kmsg_clean_body(kmsg, body->msgh_descriptor_count,
1456 (mach_msg_descriptor_t *)(body + 1));
1457 }
1458 }
1459
1460 /*
1461 * Routine: ipc_kmsg_set_prealloc
1462 * Purpose:
1463 * Assign a kmsg as a preallocated message buffer to a port.
1464 * Conditions:
1465 * port locked.
1466 */
1467
1468 void
1469 ipc_kmsg_set_prealloc(
1470 ipc_kmsg_t kmsg,
1471 ipc_port_t port)
1472 {
1473 assert(kmsg->ikm_prealloc == IP_NULL);
1474
1475 kmsg->ikm_prealloc = IP_NULL;
1476
1477 assert(port_send_turnstile(port) == TURNSTILE_NULL);
1478 kmsg->ikm_turnstile = TURNSTILE_NULL;
1479 IP_SET_PREALLOC(port, kmsg);
1480 }
1481
1482 /*
1483 * Routine: ipc_kmsg_clear_prealloc
1484 * Purpose:
1485 * Release the Assignment of a preallocated message buffer from a port.
1486 * Conditions:
1487 * port locked.
1488 */
1489 void
1490 ipc_kmsg_clear_prealloc(
1491 ipc_kmsg_t kmsg,
1492 ipc_port_t port)
1493 {
1494 /* take the mqueue lock since the turnstile is protected under it */
1495 imq_lock(&port->ip_messages);
1496
1497 IP_CLEAR_PREALLOC(port, kmsg);
1498 set_port_send_turnstile(port, kmsg->ikm_turnstile);
1499 imq_unlock(&port->ip_messages);
1500 }
1501
1502 /*
1503 * Routine: ipc_kmsg_prealloc
1504 * Purpose:
1505 * Wraper to ipc_kmsg_alloc() to account for
1506 * header expansion requirements.
1507 */
1508 ipc_kmsg_t
1509 ipc_kmsg_prealloc(mach_msg_size_t size)
1510 {
1511 #if defined(__LP64__)
1512 if (size > MACH_MSG_SIZE_MAX - LEGACY_HEADER_SIZE_DELTA) {
1513 return IKM_NULL;
1514 }
1515
1516 size += LEGACY_HEADER_SIZE_DELTA;
1517 #endif
1518 return ipc_kmsg_alloc(size);
1519 }
1520
1521
1522 /*
1523 * Routine: ipc_kmsg_get
1524 * Purpose:
1525 * Allocates a kernel message buffer.
1526 * Copies a user message to the message buffer.
1527 * Conditions:
1528 * Nothing locked.
1529 * Returns:
1530 * MACH_MSG_SUCCESS Acquired a message buffer.
1531 * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
1532 * MACH_SEND_MSG_TOO_SMALL Message size not long-word multiple.
1533 * MACH_SEND_TOO_LARGE Message too large to ever be sent.
1534 * MACH_SEND_NO_BUFFER Couldn't allocate a message buffer.
1535 * MACH_SEND_INVALID_DATA Couldn't copy message data.
1536 */
1537
1538 mach_msg_return_t
1539 ipc_kmsg_get(
1540 mach_vm_address_t msg_addr,
1541 mach_msg_size_t size,
1542 ipc_kmsg_t *kmsgp)
1543 {
1544 mach_msg_size_t msg_and_trailer_size;
1545 ipc_kmsg_t kmsg;
1546 mach_msg_max_trailer_t *trailer;
1547 mach_msg_legacy_base_t legacy_base;
1548 mach_msg_size_t len_copied;
1549 legacy_base.body.msgh_descriptor_count = 0;
1550
1551 if ((size < sizeof(mach_msg_legacy_header_t)) || (size & 3)) {
1552 return MACH_SEND_MSG_TOO_SMALL;
1553 }
1554
1555 if (size > ipc_kmsg_max_body_space) {
1556 return MACH_SEND_TOO_LARGE;
1557 }
1558
1559 if (size == sizeof(mach_msg_legacy_header_t)) {
1560 len_copied = sizeof(mach_msg_legacy_header_t);
1561 } else {
1562 len_copied = sizeof(mach_msg_legacy_base_t);
1563 }
1564
1565 if (copyinmsg(msg_addr, (char *)&legacy_base, len_copied)) {
1566 return MACH_SEND_INVALID_DATA;
1567 }
1568
1569 /*
1570 * If the message claims to be complex, it must at least
1571 * have the length of a "base" message (header + dsc_count).
1572 */
1573 if (len_copied < sizeof(mach_msg_legacy_base_t) &&
1574 (legacy_base.header.msgh_bits & MACH_MSGH_BITS_COMPLEX)) {
1575 return MACH_SEND_MSG_TOO_SMALL;
1576 }
1577
1578 msg_addr += sizeof(legacy_base.header);
1579 #if defined(__LP64__)
1580 size += LEGACY_HEADER_SIZE_DELTA;
1581 #endif
1582 /* unreachable if !DEBUG */
1583 __unreachable_ok_push
1584 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) {
1585 unsigned int j;
1586 for (j = 0; j < sizeof(legacy_base.header); j++) {
1587 kprintf("%02x\n", ((unsigned char*)&legacy_base.header)[j]);
1588 }
1589 }
1590 __unreachable_ok_pop
1591
1592 msg_and_trailer_size = size + MAX_TRAILER_SIZE;
1593 kmsg = ipc_kmsg_alloc(msg_and_trailer_size);
1594 if (kmsg == IKM_NULL) {
1595 return MACH_SEND_NO_BUFFER;
1596 }
1597
1598 kmsg->ikm_header->msgh_size = size;
1599 kmsg->ikm_header->msgh_bits = legacy_base.header.msgh_bits;
1600 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(legacy_base.header.msgh_remote_port);
1601 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(legacy_base.header.msgh_local_port);
1602 kmsg->ikm_header->msgh_voucher_port = legacy_base.header.msgh_voucher_port;
1603 kmsg->ikm_header->msgh_id = legacy_base.header.msgh_id;
1604
1605 DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_get header:\n"
1606 " size: 0x%.8x\n"
1607 " bits: 0x%.8x\n"
1608 " remote_port: %p\n"
1609 " local_port: %p\n"
1610 " voucher_port: 0x%.8x\n"
1611 " id: %.8d\n",
1612 kmsg->ikm_header->msgh_size,
1613 kmsg->ikm_header->msgh_bits,
1614 kmsg->ikm_header->msgh_remote_port,
1615 kmsg->ikm_header->msgh_local_port,
1616 kmsg->ikm_header->msgh_voucher_port,
1617 kmsg->ikm_header->msgh_id);
1618
1619 if (copyinmsg(msg_addr, (char *)(kmsg->ikm_header + 1), size - (mach_msg_size_t)sizeof(mach_msg_header_t))) {
1620 ipc_kmsg_free(kmsg);
1621 return MACH_SEND_INVALID_DATA;
1622 }
1623
1624 /* unreachable if !DEBUG */
1625 __unreachable_ok_push
1626 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) {
1627 kprintf("body: size: %lu\n", (size - sizeof(mach_msg_header_t)));
1628 uint32_t i;
1629 for (i = 0; i * 4 < (size - sizeof(mach_msg_header_t)); i++) {
1630 kprintf("%.4x\n", ((uint32_t *)(kmsg->ikm_header + 1))[i]);
1631 }
1632 }
1633 __unreachable_ok_pop
1634 DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_get()");
1635
1636 /*
1637 * I reserve for the trailer the largest space (MAX_TRAILER_SIZE)
1638 * However, the internal size field of the trailer (msgh_trailer_size)
1639 * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to optimize
1640 * the cases where no implicit data is requested.
1641 */
1642 trailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + size);
1643 trailer->msgh_sender = current_thread()->task->sec_token;
1644 trailer->msgh_audit = current_thread()->task->audit_token;
1645 trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
1646 trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
1647
1648 #ifdef ppc
1649 if (trcWork.traceMask) {
1650 dbgTrace(0x1100, (unsigned int)kmsg->ikm_header->msgh_id,
1651 (unsigned int)kmsg->ikm_header->msgh_remote_port,
1652 (unsigned int)kmsg->ikm_header->msgh_local_port, 0);
1653 }
1654 #endif
1655
1656 trailer->msgh_labels.sender = 0;
1657 *kmsgp = kmsg;
1658 return MACH_MSG_SUCCESS;
1659 }
1660
1661 /*
1662 * Routine: ipc_kmsg_get_from_kernel
1663 * Purpose:
1664 * First checks for a preallocated message
1665 * reserved for kernel clients. If not found -
1666 * allocates a new kernel message buffer.
1667 * Copies a kernel message to the message buffer.
1668 * Only resource errors are allowed.
1669 * Conditions:
1670 * Nothing locked.
1671 * Ports in header are ipc_port_t.
1672 * Returns:
1673 * MACH_MSG_SUCCESS Acquired a message buffer.
1674 * MACH_SEND_NO_BUFFER Couldn't allocate a message buffer.
1675 */
1676
1677 mach_msg_return_t
1678 ipc_kmsg_get_from_kernel(
1679 mach_msg_header_t *msg,
1680 mach_msg_size_t size,
1681 ipc_kmsg_t *kmsgp)
1682 {
1683 ipc_kmsg_t kmsg;
1684 mach_msg_size_t msg_and_trailer_size;
1685 mach_msg_max_trailer_t *trailer;
1686 ipc_port_t dest_port;
1687
1688 assert(size >= sizeof(mach_msg_header_t));
1689 assert((size & 3) == 0);
1690
1691 dest_port = (ipc_port_t)msg->msgh_remote_port;
1692
1693 msg_and_trailer_size = size + MAX_TRAILER_SIZE;
1694
1695 /*
1696 * See if the port has a pre-allocated kmsg for kernel
1697 * clients. These are set up for those kernel clients
1698 * which cannot afford to wait.
1699 */
1700 if (IP_VALID(dest_port) && IP_PREALLOC(dest_port)) {
1701 mach_msg_size_t max_desc = 0;
1702
1703 ip_lock(dest_port);
1704 if (!ip_active(dest_port)) {
1705 ip_unlock(dest_port);
1706 return MACH_SEND_NO_BUFFER;
1707 }
1708 assert(IP_PREALLOC(dest_port));
1709 kmsg = dest_port->ip_premsg;
1710 if (ikm_prealloc_inuse(kmsg)) {
1711 ip_unlock(dest_port);
1712 return MACH_SEND_NO_BUFFER;
1713 }
1714 #if !defined(__LP64__)
1715 if (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
1716 assert(size > sizeof(mach_msg_base_t));
1717 max_desc = ((mach_msg_base_t *)msg)->body.msgh_descriptor_count *
1718 DESC_SIZE_ADJUSTMENT;
1719 }
1720 #endif
1721 if (msg_and_trailer_size > kmsg->ikm_size - max_desc) {
1722 ip_unlock(dest_port);
1723 return MACH_SEND_TOO_LARGE;
1724 }
1725 ikm_prealloc_set_inuse(kmsg, dest_port);
1726 ikm_set_header(kmsg, msg_and_trailer_size);
1727 ip_unlock(dest_port);
1728 } else {
1729 kmsg = ipc_kmsg_alloc(msg_and_trailer_size);
1730 if (kmsg == IKM_NULL) {
1731 return MACH_SEND_NO_BUFFER;
1732 }
1733 }
1734
1735 (void) memcpy((void *) kmsg->ikm_header, (const void *) msg, size);
1736
1737 ikm_qos_init(kmsg);
1738
1739 kmsg->ikm_header->msgh_size = size;
1740
1741 /*
1742 * I reserve for the trailer the largest space (MAX_TRAILER_SIZE)
1743 * However, the internal size field of the trailer (msgh_trailer_size)
1744 * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to
1745 * optimize the cases where no implicit data is requested.
1746 */
1747 trailer = (mach_msg_max_trailer_t *)
1748 ((vm_offset_t)kmsg->ikm_header + size);
1749 trailer->msgh_sender = KERNEL_SECURITY_TOKEN;
1750 trailer->msgh_audit = KERNEL_AUDIT_TOKEN;
1751 trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
1752 trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
1753
1754 trailer->msgh_labels.sender = 0;
1755
1756 *kmsgp = kmsg;
1757 return MACH_MSG_SUCCESS;
1758 }
1759
1760 /*
1761 * Routine: ipc_kmsg_send
1762 * Purpose:
1763 * Send a message. The message holds a reference
1764 * for the destination port in the msgh_remote_port field.
1765 *
1766 * If unsuccessful, the caller still has possession of
1767 * the message and must do something with it. If successful,
1768 * the message is queued, given to a receiver, destroyed,
1769 * or handled directly by the kernel via mach_msg.
1770 * Conditions:
1771 * Nothing locked.
1772 * Returns:
1773 * MACH_MSG_SUCCESS The message was accepted.
1774 * MACH_SEND_TIMED_OUT Caller still has message.
1775 * MACH_SEND_INTERRUPTED Caller still has message.
1776 * MACH_SEND_INVALID_DEST Caller still has message.
1777 */
1778 mach_msg_return_t
1779 ipc_kmsg_send(
1780 ipc_kmsg_t kmsg,
1781 mach_msg_option_t option,
1782 mach_msg_timeout_t send_timeout)
1783 {
1784 ipc_port_t port;
1785 thread_t th = current_thread();
1786 mach_msg_return_t error = MACH_MSG_SUCCESS;
1787 boolean_t kernel_reply = FALSE;
1788
1789 /* Check if honor qlimit flag is set on thread. */
1790 if ((th->options & TH_OPT_HONOR_QLIMIT) == TH_OPT_HONOR_QLIMIT) {
1791 /* Remove the MACH_SEND_ALWAYS flag to honor queue limit. */
1792 option &= (~MACH_SEND_ALWAYS);
1793 /* Add the timeout flag since the message queue might be full. */
1794 option |= MACH_SEND_TIMEOUT;
1795 th->options &= (~TH_OPT_HONOR_QLIMIT);
1796 }
1797
1798 #if IMPORTANCE_INHERITANCE
1799 bool did_importance = false;
1800 #if IMPORTANCE_TRACE
1801 mach_msg_id_t imp_msgh_id = -1;
1802 int sender_pid = -1;
1803 #endif /* IMPORTANCE_TRACE */
1804 #endif /* IMPORTANCE_INHERITANCE */
1805
1806 /* don't allow the creation of a circular loop */
1807 if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_CIRCULAR) {
1808 ipc_kmsg_destroy(kmsg);
1809 KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_MSGH_BITS_CIRCULAR);
1810 return MACH_MSG_SUCCESS;
1811 }
1812
1813 ipc_voucher_send_preprocessing(kmsg);
1814
1815 port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port;
1816 assert(IP_VALID(port));
1817 ip_lock(port);
1818
1819 #if IMPORTANCE_INHERITANCE
1820 retry:
1821 #endif /* IMPORTANCE_INHERITANCE */
1822 /*
1823 * Can't deliver to a dead port.
1824 * However, we can pretend it got sent
1825 * and was then immediately destroyed.
1826 */
1827 if (!ip_active(port)) {
1828 ip_unlock(port);
1829 #if MACH_FLIPC
1830 if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) {
1831 flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
1832 }
1833 #endif
1834 if (did_importance) {
1835 /*
1836 * We're going to pretend we delivered this message
1837 * successfully, and just eat the kmsg. However, the
1838 * kmsg is actually visible via the importance_task!
1839 * We need to cleanup this linkage before we destroy
1840 * the message, and more importantly before we set the
1841 * msgh_remote_port to NULL. See: 34302571
1842 */
1843 ipc_importance_clean(kmsg);
1844 }
1845 ip_release(port); /* JMM - Future: release right, not just ref */
1846 kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
1847 ipc_kmsg_destroy(kmsg);
1848 KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST);
1849 return MACH_MSG_SUCCESS;
1850 }
1851
1852 if (port->ip_receiver == ipc_space_kernel) {
1853 /*
1854 * We can check ip_receiver == ipc_space_kernel
1855 * before checking that the port is active because
1856 * ipc_port_dealloc_kernel clears ip_receiver
1857 * before destroying a kernel port.
1858 */
1859 assert(ip_active(port));
1860 port->ip_messages.imq_seqno++;
1861 ip_unlock(port);
1862
1863 current_task()->messages_sent++;
1864
1865 /*
1866 * Call the server routine, and get the reply message to send.
1867 */
1868 kmsg = ipc_kobject_server(kmsg, option);
1869 if (kmsg == IKM_NULL) {
1870 return MACH_MSG_SUCCESS;
1871 }
1872
1873 /* restart the KMSG_INFO tracing for the reply message */
1874 KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_START);
1875 port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port;
1876 assert(IP_VALID(port));
1877 ip_lock(port);
1878 /* fall thru with reply - same options */
1879 kernel_reply = TRUE;
1880 if (!ip_active(port)) {
1881 error = MACH_SEND_INVALID_DEST;
1882 }
1883 }
1884
1885 #if IMPORTANCE_INHERITANCE
1886 /*
1887 * Need to see if this message needs importance donation and/or
1888 * propagation. That routine can drop the port lock temporarily.
1889 * If it does we'll have to revalidate the destination.
1890 */
1891 if (!did_importance) {
1892 did_importance = true;
1893 if (ipc_importance_send(kmsg, option)) {
1894 goto retry;
1895 }
1896 }
1897 #endif /* IMPORTANCE_INHERITANCE */
1898
1899 if (error != MACH_MSG_SUCCESS) {
1900 ip_unlock(port);
1901 } else {
1902 /*
1903 * We have a valid message and a valid reference on the port.
1904 * we can unlock the port and call mqueue_send() on its message
1905 * queue. Lock message queue while port is locked.
1906 */
1907 imq_lock(&port->ip_messages);
1908
1909 set_ip_srp_msg_sent(port);
1910
1911 ip_unlock(port);
1912
1913 error = ipc_mqueue_send(&port->ip_messages, kmsg, option,
1914 send_timeout);
1915 }
1916
1917 #if IMPORTANCE_INHERITANCE
1918 if (did_importance) {
1919 __unused int importance_cleared = 0;
1920 switch (error) {
1921 case MACH_SEND_TIMED_OUT:
1922 case MACH_SEND_NO_BUFFER:
1923 case MACH_SEND_INTERRUPTED:
1924 case MACH_SEND_INVALID_DEST:
1925 /*
1926 * We still have the kmsg and its
1927 * reference on the port. But we
1928 * have to back out the importance
1929 * boost.
1930 *
1931 * The port could have changed hands,
1932 * be inflight to another destination,
1933 * etc... But in those cases our
1934 * back-out will find the new owner
1935 * (and all the operations that
1936 * transferred the right should have
1937 * applied their own boost adjustments
1938 * to the old owner(s)).
1939 */
1940 importance_cleared = 1;
1941 ipc_importance_clean(kmsg);
1942 break;
1943
1944 case MACH_MSG_SUCCESS:
1945 default:
1946 break;
1947 }
1948 #if IMPORTANCE_TRACE
1949 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_END,
1950 task_pid(current_task()), sender_pid, imp_msgh_id, importance_cleared, 0);
1951 #endif /* IMPORTANCE_TRACE */
1952 }
1953 #endif /* IMPORTANCE_INHERITANCE */
1954
1955 /*
1956 * If the port has been destroyed while we wait, treat the message
1957 * as a successful delivery (like we do for an inactive port).
1958 */
1959 if (error == MACH_SEND_INVALID_DEST) {
1960 #if MACH_FLIPC
1961 if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) {
1962 flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
1963 }
1964 #endif
1965 ip_release(port); /* JMM - Future: release right, not just ref */
1966 kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
1967 ipc_kmsg_destroy(kmsg);
1968 KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST);
1969 return MACH_MSG_SUCCESS;
1970 }
1971
1972 if (error != MACH_MSG_SUCCESS && kernel_reply) {
1973 /*
1974 * Kernel reply messages that fail can't be allowed to
1975 * pseudo-receive on error conditions. We need to just treat
1976 * the message as a successful delivery.
1977 */
1978 #if MACH_FLIPC
1979 if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) {
1980 flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
1981 }
1982 #endif
1983 ip_release(port); /* JMM - Future: release right, not just ref */
1984 kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
1985 ipc_kmsg_destroy(kmsg);
1986 KDBG(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_INFO) | DBG_FUNC_END, error);
1987 return MACH_MSG_SUCCESS;
1988 }
1989 return error;
1990 }
1991
1992 /*
1993 * Routine: ipc_kmsg_put
1994 * Purpose:
1995 * Copies a message buffer to a user message.
1996 * Copies only the specified number of bytes.
1997 * Frees the message buffer.
1998 * Conditions:
1999 * Nothing locked. The message buffer must have clean
2000 * header fields.
2001 * Returns:
2002 * MACH_MSG_SUCCESS Copied data out of message buffer.
2003 * MACH_RCV_INVALID_DATA Couldn't copy to user message.
2004 */
2005
2006 mach_msg_return_t
2007 ipc_kmsg_put(
2008 ipc_kmsg_t kmsg,
2009 mach_msg_option_t option,
2010 mach_vm_address_t rcv_addr,
2011 mach_msg_size_t rcv_size,
2012 mach_msg_size_t trailer_size,
2013 mach_msg_size_t *sizep)
2014 {
2015 mach_msg_size_t size = kmsg->ikm_header->msgh_size + trailer_size;
2016 mach_msg_return_t mr;
2017
2018 DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_put()");
2019
2020
2021 DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_put header:\n"
2022 " size: 0x%.8x\n"
2023 " bits: 0x%.8x\n"
2024 " remote_port: %p\n"
2025 " local_port: %p\n"
2026 " voucher_port: 0x%.8x\n"
2027 " id: %.8d\n",
2028 kmsg->ikm_header->msgh_size,
2029 kmsg->ikm_header->msgh_bits,
2030 kmsg->ikm_header->msgh_remote_port,
2031 kmsg->ikm_header->msgh_local_port,
2032 kmsg->ikm_header->msgh_voucher_port,
2033 kmsg->ikm_header->msgh_id);
2034
2035 #if defined(__LP64__)
2036 if (current_task() != kernel_task) { /* don't if receiver expects fully-cooked in-kernel msg; */
2037 mach_msg_legacy_header_t *legacy_header =
2038 (mach_msg_legacy_header_t *)((vm_offset_t)(kmsg->ikm_header) + LEGACY_HEADER_SIZE_DELTA);
2039
2040 mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits;
2041 mach_msg_size_t msg_size = kmsg->ikm_header->msgh_size;
2042 mach_port_name_t remote_port = CAST_MACH_PORT_TO_NAME(kmsg->ikm_header->msgh_remote_port);
2043 mach_port_name_t local_port = CAST_MACH_PORT_TO_NAME(kmsg->ikm_header->msgh_local_port);
2044 mach_port_name_t voucher_port = kmsg->ikm_header->msgh_voucher_port;
2045 mach_msg_id_t id = kmsg->ikm_header->msgh_id;
2046
2047 legacy_header->msgh_id = id;
2048 legacy_header->msgh_local_port = local_port;
2049 legacy_header->msgh_remote_port = remote_port;
2050 legacy_header->msgh_voucher_port = voucher_port;
2051 legacy_header->msgh_size = msg_size - LEGACY_HEADER_SIZE_DELTA;
2052 legacy_header->msgh_bits = bits;
2053
2054 size -= LEGACY_HEADER_SIZE_DELTA;
2055 kmsg->ikm_header = (mach_msg_header_t *)legacy_header;
2056 }
2057 #endif
2058
2059 /* unreachable if !DEBUG */
2060 __unreachable_ok_push
2061 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) {
2062 kprintf("ipc_kmsg_put header+body: %d\n", (size));
2063 uint32_t i;
2064 for (i = 0; i * 4 < size; i++) {
2065 kprintf("%.4x\n", ((uint32_t *)kmsg->ikm_header)[i]);
2066 }
2067 kprintf("type: %d\n", ((mach_msg_type_descriptor_t *)(((mach_msg_base_t *)kmsg->ikm_header) + 1))->type);
2068 }
2069 __unreachable_ok_pop
2070
2071 /* Re-Compute target address if using stack-style delivery */
2072 if (option & MACH_RCV_STACK) {
2073 rcv_addr += rcv_size - size;
2074 }
2075
2076 if (copyoutmsg((const char *) kmsg->ikm_header, rcv_addr, size)) {
2077 mr = MACH_RCV_INVALID_DATA;
2078 size = 0;
2079 } else {
2080 mr = MACH_MSG_SUCCESS;
2081 }
2082
2083 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_KMSG_LINK) | DBG_FUNC_NONE,
2084 (rcv_addr >= VM_MIN_KERNEL_AND_KEXT_ADDRESS ||
2085 rcv_addr + size >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) ? (uintptr_t)0 : (uintptr_t)rcv_addr,
2086 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
2087 1 /* this is on the receive/copyout path */,
2088 0,
2089 0);
2090 ipc_kmsg_free(kmsg);
2091
2092 if (sizep) {
2093 *sizep = size;
2094 }
2095 return mr;
2096 }
2097
2098 /*
2099 * Routine: ipc_kmsg_put_to_kernel
2100 * Purpose:
2101 * Copies a message buffer to a kernel message.
2102 * Frees the message buffer.
2103 * No errors allowed.
2104 * Conditions:
2105 * Nothing locked.
2106 */
2107
2108 void
2109 ipc_kmsg_put_to_kernel(
2110 mach_msg_header_t *msg,
2111 ipc_kmsg_t kmsg,
2112 mach_msg_size_t size)
2113 {
2114 (void) memcpy((void *) msg, (const void *) kmsg->ikm_header, size);
2115
2116 ipc_kmsg_free(kmsg);
2117 }
2118
2119 static mach_msg_priority_t
2120 ipc_get_current_thread_priority(void)
2121 {
2122 thread_t thread = current_thread();
2123 thread_qos_t qos;
2124 int relpri;
2125
2126 qos = thread_get_requested_qos(thread, &relpri);
2127 if (!qos) {
2128 qos = thread_user_promotion_qos_for_pri(thread->base_pri);
2129 relpri = 0;
2130 }
2131 return (mach_msg_priority_t)_pthread_priority_make_from_thread_qos(qos, relpri, 0);
2132 }
2133
2134 static kern_return_t
2135 ipc_kmsg_set_qos(
2136 ipc_kmsg_t kmsg,
2137 mach_msg_option_t options,
2138 mach_msg_priority_t override)
2139 {
2140 kern_return_t kr;
2141 ipc_port_t special_reply_port = kmsg->ikm_header->msgh_local_port;
2142 ipc_port_t dest_port = kmsg->ikm_header->msgh_remote_port;
2143
2144 kr = ipc_get_pthpriority_from_kmsg_voucher(kmsg, &kmsg->ikm_qos);
2145 if (kr != KERN_SUCCESS) {
2146 if (options & MACH_SEND_PROPAGATE_QOS) {
2147 kmsg->ikm_qos = ipc_get_current_thread_priority();
2148 } else {
2149 kmsg->ikm_qos = MACH_MSG_PRIORITY_UNSPECIFIED;
2150 }
2151 }
2152 kmsg->ikm_qos_override = kmsg->ikm_qos;
2153
2154 if (options & MACH_SEND_OVERRIDE) {
2155 pthread_priority_t pp = _pthread_priority_normalize_for_ipc(override);
2156 if (pp > kmsg->ikm_qos) {
2157 kmsg->ikm_qos_override = (mach_msg_priority_t)pp;
2158 }
2159 }
2160
2161 kr = KERN_SUCCESS;
2162 if ((options & MACH_SEND_SYNC_OVERRIDE)) {
2163 if (IP_VALID(special_reply_port) &&
2164 MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits) == MACH_MSG_TYPE_PORT_SEND_ONCE) {
2165 /*
2166 * Link the destination port to special reply port and make sure that
2167 * dest port has a send turnstile, else allocate one.
2168 */
2169 ipc_port_link_special_reply_port(special_reply_port, dest_port);
2170 }
2171 }
2172 return kr;
2173 }
2174
2175 /*
2176 * Routine: ipc_kmsg_copyin_header
2177 * Purpose:
2178 * "Copy-in" port rights in the header of a message.
2179 * Operates atomically; if it doesn't succeed the
2180 * message header and the space are left untouched.
2181 * If it does succeed the remote/local port fields
2182 * contain object pointers instead of port names,
2183 * and the bits field is updated. The destination port
2184 * will be a valid port pointer.
2185 *
2186 * Conditions:
2187 * Nothing locked.
2188 * Returns:
2189 * MACH_MSG_SUCCESS Successful copyin.
2190 * MACH_SEND_INVALID_HEADER
2191 * Illegal value in the message header bits.
2192 * MACH_SEND_INVALID_DEST The space is dead.
2193 * MACH_SEND_INVALID_DEST Can't copyin destination port.
2194 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
2195 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
2196 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
2197 */
2198
2199 mach_msg_return_t
2200 ipc_kmsg_copyin_header(
2201 ipc_kmsg_t kmsg,
2202 ipc_space_t space,
2203 mach_msg_priority_t override,
2204 mach_msg_option_t *optionp)
2205 {
2206 mach_msg_header_t *msg = kmsg->ikm_header;
2207 mach_msg_bits_t mbits = msg->msgh_bits & MACH_MSGH_BITS_USER;
2208 mach_port_name_t dest_name = CAST_MACH_PORT_TO_NAME(msg->msgh_remote_port);
2209 mach_port_name_t reply_name = CAST_MACH_PORT_TO_NAME(msg->msgh_local_port);
2210 mach_port_name_t voucher_name = MACH_PORT_NULL;
2211 kern_return_t kr;
2212
2213 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
2214 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
2215 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
2216 ipc_object_t dest_port = IO_NULL;
2217 ipc_object_t reply_port = IO_NULL;
2218 ipc_port_t dest_soright = IP_NULL;
2219 ipc_port_t reply_soright = IP_NULL;
2220 ipc_port_t voucher_soright = IP_NULL;
2221 ipc_port_t release_port = IP_NULL;
2222 ipc_port_t voucher_port = IP_NULL;
2223 ipc_port_t voucher_release_port = IP_NULL;
2224 ipc_entry_t dest_entry = IE_NULL;
2225 ipc_entry_t reply_entry = IE_NULL;
2226 ipc_entry_t voucher_entry = IE_NULL;
2227
2228 int assertcnt = 0;
2229 #if IMPORTANCE_INHERITANCE
2230 boolean_t needboost = FALSE;
2231 #endif /* IMPORTANCE_INHERITANCE */
2232
2233 if ((mbits != msg->msgh_bits) ||
2234 (!MACH_MSG_TYPE_PORT_ANY_SEND(dest_type)) ||
2235 ((reply_type == 0) ?
2236 (reply_name != MACH_PORT_NULL) :
2237 !MACH_MSG_TYPE_PORT_ANY_SEND(reply_type))) {
2238 return MACH_SEND_INVALID_HEADER;
2239 }
2240
2241 if (!MACH_PORT_VALID(dest_name)) {
2242 return MACH_SEND_INVALID_DEST;
2243 }
2244
2245 is_write_lock(space);
2246 if (!is_active(space)) {
2247 is_write_unlock(space);
2248 return MACH_SEND_INVALID_DEST;
2249 }
2250 /* space locked and active */
2251
2252 /*
2253 * If there is a voucher specified, make sure the disposition is
2254 * valid and the entry actually refers to a voucher port. Don't
2255 * actually copy in until we validate destination and reply.
2256 */
2257 if (voucher_type != MACH_MSGH_BITS_ZERO) {
2258 voucher_name = msg->msgh_voucher_port;
2259
2260 if (voucher_name == MACH_PORT_DEAD ||
2261 (voucher_type != MACH_MSG_TYPE_MOVE_SEND &&
2262 voucher_type != MACH_MSG_TYPE_COPY_SEND)) {
2263 is_write_unlock(space);
2264 if ((*optionp & MACH_SEND_KERNEL) == 0) {
2265 mach_port_guard_exception(voucher_name, 0, 0, kGUARD_EXC_SEND_INVALID_VOUCHER);
2266 }
2267 return MACH_SEND_INVALID_VOUCHER;
2268 }
2269
2270 if (voucher_name != MACH_PORT_NULL) {
2271 voucher_entry = ipc_entry_lookup(space, voucher_name);
2272 if (voucher_entry == IE_NULL ||
2273 (voucher_entry->ie_bits & MACH_PORT_TYPE_SEND) == 0 ||
2274 io_kotype(voucher_entry->ie_object) != IKOT_VOUCHER) {
2275 is_write_unlock(space);
2276 if ((*optionp & MACH_SEND_KERNEL) == 0) {
2277 mach_port_guard_exception(voucher_name, 0, 0, kGUARD_EXC_SEND_INVALID_VOUCHER);
2278 }
2279 return MACH_SEND_INVALID_VOUCHER;
2280 }
2281 } else {
2282 voucher_type = MACH_MSG_TYPE_MOVE_SEND;
2283 }
2284 }
2285
2286 /*
2287 * Handle combinations of validating destination and reply; along
2288 * with copying in destination, reply, and voucher in an atomic way.
2289 */
2290
2291 if (dest_name == voucher_name) {
2292 /*
2293 * If the destination name is the same as the voucher name,
2294 * the voucher_entry must already be known. Either that or
2295 * the destination name is MACH_PORT_NULL (i.e. invalid).
2296 */
2297 dest_entry = voucher_entry;
2298 if (dest_entry == IE_NULL) {
2299 goto invalid_dest;
2300 }
2301
2302 /*
2303 * Make sure a future copyin of the reply port will succeed.
2304 * Once we start copying in the dest/voucher pair, we can't
2305 * back out.
2306 */
2307 if (MACH_PORT_VALID(reply_name)) {
2308 assert(reply_type != 0); /* because reply_name not null */
2309
2310 /* It is just WRONG if dest, voucher, and reply are all the same. */
2311 if (voucher_name == reply_name) {
2312 goto invalid_reply;
2313 }
2314 reply_entry = ipc_entry_lookup(space, reply_name);
2315 if (reply_entry == IE_NULL) {
2316 goto invalid_reply;
2317 }
2318 assert(dest_entry != reply_entry); /* names are not equal */
2319 if (!ipc_right_copyin_check(space, reply_name, reply_entry, reply_type)) {
2320 goto invalid_reply;
2321 }
2322 }
2323
2324 /*
2325 * Do the joint copyin of the dest disposition and
2326 * voucher disposition from the one entry/port. We
2327 * already validated that the voucher copyin would
2328 * succeed (above). So, any failure in combining
2329 * the copyins can be blamed on the destination.
2330 */
2331 kr = ipc_right_copyin_two(space, dest_name, dest_entry,
2332 dest_type, voucher_type,
2333 &dest_port, &dest_soright,
2334 &release_port);
2335 if (kr != KERN_SUCCESS) {
2336 assert(kr != KERN_INVALID_CAPABILITY);
2337 goto invalid_dest;
2338 }
2339 voucher_port = (ipc_port_t)dest_port;
2340
2341 /*
2342 * could not have been one of these dispositions,
2343 * validated the port was a true kernel voucher port above,
2344 * AND was successfully able to copyin both dest and voucher.
2345 */
2346 assert(dest_type != MACH_MSG_TYPE_MAKE_SEND);
2347 assert(dest_type != MACH_MSG_TYPE_MAKE_SEND_ONCE);
2348 assert(dest_type != MACH_MSG_TYPE_MOVE_SEND_ONCE);
2349
2350 /*
2351 * Perform the delayed reply right copyin (guaranteed success).
2352 */
2353 if (reply_entry != IE_NULL) {
2354 kr = ipc_right_copyin(space, reply_name, reply_entry,
2355 reply_type, IPC_RIGHT_COPYIN_FLAGS_DEADOK,
2356 &reply_port, &reply_soright,
2357 &release_port, &assertcnt);
2358 assert(assertcnt == 0);
2359 assert(kr == KERN_SUCCESS);
2360 }
2361 } else {
2362 if (dest_name == reply_name) {
2363 /*
2364 * Destination and reply ports are the same!
2365 * This is very similar to the case where the
2366 * destination and voucher ports were the same
2367 * (except the reply port disposition is not
2368 * previously validated).
2369 */
2370 dest_entry = ipc_entry_lookup(space, dest_name);
2371 if (dest_entry == IE_NULL) {
2372 goto invalid_dest;
2373 }
2374 reply_entry = dest_entry;
2375 assert(reply_type != 0); /* because name not null */
2376
2377 /*
2378 * Do the joint copyin of the dest disposition and
2379 * reply disposition from the one entry/port.
2380 */
2381 kr = ipc_right_copyin_two(space, dest_name, dest_entry,
2382 dest_type, reply_type,
2383 &dest_port, &dest_soright,
2384 &release_port);
2385 if (kr == KERN_INVALID_CAPABILITY) {
2386 goto invalid_reply;
2387 } else if (kr != KERN_SUCCESS) {
2388 goto invalid_dest;
2389 }
2390 reply_port = dest_port;
2391 } else {
2392 /*
2393 * Handle destination and reply independently, as
2394 * they are independent entries (even if the entries
2395 * refer to the same port).
2396 *
2397 * This can be the tough case to make atomic.
2398 *
2399 * The difficult problem is serializing with port death.
2400 * The bad case is when dest_port dies after its copyin,
2401 * reply_port dies before its copyin, and dest_port dies before
2402 * reply_port. Then the copyins operated as if dest_port was
2403 * alive and reply_port was dead, which shouldn't have happened
2404 * because they died in the other order.
2405 *
2406 * Note that it is easy for a user task to tell if
2407 * a copyin happened before or after a port died.
2408 * If a port dies before copyin, a dead-name notification
2409 * is generated and the dead name's urefs are incremented,
2410 * and if the copyin happens first, a port-deleted
2411 * notification is generated.
2412 *
2413 * Even so, avoiding that potentially detectable race is too
2414 * expensive - and no known code cares about it. So, we just
2415 * do the expedient thing and copy them in one after the other.
2416 */
2417
2418 dest_entry = ipc_entry_lookup(space, dest_name);
2419 if (dest_entry == IE_NULL) {
2420 goto invalid_dest;
2421 }
2422 assert(dest_entry != voucher_entry);
2423
2424 /*
2425 * Make sure reply port entry is valid before dest copyin.
2426 */
2427 if (MACH_PORT_VALID(reply_name)) {
2428 if (reply_name == voucher_name) {
2429 goto invalid_reply;
2430 }
2431 reply_entry = ipc_entry_lookup(space, reply_name);
2432 if (reply_entry == IE_NULL) {
2433 goto invalid_reply;
2434 }
2435 assert(dest_entry != reply_entry); /* names are not equal */
2436 assert(reply_type != 0); /* because reply_name not null */
2437
2438 if (!ipc_right_copyin_check(space, reply_name, reply_entry, reply_type)) {
2439 goto invalid_reply;
2440 }
2441 }
2442
2443 /*
2444 * copyin the destination.
2445 */
2446 kr = ipc_right_copyin(space, dest_name, dest_entry,
2447 dest_type, IPC_RIGHT_COPYIN_FLAGS_ALLOW_DEAD_SEND_ONCE,
2448 &dest_port, &dest_soright,
2449 &release_port, &assertcnt);
2450 assert(assertcnt == 0);
2451 if (kr != KERN_SUCCESS) {
2452 goto invalid_dest;
2453 }
2454 assert(IO_VALID(dest_port));
2455 assert(!IP_VALID(release_port));
2456
2457 /*
2458 * Copyin the pre-validated reply right.
2459 * It's OK if the reply right has gone dead in the meantime.
2460 */
2461 if (MACH_PORT_VALID(reply_name)) {
2462 kr = ipc_right_copyin(space, reply_name, reply_entry,
2463 reply_type, IPC_RIGHT_COPYIN_FLAGS_DEADOK,
2464 &reply_port, &reply_soright,
2465 &release_port, &assertcnt);
2466 assert(assertcnt == 0);
2467 assert(kr == KERN_SUCCESS);
2468 } else {
2469 /* convert invalid name to equivalent ipc_object type */
2470 reply_port = (ipc_object_t)CAST_MACH_NAME_TO_PORT(reply_name);
2471 }
2472 }
2473
2474 /*
2475 * Finally can copyin the voucher right now that dest and reply
2476 * are fully copied in (guaranteed success).
2477 */
2478 if (IE_NULL != voucher_entry) {
2479 kr = ipc_right_copyin(space, voucher_name, voucher_entry,
2480 voucher_type, IPC_RIGHT_COPYIN_FLAGS_NONE,
2481 (ipc_object_t *)&voucher_port,
2482 &voucher_soright,
2483 &voucher_release_port,
2484 &assertcnt);
2485 assert(assertcnt == 0);
2486 assert(KERN_SUCCESS == kr);
2487 assert(IP_VALID(voucher_port));
2488 assert(ip_active(voucher_port));
2489 }
2490 }
2491
2492 /*
2493 * The entries might need to be deallocated.
2494 *
2495 * Each entry should be deallocated only once,
2496 * even if it was specified in more than one slot in the header.
2497 * Note that dest can be the same entry as reply or voucher,
2498 * but reply and voucher must be distinct entries.
2499 */
2500 assert(IE_NULL != dest_entry);
2501 if (IE_NULL != reply_entry) {
2502 assert(reply_entry != voucher_entry);
2503 }
2504
2505 if (IE_BITS_TYPE(dest_entry->ie_bits) == MACH_PORT_TYPE_NONE) {
2506 ipc_entry_dealloc(space, dest_name, dest_entry);
2507
2508 if (dest_entry == reply_entry) {
2509 reply_entry = IE_NULL;
2510 }
2511
2512 if (dest_entry == voucher_entry) {
2513 voucher_entry = IE_NULL;
2514 }
2515
2516 dest_entry = IE_NULL;
2517 }
2518 if (IE_NULL != reply_entry &&
2519 IE_BITS_TYPE(reply_entry->ie_bits) == MACH_PORT_TYPE_NONE) {
2520 ipc_entry_dealloc(space, reply_name, reply_entry);
2521 reply_entry = IE_NULL;
2522 }
2523 if (IE_NULL != voucher_entry &&
2524 IE_BITS_TYPE(voucher_entry->ie_bits) == MACH_PORT_TYPE_NONE) {
2525 ipc_entry_dealloc(space, voucher_name, voucher_entry);
2526 voucher_entry = IE_NULL;
2527 }
2528
2529 dest_type = ipc_object_copyin_type(dest_type);
2530 reply_type = ipc_object_copyin_type(reply_type);
2531
2532 /*
2533 * JMM - Without rdar://problem/6275821, this is the last place we can
2534 * re-arm the send-possible notifications. It may trigger unexpectedly
2535 * early (send may NOT have failed), but better than missing. We assure
2536 * we won't miss by forcing MACH_SEND_ALWAYS if we got past arming.
2537 */
2538 if (((*optionp & MACH_SEND_NOTIFY) != 0) &&
2539 dest_type != MACH_MSG_TYPE_PORT_SEND_ONCE &&
2540 dest_entry != IE_NULL && dest_entry->ie_request != IE_REQ_NONE) {
2541 ipc_port_t dport = (ipc_port_t)dest_port;
2542
2543 assert(dport != IP_NULL);
2544 ip_lock(dport);
2545 if (ip_active(dport) && dport->ip_receiver != ipc_space_kernel) {
2546 if (ip_full(dport)) {
2547 #if IMPORTANCE_INHERITANCE
2548 needboost = ipc_port_request_sparm(dport, dest_name,
2549 dest_entry->ie_request,
2550 *optionp,
2551 override);
2552 if (needboost == FALSE) {
2553 ip_unlock(dport);
2554 }
2555 #else
2556 ipc_port_request_sparm(dport, dest_name,
2557 dest_entry->ie_request,
2558 *optionp,
2559 override);
2560 ip_unlock(dport);
2561 #endif /* IMPORTANCE_INHERITANCE */
2562 } else {
2563 *optionp |= MACH_SEND_ALWAYS;
2564 ip_unlock(dport);
2565 }
2566 } else {
2567 ip_unlock(dport);
2568 }
2569 }
2570
2571 is_write_unlock(space);
2572
2573 #if IMPORTANCE_INHERITANCE
2574 /*
2575 * If our request is the first boosting send-possible
2576 * notification this cycle, push the boost down the
2577 * destination port.
2578 */
2579 if (needboost == TRUE) {
2580 ipc_port_t dport = (ipc_port_t)dest_port;
2581
2582 /* dport still locked from above */
2583 if (ipc_port_importance_delta(dport, IPID_OPTION_SENDPOSSIBLE, 1) == FALSE) {
2584 ip_unlock(dport);
2585 }
2586 }
2587 #endif /* IMPORTANCE_INHERITANCE */
2588
2589 if (dest_soright != IP_NULL) {
2590 ipc_notify_port_deleted(dest_soright, dest_name);
2591 }
2592 if (reply_soright != IP_NULL) {
2593 ipc_notify_port_deleted(reply_soright, reply_name);
2594 }
2595 if (voucher_soright != IP_NULL) {
2596 ipc_notify_port_deleted(voucher_soright, voucher_name);
2597 }
2598
2599 /*
2600 * No room to store voucher port in in-kernel msg header,
2601 * so we store it back in the kmsg itself. Extract the
2602 * qos, and apply any override before we enqueue the kmsg.
2603 */
2604 if (IP_VALID(voucher_port)) {
2605 kmsg->ikm_voucher = voucher_port;
2606 voucher_type = MACH_MSG_TYPE_MOVE_SEND;
2607 }
2608
2609 msg->msgh_bits = MACH_MSGH_BITS_SET(dest_type, reply_type, voucher_type, mbits);
2610 msg->msgh_remote_port = (ipc_port_t)dest_port;
2611 msg->msgh_local_port = (ipc_port_t)reply_port;
2612
2613 /* capture the qos value(s) for the kmsg */
2614 ipc_kmsg_set_qos(kmsg, *optionp, override);
2615
2616 if (release_port != IP_NULL) {
2617 ip_release(release_port);
2618 }
2619
2620 if (voucher_release_port != IP_NULL) {
2621 ip_release(voucher_release_port);
2622 }
2623
2624 return MACH_MSG_SUCCESS;
2625
2626 invalid_reply:
2627 is_write_unlock(space);
2628
2629 if (release_port != IP_NULL) {
2630 ip_release(release_port);
2631 }
2632
2633 assert(voucher_port == IP_NULL);
2634 assert(voucher_soright == IP_NULL);
2635
2636 if ((*optionp & MACH_SEND_KERNEL) == 0) {
2637 mach_port_guard_exception(reply_name, 0, 0, kGUARD_EXC_SEND_INVALID_REPLY);
2638 }
2639 return MACH_SEND_INVALID_REPLY;
2640
2641 invalid_dest:
2642 is_write_unlock(space);
2643
2644 if (release_port != IP_NULL) {
2645 ip_release(release_port);
2646 }
2647
2648 if (reply_soright != IP_NULL) {
2649 ipc_notify_port_deleted(reply_soright, reply_name);
2650 }
2651
2652 assert(voucher_port == IP_NULL);
2653 assert(voucher_soright == IP_NULL);
2654
2655 return MACH_SEND_INVALID_DEST;
2656 }
2657
2658 mach_msg_descriptor_t *ipc_kmsg_copyin_port_descriptor(
2659 volatile mach_msg_port_descriptor_t *dsc,
2660 mach_msg_legacy_port_descriptor_t *user_dsc,
2661 ipc_space_t space,
2662 ipc_object_t dest,
2663 ipc_kmsg_t kmsg,
2664 mach_msg_option_t *optionp,
2665 mach_msg_return_t *mr);
2666
2667 void ipc_print_type_name(
2668 int type_name);
2669
2670 mach_msg_descriptor_t *
2671 ipc_kmsg_copyin_port_descriptor(
2672 volatile mach_msg_port_descriptor_t *dsc,
2673 mach_msg_legacy_port_descriptor_t *user_dsc_in,
2674 ipc_space_t space,
2675 ipc_object_t dest,
2676 ipc_kmsg_t kmsg,
2677 mach_msg_option_t *optionp,
2678 mach_msg_return_t *mr)
2679 {
2680 volatile mach_msg_legacy_port_descriptor_t *user_dsc = user_dsc_in;
2681 mach_msg_type_name_t user_disp;
2682 mach_msg_type_name_t result_disp;
2683 mach_port_name_t name;
2684 ipc_object_t object;
2685
2686 user_disp = user_dsc->disposition;
2687 result_disp = ipc_object_copyin_type(user_disp);
2688
2689 name = (mach_port_name_t)user_dsc->name;
2690 if (MACH_PORT_VALID(name)) {
2691 kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object);
2692 if (kr != KERN_SUCCESS) {
2693 if ((*optionp & MACH_SEND_KERNEL) == 0) {
2694 mach_port_guard_exception(name, 0, 0, kGUARD_EXC_SEND_INVALID_RIGHT);
2695 }
2696 *mr = MACH_SEND_INVALID_RIGHT;
2697 return NULL;
2698 }
2699
2700 if ((result_disp == MACH_MSG_TYPE_PORT_RECEIVE) &&
2701 ipc_port_check_circularity((ipc_port_t) object,
2702 (ipc_port_t) dest)) {
2703 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
2704 }
2705 dsc->name = (ipc_port_t) object;
2706 } else {
2707 dsc->name = CAST_MACH_NAME_TO_PORT(name);
2708 }
2709 dsc->disposition = result_disp;
2710 dsc->type = MACH_MSG_PORT_DESCRIPTOR;
2711
2712 dsc->pad_end = 0; // debug, unnecessary
2713
2714 return (mach_msg_descriptor_t *)(user_dsc_in + 1);
2715 }
2716
2717 mach_msg_descriptor_t * ipc_kmsg_copyin_ool_descriptor(
2718 mach_msg_ool_descriptor_t *dsc,
2719 mach_msg_descriptor_t *user_dsc,
2720 int is_64bit,
2721 vm_offset_t *paddr,
2722 vm_map_copy_t *copy,
2723 vm_size_t *space_needed,
2724 vm_map_t map,
2725 mach_msg_option_t *optionp,
2726 mach_msg_return_t *mr);
2727
2728 mach_msg_descriptor_t *
2729 ipc_kmsg_copyin_ool_descriptor(
2730 mach_msg_ool_descriptor_t *dsc,
2731 mach_msg_descriptor_t *user_dsc,
2732 int is_64bit,
2733 vm_offset_t *paddr,
2734 vm_map_copy_t *copy,
2735 vm_size_t *space_needed,
2736 vm_map_t map,
2737 __unused mach_msg_option_t *optionp,
2738 mach_msg_return_t *mr)
2739 {
2740 vm_size_t length;
2741 boolean_t dealloc;
2742 mach_msg_copy_options_t copy_options;
2743 mach_vm_offset_t addr;
2744 mach_msg_descriptor_type_t dsc_type;
2745
2746 if (is_64bit) {
2747 mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
2748
2749 addr = (mach_vm_offset_t) user_ool_dsc->address;
2750 length = user_ool_dsc->size;
2751 dealloc = user_ool_dsc->deallocate;
2752 copy_options = user_ool_dsc->copy;
2753 dsc_type = user_ool_dsc->type;
2754
2755 user_dsc = (typeof(user_dsc))(user_ool_dsc + 1);
2756 } else {
2757 mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
2758
2759 addr = CAST_USER_ADDR_T(user_ool_dsc->address);
2760 dealloc = user_ool_dsc->deallocate;
2761 copy_options = user_ool_dsc->copy;
2762 dsc_type = user_ool_dsc->type;
2763 length = user_ool_dsc->size;
2764
2765 user_dsc = (typeof(user_dsc))(user_ool_dsc + 1);
2766 }
2767
2768 dsc->size = (mach_msg_size_t)length;
2769 dsc->deallocate = dealloc;
2770 dsc->copy = copy_options;
2771 dsc->type = dsc_type;
2772
2773 if (length == 0) {
2774 dsc->address = NULL;
2775 } else if ((length >= MSG_OOL_SIZE_SMALL) &&
2776 (copy_options == MACH_MSG_PHYSICAL_COPY) && !dealloc) {
2777 /*
2778 * If the request is a physical copy and the source
2779 * is not being deallocated, then allocate space
2780 * in the kernel's pageable ipc copy map and copy
2781 * the data in. The semantics guarantee that the
2782 * data will have been physically copied before
2783 * the send operation terminates. Thus if the data
2784 * is not being deallocated, we must be prepared
2785 * to page if the region is sufficiently large.
2786 */
2787 if (copyin(addr, (char *)*paddr, length)) {
2788 *mr = MACH_SEND_INVALID_MEMORY;
2789 return NULL;
2790 }
2791
2792 /*
2793 * The kernel ipc copy map is marked no_zero_fill.
2794 * If the transfer is not a page multiple, we need
2795 * to zero fill the balance.
2796 */
2797 if (!page_aligned(length)) {
2798 (void) memset((void *) (*paddr + length), 0,
2799 round_page(length) - length);
2800 }
2801 if (vm_map_copyin(ipc_kernel_copy_map, (vm_map_address_t)*paddr,
2802 (vm_map_size_t)length, TRUE, copy) != KERN_SUCCESS) {
2803 *mr = MACH_MSG_VM_KERNEL;
2804 return NULL;
2805 }
2806 dsc->address = (void *)*copy;
2807 *paddr += round_page(length);
2808 *space_needed -= round_page(length);
2809 } else {
2810 /*
2811 * Make a vm_map_copy_t of the of the data. If the
2812 * data is small, this will do an optimized physical
2813 * copy. Otherwise, it will do a virtual copy.
2814 *
2815 * NOTE: A virtual copy is OK if the original is being
2816 * deallocted, even if a physical copy was requested.
2817 */
2818 kern_return_t kr = vm_map_copyin(map, addr,
2819 (vm_map_size_t)length, dealloc, copy);
2820 if (kr != KERN_SUCCESS) {
2821 *mr = (kr == KERN_RESOURCE_SHORTAGE) ?
2822 MACH_MSG_VM_KERNEL :
2823 MACH_SEND_INVALID_MEMORY;
2824 return NULL;
2825 }
2826 dsc->address = (void *)*copy;
2827 }
2828 return user_dsc;
2829 }
2830
2831 mach_msg_descriptor_t * ipc_kmsg_copyin_ool_ports_descriptor(
2832 mach_msg_ool_ports_descriptor_t *dsc,
2833 mach_msg_descriptor_t *user_dsc,
2834 int is_64bit,
2835 vm_map_t map,
2836 ipc_space_t space,
2837 ipc_object_t dest,
2838 ipc_kmsg_t kmsg,
2839 mach_msg_option_t *optionp,
2840 mach_msg_return_t *mr);
2841
2842 mach_msg_descriptor_t *
2843 ipc_kmsg_copyin_ool_ports_descriptor(
2844 mach_msg_ool_ports_descriptor_t *dsc,
2845 mach_msg_descriptor_t *user_dsc,
2846 int is_64bit,
2847 vm_map_t map,
2848 ipc_space_t space,
2849 ipc_object_t dest,
2850 ipc_kmsg_t kmsg,
2851 mach_msg_option_t *optionp,
2852 mach_msg_return_t *mr)
2853 {
2854 void *data;
2855 ipc_object_t *objects;
2856 unsigned int i;
2857 mach_vm_offset_t addr;
2858 mach_msg_type_name_t user_disp;
2859 mach_msg_type_name_t result_disp;
2860 mach_msg_type_number_t count;
2861 mach_msg_copy_options_t copy_option;
2862 boolean_t deallocate;
2863 mach_msg_descriptor_type_t type;
2864 vm_size_t ports_length, names_length;
2865
2866 if (is_64bit) {
2867 mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
2868
2869 addr = (mach_vm_offset_t)user_ool_dsc->address;
2870 count = user_ool_dsc->count;
2871 deallocate = user_ool_dsc->deallocate;
2872 copy_option = user_ool_dsc->copy;
2873 user_disp = user_ool_dsc->disposition;
2874 type = user_ool_dsc->type;
2875
2876 user_dsc = (typeof(user_dsc))(user_ool_dsc + 1);
2877 } else {
2878 mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
2879
2880 addr = CAST_USER_ADDR_T(user_ool_dsc->address);
2881 count = user_ool_dsc->count;
2882 deallocate = user_ool_dsc->deallocate;
2883 copy_option = user_ool_dsc->copy;
2884 user_disp = user_ool_dsc->disposition;
2885 type = user_ool_dsc->type;
2886
2887 user_dsc = (typeof(user_dsc))(user_ool_dsc + 1);
2888 }
2889
2890 dsc->deallocate = deallocate;
2891 dsc->copy = copy_option;
2892 dsc->type = type;
2893 dsc->count = count;
2894 dsc->address = NULL; /* for now */
2895
2896 result_disp = ipc_object_copyin_type(user_disp);
2897 dsc->disposition = result_disp;
2898
2899 /* We always do a 'physical copy', but you have to specify something valid */
2900 if (copy_option != MACH_MSG_PHYSICAL_COPY &&
2901 copy_option != MACH_MSG_VIRTUAL_COPY) {
2902 *mr = MACH_SEND_INVALID_TYPE;
2903 return NULL;
2904 }
2905
2906 /* calculate length of data in bytes, rounding up */
2907
2908 if (os_mul_overflow(count, sizeof(mach_port_t), &ports_length)) {
2909 *mr = MACH_SEND_TOO_LARGE;
2910 return NULL;
2911 }
2912
2913 if (os_mul_overflow(count, sizeof(mach_port_name_t), &names_length)) {
2914 *mr = MACH_SEND_TOO_LARGE;
2915 return NULL;
2916 }
2917
2918 if (ports_length == 0) {
2919 return user_dsc;
2920 }
2921
2922 data = kalloc(ports_length);
2923
2924 if (data == NULL) {
2925 *mr = MACH_SEND_NO_BUFFER;
2926 return NULL;
2927 }
2928
2929 #ifdef __LP64__
2930 mach_port_name_t *names = &((mach_port_name_t *)data)[count];
2931 #else
2932 mach_port_name_t *names = ((mach_port_name_t *)data);
2933 #endif
2934
2935 if (copyinmap(map, addr, names, names_length) != KERN_SUCCESS) {
2936 kfree(data, ports_length);
2937 *mr = MACH_SEND_INVALID_MEMORY;
2938 return NULL;
2939 }
2940
2941 if (deallocate) {
2942 (void) mach_vm_deallocate(map, addr, (mach_vm_size_t)ports_length);
2943 }
2944
2945 objects = (ipc_object_t *) data;
2946 dsc->address = data;
2947
2948 for (i = 0; i < count; i++) {
2949 mach_port_name_t name = names[i];
2950 ipc_object_t object;
2951
2952 if (!MACH_PORT_VALID(name)) {
2953 objects[i] = (ipc_object_t)CAST_MACH_NAME_TO_PORT(name);
2954 continue;
2955 }
2956
2957 kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object);
2958
2959 if (kr != KERN_SUCCESS) {
2960 unsigned int j;
2961
2962 for (j = 0; j < i; j++) {
2963 object = objects[j];
2964 if (IPC_OBJECT_VALID(object)) {
2965 ipc_object_destroy(object, result_disp);
2966 }
2967 }
2968 kfree(data, ports_length);
2969 dsc->address = NULL;
2970 if ((*optionp & MACH_SEND_KERNEL) == 0) {
2971 mach_port_guard_exception(name, 0, 0, kGUARD_EXC_SEND_INVALID_RIGHT);
2972 }
2973 *mr = MACH_SEND_INVALID_RIGHT;
2974 return NULL;
2975 }
2976
2977 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
2978 ipc_port_check_circularity(
2979 (ipc_port_t) object,
2980 (ipc_port_t) dest)) {
2981 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
2982 }
2983
2984 objects[i] = object;
2985 }
2986
2987 return user_dsc;
2988 }
2989
2990 /*
2991 * Routine: ipc_kmsg_copyin_body
2992 * Purpose:
2993 * "Copy-in" port rights and out-of-line memory
2994 * in the message body.
2995 *
2996 * In all failure cases, the message is left holding
2997 * no rights or memory. However, the message buffer
2998 * is not deallocated. If successful, the message
2999 * contains a valid destination port.
3000 * Conditions:
3001 * Nothing locked.
3002 * Returns:
3003 * MACH_MSG_SUCCESS Successful copyin.
3004 * MACH_SEND_INVALID_MEMORY Can't grab out-of-line memory.
3005 * MACH_SEND_INVALID_RIGHT Can't copyin port right in body.
3006 * MACH_SEND_INVALID_TYPE Bad type specification.
3007 * MACH_SEND_MSG_TOO_SMALL Body is too small for types/data.
3008 * MACH_SEND_INVALID_RT_OOL_SIZE OOL Buffer too large for RT
3009 * MACH_MSG_INVALID_RT_DESCRIPTOR Dealloc and RT are incompatible
3010 */
3011
3012 mach_msg_return_t
3013 ipc_kmsg_copyin_body(
3014 ipc_kmsg_t kmsg,
3015 ipc_space_t space,
3016 vm_map_t map,
3017 mach_msg_option_t *optionp)
3018 {
3019 ipc_object_t dest;
3020 mach_msg_body_t *body;
3021 mach_msg_descriptor_t *daddr, *naddr;
3022 mach_msg_descriptor_t *user_addr, *kern_addr;
3023 mach_msg_type_number_t dsc_count;
3024 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
3025 boolean_t complex = FALSE;
3026 vm_size_t space_needed = 0;
3027 vm_offset_t paddr = 0;
3028 vm_map_copy_t copy = VM_MAP_COPY_NULL;
3029 mach_msg_type_number_t i;
3030 mach_msg_return_t mr = MACH_MSG_SUCCESS;
3031
3032 vm_size_t descriptor_size = 0;
3033
3034 mach_msg_type_number_t total_ool_port_count = 0;
3035
3036 /*
3037 * Determine if the target is a kernel port.
3038 */
3039 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
3040 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
3041 naddr = (mach_msg_descriptor_t *) (body + 1);
3042
3043 dsc_count = body->msgh_descriptor_count;
3044 if (dsc_count == 0) {
3045 return MACH_MSG_SUCCESS;
3046 }
3047
3048 /*
3049 * Make an initial pass to determine kernal VM space requirements for
3050 * physical copies and possible contraction of the descriptors from
3051 * processes with pointers larger than the kernel's.
3052 */
3053 daddr = NULL;
3054 for (i = 0; i < dsc_count; i++) {
3055 mach_msg_size_t size;
3056 mach_msg_type_number_t ool_port_count = 0;
3057
3058 daddr = naddr;
3059
3060 /* make sure the descriptor fits in the message */
3061 if (is_task_64bit) {
3062 switch (daddr->type.type) {
3063 case MACH_MSG_OOL_DESCRIPTOR:
3064 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3065 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
3066 descriptor_size += 16;
3067 naddr = (typeof(naddr))((vm_offset_t)daddr + 16);
3068 break;
3069 default:
3070 descriptor_size += 12;
3071 naddr = (typeof(naddr))((vm_offset_t)daddr + 12);
3072 break;
3073 }
3074 } else {
3075 descriptor_size += 12;
3076 naddr = (typeof(naddr))((vm_offset_t)daddr + 12);
3077 }
3078
3079 if (naddr > (mach_msg_descriptor_t *)
3080 ((vm_offset_t)kmsg->ikm_header + kmsg->ikm_header->msgh_size)) {
3081 mr = MACH_SEND_MSG_TOO_SMALL;
3082 goto clean_message;
3083 }
3084
3085 switch (daddr->type.type) {
3086 case MACH_MSG_OOL_DESCRIPTOR:
3087 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3088 size = (is_task_64bit) ?
3089 ((mach_msg_ool_descriptor64_t *)daddr)->size :
3090 daddr->out_of_line.size;
3091
3092 if (daddr->out_of_line.copy != MACH_MSG_PHYSICAL_COPY &&
3093 daddr->out_of_line.copy != MACH_MSG_VIRTUAL_COPY) {
3094 /*
3095 * Invalid copy option
3096 */
3097 mr = MACH_SEND_INVALID_TYPE;
3098 goto clean_message;
3099 }
3100
3101 if ((size >= MSG_OOL_SIZE_SMALL) &&
3102 (daddr->out_of_line.copy == MACH_MSG_PHYSICAL_COPY) &&
3103 !(daddr->out_of_line.deallocate)) {
3104 /*
3105 * Out-of-line memory descriptor, accumulate kernel
3106 * memory requirements
3107 */
3108 if (space_needed + round_page(size) <= space_needed) {
3109 /* Overflow dectected */
3110 mr = MACH_MSG_VM_KERNEL;
3111 goto clean_message;
3112 }
3113
3114 space_needed += round_page(size);
3115 if (space_needed > ipc_kmsg_max_vm_space) {
3116 /* Per message kernel memory limit exceeded */
3117 mr = MACH_MSG_VM_KERNEL;
3118 goto clean_message;
3119 }
3120 }
3121 break;
3122 case MACH_MSG_PORT_DESCRIPTOR:
3123 if (os_add_overflow(total_ool_port_count, 1, &total_ool_port_count)) {
3124 /* Overflow detected */
3125 mr = MACH_SEND_TOO_LARGE;
3126 goto clean_message;
3127 }
3128 break;
3129 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
3130 ool_port_count = (is_task_64bit) ?
3131 ((mach_msg_ool_ports_descriptor64_t *)daddr)->count :
3132 daddr->ool_ports.count;
3133
3134 if (os_add_overflow(total_ool_port_count, ool_port_count, &total_ool_port_count)) {
3135 /* Overflow detected */
3136 mr = MACH_SEND_TOO_LARGE;
3137 goto clean_message;
3138 }
3139
3140 if (ool_port_count > (ipc_kmsg_max_vm_space / sizeof(mach_port_t))) {
3141 /* Per message kernel memory limit exceeded */
3142 mr = MACH_SEND_TOO_LARGE;
3143 goto clean_message;
3144 }
3145 break;
3146 }
3147 }
3148
3149 /* Sending more than 16383 rights in one message seems crazy */
3150 if (total_ool_port_count >= (MACH_PORT_UREFS_MAX / 4)) {
3151 mr = MACH_SEND_TOO_LARGE;
3152 goto clean_message;
3153 }
3154
3155 /*
3156 * Allocate space in the pageable kernel ipc copy map for all the
3157 * ool data that is to be physically copied. Map is marked wait for
3158 * space.
3159 */
3160 if (space_needed) {
3161 if (vm_allocate_kernel(ipc_kernel_copy_map, &paddr, space_needed,
3162 VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC) != KERN_SUCCESS) {
3163 mr = MACH_MSG_VM_KERNEL;
3164 goto clean_message;
3165 }
3166 }
3167
3168 /* user_addr = just after base as it was copied in */
3169 user_addr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t));
3170
3171 /* Shift the mach_msg_base_t down to make room for dsc_count*16bytes of descriptors */
3172 if (descriptor_size != 16 * dsc_count) {
3173 vm_offset_t dsc_adjust = 16 * dsc_count - descriptor_size;
3174
3175 memmove((char *)(((vm_offset_t)kmsg->ikm_header) - dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t));
3176 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header - dsc_adjust);
3177
3178 /* Update the message size for the larger in-kernel representation */
3179 kmsg->ikm_header->msgh_size += (mach_msg_size_t)dsc_adjust;
3180 }
3181
3182
3183 /* kern_addr = just after base after it has been (conditionally) moved */
3184 kern_addr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t));
3185
3186 /* handle the OOL regions and port descriptors. */
3187 for (i = 0; i < dsc_count; i++) {
3188 switch (user_addr->type.type) {
3189 case MACH_MSG_PORT_DESCRIPTOR:
3190 user_addr = ipc_kmsg_copyin_port_descriptor((mach_msg_port_descriptor_t *)kern_addr,
3191 (mach_msg_legacy_port_descriptor_t *)user_addr, space, dest, kmsg, optionp, &mr);
3192 kern_addr++;
3193 complex = TRUE;
3194 break;
3195 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3196 case MACH_MSG_OOL_DESCRIPTOR:
3197 user_addr = ipc_kmsg_copyin_ool_descriptor((mach_msg_ool_descriptor_t *)kern_addr,
3198 user_addr, is_task_64bit, &paddr, &copy, &space_needed, map, optionp, &mr);
3199 kern_addr++;
3200 complex = TRUE;
3201 break;
3202 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
3203 user_addr = ipc_kmsg_copyin_ool_ports_descriptor((mach_msg_ool_ports_descriptor_t *)kern_addr,
3204 user_addr, is_task_64bit, map, space, dest, kmsg, optionp, &mr);
3205 kern_addr++;
3206 complex = TRUE;
3207 break;
3208 default:
3209 /* Invalid descriptor */
3210 mr = MACH_SEND_INVALID_TYPE;
3211 break;
3212 }
3213
3214 if (MACH_MSG_SUCCESS != mr) {
3215 /* clean from start of message descriptors to i */
3216 ipc_kmsg_clean_partial(kmsg, i,
3217 (mach_msg_descriptor_t *)((mach_msg_base_t *)kmsg->ikm_header + 1),
3218 paddr, space_needed);
3219 goto out;
3220 }
3221 } /* End of loop */
3222
3223 if (!complex) {
3224 kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_COMPLEX;
3225 }
3226 out:
3227 return mr;
3228
3229 clean_message:
3230 /* no descriptors have been copied in yet */
3231 ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0);
3232 return mr;
3233 }
3234
3235
3236 /*
3237 * Routine: ipc_kmsg_copyin
3238 * Purpose:
3239 * "Copy-in" port rights and out-of-line memory
3240 * in the message.
3241 *
3242 * In all failure cases, the message is left holding
3243 * no rights or memory. However, the message buffer
3244 * is not deallocated. If successful, the message
3245 * contains a valid destination port.
3246 * Conditions:
3247 * Nothing locked.
3248 * Returns:
3249 * MACH_MSG_SUCCESS Successful copyin.
3250 * MACH_SEND_INVALID_HEADER
3251 * Illegal value in the message header bits.
3252 * MACH_SEND_INVALID_DEST Can't copyin destination port.
3253 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
3254 * MACH_SEND_INVALID_MEMORY Can't grab out-of-line memory.
3255 * MACH_SEND_INVALID_RIGHT Can't copyin port right in body.
3256 * MACH_SEND_INVALID_TYPE Bad type specification.
3257 * MACH_SEND_MSG_TOO_SMALL Body is too small for types/data.
3258 */
3259
3260 mach_msg_return_t
3261 ipc_kmsg_copyin(
3262 ipc_kmsg_t kmsg,
3263 ipc_space_t space,
3264 vm_map_t map,
3265 mach_msg_priority_t override,
3266 mach_msg_option_t *optionp)
3267 {
3268 mach_msg_return_t mr;
3269
3270 kmsg->ikm_header->msgh_bits &= MACH_MSGH_BITS_USER;
3271
3272 mr = ipc_kmsg_copyin_header(kmsg, space, override, optionp);
3273
3274 if (mr != MACH_MSG_SUCCESS) {
3275 return mr;
3276 }
3277
3278 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_SEND) | DBG_FUNC_NONE,
3279 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3280 (uintptr_t)kmsg->ikm_header->msgh_bits,
3281 (uintptr_t)kmsg->ikm_header->msgh_id,
3282 VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(kmsg->ikm_voucher)),
3283 0);
3284
3285 DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_copyin header:\n%.8x\n%.8x\n%p\n%p\n%p\n%.8x\n",
3286 kmsg->ikm_header->msgh_size,
3287 kmsg->ikm_header->msgh_bits,
3288 kmsg->ikm_header->msgh_remote_port,
3289 kmsg->ikm_header->msgh_local_port,
3290 kmsg->ikm_voucher,
3291 kmsg->ikm_header->msgh_id);
3292
3293 if ((kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) == 0) {
3294 return MACH_MSG_SUCCESS;
3295 }
3296
3297 mr = ipc_kmsg_copyin_body( kmsg, space, map, optionp);
3298
3299 /* unreachable if !DEBUG */
3300 __unreachable_ok_push
3301 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) {
3302 kprintf("body:\n");
3303 uint32_t i;
3304 for (i = 0; i * 4 < (kmsg->ikm_header->msgh_size - sizeof(mach_msg_header_t)); i++) {
3305 kprintf("%.4x\n", ((uint32_t *)(kmsg->ikm_header + 1))[i]);
3306 }
3307 }
3308 __unreachable_ok_pop
3309
3310 return mr;
3311 }
3312
3313 /*
3314 * Routine: ipc_kmsg_copyin_from_kernel
3315 * Purpose:
3316 * "Copy-in" port rights and out-of-line memory
3317 * in a message sent from the kernel.
3318 *
3319 * Because the message comes from the kernel,
3320 * the implementation assumes there are no errors
3321 * or peculiarities in the message.
3322 * Conditions:
3323 * Nothing locked.
3324 */
3325
3326 mach_msg_return_t
3327 ipc_kmsg_copyin_from_kernel(
3328 ipc_kmsg_t kmsg)
3329 {
3330 mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits;
3331 mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits);
3332 mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits);
3333 ipc_object_t remote = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
3334 ipc_object_t local = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
3335
3336 /* translate the destination and reply ports */
3337 if (!IO_VALID(remote)) {
3338 return MACH_SEND_INVALID_DEST;
3339 }
3340
3341 ipc_object_copyin_from_kernel(remote, rname);
3342 if (IO_VALID(local)) {
3343 ipc_object_copyin_from_kernel(local, lname);
3344 }
3345
3346 /*
3347 * The common case is a complex message with no reply port,
3348 * because that is what the memory_object interface uses.
3349 */
3350
3351 if (bits == (MACH_MSGH_BITS_COMPLEX |
3352 MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) {
3353 bits = (MACH_MSGH_BITS_COMPLEX |
3354 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0));
3355
3356 kmsg->ikm_header->msgh_bits = bits;
3357 } else {
3358 bits = (MACH_MSGH_BITS_OTHER(bits) |
3359 MACH_MSGH_BITS(ipc_object_copyin_type(rname),
3360 ipc_object_copyin_type(lname)));
3361
3362 kmsg->ikm_header->msgh_bits = bits;
3363 if ((bits & MACH_MSGH_BITS_COMPLEX) == 0) {
3364 return MACH_MSG_SUCCESS;
3365 }
3366 }
3367 {
3368 mach_msg_descriptor_t *saddr;
3369 mach_msg_body_t *body;
3370 mach_msg_type_number_t i, count;
3371
3372 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
3373 saddr = (mach_msg_descriptor_t *) (body + 1);
3374 count = body->msgh_descriptor_count;
3375
3376 for (i = 0; i < count; i++, saddr++) {
3377 switch (saddr->type.type) {
3378 case MACH_MSG_PORT_DESCRIPTOR: {
3379 mach_msg_type_name_t name;
3380 ipc_object_t object;
3381 mach_msg_port_descriptor_t *dsc;
3382
3383 dsc = &saddr->port;
3384
3385 /* this is really the type SEND, SEND_ONCE, etc. */
3386 name = dsc->disposition;
3387 object = (ipc_object_t) dsc->name;
3388 dsc->disposition = ipc_object_copyin_type(name);
3389
3390 if (!IO_VALID(object)) {
3391 break;
3392 }
3393
3394 ipc_object_copyin_from_kernel(object, name);
3395
3396 /* CDY avoid circularity when the destination is also */
3397 /* the kernel. This check should be changed into an */
3398 /* assert when the new kobject model is in place since*/
3399 /* ports will not be used in kernel to kernel chats */
3400
3401 if (((ipc_port_t)remote)->ip_receiver != ipc_space_kernel) {
3402 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3403 ipc_port_check_circularity((ipc_port_t) object,
3404 (ipc_port_t) remote)) {
3405 kmsg->ikm_header->msgh_bits |=
3406 MACH_MSGH_BITS_CIRCULAR;
3407 }
3408 }
3409 break;
3410 }
3411 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3412 case MACH_MSG_OOL_DESCRIPTOR: {
3413 /*
3414 * The sender should supply ready-made memory, i.e.
3415 * a vm_map_copy_t, so we don't need to do anything.
3416 */
3417 break;
3418 }
3419 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
3420 ipc_object_t *objects;
3421 unsigned int j;
3422 mach_msg_type_name_t name;
3423 mach_msg_ool_ports_descriptor_t *dsc;
3424
3425 dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports;
3426
3427 /* this is really the type SEND, SEND_ONCE, etc. */
3428 name = dsc->disposition;
3429 dsc->disposition = ipc_object_copyin_type(name);
3430
3431 objects = (ipc_object_t *) dsc->address;
3432
3433 for (j = 0; j < dsc->count; j++) {
3434 ipc_object_t object = objects[j];
3435
3436 if (!IO_VALID(object)) {
3437 continue;
3438 }
3439
3440 ipc_object_copyin_from_kernel(object, name);
3441
3442 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3443 ipc_port_check_circularity(
3444 (ipc_port_t) object,
3445 (ipc_port_t) remote)) {
3446 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
3447 }
3448 }
3449 break;
3450 }
3451 default: {
3452 #if MACH_ASSERT
3453 panic("ipc_kmsg_copyin_from_kernel: bad descriptor");
3454 #endif /* MACH_ASSERT */
3455 }
3456 }
3457 }
3458 }
3459 return MACH_MSG_SUCCESS;
3460 }
3461
3462 #if IKM_SUPPORT_LEGACY
3463 mach_msg_return_t
3464 ipc_kmsg_copyin_from_kernel_legacy(
3465 ipc_kmsg_t kmsg)
3466 {
3467 mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits;
3468 mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits);
3469 mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits);
3470 ipc_object_t remote = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
3471 ipc_object_t local = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
3472
3473 /* translate the destination and reply ports */
3474 if (!IO_VALID(remote)) {
3475 return MACH_SEND_INVALID_DEST;
3476 }
3477
3478 ipc_object_copyin_from_kernel(remote, rname);
3479 if (IO_VALID(local)) {
3480 ipc_object_copyin_from_kernel(local, lname);
3481 }
3482
3483 /*
3484 * The common case is a complex message with no reply port,
3485 * because that is what the memory_object interface uses.
3486 */
3487
3488 if (bits == (MACH_MSGH_BITS_COMPLEX |
3489 MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) {
3490 bits = (MACH_MSGH_BITS_COMPLEX |
3491 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0));
3492
3493 kmsg->ikm_header->msgh_bits = bits;
3494 } else {
3495 bits = (MACH_MSGH_BITS_OTHER(bits) |
3496 MACH_MSGH_BITS(ipc_object_copyin_type(rname),
3497 ipc_object_copyin_type(lname)));
3498
3499 kmsg->ikm_header->msgh_bits = bits;
3500 if ((bits & MACH_MSGH_BITS_COMPLEX) == 0) {
3501 return MACH_MSG_SUCCESS;
3502 }
3503 }
3504 {
3505 mach_msg_legacy_descriptor_t *saddr;
3506 mach_msg_descriptor_t *daddr;
3507 mach_msg_body_t *body;
3508 mach_msg_type_number_t i, count;
3509
3510 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
3511 saddr = (typeof(saddr))(body + 1);
3512 count = body->msgh_descriptor_count;
3513
3514 if (count) {
3515 vm_offset_t dsc_adjust = 4 * count;
3516 memmove((char *)(((vm_offset_t)kmsg->ikm_header) - dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t));
3517 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header - dsc_adjust);
3518 /* Update the message size for the larger in-kernel representation */
3519 kmsg->ikm_header->msgh_size += dsc_adjust;
3520 }
3521 daddr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t));
3522
3523 for (i = 0; i < count; i++, saddr++, daddr++) {
3524 switch (saddr->type.type) {
3525 case MACH_MSG_PORT_DESCRIPTOR: {
3526 mach_msg_type_name_t name;
3527 ipc_object_t object;
3528 mach_msg_legacy_port_descriptor_t *dsc;
3529 mach_msg_port_descriptor_t *dest_dsc;
3530
3531 dsc = (typeof(dsc)) & saddr->port;
3532 dest_dsc = &daddr->port;
3533
3534 /* this is really the type SEND, SEND_ONCE, etc. */
3535 name = dsc->disposition;
3536 object = (ipc_object_t) CAST_MACH_NAME_TO_PORT(dsc->name);
3537 dest_dsc->disposition = ipc_object_copyin_type(name);
3538 dest_dsc->name = (mach_port_t)object;
3539 dest_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
3540
3541 if (!IO_VALID(object)) {
3542 break;
3543 }
3544
3545 ipc_object_copyin_from_kernel(object, name);
3546
3547 /* CDY avoid circularity when the destination is also */
3548 /* the kernel. This check should be changed into an */
3549 /* assert when the new kobject model is in place since*/
3550 /* ports will not be used in kernel to kernel chats */
3551
3552 if (((ipc_port_t)remote)->ip_receiver != ipc_space_kernel) {
3553 if ((dest_dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3554 ipc_port_check_circularity((ipc_port_t) object,
3555 (ipc_port_t) remote)) {
3556 kmsg->ikm_header->msgh_bits |=
3557 MACH_MSGH_BITS_CIRCULAR;
3558 }
3559 }
3560 break;
3561 }
3562 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3563 case MACH_MSG_OOL_DESCRIPTOR: {
3564 /* The sender should supply ready-made memory, i.e. a vm_map_copy_t
3565 * so we don't need to do anything special. */
3566
3567 mach_msg_ool_descriptor32_t *source_dsc = &saddr->out_of_line32;
3568 mach_msg_ool_descriptor_t *dest_dsc = (typeof(dest_dsc)) & daddr->out_of_line;
3569
3570 vm_offset_t address = source_dsc->address;
3571 vm_size_t size = source_dsc->size;
3572 boolean_t deallocate = source_dsc->deallocate;
3573 mach_msg_copy_options_t copy = source_dsc->copy;
3574 mach_msg_descriptor_type_t type = source_dsc->type;
3575
3576 dest_dsc->address = (void *)address;
3577 dest_dsc->size = size;
3578 dest_dsc->deallocate = deallocate;
3579 dest_dsc->copy = copy;
3580 dest_dsc->type = type;
3581 break;
3582 }
3583 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
3584 ipc_object_t *objects;
3585 unsigned int j;
3586 mach_msg_type_name_t name;
3587 mach_msg_ool_ports_descriptor_t *dest_dsc;
3588
3589 mach_msg_ool_ports_descriptor32_t *source_dsc = &saddr->ool_ports32;
3590 dest_dsc = (typeof(dest_dsc)) & daddr->ool_ports;
3591
3592 boolean_t deallocate = source_dsc->deallocate;
3593 mach_msg_copy_options_t copy = source_dsc->copy;
3594 mach_msg_size_t port_count = source_dsc->count;
3595 mach_msg_type_name_t disposition = source_dsc->disposition;
3596
3597 /* this is really the type SEND, SEND_ONCE, etc. */
3598 name = disposition;
3599 disposition = ipc_object_copyin_type(name);
3600
3601 objects = (ipc_object_t *) (uintptr_t)source_dsc->address;
3602
3603 for (j = 0; j < port_count; j++) {
3604 ipc_object_t object = objects[j];
3605
3606 if (!IO_VALID(object)) {
3607 continue;
3608 }
3609
3610 ipc_object_copyin_from_kernel(object, name);
3611
3612 if ((disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3613 ipc_port_check_circularity(
3614 (ipc_port_t) object,
3615 (ipc_port_t) remote)) {
3616 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
3617 }
3618 }
3619
3620 dest_dsc->address = objects;
3621 dest_dsc->deallocate = deallocate;
3622 dest_dsc->copy = copy;
3623 dest_dsc->disposition = disposition;
3624 dest_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
3625 dest_dsc->count = port_count;
3626 break;
3627 }
3628 default: {
3629 #if MACH_ASSERT
3630 panic("ipc_kmsg_copyin_from_kernel: bad descriptor");
3631 #endif /* MACH_ASSERT */
3632 }
3633 }
3634 }
3635 }
3636 return MACH_MSG_SUCCESS;
3637 }
3638 #endif /* IKM_SUPPORT_LEGACY */
3639
3640 /*
3641 * Routine: ipc_kmsg_copyout_header
3642 * Purpose:
3643 * "Copy-out" port rights in the header of a message.
3644 * Operates atomically; if it doesn't succeed the
3645 * message header and the space are left untouched.
3646 * If it does succeed the remote/local port fields
3647 * contain port names instead of object pointers,
3648 * and the bits field is updated.
3649 * Conditions:
3650 * Nothing locked.
3651 * Returns:
3652 * MACH_MSG_SUCCESS Copied out port rights.
3653 * MACH_RCV_INVALID_NOTIFY
3654 * Notify is non-null and doesn't name a receive right.
3655 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
3656 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE
3657 * The space is dead.
3658 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE
3659 * No room in space for another name.
3660 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_KERNEL
3661 * Couldn't allocate memory for the reply port.
3662 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_KERNEL
3663 * Couldn't allocate memory for the dead-name request.
3664 */
3665
3666 mach_msg_return_t
3667 ipc_kmsg_copyout_header(
3668 ipc_kmsg_t kmsg,
3669 ipc_space_t space,
3670 mach_msg_option_t option)
3671 {
3672 mach_msg_header_t *msg = kmsg->ikm_header;
3673 mach_msg_bits_t mbits = msg->msgh_bits;
3674 ipc_port_t dest = (ipc_port_t) msg->msgh_remote_port;
3675
3676 assert(IP_VALID(dest));
3677
3678 /*
3679 * While we still hold a reference on the received-from port,
3680 * process all send-possible notfications we received along with
3681 * the message.
3682 */
3683 ipc_port_spnotify(dest);
3684
3685 {
3686 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
3687 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
3688 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
3689 ipc_port_t reply = msg->msgh_local_port;
3690 ipc_port_t release_reply_port = IP_NULL;
3691 mach_port_name_t dest_name, reply_name;
3692
3693 ipc_port_t voucher = kmsg->ikm_voucher;
3694 ipc_port_t release_voucher_port = IP_NULL;
3695 mach_port_name_t voucher_name;
3696
3697 uint32_t entries_held = 0;
3698 boolean_t need_write_lock = FALSE;
3699 kern_return_t kr;
3700
3701 /*
3702 * Reserve any potentially needed entries in the target space.
3703 * We'll free any unused before unlocking the space.
3704 */
3705 if (IP_VALID(reply)) {
3706 entries_held++;
3707 need_write_lock = TRUE;
3708 }
3709 if (IP_VALID(voucher)) {
3710 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
3711
3712 if ((option & MACH_RCV_VOUCHER) != 0) {
3713 entries_held++;
3714 }
3715 need_write_lock = TRUE;
3716 }
3717
3718 if (need_write_lock) {
3719 is_write_lock(space);
3720
3721 while (entries_held) {
3722 if (!is_active(space)) {
3723 is_write_unlock(space);
3724 return MACH_RCV_HEADER_ERROR |
3725 MACH_MSG_IPC_SPACE;
3726 }
3727
3728 kr = ipc_entries_hold(space, entries_held);
3729 if (KERN_SUCCESS == kr) {
3730 break;
3731 }
3732
3733 kr = ipc_entry_grow_table(space, ITS_SIZE_NONE);
3734 if (KERN_SUCCESS != kr) {
3735 return MACH_RCV_HEADER_ERROR |
3736 MACH_MSG_IPC_SPACE;
3737 }
3738 /* space was unlocked and relocked - retry */
3739 }
3740
3741 /* Handle reply port. */
3742 if (IP_VALID(reply)) {
3743 ipc_entry_t entry;
3744
3745 /* Is there already an entry we can use? */
3746 if ((reply_type != MACH_MSG_TYPE_PORT_SEND_ONCE) &&
3747 ipc_right_reverse(space, (ipc_object_t) reply, &reply_name, &entry)) {
3748 /* reply port is locked and active */
3749 assert(entry->ie_bits & MACH_PORT_TYPE_SEND_RECEIVE);
3750 } else {
3751 ip_lock(reply);
3752 if (!ip_active(reply)) {
3753 ip_unlock(reply);
3754
3755 release_reply_port = reply;
3756 reply = IP_DEAD;
3757 reply_name = MACH_PORT_DEAD;
3758 goto done_with_reply;
3759 }
3760
3761 /* claim a held entry for the reply port */
3762 assert(entries_held > 0);
3763 entries_held--;
3764 ipc_entry_claim(space, &reply_name, &entry);
3765 assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE);
3766 assert(entry->ie_object == IO_NULL);
3767 entry->ie_object = (ipc_object_t) reply;
3768 }
3769
3770 /* space and reply port are locked and active */
3771 ip_reference(reply); /* hold onto the reply port */
3772
3773 kr = ipc_right_copyout(space, reply_name, entry,
3774 reply_type, TRUE, (ipc_object_t) reply);
3775 assert(kr == KERN_SUCCESS);
3776 /* reply port is unlocked */
3777 } else {
3778 reply_name = CAST_MACH_PORT_TO_NAME(reply);
3779 }
3780
3781 done_with_reply:
3782
3783 /* Handle voucher port. */
3784 if (voucher_type != MACH_MSGH_BITS_ZERO) {
3785 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
3786
3787 if (!IP_VALID(voucher)) {
3788 if ((option & MACH_RCV_VOUCHER) == 0) {
3789 voucher_type = MACH_MSGH_BITS_ZERO;
3790 }
3791 voucher_name = MACH_PORT_NULL;
3792 goto done_with_voucher;
3793 }
3794
3795 /* clear voucher from its hiding place back in the kmsg */
3796 kmsg->ikm_voucher = IP_NULL;
3797
3798 if ((option & MACH_RCV_VOUCHER) != 0) {
3799 ipc_entry_t entry;
3800
3801 if (ipc_right_reverse(space, (ipc_object_t) voucher,
3802 &voucher_name, &entry)) {
3803 /* voucher port locked */
3804 assert(entry->ie_bits & MACH_PORT_TYPE_SEND);
3805 } else {
3806 assert(entries_held > 0);
3807 entries_held--;
3808 ipc_entry_claim(space, &voucher_name, &entry);
3809 assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE);
3810 assert(entry->ie_object == IO_NULL);
3811 entry->ie_object = (ipc_object_t) voucher;
3812 ip_lock(voucher);
3813 }
3814 /* space is locked and active */
3815
3816 assert(ip_active(voucher));
3817 assert(ip_kotype(voucher) == IKOT_VOUCHER);
3818 kr = ipc_right_copyout(space, voucher_name, entry,
3819 MACH_MSG_TYPE_MOVE_SEND, TRUE,
3820 (ipc_object_t) voucher);
3821 /* voucher port is unlocked */
3822 } else {
3823 voucher_type = MACH_MSGH_BITS_ZERO;
3824 release_voucher_port = voucher;
3825 voucher_name = MACH_PORT_NULL;
3826 }
3827 } else {
3828 voucher_name = msg->msgh_voucher_port;
3829 }
3830
3831 done_with_voucher:
3832
3833 ip_lock(dest);
3834 is_write_unlock(space);
3835 } else {
3836 /*
3837 * No reply or voucher port! This is an easy case.
3838 * We only need to have the space locked
3839 * when locking the destination.
3840 */
3841
3842 is_read_lock(space);
3843 if (!is_active(space)) {
3844 is_read_unlock(space);
3845 return MACH_RCV_HEADER_ERROR | MACH_MSG_IPC_SPACE;
3846 }
3847
3848 ip_lock(dest);
3849 is_read_unlock(space);
3850
3851 reply_name = CAST_MACH_PORT_TO_NAME(reply);
3852
3853 if (voucher_type != MACH_MSGH_BITS_ZERO) {
3854 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
3855 if ((option & MACH_RCV_VOUCHER) == 0) {
3856 voucher_type = MACH_MSGH_BITS_ZERO;
3857 }
3858 voucher_name = MACH_PORT_NULL;
3859 } else {
3860 voucher_name = msg->msgh_voucher_port;
3861 }
3862 }
3863
3864 /*
3865 * At this point, the space is unlocked and the destination
3866 * port is locked. (Lock taken while space was locked.)
3867 * reply_name is taken care of; we still need dest_name.
3868 * We still hold a ref for reply (if it is valid).
3869 *
3870 * If the space holds receive rights for the destination,
3871 * we return its name for the right. Otherwise the task
3872 * managed to destroy or give away the receive right between
3873 * receiving the message and this copyout. If the destination
3874 * is dead, return MACH_PORT_DEAD, and if the receive right
3875 * exists somewhere else (another space, in transit)
3876 * return MACH_PORT_NULL.
3877 *
3878 * Making this copyout operation atomic with the previous
3879 * copyout of the reply port is a bit tricky. If there was
3880 * no real reply port (it wasn't IP_VALID) then this isn't
3881 * an issue. If the reply port was dead at copyout time,
3882 * then we are OK, because if dest is dead we serialize
3883 * after the death of both ports and if dest is alive
3884 * we serialize after reply died but before dest's (later) death.
3885 * So assume reply was alive when we copied it out. If dest
3886 * is alive, then we are OK because we serialize before
3887 * the ports' deaths. So assume dest is dead when we look at it.
3888 * If reply dies/died after dest, then we are OK because
3889 * we serialize after dest died but before reply dies.
3890 * So the hard case is when reply is alive at copyout,
3891 * dest is dead at copyout, and reply died before dest died.
3892 * In this case pretend that dest is still alive, so
3893 * we serialize while both ports are alive.
3894 *
3895 * Because the space lock is held across the copyout of reply
3896 * and locking dest, the receive right for dest can't move
3897 * in or out of the space while the copyouts happen, so
3898 * that isn't an atomicity problem. In the last hard case
3899 * above, this implies that when dest is dead that the
3900 * space couldn't have had receive rights for dest at
3901 * the time reply was copied-out, so when we pretend
3902 * that dest is still alive, we can return MACH_PORT_NULL.
3903 *
3904 * If dest == reply, then we have to make it look like
3905 * either both copyouts happened before the port died,
3906 * or both happened after the port died. This special
3907 * case works naturally if the timestamp comparison
3908 * is done correctly.
3909 */
3910
3911 if (ip_active(dest)) {
3912 ipc_object_copyout_dest(space, (ipc_object_t) dest,
3913 dest_type, &dest_name);
3914 /* dest is unlocked */
3915 } else {
3916 ipc_port_timestamp_t timestamp;
3917
3918 timestamp = dest->ip_timestamp;
3919 ip_unlock(dest);
3920 ip_release(dest);
3921
3922 if (IP_VALID(reply)) {
3923 ip_lock(reply);
3924 if (ip_active(reply) ||
3925 IP_TIMESTAMP_ORDER(timestamp,
3926 reply->ip_timestamp)) {
3927 dest_name = MACH_PORT_DEAD;
3928 } else {
3929 dest_name = MACH_PORT_NULL;
3930 }
3931 ip_unlock(reply);
3932 } else {
3933 dest_name = MACH_PORT_DEAD;
3934 }
3935 }
3936
3937 if (IP_VALID(reply)) {
3938 ip_release(reply);
3939 }
3940
3941 if (IP_VALID(release_reply_port)) {
3942 if (reply_type == MACH_MSG_TYPE_PORT_SEND_ONCE) {
3943 ipc_port_release_sonce(release_reply_port);
3944 } else {
3945 ipc_port_release_send(release_reply_port);
3946 }
3947 }
3948
3949 if (IP_VALID(release_voucher_port)) {
3950 ipc_port_release_send(release_voucher_port);
3951 }
3952
3953
3954 if ((option & MACH_RCV_VOUCHER) != 0) {
3955 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV) | DBG_FUNC_NONE,
3956 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3957 (uintptr_t)kmsg->ikm_header->msgh_bits,
3958 (uintptr_t)kmsg->ikm_header->msgh_id,
3959 VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(voucher)),
3960 0);
3961 } else {
3962 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV_VOUCHER_REFUSED) | DBG_FUNC_NONE,
3963 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3964 (uintptr_t)kmsg->ikm_header->msgh_bits,
3965 (uintptr_t)kmsg->ikm_header->msgh_id,
3966 VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(voucher)),
3967 0);
3968 }
3969
3970 msg->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type,
3971 voucher_type, mbits);
3972 msg->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
3973 msg->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
3974 msg->msgh_voucher_port = voucher_name;
3975 }
3976
3977 return MACH_MSG_SUCCESS;
3978 }
3979
3980 /*
3981 * Routine: ipc_kmsg_copyout_object
3982 * Purpose:
3983 * Copy-out a port right. Always returns a name,
3984 * even for unsuccessful return codes. Always
3985 * consumes the supplied object.
3986 * Conditions:
3987 * Nothing locked.
3988 * Returns:
3989 * MACH_MSG_SUCCESS The space acquired the right
3990 * (name is valid) or the object is dead (MACH_PORT_DEAD).
3991 * MACH_MSG_IPC_SPACE No room in space for the right,
3992 * or the space is dead. (Name is MACH_PORT_NULL.)
3993 * MACH_MSG_IPC_KERNEL Kernel resource shortage.
3994 * (Name is MACH_PORT_NULL.)
3995 */
3996
3997 mach_msg_return_t
3998 ipc_kmsg_copyout_object(
3999 ipc_space_t space,
4000 ipc_object_t object,
4001 mach_msg_type_name_t msgt_name,
4002 mach_port_name_t *namep)
4003 {
4004 kern_return_t kr;
4005
4006 if (!IO_VALID(object)) {
4007 *namep = CAST_MACH_PORT_TO_NAME(object);
4008 return MACH_MSG_SUCCESS;
4009 }
4010
4011 kr = ipc_object_copyout(space, object, msgt_name, TRUE, namep);
4012 if (kr != KERN_SUCCESS) {
4013 ipc_object_destroy(object, msgt_name);
4014
4015 if (kr == KERN_INVALID_CAPABILITY) {
4016 *namep = MACH_PORT_DEAD;
4017 } else {
4018 *namep = MACH_PORT_NULL;
4019
4020 if (kr == KERN_RESOURCE_SHORTAGE) {
4021 return MACH_MSG_IPC_KERNEL;
4022 } else {
4023 return MACH_MSG_IPC_SPACE;
4024 }
4025 }
4026 }
4027
4028 return MACH_MSG_SUCCESS;
4029 }
4030
4031 mach_msg_descriptor_t *
4032 ipc_kmsg_copyout_port_descriptor(mach_msg_descriptor_t *dsc,
4033 mach_msg_descriptor_t *user_dsc,
4034 ipc_space_t space,
4035 kern_return_t *mr);
4036 mach_msg_descriptor_t *
4037 ipc_kmsg_copyout_port_descriptor(mach_msg_descriptor_t *dsc,
4038 mach_msg_descriptor_t *dest_dsc,
4039 ipc_space_t space,
4040 kern_return_t *mr)
4041 {
4042 mach_port_t port;
4043 mach_port_name_t name;
4044 mach_msg_type_name_t disp;
4045
4046
4047 /* Copyout port right carried in the message */
4048 port = dsc->port.name;
4049 disp = dsc->port.disposition;
4050 *mr |= ipc_kmsg_copyout_object(space,
4051 (ipc_object_t)port,
4052 disp,
4053 &name);
4054
4055 if (current_task() == kernel_task) {
4056 mach_msg_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc;
4057 user_dsc--; // point to the start of this port descriptor
4058 bzero((void *)user_dsc, sizeof(*user_dsc));
4059 user_dsc->name = CAST_MACH_NAME_TO_PORT(name);
4060 user_dsc->disposition = disp;
4061 user_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
4062 dest_dsc = (typeof(dest_dsc))user_dsc;
4063 } else {
4064 mach_msg_legacy_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc;
4065 user_dsc--; // point to the start of this port descriptor
4066 bzero((void *)user_dsc, sizeof(*user_dsc));
4067 user_dsc->name = CAST_MACH_PORT_TO_NAME(name);
4068 user_dsc->disposition = disp;
4069 user_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
4070 dest_dsc = (typeof(dest_dsc))user_dsc;
4071 }
4072
4073 return (mach_msg_descriptor_t *)dest_dsc;
4074 }
4075
4076 mach_msg_descriptor_t *
4077 ipc_kmsg_copyout_ool_descriptor(mach_msg_ool_descriptor_t *dsc, mach_msg_descriptor_t *user_dsc, int is_64bit, vm_map_t map, mach_msg_return_t *mr);
4078 mach_msg_descriptor_t *
4079 ipc_kmsg_copyout_ool_descriptor(mach_msg_ool_descriptor_t *dsc, mach_msg_descriptor_t *user_dsc, int is_64bit, vm_map_t map, mach_msg_return_t *mr)
4080 {
4081 vm_map_copy_t copy;
4082 vm_map_address_t rcv_addr;
4083 mach_msg_copy_options_t copy_options;
4084 vm_map_size_t size;
4085 mach_msg_descriptor_type_t dsc_type;
4086
4087 //SKIP_PORT_DESCRIPTORS(saddr, sdsc_count);
4088
4089 copy = (vm_map_copy_t)dsc->address;
4090 size = (vm_map_size_t)dsc->size;
4091 copy_options = dsc->copy;
4092 assert(copy_options != MACH_MSG_KALLOC_COPY_T);
4093 dsc_type = dsc->type;
4094
4095 if (copy != VM_MAP_COPY_NULL) {
4096 kern_return_t kr;
4097
4098 rcv_addr = 0;
4099 if (vm_map_copy_validate_size(map, copy, &size) == FALSE) {
4100 panic("Inconsistent OOL/copyout size on %p: expected %d, got %lld @%p",
4101 dsc, dsc->size, (unsigned long long)copy->size, copy);
4102 }
4103 kr = vm_map_copyout_size(map, &rcv_addr, copy, size);
4104 if (kr != KERN_SUCCESS) {
4105 if (kr == KERN_RESOURCE_SHORTAGE) {
4106 *mr |= MACH_MSG_VM_KERNEL;
4107 } else {
4108 *mr |= MACH_MSG_VM_SPACE;
4109 }
4110 vm_map_copy_discard(copy);
4111 rcv_addr = 0;
4112 size = 0;
4113 }
4114 } else {
4115 rcv_addr = 0;
4116 size = 0;
4117 }
4118
4119 /*
4120 * Now update the descriptor as the user would see it.
4121 * This may require expanding the descriptor to the user
4122 * visible size. There is already space allocated for
4123 * this in what naddr points to.
4124 */
4125 if (current_task() == kernel_task) {
4126 mach_msg_ool_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4127 user_ool_dsc--;
4128 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4129
4130 user_ool_dsc->address = (void *)(uintptr_t)rcv_addr;
4131 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4132 TRUE : FALSE;
4133 user_ool_dsc->copy = copy_options;
4134 user_ool_dsc->type = dsc_type;
4135 user_ool_dsc->size = (mach_msg_size_t)size;
4136
4137 user_dsc = (typeof(user_dsc))user_ool_dsc;
4138 } else if (is_64bit) {
4139 mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4140 user_ool_dsc--;
4141 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4142
4143 user_ool_dsc->address = rcv_addr;
4144 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4145 TRUE : FALSE;
4146 user_ool_dsc->copy = copy_options;
4147 user_ool_dsc->type = dsc_type;
4148 user_ool_dsc->size = (mach_msg_size_t)size;
4149
4150 user_dsc = (typeof(user_dsc))user_ool_dsc;
4151 } else {
4152 mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4153 user_ool_dsc--;
4154 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4155
4156 user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr);
4157 user_ool_dsc->size = (mach_msg_size_t)size;
4158 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4159 TRUE : FALSE;
4160 user_ool_dsc->copy = copy_options;
4161 user_ool_dsc->type = dsc_type;
4162
4163 user_dsc = (typeof(user_dsc))user_ool_dsc;
4164 }
4165 return user_dsc;
4166 }
4167
4168 mach_msg_descriptor_t *
4169 ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc,
4170 mach_msg_descriptor_t *user_dsc,
4171 int is_64bit,
4172 vm_map_t map,
4173 ipc_space_t space,
4174 ipc_kmsg_t kmsg,
4175 mach_msg_return_t *mr);
4176 mach_msg_descriptor_t *
4177 ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc,
4178 mach_msg_descriptor_t *user_dsc,
4179 int is_64bit,
4180 vm_map_t map,
4181 ipc_space_t space,
4182 ipc_kmsg_t kmsg,
4183 mach_msg_return_t *mr)
4184 {
4185 mach_vm_offset_t rcv_addr = 0;
4186 mach_msg_type_name_t disp;
4187 mach_msg_type_number_t count, i;
4188 vm_size_t ports_length, names_length;
4189
4190 mach_msg_copy_options_t copy_options = MACH_MSG_VIRTUAL_COPY;
4191
4192 //SKIP_PORT_DESCRIPTORS(saddr, sdsc_count);
4193
4194 count = dsc->count;
4195 disp = dsc->disposition;
4196 ports_length = count * sizeof(mach_port_t);
4197 names_length = count * sizeof(mach_port_name_t);
4198
4199 if (ports_length != 0 && dsc->address != 0) {
4200 /*
4201 * Check to see if there is an overwrite descriptor
4202 * specified in the scatter list for this ool data.
4203 * The descriptor has already been verified.
4204 */
4205 #if 0
4206 if (saddr != MACH_MSG_DESCRIPTOR_NULL) {
4207 if (differs) {
4208 OTHER_OOL_DESCRIPTOR *scatter_dsc;
4209
4210 scatter_dsc = (OTHER_OOL_DESCRIPTOR *)saddr;
4211 rcv_addr = (mach_vm_offset_t) scatter_dsc->address;
4212 copy_options = scatter_dsc->copy;
4213 } else {
4214 mach_msg_ool_descriptor_t *scatter_dsc;
4215
4216 scatter_dsc = &saddr->out_of_line;
4217 rcv_addr = CAST_USER_ADDR_T(scatter_dsc->address);
4218 copy_options = scatter_dsc->copy;
4219 }
4220 INCREMENT_SCATTER(saddr, sdsc_count, differs);
4221 }
4222 #endif
4223
4224 if (copy_options == MACH_MSG_VIRTUAL_COPY) {
4225 /*
4226 * Dynamically allocate the region
4227 */
4228 vm_tag_t tag;
4229 if (vm_kernel_map_is_kernel(map)) {
4230 tag = VM_KERN_MEMORY_IPC;
4231 } else {
4232 tag = VM_MEMORY_MACH_MSG;
4233 }
4234
4235 kern_return_t kr;
4236 if ((kr = mach_vm_allocate_kernel(map, &rcv_addr,
4237 (mach_vm_size_t)names_length,
4238 VM_FLAGS_ANYWHERE, tag)) != KERN_SUCCESS) {
4239 ipc_kmsg_clean_body(kmsg, 1, (mach_msg_descriptor_t *)dsc);
4240 rcv_addr = 0;
4241
4242 if (kr == KERN_RESOURCE_SHORTAGE) {
4243 *mr |= MACH_MSG_VM_KERNEL;
4244 } else {
4245 *mr |= MACH_MSG_VM_SPACE;
4246 }
4247 }
4248 }
4249
4250 /*
4251 * Handle the port rights and copy out the names
4252 * for those rights out to user-space.
4253 */
4254 if (rcv_addr != 0) {
4255 mach_port_t *objects = (mach_port_t *) dsc->address;
4256 mach_port_name_t *names = (mach_port_name_t *) dsc->address;
4257
4258 /* copyout port rights carried in the message */
4259
4260 for (i = 0; i < count; i++) {
4261 ipc_object_t object = (ipc_object_t)objects[i];
4262
4263 *mr |= ipc_kmsg_copyout_object(space, object,
4264 disp, &names[i]);
4265 }
4266
4267 /* copyout to memory allocated above */
4268 void *data = dsc->address;
4269 if (copyoutmap(map, data, rcv_addr, names_length) != KERN_SUCCESS) {
4270 *mr |= MACH_MSG_VM_SPACE;
4271 }
4272 kfree(data, ports_length);
4273 }
4274 } else {
4275 rcv_addr = 0;
4276 }
4277
4278 /*
4279 * Now update the descriptor based on the information
4280 * calculated above.
4281 */
4282 if (current_task() == kernel_task) {
4283 mach_msg_ool_ports_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4284 user_ool_dsc--;
4285 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4286
4287 user_ool_dsc->address = (void *)(uintptr_t)rcv_addr;
4288 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4289 TRUE : FALSE;
4290 user_ool_dsc->copy = copy_options;
4291 user_ool_dsc->disposition = disp;
4292 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
4293 user_ool_dsc->count = count;
4294
4295 user_dsc = (typeof(user_dsc))user_ool_dsc;
4296 } else if (is_64bit) {
4297 mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4298 user_ool_dsc--;
4299 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4300
4301 user_ool_dsc->address = rcv_addr;
4302 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4303 TRUE : FALSE;
4304 user_ool_dsc->copy = copy_options;
4305 user_ool_dsc->disposition = disp;
4306 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
4307 user_ool_dsc->count = count;
4308
4309 user_dsc = (typeof(user_dsc))user_ool_dsc;
4310 } else {
4311 mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4312 user_ool_dsc--;
4313 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4314
4315 user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr);
4316 user_ool_dsc->count = count;
4317 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4318 TRUE : FALSE;
4319 user_ool_dsc->copy = copy_options;
4320 user_ool_dsc->disposition = disp;
4321 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
4322
4323 user_dsc = (typeof(user_dsc))user_ool_dsc;
4324 }
4325 return user_dsc;
4326 }
4327
4328 /*
4329 * Routine: ipc_kmsg_copyout_body
4330 * Purpose:
4331 * "Copy-out" port rights and out-of-line memory
4332 * in the body of a message.
4333 *
4334 * The error codes are a combination of special bits.
4335 * The copyout proceeds despite errors.
4336 * Conditions:
4337 * Nothing locked.
4338 * Returns:
4339 * MACH_MSG_SUCCESS Successful copyout.
4340 * MACH_MSG_IPC_SPACE No room for port right in name space.
4341 * MACH_MSG_VM_SPACE No room for memory in address space.
4342 * MACH_MSG_IPC_KERNEL Resource shortage handling port right.
4343 * MACH_MSG_VM_KERNEL Resource shortage handling memory.
4344 * MACH_MSG_INVALID_RT_DESCRIPTOR Descriptor incompatible with RT
4345 */
4346
4347 mach_msg_return_t
4348 ipc_kmsg_copyout_body(
4349 ipc_kmsg_t kmsg,
4350 ipc_space_t space,
4351 vm_map_t map,
4352 mach_msg_body_t *slist)
4353 {
4354 mach_msg_body_t *body;
4355 mach_msg_descriptor_t *kern_dsc, *user_dsc;
4356 mach_msg_descriptor_t *saddr;
4357 mach_msg_type_number_t dsc_count, sdsc_count;
4358 int i;
4359 mach_msg_return_t mr = MACH_MSG_SUCCESS;
4360 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
4361
4362 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
4363 dsc_count = body->msgh_descriptor_count;
4364 kern_dsc = (mach_msg_descriptor_t *) (body + 1);
4365 /* Point user_dsc just after the end of all the descriptors */
4366 user_dsc = &kern_dsc[dsc_count];
4367
4368 /* Do scatter list setup */
4369 if (slist != MACH_MSG_BODY_NULL) {
4370 panic("Scatter lists disabled");
4371 saddr = (mach_msg_descriptor_t *) (slist + 1);
4372 sdsc_count = slist->msgh_descriptor_count;
4373 } else {
4374 saddr = MACH_MSG_DESCRIPTOR_NULL;
4375 sdsc_count = 0;
4376 }
4377
4378 /* Now process the descriptors */
4379 for (i = dsc_count - 1; i >= 0; i--) {
4380 switch (kern_dsc[i].type.type) {
4381 case MACH_MSG_PORT_DESCRIPTOR:
4382 user_dsc = ipc_kmsg_copyout_port_descriptor(&kern_dsc[i], user_dsc, space, &mr);
4383 break;
4384 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
4385 case MACH_MSG_OOL_DESCRIPTOR:
4386 user_dsc = ipc_kmsg_copyout_ool_descriptor(
4387 (mach_msg_ool_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, map, &mr);
4388 break;
4389 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
4390 user_dsc = ipc_kmsg_copyout_ool_ports_descriptor(
4391 (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, map, space, kmsg, &mr);
4392 break;
4393 default: {
4394 panic("untyped IPC copyout body: invalid message descriptor");
4395 }
4396 }
4397 }
4398
4399 if (user_dsc != kern_dsc) {
4400 vm_offset_t dsc_adjust = (vm_offset_t)user_dsc - (vm_offset_t)kern_dsc;
4401 memmove((char *)((vm_offset_t)kmsg->ikm_header + dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t));
4402 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header + dsc_adjust);
4403 /* Update the message size for the smaller user representation */
4404 kmsg->ikm_header->msgh_size -= (mach_msg_size_t)dsc_adjust;
4405 }
4406
4407 return mr;
4408 }
4409
4410 /*
4411 * Routine: ipc_kmsg_copyout_size
4412 * Purpose:
4413 * Compute the size of the message as copied out to the given
4414 * map. If the destination map's pointers are a different size
4415 * than the kernel's, we have to allow for expansion/
4416 * contraction of the descriptors as appropriate.
4417 * Conditions:
4418 * Nothing locked.
4419 * Returns:
4420 * size of the message as it would be received.
4421 */
4422
4423 mach_msg_size_t
4424 ipc_kmsg_copyout_size(
4425 ipc_kmsg_t kmsg,
4426 vm_map_t map)
4427 {
4428 mach_msg_size_t send_size;
4429
4430 send_size = kmsg->ikm_header->msgh_size;
4431
4432 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
4433
4434 #if defined(__LP64__)
4435 send_size -= LEGACY_HEADER_SIZE_DELTA;
4436 #endif
4437
4438 if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
4439 mach_msg_body_t *body;
4440 mach_msg_descriptor_t *saddr, *eaddr;
4441
4442 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
4443 saddr = (mach_msg_descriptor_t *) (body + 1);
4444 eaddr = saddr + body->msgh_descriptor_count;
4445
4446 for (; saddr < eaddr; saddr++) {
4447 switch (saddr->type.type) {
4448 case MACH_MSG_OOL_DESCRIPTOR:
4449 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
4450 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
4451 if (!is_task_64bit) {
4452 send_size -= DESC_SIZE_ADJUSTMENT;
4453 }
4454 break;
4455 case MACH_MSG_PORT_DESCRIPTOR:
4456 send_size -= DESC_SIZE_ADJUSTMENT;
4457 break;
4458 default:
4459 break;
4460 }
4461 }
4462 }
4463 return send_size;
4464 }
4465
4466 /*
4467 * Routine: ipc_kmsg_copyout
4468 * Purpose:
4469 * "Copy-out" port rights and out-of-line memory
4470 * in the message.
4471 * Conditions:
4472 * Nothing locked.
4473 * Returns:
4474 * MACH_MSG_SUCCESS Copied out all rights and memory.
4475 * MACH_RCV_HEADER_ERROR + special bits
4476 * Rights and memory in the message are intact.
4477 * MACH_RCV_BODY_ERROR + special bits
4478 * The message header was successfully copied out.
4479 * As much of the body was handled as possible.
4480 */
4481
4482 mach_msg_return_t
4483 ipc_kmsg_copyout(
4484 ipc_kmsg_t kmsg,
4485 ipc_space_t space,
4486 vm_map_t map,
4487 mach_msg_body_t *slist,
4488 mach_msg_option_t option)
4489 {
4490 mach_msg_return_t mr;
4491
4492 mr = ipc_kmsg_copyout_header(kmsg, space, option);
4493 if (mr != MACH_MSG_SUCCESS) {
4494 return mr;
4495 }
4496
4497 if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
4498 mr = ipc_kmsg_copyout_body(kmsg, space, map, slist);
4499
4500 if (mr != MACH_MSG_SUCCESS) {
4501 mr |= MACH_RCV_BODY_ERROR;
4502 }
4503 }
4504
4505 return mr;
4506 }
4507
4508 /*
4509 * Routine: ipc_kmsg_copyout_pseudo
4510 * Purpose:
4511 * Does a pseudo-copyout of the message.
4512 * This is like a regular copyout, except
4513 * that the ports in the header are handled
4514 * as if they are in the body. They aren't reversed.
4515 *
4516 * The error codes are a combination of special bits.
4517 * The copyout proceeds despite errors.
4518 * Conditions:
4519 * Nothing locked.
4520 * Returns:
4521 * MACH_MSG_SUCCESS Successful copyout.
4522 * MACH_MSG_IPC_SPACE No room for port right in name space.
4523 * MACH_MSG_VM_SPACE No room for memory in address space.
4524 * MACH_MSG_IPC_KERNEL Resource shortage handling port right.
4525 * MACH_MSG_VM_KERNEL Resource shortage handling memory.
4526 */
4527
4528 mach_msg_return_t
4529 ipc_kmsg_copyout_pseudo(
4530 ipc_kmsg_t kmsg,
4531 ipc_space_t space,
4532 vm_map_t map,
4533 mach_msg_body_t *slist)
4534 {
4535 mach_msg_bits_t mbits = kmsg->ikm_header->msgh_bits;
4536 ipc_object_t dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4537 ipc_object_t reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
4538 ipc_object_t voucher = (ipc_object_t) kmsg->ikm_voucher;
4539 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
4540 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
4541 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
4542 mach_port_name_t voucher_name = kmsg->ikm_header->msgh_voucher_port;
4543 mach_port_name_t dest_name, reply_name;
4544 mach_msg_return_t mr;
4545
4546 /* Set ith_knote to ITH_KNOTE_PSEUDO */
4547 current_thread()->ith_knote = ITH_KNOTE_PSEUDO;
4548
4549 assert(IO_VALID(dest));
4550
4551 #if 0
4552 /*
4553 * If we did this here, it looks like we wouldn't need the undo logic
4554 * at the end of ipc_kmsg_send() in the error cases. Not sure which
4555 * would be more elegant to keep.
4556 */
4557 ipc_importance_clean(kmsg);
4558 #else
4559 /* just assert it is already clean */
4560 ipc_importance_assert_clean(kmsg);
4561 #endif
4562
4563 mr = (ipc_kmsg_copyout_object(space, dest, dest_type, &dest_name) |
4564 ipc_kmsg_copyout_object(space, reply, reply_type, &reply_name));
4565
4566 kmsg->ikm_header->msgh_bits = mbits & MACH_MSGH_BITS_USER;
4567 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(dest_name);
4568 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(reply_name);
4569
4570 if (IO_VALID(voucher)) {
4571 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
4572
4573 kmsg->ikm_voucher = IP_NULL;
4574 mr |= ipc_kmsg_copyout_object(space, voucher, voucher_type, &voucher_name);
4575 kmsg->ikm_header->msgh_voucher_port = voucher_name;
4576 }
4577
4578 if (mbits & MACH_MSGH_BITS_COMPLEX) {
4579 mr |= ipc_kmsg_copyout_body(kmsg, space, map, slist);
4580 }
4581
4582 return mr;
4583 }
4584
4585 /*
4586 * Routine: ipc_kmsg_copyout_dest
4587 * Purpose:
4588 * Copies out the destination port in the message.
4589 * Destroys all other rights and memory in the message.
4590 * Conditions:
4591 * Nothing locked.
4592 */
4593
4594 void
4595 ipc_kmsg_copyout_dest(
4596 ipc_kmsg_t kmsg,
4597 ipc_space_t space)
4598 {
4599 mach_msg_bits_t mbits;
4600 ipc_object_t dest;
4601 ipc_object_t reply;
4602 ipc_object_t voucher;
4603 mach_msg_type_name_t dest_type;
4604 mach_msg_type_name_t reply_type;
4605 mach_msg_type_name_t voucher_type;
4606 mach_port_name_t dest_name, reply_name, voucher_name;
4607
4608 mbits = kmsg->ikm_header->msgh_bits;
4609 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4610 reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
4611 voucher = (ipc_object_t) kmsg->ikm_voucher;
4612 voucher_name = kmsg->ikm_header->msgh_voucher_port;
4613 dest_type = MACH_MSGH_BITS_REMOTE(mbits);
4614 reply_type = MACH_MSGH_BITS_LOCAL(mbits);
4615 voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
4616
4617 assert(IO_VALID(dest));
4618
4619 ipc_importance_assert_clean(kmsg);
4620
4621 io_lock(dest);
4622 if (io_active(dest)) {
4623 ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
4624 /* dest is unlocked */
4625 } else {
4626 io_unlock(dest);
4627 io_release(dest);
4628 dest_name = MACH_PORT_DEAD;
4629 }
4630
4631 if (IO_VALID(reply)) {
4632 ipc_object_destroy(reply, reply_type);
4633 reply_name = MACH_PORT_NULL;
4634 } else {
4635 reply_name = CAST_MACH_PORT_TO_NAME(reply);
4636 }
4637
4638 if (IO_VALID(voucher)) {
4639 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
4640
4641 kmsg->ikm_voucher = IP_NULL;
4642 ipc_object_destroy((ipc_object_t)voucher, voucher_type);
4643 voucher_name = MACH_PORT_NULL;
4644 }
4645
4646 kmsg->ikm_header->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type,
4647 voucher_type, mbits);
4648 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
4649 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
4650 kmsg->ikm_header->msgh_voucher_port = voucher_name;
4651
4652 if (mbits & MACH_MSGH_BITS_COMPLEX) {
4653 mach_msg_body_t *body;
4654
4655 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
4656 ipc_kmsg_clean_body(kmsg, body->msgh_descriptor_count,
4657 (mach_msg_descriptor_t *)(body + 1));
4658 }
4659 }
4660
4661 /*
4662 * Routine: ipc_kmsg_copyout_to_kernel
4663 * Purpose:
4664 * Copies out the destination and reply ports in the message.
4665 * Leaves all other rights and memory in the message alone.
4666 * Conditions:
4667 * Nothing locked.
4668 *
4669 * Derived from ipc_kmsg_copyout_dest.
4670 * Use by mach_msg_rpc_from_kernel (which used to use copyout_dest).
4671 * We really do want to save rights and memory.
4672 */
4673
4674 void
4675 ipc_kmsg_copyout_to_kernel(
4676 ipc_kmsg_t kmsg,
4677 ipc_space_t space)
4678 {
4679 ipc_object_t dest;
4680 mach_port_t reply;
4681 mach_msg_type_name_t dest_type;
4682 mach_msg_type_name_t reply_type;
4683 mach_port_name_t dest_name;
4684
4685 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4686 reply = kmsg->ikm_header->msgh_local_port;
4687 dest_type = MACH_MSGH_BITS_REMOTE(kmsg->ikm_header->msgh_bits);
4688 reply_type = MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits);
4689
4690 assert(IO_VALID(dest));
4691
4692 io_lock(dest);
4693 if (io_active(dest)) {
4694 ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
4695 /* dest is unlocked */
4696 } else {
4697 io_unlock(dest);
4698 io_release(dest);
4699 dest_name = MACH_PORT_DEAD;
4700 }
4701
4702 /*
4703 * While MIG kernel users don't receive vouchers, the
4704 * msgh_voucher_port field is intended to be round-tripped through the
4705 * kernel if there is no voucher disposition set. Here we check for a
4706 * non-zero voucher disposition, and consume the voucher send right as
4707 * there is no possible way to specify MACH_RCV_VOUCHER semantics.
4708 */
4709 mach_msg_type_name_t voucher_type;
4710 voucher_type = MACH_MSGH_BITS_VOUCHER(kmsg->ikm_header->msgh_bits);
4711 if (voucher_type != MACH_MSGH_BITS_ZERO) {
4712 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
4713 /*
4714 * someone managed to send this kernel routine a message with
4715 * a voucher in it. Cleanup the reference in
4716 * kmsg->ikm_voucher.
4717 */
4718 if (IP_VALID(kmsg->ikm_voucher)) {
4719 ipc_port_release_send(kmsg->ikm_voucher);
4720 }
4721 kmsg->ikm_voucher = IP_NULL;
4722 kmsg->ikm_header->msgh_voucher_port = 0;
4723 }
4724
4725 kmsg->ikm_header->msgh_bits =
4726 (MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) |
4727 MACH_MSGH_BITS(reply_type, dest_type));
4728 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
4729 kmsg->ikm_header->msgh_remote_port = reply;
4730 }
4731
4732 #if IKM_SUPPORT_LEGACY
4733 void
4734 ipc_kmsg_copyout_to_kernel_legacy(
4735 ipc_kmsg_t kmsg,
4736 ipc_space_t space)
4737 {
4738 ipc_object_t dest;
4739 mach_port_t reply;
4740 mach_msg_type_name_t dest_type;
4741 mach_msg_type_name_t reply_type;
4742 mach_port_name_t dest_name;
4743
4744 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4745 reply = kmsg->ikm_header->msgh_local_port;
4746 dest_type = MACH_MSGH_BITS_REMOTE(kmsg->ikm_header->msgh_bits);
4747 reply_type = MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits);
4748
4749 assert(IO_VALID(dest));
4750
4751 io_lock(dest);
4752 if (io_active(dest)) {
4753 ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
4754 /* dest is unlocked */
4755 } else {
4756 io_unlock(dest);
4757 io_release(dest);
4758 dest_name = MACH_PORT_DEAD;
4759 }
4760
4761 mach_msg_type_name_t voucher_type;
4762 voucher_type = MACH_MSGH_BITS_VOUCHER(kmsg->ikm_header->msgh_bits);
4763 if (voucher_type != MACH_MSGH_BITS_ZERO) {
4764 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
4765 assert(IP_VALID(kmsg->ikm_voucher));
4766 /*
4767 * someone managed to send this kernel routine a message with
4768 * a voucher in it. Cleanup the reference in
4769 * kmsg->ikm_voucher.
4770 */
4771 ipc_port_release_send(kmsg->ikm_voucher);
4772 kmsg->ikm_voucher = IP_NULL;
4773 kmsg->ikm_header->msgh_voucher_port = 0;
4774 }
4775
4776 kmsg->ikm_header->msgh_bits =
4777 (MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) |
4778 MACH_MSGH_BITS(reply_type, dest_type));
4779 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
4780 kmsg->ikm_header->msgh_remote_port = reply;
4781
4782 mach_msg_descriptor_t *saddr;
4783 mach_msg_legacy_descriptor_t *daddr;
4784 mach_msg_type_number_t i, count = ((mach_msg_base_t *)kmsg->ikm_header)->body.msgh_descriptor_count;
4785 saddr = (mach_msg_descriptor_t *) (((mach_msg_base_t *)kmsg->ikm_header) + 1);
4786 saddr = &saddr[count - 1];
4787 daddr = (mach_msg_legacy_descriptor_t *)&saddr[count];
4788 daddr--;
4789
4790 vm_offset_t dsc_adjust = 0;
4791
4792 for (i = 0; i < count; i++, saddr--, daddr--) {
4793 switch (saddr->type.type) {
4794 case MACH_MSG_PORT_DESCRIPTOR: {
4795 mach_msg_port_descriptor_t *dsc = &saddr->port;
4796 mach_msg_legacy_port_descriptor_t *dest_dsc = &daddr->port;
4797
4798 mach_port_t name = dsc->name;
4799 mach_msg_type_name_t disposition = dsc->disposition;
4800
4801 dest_dsc->name = CAST_MACH_PORT_TO_NAME(name);
4802 dest_dsc->disposition = disposition;
4803 dest_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
4804 break;
4805 }
4806 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
4807 case MACH_MSG_OOL_DESCRIPTOR: {
4808 /* The sender should supply ready-made memory, i.e. a vm_map_copy_t
4809 * so we don't need to do anything special. */
4810
4811 mach_msg_ool_descriptor_t *source_dsc = (typeof(source_dsc)) & saddr->out_of_line;
4812
4813 mach_msg_ool_descriptor32_t *dest_dsc = &daddr->out_of_line32;
4814
4815 vm_offset_t address = (vm_offset_t)source_dsc->address;
4816 vm_size_t size = source_dsc->size;
4817 boolean_t deallocate = source_dsc->deallocate;
4818 mach_msg_copy_options_t copy = source_dsc->copy;
4819 mach_msg_descriptor_type_t type = source_dsc->type;
4820
4821 dest_dsc->address = address;
4822 dest_dsc->size = size;
4823 dest_dsc->deallocate = deallocate;
4824 dest_dsc->copy = copy;
4825 dest_dsc->type = type;
4826 break;
4827 }
4828 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
4829 mach_msg_ool_ports_descriptor_t *source_dsc = (typeof(source_dsc)) & saddr->ool_ports;
4830
4831 mach_msg_ool_ports_descriptor32_t *dest_dsc = &daddr->ool_ports32;
4832
4833 vm_offset_t address = (vm_offset_t)source_dsc->address;
4834 vm_size_t port_count = source_dsc->count;
4835 boolean_t deallocate = source_dsc->deallocate;
4836 mach_msg_copy_options_t copy = source_dsc->copy;
4837 mach_msg_descriptor_type_t type = source_dsc->type;
4838
4839 dest_dsc->address = address;
4840 dest_dsc->count = port_count;
4841 dest_dsc->deallocate = deallocate;
4842 dest_dsc->copy = copy;
4843 dest_dsc->type = type;
4844 break;
4845 }
4846 default: {
4847 #if MACH_ASSERT
4848 panic("ipc_kmsg_copyin_from_kernel: bad descriptor");
4849 #endif /* MACH_ASSERT */
4850 }
4851 }
4852 }
4853
4854 if (count) {
4855 dsc_adjust = 4 * count;
4856 memmove((char *)((vm_offset_t)kmsg->ikm_header + dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t));
4857 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header + dsc_adjust);
4858 /* Update the message size for the smaller user representation */
4859 kmsg->ikm_header->msgh_size -= dsc_adjust;
4860 }
4861 }
4862 #endif /* IKM_SUPPORT_LEGACY */
4863
4864 #ifdef __arm64__
4865 /*
4866 * Just sets those parts of the trailer that aren't set up at allocation time.
4867 */
4868 static void
4869 ipc_kmsg_munge_trailer(mach_msg_max_trailer_t *in, void *_out, boolean_t is64bit)
4870 {
4871 if (is64bit) {
4872 mach_msg_max_trailer64_t *out = (mach_msg_max_trailer64_t*)_out;
4873 out->msgh_seqno = in->msgh_seqno;
4874 out->msgh_context = in->msgh_context;
4875 out->msgh_trailer_size = in->msgh_trailer_size;
4876 out->msgh_ad = in->msgh_ad;
4877 } else {
4878 mach_msg_max_trailer32_t *out = (mach_msg_max_trailer32_t*)_out;
4879 out->msgh_seqno = in->msgh_seqno;
4880 out->msgh_context = (mach_port_context32_t)in->msgh_context;
4881 out->msgh_trailer_size = in->msgh_trailer_size;
4882 out->msgh_ad = in->msgh_ad;
4883 }
4884 }
4885 #endif /* __arm64__ */
4886
4887 mach_msg_trailer_size_t
4888 ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused,
4889 mach_msg_option_t option, thread_t thread,
4890 mach_port_seqno_t seqno, boolean_t minimal_trailer,
4891 mach_vm_offset_t context)
4892 {
4893 mach_msg_max_trailer_t *trailer;
4894
4895 #ifdef __arm64__
4896 mach_msg_max_trailer_t tmp_trailer; /* This accommodates U64, and we'll munge */
4897 void *real_trailer_out = (void*)(mach_msg_max_trailer_t *)
4898 ((vm_offset_t)kmsg->ikm_header +
4899 round_msg(kmsg->ikm_header->msgh_size));
4900
4901 /*
4902 * Populate scratch with initial values set up at message allocation time.
4903 * After, we reinterpret the space in the message as the right type
4904 * of trailer for the address space in question.
4905 */
4906 bcopy(real_trailer_out, &tmp_trailer, MAX_TRAILER_SIZE);
4907 trailer = &tmp_trailer;
4908 #else /* __arm64__ */
4909 (void)thread;
4910 trailer = (mach_msg_max_trailer_t *)
4911 ((vm_offset_t)kmsg->ikm_header +
4912 round_msg(kmsg->ikm_header->msgh_size));
4913 #endif /* __arm64__ */
4914
4915 if (!(option & MACH_RCV_TRAILER_MASK)) {
4916 return trailer->msgh_trailer_size;
4917 }
4918
4919 trailer->msgh_seqno = seqno;
4920 trailer->msgh_context = context;
4921 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(thread_is_64bit_addr(thread), option);
4922
4923 if (minimal_trailer) {
4924 goto done;
4925 }
4926
4927 if (GET_RCV_ELEMENTS(option) >= MACH_RCV_TRAILER_AV) {
4928 trailer->msgh_ad = 0;
4929 }
4930
4931 /*
4932 * The ipc_kmsg_t holds a reference to the label of a label
4933 * handle, not the port. We must get a reference to the port
4934 * and a send right to copyout to the receiver.
4935 */
4936
4937 if (option & MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_LABELS)) {
4938 trailer->msgh_labels.sender = 0;
4939 }
4940
4941 done:
4942 #ifdef __arm64__
4943 ipc_kmsg_munge_trailer(trailer, real_trailer_out, thread_is_64bit_addr(thread));
4944 #endif /* __arm64__ */
4945
4946 return trailer->msgh_trailer_size;
4947 }
mirror of XNU