]> git.saurik.com Git - apple/xnu.git/blob - osfmk/ipc/ipc_kmsg.c
projects / apple / xnu.git / blob
? search:


7955cf45de23d3250d65dab8a6031ad813de394f
[apple/xnu.git] / osfmk / ipc / ipc_kmsg.c
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 * NOTICE: This file was modified by McAfee Research in 2004 to introduce
58 * support for mandatory and extensible security protections. This notice
59 * is included in support of clause 2.2 (b) of the Apple Public License,
60 * Version 2.0.
61 * Copyright (c) 2005 SPARTA, Inc.
62 */
63 /*
64 */
65 /*
66 * File: ipc/ipc_kmsg.c
67 * Author: Rich Draves
68 * Date: 1989
69 *
70 * Operations on kernel messages.
71 */
72
73
74 #include <mach/mach_types.h>
75 #include <mach/boolean.h>
76 #include <mach/kern_return.h>
77 #include <mach/message.h>
78 #include <mach/port.h>
79 #include <mach/vm_map.h>
80 #include <mach/mach_vm.h>
81 #include <mach/vm_statistics.h>
82
83 #include <kern/kern_types.h>
84 #include <kern/assert.h>
85 #include <kern/debug.h>
86 #include <kern/ipc_kobject.h>
87 #include <kern/kalloc.h>
88 #include <kern/zalloc.h>
89 #include <kern/processor.h>
90 #include <kern/thread.h>
91 #include <kern/sched_prim.h>
92 #include <kern/misc_protos.h>
93 #include <kern/counters.h>
94 #include <kern/cpu_data.h>
95 #include <kern/policy_internal.h>
96
97 #include <pthread/priority_private.h>
98
99 #include <machine/machlimits.h>
100
101 #include <vm/vm_map.h>
102 #include <vm/vm_object.h>
103 #include <vm/vm_kern.h>
104
105 #include <ipc/port.h>
106 #include <ipc/ipc_types.h>
107 #include <ipc/ipc_entry.h>
108 #include <ipc/ipc_kmsg.h>
109 #include <ipc/ipc_notify.h>
110 #include <ipc/ipc_object.h>
111 #include <ipc/ipc_space.h>
112 #include <ipc/ipc_port.h>
113 #include <ipc/ipc_right.h>
114 #include <ipc/ipc_hash.h>
115 #include <ipc/ipc_table.h>
116 #include <ipc/ipc_importance.h>
117 #if MACH_FLIPC
118 #include <kern/mach_node.h>
119 #include <ipc/flipc.h>
120 #endif
121
122 #include <os/overflow.h>
123
124 #include <security/mac_mach_internal.h>
125
126 #include <device/device_server.h>
127
128 #include <string.h>
129
130 #ifdef ppc
131 #include <ppc/Firmware.h>
132 #include <ppc/low_trace.h>
133 #endif
134
135 #if DEBUG
136 #define DEBUG_MSGS_K64 1
137 #endif
138
139 #include <sys/kdebug.h>
140 #include <libkern/OSAtomic.h>
141
142 #pragma pack(4)
143
144 typedef struct
145 {
146 mach_msg_bits_t msgh_bits;
147 mach_msg_size_t msgh_size;
148 mach_port_name_t msgh_remote_port;
149 mach_port_name_t msgh_local_port;
150 mach_port_name_t msgh_voucher_port;
151 mach_msg_id_t msgh_id;
152 } mach_msg_legacy_header_t;
153
154 typedef struct
155 {
156 mach_msg_legacy_header_t header;
157 mach_msg_body_t body;
158 } mach_msg_legacy_base_t;
159
160 typedef struct
161 {
162 mach_port_name_t name;
163 mach_msg_size_t pad1;
164 uint32_t pad2 : 16;
165 mach_msg_type_name_t disposition : 8;
166 mach_msg_descriptor_type_t type : 8;
167 } mach_msg_legacy_port_descriptor_t;
168
169
170 typedef union
171 {
172 mach_msg_legacy_port_descriptor_t port;
173 mach_msg_ool_descriptor32_t out_of_line32;
174 mach_msg_ool_ports_descriptor32_t ool_ports32;
175 mach_msg_type_descriptor_t type;
176 } mach_msg_legacy_descriptor_t;
177
178 #pragma pack()
179
180 #define LEGACY_HEADER_SIZE_DELTA ((mach_msg_size_t)(sizeof(mach_msg_header_t) - sizeof(mach_msg_legacy_header_t)))
181
182 // END LP64 fixes
183
184
185 #if DEBUG_MSGS_K64
186 extern void ipc_pset_print64(
187 ipc_pset_t pset);
188
189 extern void ipc_kmsg_print64(
190 ipc_kmsg_t kmsg,
191 const char *str);
192
193 extern void ipc_msg_print64(
194 mach_msg_header_t *msgh);
195
196 extern ipc_port_t ipc_name_to_data64(
197 task_t task,
198 mach_port_name_t name);
199
200 /*
201 * Forward declarations
202 */
203 void ipc_msg_print_untyped64(
204 mach_msg_body_t *body);
205
206 const char * ipc_type_name64(
207 int type_name,
208 boolean_t received);
209
210 void ipc_print_type_name64(
211 int type_name);
212
213 const char *
214 msgh_bit_decode64(
215 mach_msg_bits_t bit);
216
217 const char *
218 mm_copy_options_string64(
219 mach_msg_copy_options_t option);
220
221 void db_print_msg_uid64(mach_msg_header_t *);
222
223 static void
224 ipc_msg_body_print64(void *body, int size)
225 {
226 uint32_t *word = (uint32_t *) body;
227 uint32_t *end = (uint32_t *)(((uintptr_t) body) + size
228 - sizeof(mach_msg_header_t));
229 int i;
230
231 kprintf(" body(%p-%p):\n %p: ", body, end, word);
232 for (;;) {
233 for (i = 0; i < 8; i++, word++) {
234 if (word >= end) {
235 kprintf("\n");
236 return;
237 }
238 kprintf("%08x ", *word);
239 }
240 kprintf("\n %p: ", word);
241 }
242 }
243
244
245 const char *
246 ipc_type_name64(
247 int type_name,
248 boolean_t received)
249 {
250 switch (type_name) {
251 case MACH_MSG_TYPE_PORT_NAME:
252 return "port_name";
253
254 case MACH_MSG_TYPE_MOVE_RECEIVE:
255 if (received) {
256 return "port_receive";
257 } else {
258 return "move_receive";
259 }
260
261 case MACH_MSG_TYPE_MOVE_SEND:
262 if (received) {
263 return "port_send";
264 } else {
265 return "move_send";
266 }
267
268 case MACH_MSG_TYPE_MOVE_SEND_ONCE:
269 if (received) {
270 return "port_send_once";
271 } else {
272 return "move_send_once";
273 }
274
275 case MACH_MSG_TYPE_COPY_SEND:
276 return "copy_send";
277
278 case MACH_MSG_TYPE_MAKE_SEND:
279 return "make_send";
280
281 case MACH_MSG_TYPE_MAKE_SEND_ONCE:
282 return "make_send_once";
283
284 default:
285 return (char *) 0;
286 }
287 }
288
289 void
290 ipc_print_type_name64(
291 int type_name)
292 {
293 const char *name = ipc_type_name64(type_name, TRUE);
294 if (name) {
295 kprintf("%s", name);
296 } else {
297 kprintf("type%d", type_name);
298 }
299 }
300
301 /*
302 * ipc_kmsg_print64 [ debug ]
303 */
304 void
305 ipc_kmsg_print64(
306 ipc_kmsg_t kmsg,
307 const char *str)
308 {
309 kprintf("%s kmsg=%p:\n", str, kmsg);
310 kprintf(" next=%p, prev=%p, size=%d",
311 kmsg->ikm_next,
312 kmsg->ikm_prev,
313 kmsg->ikm_size);
314 kprintf("\n");
315 ipc_msg_print64(kmsg->ikm_header);
316 }
317
318 const char *
319 msgh_bit_decode64(
320 mach_msg_bits_t bit)
321 {
322 switch (bit) {
323 case MACH_MSGH_BITS_COMPLEX: return "complex";
324 case MACH_MSGH_BITS_CIRCULAR: return "circular";
325 default: return (char *) 0;
326 }
327 }
328
329 /*
330 * ipc_msg_print64 [ debug ]
331 */
332 void
333 ipc_msg_print64(
334 mach_msg_header_t *msgh)
335 {
336 mach_msg_bits_t mbits;
337 unsigned int bit, i;
338 const char *bit_name;
339 int needs_comma;
340
341 mbits = msgh->msgh_bits;
342 kprintf(" msgh_bits=0x%x: l=0x%x,r=0x%x\n",
343 mbits,
344 MACH_MSGH_BITS_LOCAL(msgh->msgh_bits),
345 MACH_MSGH_BITS_REMOTE(msgh->msgh_bits));
346
347 mbits = MACH_MSGH_BITS_OTHER(mbits) & MACH_MSGH_BITS_USED;
348 kprintf(" decoded bits: ");
349 needs_comma = 0;
350 for (i = 0, bit = 1; i < sizeof(mbits) * 8; ++i, bit <<= 1) {
351 if ((mbits & bit) == 0)
352 continue;
353 bit_name = msgh_bit_decode64((mach_msg_bits_t)bit);
354 if (bit_name)
355 kprintf("%s%s", needs_comma ? "," : "", bit_name);
356 else
357 kprintf("%sunknown(0x%x),", needs_comma ? "," : "", bit);
358 ++needs_comma;
359 }
360 if (msgh->msgh_bits & ~MACH_MSGH_BITS_USED) {
361 kprintf("%sunused=0x%x,", needs_comma ? "," : "",
362 msgh->msgh_bits & ~MACH_MSGH_BITS_USED);
363 }
364 kprintf("\n");
365
366 needs_comma = 1;
367 if (msgh->msgh_remote_port) {
368 kprintf(" remote=%p(", msgh->msgh_remote_port);
369 ipc_print_type_name64(MACH_MSGH_BITS_REMOTE(msgh->msgh_bits));
370 kprintf(")");
371 } else {
372 kprintf(" remote=null");
373 }
374
375 if (msgh->msgh_local_port) {
376 kprintf("%slocal=%p(", needs_comma ? "," : "",
377 msgh->msgh_local_port);
378 ipc_print_type_name64(MACH_MSGH_BITS_LOCAL(msgh->msgh_bits));
379 kprintf(")\n");
380 } else {
381 kprintf("local=null\n");
382 }
383
384 kprintf(" msgh_id=%d, size=%d\n",
385 msgh->msgh_id,
386 msgh->msgh_size);
387
388 if (mbits & MACH_MSGH_BITS_COMPLEX) {
389 ipc_msg_print_untyped64((mach_msg_body_t *) (msgh + 1));
390 }
391
392 ipc_msg_body_print64((void *)(msgh + 1), msgh->msgh_size);
393 }
394
395
396 const char *
397 mm_copy_options_string64(
398 mach_msg_copy_options_t option)
399 {
400 const char *name;
401
402 switch (option) {
403 case MACH_MSG_PHYSICAL_COPY:
404 name = "PHYSICAL";
405 break;
406 case MACH_MSG_VIRTUAL_COPY:
407 name = "VIRTUAL";
408 break;
409 case MACH_MSG_OVERWRITE:
410 name = "OVERWRITE(DEPRECATED)";
411 break;
412 case MACH_MSG_ALLOCATE:
413 name = "ALLOCATE";
414 break;
415 case MACH_MSG_KALLOC_COPY_T:
416 name = "KALLOC_COPY_T";
417 break;
418 default:
419 name = "unknown";
420 break;
421 }
422 return name;
423 }
424
425 void
426 ipc_msg_print_untyped64(
427 mach_msg_body_t *body)
428 {
429 mach_msg_descriptor_t *saddr, *send;
430 mach_msg_descriptor_type_t type;
431
432 kprintf(" %d descriptors: \n", body->msgh_descriptor_count);
433
434 saddr = (mach_msg_descriptor_t *) (body + 1);
435 send = saddr + body->msgh_descriptor_count;
436
437 for ( ; saddr < send; saddr++ ) {
438
439 type = saddr->type.type;
440
441 switch (type) {
442
443 case MACH_MSG_PORT_DESCRIPTOR: {
444 mach_msg_port_descriptor_t *dsc;
445
446 dsc = &saddr->port;
447 kprintf(" PORT name = %p disp = ", dsc->name);
448 ipc_print_type_name64(dsc->disposition);
449 kprintf("\n");
450 break;
451 }
452 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
453 case MACH_MSG_OOL_DESCRIPTOR: {
454 mach_msg_ool_descriptor_t *dsc;
455
456 dsc = (mach_msg_ool_descriptor_t *) &saddr->out_of_line;
457 kprintf(" OOL%s addr = %p size = 0x%x copy = %s %s\n",
458 type == MACH_MSG_OOL_DESCRIPTOR ? "" : " VOLATILE",
459 dsc->address, dsc->size,
460 mm_copy_options_string64(dsc->copy),
461 dsc->deallocate ? "DEALLOC" : "");
462 break;
463 }
464 case MACH_MSG_OOL_PORTS_DESCRIPTOR : {
465 mach_msg_ool_ports_descriptor_t *dsc;
466
467 dsc = (mach_msg_ool_ports_descriptor_t *) &saddr->ool_ports;
468
469 kprintf(" OOL_PORTS addr = %p count = 0x%x ",
470 dsc->address, dsc->count);
471 kprintf("disp = ");
472 ipc_print_type_name64(dsc->disposition);
473 kprintf(" copy = %s %s\n",
474 mm_copy_options_string64(dsc->copy),
475 dsc->deallocate ? "DEALLOC" : "");
476 break;
477 }
478
479 default: {
480 kprintf(" UNKNOWN DESCRIPTOR 0x%x\n", type);
481 break;
482 }
483 }
484 }
485 }
486
487 #define DEBUG_IPC_KMSG_PRINT(kmsg,string) \
488 __unreachable_ok_push \
489 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \
490 ipc_kmsg_print64(kmsg, string); \
491 } \
492 __unreachable_ok_pop
493
494 #define DEBUG_IPC_MSG_BODY_PRINT(body,size) \
495 __unreachable_ok_push \
496 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \
497 ipc_msg_body_print64(body,size);\
498 } \
499 __unreachable_ok_pop
500 #else /* !DEBUG_MSGS_K64 */
501 #define DEBUG_IPC_KMSG_PRINT(kmsg,string)
502 #define DEBUG_IPC_MSG_BODY_PRINT(body,size)
503 #endif /* !DEBUG_MSGS_K64 */
504
505 extern vm_map_t ipc_kernel_copy_map;
506 extern vm_size_t ipc_kmsg_max_space;
507 extern vm_size_t ipc_kmsg_max_vm_space;
508 extern vm_size_t ipc_kmsg_max_body_space;
509 extern vm_size_t msg_ool_size_small;
510
511 #define MSG_OOL_SIZE_SMALL msg_ool_size_small
512
513 #if defined(__LP64__)
514 #define MAP_SIZE_DIFFERS(map) (map->max_offset < MACH_VM_MAX_ADDRESS)
515 #define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor32_t
516 #define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor32_t
517 #else
518 #define MAP_SIZE_DIFFERS(map) (map->max_offset > VM_MAX_ADDRESS)
519 #define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor64_t
520 #define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor64_t
521 #endif
522
523 #define DESC_SIZE_ADJUSTMENT ((mach_msg_size_t)(sizeof(mach_msg_ool_descriptor64_t) - \
524 sizeof(mach_msg_ool_descriptor32_t)))
525
526 /* scatter list macros */
527
528 #define SKIP_PORT_DESCRIPTORS(s, c) \
529 MACRO_BEGIN \
530 if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \
531 while ((c) > 0) { \
532 if ((s)->type.type != MACH_MSG_PORT_DESCRIPTOR) \
533 break; \
534 (s)++; (c)--; \
535 } \
536 if (c == 0) \
537 (s) = MACH_MSG_DESCRIPTOR_NULL; \
538 } \
539 MACRO_END
540
541 #define INCREMENT_SCATTER(s, c, d) \
542 MACRO_BEGIN \
543 if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \
544 s = (d) ? (mach_msg_descriptor_t *) \
545 ((OTHER_OOL_DESCRIPTOR *)(s) + 1) : \
546 (s + 1); \
547 (c)--; \
548 } \
549 MACRO_END
550
551 #define KMSG_TRACE_FLAG_TRACED 0x000001
552 #define KMSG_TRACE_FLAG_COMPLEX 0x000002
553 #define KMSG_TRACE_FLAG_OOLMEM 0x000004
554 #define KMSG_TRACE_FLAG_VCPY 0x000008
555 #define KMSG_TRACE_FLAG_PCPY 0x000010
556 #define KMSG_TRACE_FLAG_SND64 0x000020
557 #define KMSG_TRACE_FLAG_RAISEIMP 0x000040
558 #define KMSG_TRACE_FLAG_APP_SRC 0x000080
559 #define KMSG_TRACE_FLAG_APP_DST 0x000100
560 #define KMSG_TRACE_FLAG_DAEMON_SRC 0x000200
561 #define KMSG_TRACE_FLAG_DAEMON_DST 0x000400
562 #define KMSG_TRACE_FLAG_DST_NDFLTQ 0x000800
563 #define KMSG_TRACE_FLAG_SRC_NDFLTQ 0x001000
564 #define KMSG_TRACE_FLAG_DST_SONCE 0x002000
565 #define KMSG_TRACE_FLAG_SRC_SONCE 0x004000
566 #define KMSG_TRACE_FLAG_CHECKIN 0x008000
567 #define KMSG_TRACE_FLAG_ONEWAY 0x010000
568 #define KMSG_TRACE_FLAG_IOKIT 0x020000
569 #define KMSG_TRACE_FLAG_SNDRCV 0x040000
570 #define KMSG_TRACE_FLAG_DSTQFULL 0x080000
571 #define KMSG_TRACE_FLAG_VOUCHER 0x100000
572 #define KMSG_TRACE_FLAG_TIMER 0x200000
573 #define KMSG_TRACE_FLAG_SEMA 0x400000
574 #define KMSG_TRACE_FLAG_DTMPOWNER 0x800000
575
576 #define KMSG_TRACE_FLAGS_MASK 0xffffff
577 #define KMSG_TRACE_FLAGS_SHIFT 8
578
579 #define KMSG_TRACE_PORTS_MASK 0xff
580 #define KMSG_TRACE_PORTS_SHIFT 0
581
582 #if (KDEBUG_LEVEL >= KDEBUG_LEVEL_STANDARD)
583 #include <stdint.h>
584 extern boolean_t kdebug_debugid_enabled(uint32_t debugid);
585
586 void ipc_kmsg_trace_send(ipc_kmsg_t kmsg,
587 mach_msg_option_t option)
588 {
589 task_t send_task = TASK_NULL;
590 ipc_port_t dst_port, src_port;
591 boolean_t is_task_64bit;
592 mach_msg_header_t *msg;
593 mach_msg_trailer_t *trailer;
594
595 int kotype = 0;
596 uint32_t msg_size = 0;
597 uint32_t msg_flags = KMSG_TRACE_FLAG_TRACED;
598 uint32_t num_ports = 0;
599 uint32_t send_pid, dst_pid;
600
601 /*
602 * check to see not only if ktracing is enabled, but if we will
603 * _actually_ emit the KMSG_INFO tracepoint. This saves us a
604 * significant amount of processing (and a port lock hold) in
605 * the non-tracing case.
606 */
607 if (__probable((kdebug_enable & KDEBUG_TRACE) == 0))
608 return;
609 if (!kdebug_debugid_enabled(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO)))
610 return;
611
612 msg = kmsg->ikm_header;
613
614 dst_port = (ipc_port_t)(msg->msgh_remote_port);
615 if (!IPC_PORT_VALID(dst_port))
616 return;
617
618 /*
619 * Message properties / options
620 */
621 if ((option & (MACH_SEND_MSG|MACH_RCV_MSG)) == (MACH_SEND_MSG|MACH_RCV_MSG))
622 msg_flags |= KMSG_TRACE_FLAG_SNDRCV;
623
624 if (msg->msgh_id >= is_iokit_subsystem.start &&
625 msg->msgh_id < is_iokit_subsystem.end + 100)
626 msg_flags |= KMSG_TRACE_FLAG_IOKIT;
627 /* magic XPC checkin message id (XPC_MESSAGE_ID_CHECKIN) from libxpc */
628 else if (msg->msgh_id == 0x77303074u /* w00t */)
629 msg_flags |= KMSG_TRACE_FLAG_CHECKIN;
630
631 if (msg->msgh_bits & MACH_MSGH_BITS_RAISEIMP)
632 msg_flags |= KMSG_TRACE_FLAG_RAISEIMP;
633
634 if (unsafe_convert_port_to_voucher(kmsg->ikm_voucher))
635 msg_flags |= KMSG_TRACE_FLAG_VOUCHER;
636
637 /*
638 * Sending task / port
639 */
640 send_task = current_task();
641 send_pid = task_pid(send_task);
642
643 if (send_pid != 0) {
644 if (task_is_daemon(send_task))
645 msg_flags |= KMSG_TRACE_FLAG_DAEMON_SRC;
646 else if (task_is_app(send_task))
647 msg_flags |= KMSG_TRACE_FLAG_APP_SRC;
648 }
649
650 is_task_64bit = (send_task->map->max_offset > VM_MAX_ADDRESS);
651 if (is_task_64bit)
652 msg_flags |= KMSG_TRACE_FLAG_SND64;
653
654 src_port = (ipc_port_t)(msg->msgh_local_port);
655 if (src_port) {
656 if (src_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT)
657 msg_flags |= KMSG_TRACE_FLAG_SRC_NDFLTQ;
658 switch (MACH_MSGH_BITS_LOCAL(msg->msgh_bits)) {
659 case MACH_MSG_TYPE_MOVE_SEND_ONCE:
660 msg_flags |= KMSG_TRACE_FLAG_SRC_SONCE;
661 break;
662 default:
663 break;
664 }
665 } else {
666 msg_flags |= KMSG_TRACE_FLAG_ONEWAY;
667 }
668
669
670 /*
671 * Destination task / port
672 */
673 ip_lock(dst_port);
674 if (!ip_active(dst_port)) {
675 /* dst port is being torn down */
676 dst_pid = (uint32_t)0xfffffff0;
677 } else if (dst_port->ip_tempowner) {
678 msg_flags |= KMSG_TRACE_FLAG_DTMPOWNER;
679 if (IIT_NULL != dst_port->ip_imp_task)
680 dst_pid = task_pid(dst_port->ip_imp_task->iit_task);
681 else
682 dst_pid = (uint32_t)0xfffffff1;
683 } else if (dst_port->ip_receiver_name == MACH_PORT_NULL) {
684 /* dst_port is otherwise in-transit */
685 dst_pid = (uint32_t)0xfffffff2;
686 } else {
687 if (dst_port->ip_receiver == ipc_space_kernel) {
688 dst_pid = 0;
689 } else {
690 ipc_space_t dst_space;
691 dst_space = dst_port->ip_receiver;
692 if (dst_space && is_active(dst_space)) {
693 dst_pid = task_pid(dst_space->is_task);
694 if (task_is_daemon(dst_space->is_task))
695 msg_flags |= KMSG_TRACE_FLAG_DAEMON_DST;
696 else if (task_is_app(dst_space->is_task))
697 msg_flags |= KMSG_TRACE_FLAG_APP_DST;
698 } else {
699 /* receiving task is being torn down */
700 dst_pid = (uint32_t)0xfffffff3;
701 }
702 }
703 }
704
705 if (dst_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT)
706 msg_flags |= KMSG_TRACE_FLAG_DST_NDFLTQ;
707 if (imq_full(&dst_port->ip_messages))
708 msg_flags |= KMSG_TRACE_FLAG_DSTQFULL;
709
710 kotype = ip_kotype(dst_port);
711
712 ip_unlock(dst_port);
713
714 switch (kotype) {
715 case IKOT_SEMAPHORE:
716 msg_flags |= KMSG_TRACE_FLAG_SEMA;
717 break;
718 case IKOT_TIMER:
719 case IKOT_CLOCK:
720 msg_flags |= KMSG_TRACE_FLAG_TIMER;
721 break;
722 case IKOT_MASTER_DEVICE:
723 case IKOT_IOKIT_CONNECT:
724 case IKOT_IOKIT_OBJECT:
725 case IKOT_IOKIT_IDENT:
726 msg_flags |= KMSG_TRACE_FLAG_IOKIT;
727 break;
728 default:
729 break;
730 }
731
732 switch(MACH_MSGH_BITS_REMOTE(msg->msgh_bits)) {
733 case MACH_MSG_TYPE_PORT_SEND_ONCE:
734 msg_flags |= KMSG_TRACE_FLAG_DST_SONCE;
735 break;
736 default:
737 break;
738 }
739
740
741 /*
742 * Message size / content
743 */
744 msg_size = msg->msgh_size - sizeof(mach_msg_header_t);
745
746 if (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
747 mach_msg_body_t *msg_body;
748 mach_msg_descriptor_t *kern_dsc;
749 int dsc_count;
750
751 msg_flags |= KMSG_TRACE_FLAG_COMPLEX;
752
753 msg_body = (mach_msg_body_t *)(kmsg->ikm_header + 1);
754 dsc_count = (int)msg_body->msgh_descriptor_count;
755 kern_dsc = (mach_msg_descriptor_t *)(msg_body + 1);
756
757 /* this is gross: see ipc_kmsg_copyin_body()... */
758 if (!is_task_64bit)
759 msg_size -= (dsc_count * 12);
760
761 for (int i = 0; i < dsc_count; i++) {
762 switch (kern_dsc[i].type.type) {
763 case MACH_MSG_PORT_DESCRIPTOR:
764 num_ports++;
765 if (is_task_64bit)
766 msg_size -= 12;
767 break;
768 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
769 case MACH_MSG_OOL_DESCRIPTOR: {
770 mach_msg_ool_descriptor_t *dsc;
771 dsc = (mach_msg_ool_descriptor_t *)&kern_dsc[i];
772 msg_flags |= KMSG_TRACE_FLAG_OOLMEM;
773 msg_size += dsc->size;
774 if ((dsc->size >= MSG_OOL_SIZE_SMALL) &&
775 (dsc->copy == MACH_MSG_PHYSICAL_COPY) &&
776 !dsc->deallocate)
777 msg_flags |= KMSG_TRACE_FLAG_PCPY;
778 else if (dsc->size <= MSG_OOL_SIZE_SMALL)
779 msg_flags |= KMSG_TRACE_FLAG_PCPY;
780 else
781 msg_flags |= KMSG_TRACE_FLAG_VCPY;
782 if (is_task_64bit)
783 msg_size -= 16;
784 } break;
785 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
786 mach_msg_ool_ports_descriptor_t *dsc;
787 dsc = (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i];
788 num_ports += dsc->count;
789 if (is_task_64bit)
790 msg_size -= 16;
791 } break;
792 default:
793 break;
794 }
795 }
796 }
797
798 /*
799 * Trailer contents
800 */
801 trailer = (mach_msg_trailer_t *)((vm_offset_t)msg +
802 round_msg((vm_offset_t)msg->msgh_size));
803 if (trailer->msgh_trailer_size <= sizeof(mach_msg_security_trailer_t)) {
804 extern security_token_t KERNEL_SECURITY_TOKEN;
805 mach_msg_security_trailer_t *strailer;
806 strailer = (mach_msg_security_trailer_t *)trailer;
807 /*
808 * verify the sender PID: replies from the kernel often look
809 * like self-talk because the sending port is not reset.
810 */
811 if (memcmp(&strailer->msgh_sender,
812 &KERNEL_SECURITY_TOKEN,
813 sizeof(KERNEL_SECURITY_TOKEN)) == 0) {
814 send_pid = 0;
815 msg_flags &= ~(KMSG_TRACE_FLAG_APP_SRC | KMSG_TRACE_FLAG_DAEMON_SRC);
816 }
817 }
818
819 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END,
820 (uintptr_t)send_pid,
821 (uintptr_t)dst_pid,
822 (uintptr_t)msg_size,
823 (uintptr_t)(
824 ((msg_flags & KMSG_TRACE_FLAGS_MASK) << KMSG_TRACE_FLAGS_SHIFT) |
825 ((num_ports & KMSG_TRACE_PORTS_MASK) << KMSG_TRACE_PORTS_SHIFT)
826 )
827 );
828 }
829 #endif
830
831 /* zone for cached ipc_kmsg_t structures */
832 zone_t ipc_kmsg_zone;
833
834 /*
835 * Forward declarations
836 */
837
838 void ipc_kmsg_clean(
839 ipc_kmsg_t kmsg);
840
841 void ipc_kmsg_clean_body(
842 ipc_kmsg_t kmsg,
843 mach_msg_type_number_t number,
844 mach_msg_descriptor_t *desc);
845
846 void ipc_kmsg_clean_partial(
847 ipc_kmsg_t kmsg,
848 mach_msg_type_number_t number,
849 mach_msg_descriptor_t *desc,
850 vm_offset_t paddr,
851 vm_size_t length);
852
853 mach_msg_return_t ipc_kmsg_copyin_body(
854 ipc_kmsg_t kmsg,
855 ipc_space_t space,
856 vm_map_t map,
857 mach_msg_option_t *optionp);
858
859 /*
860 * We keep a per-processor cache of kernel message buffers.
861 * The cache saves the overhead/locking of using kalloc/kfree.
862 * The per-processor cache seems to miss less than a per-thread cache,
863 * and it also uses less memory. Access to the cache doesn't
864 * require locking.
865 */
866
867 /*
868 * Routine: ipc_kmsg_alloc
869 * Purpose:
870 * Allocate a kernel message structure. If we can get one from
871 * the cache, that is best. Otherwise, allocate a new one.
872 * Conditions:
873 * Nothing locked.
874 */
875 ipc_kmsg_t
876 ipc_kmsg_alloc(
877 mach_msg_size_t msg_and_trailer_size)
878 {
879 mach_msg_size_t max_expanded_size;
880 ipc_kmsg_t kmsg;
881
882 /*
883 * LP64support -
884 * Pad the allocation in case we need to expand the
885 * message descrptors for user spaces with pointers larger than
886 * the kernel's own, or vice versa. We don't know how many descriptors
887 * there are yet, so just assume the whole body could be
888 * descriptors (if there could be any at all).
889 *
890 * The expansion space is left in front of the header,
891 * because it is easier to pull the header and descriptors
892 * forward as we process them than it is to push all the
893 * data backwards.
894 */
895 mach_msg_size_t size = msg_and_trailer_size - MAX_TRAILER_SIZE;
896
897 /* compare against implementation upper limit for the body */
898 if (size > ipc_kmsg_max_body_space)
899 return IKM_NULL;
900
901 if (size > sizeof(mach_msg_base_t)) {
902 mach_msg_size_t max_desc = (mach_msg_size_t)(((size - sizeof(mach_msg_base_t)) /
903 sizeof(mach_msg_ool_descriptor32_t)) *
904 DESC_SIZE_ADJUSTMENT);
905
906 /* make sure expansion won't cause wrap */
907 if (msg_and_trailer_size > MACH_MSG_SIZE_MAX - max_desc)
908 return IKM_NULL;
909
910 max_expanded_size = msg_and_trailer_size + max_desc;
911 } else
912 max_expanded_size = msg_and_trailer_size;
913
914 if (max_expanded_size < IKM_SAVED_MSG_SIZE)
915 max_expanded_size = IKM_SAVED_MSG_SIZE; /* round up for ikm_cache */
916
917 if (max_expanded_size == IKM_SAVED_MSG_SIZE) {
918 kmsg = (ipc_kmsg_t)zalloc(ipc_kmsg_zone);
919 } else {
920 kmsg = (ipc_kmsg_t)kalloc(ikm_plus_overhead(max_expanded_size));
921 }
922
923 if (kmsg != IKM_NULL) {
924 ikm_init(kmsg, max_expanded_size);
925 ikm_set_header(kmsg, msg_and_trailer_size);
926 }
927
928 return(kmsg);
929 }
930
931 /*
932 * Routine: ipc_kmsg_free
933 * Purpose:
934 * Free a kernel message buffer. If the kms is preallocated
935 * to a port, just "put it back (marked unused)." We have to
936 * do this with the port locked. The port may have its hold
937 * on our message released. In that case, we have to just
938 * revert the message to a traditional one and free it normally.
939 * Conditions:
940 * Nothing locked.
941 */
942
943 void
944 ipc_kmsg_free(
945 ipc_kmsg_t kmsg)
946 {
947 mach_msg_size_t size = kmsg->ikm_size;
948 ipc_port_t port;
949
950 assert(!IP_VALID(kmsg->ikm_voucher));
951
952 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_FREE) | DBG_FUNC_NONE,
953 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
954 0, 0, 0, 0);
955
956 /*
957 * Check to see if the message is bound to the port. If so,
958 * mark it not in use. If the port isn't already dead, then
959 * leave the message associated with it. Otherwise, free it.
960 */
961 port = ikm_prealloc_inuse_port(kmsg);
962 if (port != IP_NULL) {
963 ip_lock(port);
964 ikm_prealloc_clear_inuse(kmsg, port);
965 if (ip_active(port) && (port->ip_premsg == kmsg)) {
966 assert(IP_PREALLOC(port));
967 ip_unlock(port);
968 ip_release(port);
969 return;
970 }
971 ip_unlock(port);
972 ip_release(port); /* May be last reference */
973 }
974
975 if (kmsg->ikm_size == IKM_SAVED_MSG_SIZE) {
976 zfree(ipc_kmsg_zone, kmsg);
977 return;
978 }
979 kfree(kmsg, ikm_plus_overhead(size));
980 }
981
982
983 /*
984 * Routine: ipc_kmsg_enqueue
985 * Purpose:
986 * Enqueue a kmsg.
987 */
988
989 void
990 ipc_kmsg_enqueue(
991 ipc_kmsg_queue_t queue,
992 ipc_kmsg_t kmsg)
993 {
994 ipc_kmsg_t first = queue->ikmq_base;
995 ipc_kmsg_t last;
996
997 if (first == IKM_NULL) {
998 queue->ikmq_base = kmsg;
999 kmsg->ikm_next = kmsg;
1000 kmsg->ikm_prev = kmsg;
1001 } else {
1002 last = first->ikm_prev;
1003 kmsg->ikm_next = first;
1004 kmsg->ikm_prev = last;
1005 first->ikm_prev = kmsg;
1006 last->ikm_next = kmsg;
1007 }
1008 }
1009
1010 /*
1011 * Routine: ipc_kmsg_enqueue_qos
1012 * Purpose:
1013 * Enqueue a kmsg, propagating qos
1014 * overrides towards the head of the queue.
1015 *
1016 * Returns:
1017 * whether the head of the queue had
1018 * it's override-qos adjusted because
1019 * of this insertion.
1020 */
1021
1022 boolean_t
1023 ipc_kmsg_enqueue_qos(
1024 ipc_kmsg_queue_t queue,
1025 ipc_kmsg_t kmsg)
1026 {
1027 ipc_kmsg_t first = queue->ikmq_base;
1028 ipc_kmsg_t prev;
1029 mach_msg_priority_t override;
1030
1031 if (first == IKM_NULL) {
1032 /* insert a first message */
1033 queue->ikmq_base = kmsg;
1034 kmsg->ikm_next = kmsg;
1035 kmsg->ikm_prev = kmsg;
1036 return TRUE;
1037 }
1038
1039 /* insert at the tail */
1040 prev = first->ikm_prev;
1041 kmsg->ikm_next = first;
1042 kmsg->ikm_prev = prev;
1043 first->ikm_prev = kmsg;
1044 prev->ikm_next = kmsg;
1045
1046 /* apply QoS overrides towards the head */
1047 override = kmsg->ikm_qos_override;
1048 while (prev != kmsg &&
1049 override > prev->ikm_qos_override) {
1050 prev->ikm_qos_override = override;
1051 prev = prev->ikm_prev;
1052 }
1053
1054 /* did we adjust everything? */
1055 return (prev == kmsg);
1056 }
1057
1058 /*
1059 * Routine: ipc_kmsg_override_qos
1060 * Purpose:
1061 * Update the override for a given kmsg already
1062 * enqueued, propagating qos override adjustments
1063 * towards the head of the queue.
1064 *
1065 * Returns:
1066 * whether the head of the queue had
1067 * it's override-qos adjusted because
1068 * of this insertion.
1069 */
1070
1071 boolean_t
1072 ipc_kmsg_override_qos(
1073 ipc_kmsg_queue_t queue,
1074 ipc_kmsg_t kmsg,
1075 mach_msg_priority_t override)
1076 {
1077 ipc_kmsg_t first = queue->ikmq_base;
1078 ipc_kmsg_t cur = kmsg;
1079
1080 /* apply QoS overrides towards the head */
1081 while (override > cur->ikm_qos_override) {
1082 cur->ikm_qos_override = override;
1083 if (cur == first)
1084 return TRUE;
1085 cur = cur->ikm_prev;
1086 }
1087 return FALSE;
1088 }
1089
1090 /*
1091 * Routine: ipc_kmsg_dequeue
1092 * Purpose:
1093 * Dequeue and return a kmsg.
1094 */
1095
1096 ipc_kmsg_t
1097 ipc_kmsg_dequeue(
1098 ipc_kmsg_queue_t queue)
1099 {
1100 ipc_kmsg_t first;
1101
1102 first = ipc_kmsg_queue_first(queue);
1103
1104 if (first != IKM_NULL)
1105 ipc_kmsg_rmqueue(queue, first);
1106
1107 return first;
1108 }
1109
1110 /*
1111 * Routine: ipc_kmsg_rmqueue
1112 * Purpose:
1113 * Pull a kmsg out of a queue.
1114 */
1115
1116 void
1117 ipc_kmsg_rmqueue(
1118 ipc_kmsg_queue_t queue,
1119 ipc_kmsg_t kmsg)
1120 {
1121 ipc_kmsg_t next, prev;
1122
1123 assert(queue->ikmq_base != IKM_NULL);
1124
1125 next = kmsg->ikm_next;
1126 prev = kmsg->ikm_prev;
1127
1128 if (next == kmsg) {
1129 assert(prev == kmsg);
1130 assert(queue->ikmq_base == kmsg);
1131
1132 queue->ikmq_base = IKM_NULL;
1133 } else {
1134 if (__improbable(next->ikm_prev != kmsg || prev->ikm_next != kmsg)) {
1135 panic("ipc_kmsg_rmqueue: inconsistent prev/next pointers. "
1136 "(prev->next: %p, next->prev: %p, kmsg: %p)",
1137 prev->ikm_next, next->ikm_prev, kmsg);
1138 }
1139
1140 if (queue->ikmq_base == kmsg)
1141 queue->ikmq_base = next;
1142
1143 next->ikm_prev = prev;
1144 prev->ikm_next = next;
1145 }
1146 /* XXX Temporary debug logic */
1147 assert((kmsg->ikm_next = IKM_BOGUS) == IKM_BOGUS);
1148 assert((kmsg->ikm_prev = IKM_BOGUS) == IKM_BOGUS);
1149 }
1150
1151 /*
1152 * Routine: ipc_kmsg_queue_next
1153 * Purpose:
1154 * Return the kmsg following the given kmsg.
1155 * (Or IKM_NULL if it is the last one in the queue.)
1156 */
1157
1158 ipc_kmsg_t
1159 ipc_kmsg_queue_next(
1160 ipc_kmsg_queue_t queue,
1161 ipc_kmsg_t kmsg)
1162 {
1163 ipc_kmsg_t next;
1164
1165 assert(queue->ikmq_base != IKM_NULL);
1166
1167 next = kmsg->ikm_next;
1168 if (queue->ikmq_base == next)
1169 next = IKM_NULL;
1170
1171 return next;
1172 }
1173
1174 /*
1175 * Routine: ipc_kmsg_destroy
1176 * Purpose:
1177 * Destroys a kernel message. Releases all rights,
1178 * references, and memory held by the message.
1179 * Frees the message.
1180 * Conditions:
1181 * No locks held.
1182 */
1183
1184 void
1185 ipc_kmsg_destroy(
1186 ipc_kmsg_t kmsg)
1187 {
1188 /*
1189 * Destroying a message can cause more messages to be destroyed.
1190 * Curtail recursion by putting messages on the deferred
1191 * destruction queue. If this was the first message on the
1192 * queue, this instance must process the full queue.
1193 */
1194 if (ipc_kmsg_delayed_destroy(kmsg))
1195 ipc_kmsg_reap_delayed();
1196 }
1197
1198 /*
1199 * Routine: ipc_kmsg_delayed_destroy
1200 * Purpose:
1201 * Enqueues a kernel message for deferred destruction.
1202 * Returns:
1203 * Boolean indicator that the caller is responsible to reap
1204 * deferred messages.
1205 */
1206
1207 boolean_t ipc_kmsg_delayed_destroy(
1208 ipc_kmsg_t kmsg)
1209 {
1210 ipc_kmsg_queue_t queue = &(current_thread()->ith_messages);
1211 boolean_t first = ipc_kmsg_queue_empty(queue);
1212
1213 ipc_kmsg_enqueue(queue, kmsg);
1214 return first;
1215 }
1216
1217 /*
1218 * Routine: ipc_kmsg_destroy_queue
1219 * Purpose:
1220 * Destroys messages from the per-thread
1221 * deferred reaping queue.
1222 * Conditions:
1223 * No locks held.
1224 */
1225
1226 void
1227 ipc_kmsg_reap_delayed(void)
1228 {
1229 ipc_kmsg_queue_t queue = &(current_thread()->ith_messages);
1230 ipc_kmsg_t kmsg;
1231
1232 /*
1233 * must leave kmsg in queue while cleaning it to assure
1234 * no nested calls recurse into here.
1235 */
1236 while ((kmsg = ipc_kmsg_queue_first(queue)) != IKM_NULL) {
1237 ipc_kmsg_clean(kmsg);
1238 ipc_kmsg_rmqueue(queue, kmsg);
1239 ipc_kmsg_free(kmsg);
1240 }
1241 }
1242
1243 /*
1244 * Routine: ipc_kmsg_clean_body
1245 * Purpose:
1246 * Cleans the body of a kernel message.
1247 * Releases all rights, references, and memory.
1248 *
1249 * Conditions:
1250 * No locks held.
1251 */
1252 static unsigned int _ipc_kmsg_clean_invalid_desc = 0;
1253 void
1254 ipc_kmsg_clean_body(
1255 __unused ipc_kmsg_t kmsg,
1256 mach_msg_type_number_t number,
1257 mach_msg_descriptor_t *saddr)
1258 {
1259 mach_msg_type_number_t i;
1260
1261 if ( number == 0 )
1262 return;
1263
1264 for (i = 0 ; i < number; i++, saddr++ ) {
1265
1266 switch (saddr->type.type) {
1267
1268 case MACH_MSG_PORT_DESCRIPTOR: {
1269 mach_msg_port_descriptor_t *dsc;
1270
1271 dsc = &saddr->port;
1272
1273 /*
1274 * Destroy port rights carried in the message
1275 */
1276 if (!IO_VALID((ipc_object_t) dsc->name))
1277 continue;
1278 ipc_object_destroy((ipc_object_t) dsc->name, dsc->disposition);
1279 break;
1280 }
1281 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
1282 case MACH_MSG_OOL_DESCRIPTOR : {
1283 mach_msg_ool_descriptor_t *dsc;
1284
1285 dsc = (mach_msg_ool_descriptor_t *)&saddr->out_of_line;
1286
1287 /*
1288 * Destroy memory carried in the message
1289 */
1290 if (dsc->size == 0) {
1291 assert(dsc->address == (void *) 0);
1292 } else {
1293 vm_map_copy_discard((vm_map_copy_t) dsc->address);
1294 }
1295 break;
1296 }
1297 case MACH_MSG_OOL_PORTS_DESCRIPTOR : {
1298 ipc_object_t *objects;
1299 mach_msg_type_number_t j;
1300 mach_msg_ool_ports_descriptor_t *dsc;
1301
1302 dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports;
1303 objects = (ipc_object_t *) dsc->address;
1304
1305 if (dsc->count == 0) {
1306 break;
1307 }
1308
1309 assert(objects != (ipc_object_t *) 0);
1310
1311 /* destroy port rights carried in the message */
1312
1313 for (j = 0; j < dsc->count; j++) {
1314 ipc_object_t object = objects[j];
1315
1316 if (!IO_VALID(object))
1317 continue;
1318
1319 ipc_object_destroy(object, dsc->disposition);
1320 }
1321
1322 /* destroy memory carried in the message */
1323
1324 assert(dsc->count != 0);
1325
1326 kfree(dsc->address,
1327 (vm_size_t) dsc->count * sizeof(mach_port_t));
1328 break;
1329 }
1330 default : {
1331 _ipc_kmsg_clean_invalid_desc++; /* don't understand this type of descriptor */
1332 }
1333 }
1334 }
1335 }
1336
1337 /*
1338 * Routine: ipc_kmsg_clean_partial
1339 * Purpose:
1340 * Cleans a partially-acquired kernel message.
1341 * number is the index of the type descriptor
1342 * in the body of the message that contained the error.
1343 * If dolast, the memory and port rights in this last
1344 * type spec are also cleaned. In that case, number
1345 * specifies the number of port rights to clean.
1346 * Conditions:
1347 * Nothing locked.
1348 */
1349
1350 void
1351 ipc_kmsg_clean_partial(
1352 ipc_kmsg_t kmsg,
1353 mach_msg_type_number_t number,
1354 mach_msg_descriptor_t *desc,
1355 vm_offset_t paddr,
1356 vm_size_t length)
1357 {
1358 ipc_object_t object;
1359 mach_msg_bits_t mbits = kmsg->ikm_header->msgh_bits;
1360
1361 /* deal with importance chain while we still have dest and voucher references */
1362 ipc_importance_clean(kmsg);
1363
1364 object = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
1365 assert(IO_VALID(object));
1366 ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits));
1367
1368 object = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
1369 if (IO_VALID(object))
1370 ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits));
1371
1372 object = (ipc_object_t) kmsg->ikm_voucher;
1373 if (IO_VALID(object)) {
1374 assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND);
1375 ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND);
1376 kmsg->ikm_voucher = IP_NULL;
1377 }
1378
1379 if (paddr) {
1380 (void) vm_deallocate(ipc_kernel_copy_map, paddr, length);
1381 }
1382
1383 ipc_kmsg_clean_body(kmsg, number, desc);
1384 }
1385
1386 /*
1387 * Routine: ipc_kmsg_clean
1388 * Purpose:
1389 * Cleans a kernel message. Releases all rights,
1390 * references, and memory held by the message.
1391 * Conditions:
1392 * No locks held.
1393 */
1394
1395 void
1396 ipc_kmsg_clean(
1397 ipc_kmsg_t kmsg)
1398 {
1399 ipc_object_t object;
1400 mach_msg_bits_t mbits;
1401
1402 /* deal with importance chain while we still have dest and voucher references */
1403 ipc_importance_clean(kmsg);
1404
1405 mbits = kmsg->ikm_header->msgh_bits;
1406 object = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
1407 if (IO_VALID(object))
1408 ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits));
1409
1410 object = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
1411 if (IO_VALID(object))
1412 ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits));
1413
1414 object = (ipc_object_t) kmsg->ikm_voucher;
1415 if (IO_VALID(object)) {
1416 assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND);
1417 ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND);
1418 kmsg->ikm_voucher = IP_NULL;
1419 }
1420
1421 if (mbits & MACH_MSGH_BITS_COMPLEX) {
1422 mach_msg_body_t *body;
1423
1424 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
1425 ipc_kmsg_clean_body(kmsg, body->msgh_descriptor_count,
1426 (mach_msg_descriptor_t *)(body + 1));
1427 }
1428 }
1429
1430 /*
1431 * Routine: ipc_kmsg_set_prealloc
1432 * Purpose:
1433 * Assign a kmsg as a preallocated message buffer to a port.
1434 * Conditions:
1435 * port locked.
1436 */
1437
1438 void
1439 ipc_kmsg_set_prealloc(
1440 ipc_kmsg_t kmsg,
1441 ipc_port_t port)
1442 {
1443 assert(kmsg->ikm_prealloc == IP_NULL);
1444
1445 kmsg->ikm_prealloc = IP_NULL;
1446
1447 assert(port_send_turnstile(port) == TURNSTILE_NULL);
1448 kmsg->ikm_turnstile = TURNSTILE_NULL;
1449 IP_SET_PREALLOC(port, kmsg);
1450 }
1451
1452 /*
1453 * Routine: ipc_kmsg_clear_prealloc
1454 * Purpose:
1455 * Release the Assignment of a preallocated message buffer from a port.
1456 * Conditions:
1457 * port locked.
1458 */
1459 void
1460 ipc_kmsg_clear_prealloc(
1461 ipc_kmsg_t kmsg,
1462 ipc_port_t port)
1463 {
1464 /* take the mqueue lock since the turnstile is protected under it */
1465 imq_lock(&port->ip_messages);
1466
1467 IP_CLEAR_PREALLOC(port, kmsg);
1468 set_port_send_turnstile(port, kmsg->ikm_turnstile);
1469 imq_unlock(&port->ip_messages);
1470 }
1471
1472 /*
1473 * Routine: ipc_kmsg_prealloc
1474 * Purpose:
1475 * Wraper to ipc_kmsg_alloc() to account for
1476 * header expansion requirements.
1477 */
1478 ipc_kmsg_t
1479 ipc_kmsg_prealloc(mach_msg_size_t size)
1480 {
1481 #if defined(__LP64__)
1482 if (size > MACH_MSG_SIZE_MAX - LEGACY_HEADER_SIZE_DELTA)
1483 return IKM_NULL;
1484
1485 size += LEGACY_HEADER_SIZE_DELTA;
1486 #endif
1487 return ipc_kmsg_alloc(size);
1488 }
1489
1490
1491 /*
1492 * Routine: ipc_kmsg_get
1493 * Purpose:
1494 * Allocates a kernel message buffer.
1495 * Copies a user message to the message buffer.
1496 * Conditions:
1497 * Nothing locked.
1498 * Returns:
1499 * MACH_MSG_SUCCESS Acquired a message buffer.
1500 * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
1501 * MACH_SEND_MSG_TOO_SMALL Message size not long-word multiple.
1502 * MACH_SEND_TOO_LARGE Message too large to ever be sent.
1503 * MACH_SEND_NO_BUFFER Couldn't allocate a message buffer.
1504 * MACH_SEND_INVALID_DATA Couldn't copy message data.
1505 */
1506
1507 mach_msg_return_t
1508 ipc_kmsg_get(
1509 mach_vm_address_t msg_addr,
1510 mach_msg_size_t size,
1511 ipc_kmsg_t *kmsgp)
1512 {
1513 mach_msg_size_t msg_and_trailer_size;
1514 ipc_kmsg_t kmsg;
1515 mach_msg_max_trailer_t *trailer;
1516 mach_msg_legacy_base_t legacy_base;
1517 mach_msg_size_t len_copied;
1518 legacy_base.body.msgh_descriptor_count = 0;
1519
1520 if ((size < sizeof(mach_msg_legacy_header_t)) || (size & 3))
1521 return MACH_SEND_MSG_TOO_SMALL;
1522
1523 if (size > ipc_kmsg_max_body_space)
1524 return MACH_SEND_TOO_LARGE;
1525
1526 if(size == sizeof(mach_msg_legacy_header_t))
1527 len_copied = sizeof(mach_msg_legacy_header_t);
1528 else
1529 len_copied = sizeof(mach_msg_legacy_base_t);
1530
1531 if (copyinmsg(msg_addr, (char *)&legacy_base, len_copied))
1532 return MACH_SEND_INVALID_DATA;
1533
1534 /*
1535 * If the message claims to be complex, it must at least
1536 * have the length of a "base" message (header + dsc_count).
1537 */
1538 if (len_copied < sizeof(mach_msg_legacy_base_t) &&
1539 (legacy_base.header.msgh_bits & MACH_MSGH_BITS_COMPLEX))
1540 return MACH_SEND_MSG_TOO_SMALL;
1541
1542 msg_addr += sizeof(legacy_base.header);
1543 #if defined(__LP64__)
1544 size += LEGACY_HEADER_SIZE_DELTA;
1545 #endif
1546 /* unreachable if !DEBUG */
1547 __unreachable_ok_push
1548 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) {
1549 unsigned int j;
1550 for (j=0; j<sizeof(legacy_base.header); j++) {
1551 kprintf("%02x\n", ((unsigned char*)&legacy_base.header)[j]);
1552 }
1553 }
1554 __unreachable_ok_pop
1555
1556 msg_and_trailer_size = size + MAX_TRAILER_SIZE;
1557 kmsg = ipc_kmsg_alloc(msg_and_trailer_size);
1558 if (kmsg == IKM_NULL)
1559 return MACH_SEND_NO_BUFFER;
1560
1561 kmsg->ikm_header->msgh_size = size;
1562 kmsg->ikm_header->msgh_bits = legacy_base.header.msgh_bits;
1563 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(legacy_base.header.msgh_remote_port);
1564 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(legacy_base.header.msgh_local_port);
1565 kmsg->ikm_header->msgh_voucher_port = legacy_base.header.msgh_voucher_port;
1566 kmsg->ikm_header->msgh_id = legacy_base.header.msgh_id;
1567
1568 DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_get header:\n"
1569 " size: 0x%.8x\n"
1570 " bits: 0x%.8x\n"
1571 " remote_port: %p\n"
1572 " local_port: %p\n"
1573 " voucher_port: 0x%.8x\n"
1574 " id: %.8d\n",
1575 kmsg->ikm_header->msgh_size,
1576 kmsg->ikm_header->msgh_bits,
1577 kmsg->ikm_header->msgh_remote_port,
1578 kmsg->ikm_header->msgh_local_port,
1579 kmsg->ikm_header->msgh_voucher_port,
1580 kmsg->ikm_header->msgh_id);
1581
1582 if (copyinmsg(msg_addr, (char *)(kmsg->ikm_header + 1), size - (mach_msg_size_t)sizeof(mach_msg_header_t))) {
1583 ipc_kmsg_free(kmsg);
1584 return MACH_SEND_INVALID_DATA;
1585 }
1586
1587 /* unreachable if !DEBUG */
1588 __unreachable_ok_push
1589 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK))
1590 {
1591 kprintf("body: size: %lu\n", (size - sizeof(mach_msg_header_t)));
1592 uint32_t i;
1593 for(i=0;i*4 < (size - sizeof(mach_msg_header_t));i++)
1594 {
1595 kprintf("%.4x\n",((uint32_t *)(kmsg->ikm_header + 1))[i]);
1596 }
1597 }
1598 __unreachable_ok_pop
1599 DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_get()");
1600
1601 /*
1602 * I reserve for the trailer the largest space (MAX_TRAILER_SIZE)
1603 * However, the internal size field of the trailer (msgh_trailer_size)
1604 * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to optimize
1605 * the cases where no implicit data is requested.
1606 */
1607 trailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + size);
1608 trailer->msgh_sender = current_thread()->task->sec_token;
1609 trailer->msgh_audit = current_thread()->task->audit_token;
1610 trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
1611 trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
1612
1613 #ifdef ppc
1614 if(trcWork.traceMask) dbgTrace(0x1100, (unsigned int)kmsg->ikm_header->msgh_id,
1615 (unsigned int)kmsg->ikm_header->msgh_remote_port,
1616 (unsigned int)kmsg->ikm_header->msgh_local_port, 0);
1617 #endif
1618
1619 trailer->msgh_labels.sender = 0;
1620 *kmsgp = kmsg;
1621 return MACH_MSG_SUCCESS;
1622 }
1623
1624 /*
1625 * Routine: ipc_kmsg_get_from_kernel
1626 * Purpose:
1627 * First checks for a preallocated message
1628 * reserved for kernel clients. If not found -
1629 * allocates a new kernel message buffer.
1630 * Copies a kernel message to the message buffer.
1631 * Only resource errors are allowed.
1632 * Conditions:
1633 * Nothing locked.
1634 * Ports in header are ipc_port_t.
1635 * Returns:
1636 * MACH_MSG_SUCCESS Acquired a message buffer.
1637 * MACH_SEND_NO_BUFFER Couldn't allocate a message buffer.
1638 */
1639
1640 mach_msg_return_t
1641 ipc_kmsg_get_from_kernel(
1642 mach_msg_header_t *msg,
1643 mach_msg_size_t size,
1644 ipc_kmsg_t *kmsgp)
1645 {
1646 ipc_kmsg_t kmsg;
1647 mach_msg_size_t msg_and_trailer_size;
1648 mach_msg_max_trailer_t *trailer;
1649 ipc_port_t dest_port;
1650
1651 assert(size >= sizeof(mach_msg_header_t));
1652 assert((size & 3) == 0);
1653
1654 dest_port = (ipc_port_t)msg->msgh_remote_port;
1655
1656 msg_and_trailer_size = size + MAX_TRAILER_SIZE;
1657
1658 /*
1659 * See if the port has a pre-allocated kmsg for kernel
1660 * clients. These are set up for those kernel clients
1661 * which cannot afford to wait.
1662 */
1663 if (IP_VALID(dest_port) && IP_PREALLOC(dest_port)) {
1664 mach_msg_size_t max_desc = 0;
1665
1666 ip_lock(dest_port);
1667 if (!ip_active(dest_port)) {
1668 ip_unlock(dest_port);
1669 return MACH_SEND_NO_BUFFER;
1670 }
1671 assert(IP_PREALLOC(dest_port));
1672 kmsg = dest_port->ip_premsg;
1673 if (ikm_prealloc_inuse(kmsg)) {
1674 ip_unlock(dest_port);
1675 return MACH_SEND_NO_BUFFER;
1676 }
1677 #if !defined(__LP64__)
1678 if (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
1679 assert(size > sizeof(mach_msg_base_t));
1680 max_desc = ((mach_msg_base_t *)msg)->body.msgh_descriptor_count *
1681 DESC_SIZE_ADJUSTMENT;
1682 }
1683 #endif
1684 if (msg_and_trailer_size > kmsg->ikm_size - max_desc) {
1685 ip_unlock(dest_port);
1686 return MACH_SEND_TOO_LARGE;
1687 }
1688 ikm_prealloc_set_inuse(kmsg, dest_port);
1689 ikm_set_header(kmsg, msg_and_trailer_size);
1690 ip_unlock(dest_port);
1691 }
1692 else
1693 {
1694 kmsg = ipc_kmsg_alloc(msg_and_trailer_size);
1695 if (kmsg == IKM_NULL)
1696 return MACH_SEND_NO_BUFFER;
1697 }
1698
1699 (void) memcpy((void *) kmsg->ikm_header, (const void *) msg, size);
1700
1701 ikm_qos_init(kmsg);
1702
1703 kmsg->ikm_header->msgh_size = size;
1704
1705 /*
1706 * I reserve for the trailer the largest space (MAX_TRAILER_SIZE)
1707 * However, the internal size field of the trailer (msgh_trailer_size)
1708 * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to
1709 * optimize the cases where no implicit data is requested.
1710 */
1711 trailer = (mach_msg_max_trailer_t *)
1712 ((vm_offset_t)kmsg->ikm_header + size);
1713 trailer->msgh_sender = KERNEL_SECURITY_TOKEN;
1714 trailer->msgh_audit = KERNEL_AUDIT_TOKEN;
1715 trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
1716 trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
1717
1718 trailer->msgh_labels.sender = 0;
1719
1720 *kmsgp = kmsg;
1721 return MACH_MSG_SUCCESS;
1722 }
1723
1724 /*
1725 * Routine: ipc_kmsg_send
1726 * Purpose:
1727 * Send a message. The message holds a reference
1728 * for the destination port in the msgh_remote_port field.
1729 *
1730 * If unsuccessful, the caller still has possession of
1731 * the message and must do something with it. If successful,
1732 * the message is queued, given to a receiver, destroyed,
1733 * or handled directly by the kernel via mach_msg.
1734 * Conditions:
1735 * Nothing locked.
1736 * Returns:
1737 * MACH_MSG_SUCCESS The message was accepted.
1738 * MACH_SEND_TIMED_OUT Caller still has message.
1739 * MACH_SEND_INTERRUPTED Caller still has message.
1740 * MACH_SEND_INVALID_DEST Caller still has message.
1741 */
1742 mach_msg_return_t
1743 ipc_kmsg_send(
1744 ipc_kmsg_t kmsg,
1745 mach_msg_option_t option,
1746 mach_msg_timeout_t send_timeout)
1747 {
1748 ipc_port_t port;
1749 thread_t th = current_thread();
1750 mach_msg_return_t error = MACH_MSG_SUCCESS;
1751 boolean_t kernel_reply = FALSE;
1752
1753 /* Check if honor qlimit flag is set on thread. */
1754 if ((th->options & TH_OPT_HONOR_QLIMIT) == TH_OPT_HONOR_QLIMIT) {
1755 /* Remove the MACH_SEND_ALWAYS flag to honor queue limit. */
1756 option &= (~MACH_SEND_ALWAYS);
1757 /* Add the timeout flag since the message queue might be full. */
1758 option |= MACH_SEND_TIMEOUT;
1759 th->options &= (~TH_OPT_HONOR_QLIMIT);
1760 }
1761
1762 #if IMPORTANCE_INHERITANCE
1763 bool did_importance = false;
1764 #if IMPORTANCE_TRACE
1765 mach_msg_id_t imp_msgh_id = -1;
1766 int sender_pid = -1;
1767 #endif /* IMPORTANCE_TRACE */
1768 #endif /* IMPORTANCE_INHERITANCE */
1769
1770 /* don't allow the creation of a circular loop */
1771 if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_CIRCULAR) {
1772 ipc_kmsg_destroy(kmsg);
1773 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_MSGH_BITS_CIRCULAR);
1774 return MACH_MSG_SUCCESS;
1775 }
1776
1777 ipc_voucher_send_preprocessing(kmsg);
1778
1779 port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port;
1780 assert(IP_VALID(port));
1781 ip_lock(port);
1782
1783 #if IMPORTANCE_INHERITANCE
1784 retry:
1785 #endif /* IMPORTANCE_INHERITANCE */
1786 /*
1787 * Can't deliver to a dead port.
1788 * However, we can pretend it got sent
1789 * and was then immediately destroyed.
1790 */
1791 if (!ip_active(port)) {
1792 ip_unlock(port);
1793 #if MACH_FLIPC
1794 if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport))
1795 flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
1796 #endif
1797 if (did_importance) {
1798 /*
1799 * We're going to pretend we delivered this message
1800 * successfully, and just eat the kmsg. However, the
1801 * kmsg is actually visible via the importance_task!
1802 * We need to cleanup this linkage before we destroy
1803 * the message, and more importantly before we set the
1804 * msgh_remote_port to NULL. See: 34302571
1805 */
1806 ipc_importance_clean(kmsg);
1807 }
1808 ip_release(port); /* JMM - Future: release right, not just ref */
1809 kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
1810 ipc_kmsg_destroy(kmsg);
1811 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST);
1812 return MACH_MSG_SUCCESS;
1813 }
1814
1815 if (port->ip_receiver == ipc_space_kernel) {
1816
1817 /*
1818 * We can check ip_receiver == ipc_space_kernel
1819 * before checking that the port is active because
1820 * ipc_port_dealloc_kernel clears ip_receiver
1821 * before destroying a kernel port.
1822 */
1823 assert(ip_active(port));
1824 port->ip_messages.imq_seqno++;
1825 ip_unlock(port);
1826
1827 current_task()->messages_sent++;
1828
1829 /*
1830 * Call the server routine, and get the reply message to send.
1831 */
1832 kmsg = ipc_kobject_server(kmsg, option);
1833 if (kmsg == IKM_NULL)
1834 return MACH_MSG_SUCCESS;
1835
1836 /* restart the KMSG_INFO tracing for the reply message */
1837 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_START);
1838 port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port;
1839 assert(IP_VALID(port));
1840 ip_lock(port);
1841 /* fall thru with reply - same options */
1842 kernel_reply = TRUE;
1843 if (!ip_active(port))
1844 error = MACH_SEND_INVALID_DEST;
1845 }
1846
1847 #if IMPORTANCE_INHERITANCE
1848 /*
1849 * Need to see if this message needs importance donation and/or
1850 * propagation. That routine can drop the port lock temporarily.
1851 * If it does we'll have to revalidate the destination.
1852 */
1853 if (!did_importance) {
1854 did_importance = true;
1855 if (ipc_importance_send(kmsg, option))
1856 goto retry;
1857 }
1858 #endif /* IMPORTANCE_INHERITANCE */
1859
1860 if (error != MACH_MSG_SUCCESS) {
1861 ip_unlock(port);
1862 } else {
1863 /*
1864 * We have a valid message and a valid reference on the port.
1865 * we can unlock the port and call mqueue_send() on its message
1866 * queue. Lock message queue while port is locked.
1867 */
1868 imq_lock(&port->ip_messages);
1869
1870 set_ip_srp_msg_sent(port);
1871
1872 ip_unlock(port);
1873
1874 error = ipc_mqueue_send(&port->ip_messages, kmsg, option,
1875 send_timeout);
1876 }
1877
1878 #if IMPORTANCE_INHERITANCE
1879 if (did_importance) {
1880 __unused int importance_cleared = 0;
1881 switch (error) {
1882 case MACH_SEND_TIMED_OUT:
1883 case MACH_SEND_NO_BUFFER:
1884 case MACH_SEND_INTERRUPTED:
1885 case MACH_SEND_INVALID_DEST:
1886 /*
1887 * We still have the kmsg and its
1888 * reference on the port. But we
1889 * have to back out the importance
1890 * boost.
1891 *
1892 * The port could have changed hands,
1893 * be inflight to another destination,
1894 * etc... But in those cases our
1895 * back-out will find the new owner
1896 * (and all the operations that
1897 * transferred the right should have
1898 * applied their own boost adjustments
1899 * to the old owner(s)).
1900 */
1901 importance_cleared = 1;
1902 ipc_importance_clean(kmsg);
1903 break;
1904
1905 case MACH_MSG_SUCCESS:
1906 default:
1907 break;
1908 }
1909 #if IMPORTANCE_TRACE
1910 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_END,
1911 task_pid(current_task()), sender_pid, imp_msgh_id, importance_cleared, 0);
1912 #endif /* IMPORTANCE_TRACE */
1913 }
1914 #endif /* IMPORTANCE_INHERITANCE */
1915
1916 /*
1917 * If the port has been destroyed while we wait, treat the message
1918 * as a successful delivery (like we do for an inactive port).
1919 */
1920 if (error == MACH_SEND_INVALID_DEST) {
1921 #if MACH_FLIPC
1922 if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport))
1923 flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
1924 #endif
1925 ip_release(port); /* JMM - Future: release right, not just ref */
1926 kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
1927 ipc_kmsg_destroy(kmsg);
1928 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST);
1929 return MACH_MSG_SUCCESS;
1930 }
1931
1932 if (error != MACH_MSG_SUCCESS && kernel_reply) {
1933 /*
1934 * Kernel reply messages that fail can't be allowed to
1935 * pseudo-receive on error conditions. We need to just treat
1936 * the message as a successful delivery.
1937 */
1938 #if MACH_FLIPC
1939 if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport))
1940 flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
1941 #endif
1942 ip_release(port); /* JMM - Future: release right, not just ref */
1943 kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
1944 ipc_kmsg_destroy(kmsg);
1945 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, error);
1946 return MACH_MSG_SUCCESS;
1947 }
1948 return error;
1949 }
1950
1951 /*
1952 * Routine: ipc_kmsg_put
1953 * Purpose:
1954 * Copies a message buffer to a user message.
1955 * Copies only the specified number of bytes.
1956 * Frees the message buffer.
1957 * Conditions:
1958 * Nothing locked. The message buffer must have clean
1959 * header fields.
1960 * Returns:
1961 * MACH_MSG_SUCCESS Copied data out of message buffer.
1962 * MACH_RCV_INVALID_DATA Couldn't copy to user message.
1963 */
1964
1965 mach_msg_return_t
1966 ipc_kmsg_put(
1967 ipc_kmsg_t kmsg,
1968 mach_msg_option_t option,
1969 mach_vm_address_t rcv_addr,
1970 mach_msg_size_t rcv_size,
1971 mach_msg_size_t trailer_size,
1972 mach_msg_size_t *sizep)
1973 {
1974 mach_msg_size_t size = kmsg->ikm_header->msgh_size + trailer_size;
1975 mach_msg_return_t mr;
1976
1977 DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_put()");
1978
1979
1980 DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_put header:\n"
1981 " size: 0x%.8x\n"
1982 " bits: 0x%.8x\n"
1983 " remote_port: %p\n"
1984 " local_port: %p\n"
1985 " voucher_port: 0x%.8x\n"
1986 " id: %.8d\n",
1987 kmsg->ikm_header->msgh_size,
1988 kmsg->ikm_header->msgh_bits,
1989 kmsg->ikm_header->msgh_remote_port,
1990 kmsg->ikm_header->msgh_local_port,
1991 kmsg->ikm_header->msgh_voucher_port,
1992 kmsg->ikm_header->msgh_id);
1993
1994 #if defined(__LP64__)
1995 if (current_task() != kernel_task) { /* don't if receiver expects fully-cooked in-kernel msg; */
1996 mach_msg_legacy_header_t *legacy_header =
1997 (mach_msg_legacy_header_t *)((vm_offset_t)(kmsg->ikm_header) + LEGACY_HEADER_SIZE_DELTA);
1998
1999 mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits;
2000 mach_msg_size_t msg_size = kmsg->ikm_header->msgh_size;
2001 mach_port_name_t remote_port = CAST_MACH_PORT_TO_NAME(kmsg->ikm_header->msgh_remote_port);
2002 mach_port_name_t local_port = CAST_MACH_PORT_TO_NAME(kmsg->ikm_header->msgh_local_port);
2003 mach_port_name_t voucher_port = kmsg->ikm_header->msgh_voucher_port;
2004 mach_msg_id_t id = kmsg->ikm_header->msgh_id;
2005
2006 legacy_header->msgh_id = id;
2007 legacy_header->msgh_local_port = local_port;
2008 legacy_header->msgh_remote_port = remote_port;
2009 legacy_header->msgh_voucher_port = voucher_port;
2010 legacy_header->msgh_size = msg_size - LEGACY_HEADER_SIZE_DELTA;
2011 legacy_header->msgh_bits = bits;
2012
2013 size -= LEGACY_HEADER_SIZE_DELTA;
2014 kmsg->ikm_header = (mach_msg_header_t *)legacy_header;
2015 }
2016 #endif
2017
2018 /* unreachable if !DEBUG */
2019 __unreachable_ok_push
2020 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) {
2021 kprintf("ipc_kmsg_put header+body: %d\n", (size));
2022 uint32_t i;
2023 for(i=0;i*4 < size;i++)
2024 {
2025 kprintf("%.4x\n",((uint32_t *)kmsg->ikm_header)[i]);
2026 }
2027 kprintf("type: %d\n", ((mach_msg_type_descriptor_t *)(((mach_msg_base_t *)kmsg->ikm_header)+1))->type);
2028 }
2029 __unreachable_ok_pop
2030
2031 /* Re-Compute target address if using stack-style delivery */
2032 if (option & MACH_RCV_STACK) {
2033 rcv_addr += rcv_size - size;
2034 }
2035
2036 if (copyoutmsg((const char *) kmsg->ikm_header, rcv_addr, size)) {
2037 mr = MACH_RCV_INVALID_DATA;
2038 size = 0;
2039 } else
2040 mr = MACH_MSG_SUCCESS;
2041
2042 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_LINK) | DBG_FUNC_NONE,
2043 (rcv_addr >= VM_MIN_KERNEL_AND_KEXT_ADDRESS ||
2044 rcv_addr + size >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) ? (uintptr_t)0 : (uintptr_t)rcv_addr,
2045 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
2046 1 /* this is on the receive/copyout path */,
2047 0,
2048 0);
2049 ipc_kmsg_free(kmsg);
2050
2051 if (sizep)
2052 *sizep = size;
2053 return mr;
2054 }
2055
2056 /*
2057 * Routine: ipc_kmsg_put_to_kernel
2058 * Purpose:
2059 * Copies a message buffer to a kernel message.
2060 * Frees the message buffer.
2061 * No errors allowed.
2062 * Conditions:
2063 * Nothing locked.
2064 */
2065
2066 void
2067 ipc_kmsg_put_to_kernel(
2068 mach_msg_header_t *msg,
2069 ipc_kmsg_t kmsg,
2070 mach_msg_size_t size)
2071 {
2072 (void) memcpy((void *) msg, (const void *) kmsg->ikm_header, size);
2073
2074 ipc_kmsg_free(kmsg);
2075 }
2076
2077 static mach_msg_priority_t
2078 ipc_get_current_thread_priority(void)
2079 {
2080 thread_t thread = current_thread();
2081 thread_qos_t qos;
2082 int relpri;
2083
2084 qos = thread_get_requested_qos(thread, &relpri);
2085 if (!qos) {
2086 qos = thread_user_promotion_qos_for_pri(thread->base_pri);
2087 relpri = 0;
2088 }
2089 return (mach_msg_priority_t)_pthread_priority_make_from_thread_qos(qos, relpri, 0);
2090 }
2091
2092 static kern_return_t
2093 ipc_kmsg_set_qos(
2094 ipc_kmsg_t kmsg,
2095 mach_msg_option_t options,
2096 mach_msg_priority_t override)
2097 {
2098 kern_return_t kr;
2099 ipc_port_t special_reply_port = kmsg->ikm_header->msgh_local_port;
2100 ipc_port_t dest_port = kmsg->ikm_header->msgh_remote_port;
2101
2102 kr = ipc_get_pthpriority_from_kmsg_voucher(kmsg, &kmsg->ikm_qos);
2103 if (kr != KERN_SUCCESS) {
2104 if (options & MACH_SEND_PROPAGATE_QOS) {
2105 kmsg->ikm_qos = ipc_get_current_thread_priority();
2106 } else {
2107 kmsg->ikm_qos = MACH_MSG_PRIORITY_UNSPECIFIED;
2108 }
2109 }
2110 kmsg->ikm_qos_override = kmsg->ikm_qos;
2111
2112 if (options & MACH_SEND_OVERRIDE) {
2113 pthread_priority_t pp = _pthread_priority_normalize_for_ipc(override);
2114 if (pp > kmsg->ikm_qos)
2115 kmsg->ikm_qos_override = (mach_msg_priority_t)pp;
2116 }
2117
2118 kr = KERN_SUCCESS;
2119 if ((options & MACH_SEND_SYNC_OVERRIDE)) {
2120 if (IP_VALID(special_reply_port) &&
2121 MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits) == MACH_MSG_TYPE_PORT_SEND_ONCE) {
2122 /*
2123 * Link the destination port to special reply port and make sure that
2124 * dest port has a send turnstile, else allocate one.
2125 */
2126 ipc_port_link_special_reply_port(special_reply_port, dest_port);
2127 }
2128 }
2129 return kr;
2130 }
2131
2132 /*
2133 * Routine: ipc_kmsg_copyin_header
2134 * Purpose:
2135 * "Copy-in" port rights in the header of a message.
2136 * Operates atomically; if it doesn't succeed the
2137 * message header and the space are left untouched.
2138 * If it does succeed the remote/local port fields
2139 * contain object pointers instead of port names,
2140 * and the bits field is updated. The destination port
2141 * will be a valid port pointer.
2142 *
2143 * Conditions:
2144 * Nothing locked.
2145 * Returns:
2146 * MACH_MSG_SUCCESS Successful copyin.
2147 * MACH_SEND_INVALID_HEADER
2148 * Illegal value in the message header bits.
2149 * MACH_SEND_INVALID_DEST The space is dead.
2150 * MACH_SEND_INVALID_DEST Can't copyin destination port.
2151 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
2152 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
2153 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
2154 */
2155
2156 mach_msg_return_t
2157 ipc_kmsg_copyin_header(
2158 ipc_kmsg_t kmsg,
2159 ipc_space_t space,
2160 mach_msg_priority_t override,
2161 mach_msg_option_t *optionp)
2162 {
2163 mach_msg_header_t *msg = kmsg->ikm_header;
2164 mach_msg_bits_t mbits = msg->msgh_bits & MACH_MSGH_BITS_USER;
2165 mach_port_name_t dest_name = CAST_MACH_PORT_TO_NAME(msg->msgh_remote_port);
2166 mach_port_name_t reply_name = CAST_MACH_PORT_TO_NAME(msg->msgh_local_port);
2167 mach_port_name_t voucher_name = MACH_PORT_NULL;
2168 kern_return_t kr;
2169
2170 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
2171 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
2172 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
2173 ipc_object_t dest_port = IO_NULL;
2174 ipc_object_t reply_port = IO_NULL;
2175 ipc_port_t dest_soright = IP_NULL;
2176 ipc_port_t reply_soright = IP_NULL;
2177 ipc_port_t voucher_soright = IP_NULL;
2178 ipc_port_t release_port = IP_NULL;
2179 ipc_port_t voucher_port = IP_NULL;
2180 ipc_port_t voucher_release_port = IP_NULL;
2181 ipc_entry_t dest_entry = IE_NULL;
2182 ipc_entry_t reply_entry = IE_NULL;
2183 ipc_entry_t voucher_entry = IE_NULL;
2184
2185 int assertcnt = 0;
2186 #if IMPORTANCE_INHERITANCE
2187 boolean_t needboost = FALSE;
2188 #endif /* IMPORTANCE_INHERITANCE */
2189
2190 if ((mbits != msg->msgh_bits) ||
2191 (!MACH_MSG_TYPE_PORT_ANY_SEND(dest_type)) ||
2192 ((reply_type == 0) ?
2193 (reply_name != MACH_PORT_NULL) :
2194 !MACH_MSG_TYPE_PORT_ANY_SEND(reply_type)))
2195 return MACH_SEND_INVALID_HEADER;
2196
2197 if (!MACH_PORT_VALID(dest_name))
2198 return MACH_SEND_INVALID_DEST;
2199
2200 is_write_lock(space);
2201 if (!is_active(space)) {
2202 is_write_unlock(space);
2203 return MACH_SEND_INVALID_DEST;
2204 }
2205 /* space locked and active */
2206
2207 /*
2208 * If there is a voucher specified, make sure the disposition is
2209 * valid and the entry actually refers to a voucher port. Don't
2210 * actually copy in until we validate destination and reply.
2211 */
2212 if (voucher_type != MACH_MSGH_BITS_ZERO) {
2213
2214 voucher_name = msg->msgh_voucher_port;
2215
2216 if (voucher_name == MACH_PORT_DEAD ||
2217 (voucher_type != MACH_MSG_TYPE_MOVE_SEND &&
2218 voucher_type != MACH_MSG_TYPE_COPY_SEND)) {
2219 is_write_unlock(space);
2220 if ((*optionp & MACH_SEND_KERNEL) == 0) {
2221 mach_port_guard_exception(voucher_name, 0, 0, kGUARD_EXC_SEND_INVALID_VOUCHER);
2222 }
2223 return MACH_SEND_INVALID_VOUCHER;
2224 }
2225
2226 if (voucher_name != MACH_PORT_NULL) {
2227 voucher_entry = ipc_entry_lookup(space, voucher_name);
2228 if (voucher_entry == IE_NULL ||
2229 (voucher_entry->ie_bits & MACH_PORT_TYPE_SEND) == 0 ||
2230 io_kotype(voucher_entry->ie_object) != IKOT_VOUCHER) {
2231 is_write_unlock(space);
2232 if ((*optionp & MACH_SEND_KERNEL) == 0) {
2233 mach_port_guard_exception(voucher_name, 0, 0, kGUARD_EXC_SEND_INVALID_VOUCHER);
2234 }
2235 return MACH_SEND_INVALID_VOUCHER;
2236 }
2237 } else {
2238 voucher_type = MACH_MSG_TYPE_MOVE_SEND;
2239 }
2240 }
2241
2242 /*
2243 * Handle combinations of validating destination and reply; along
2244 * with copying in destination, reply, and voucher in an atomic way.
2245 */
2246
2247 if (dest_name == voucher_name) {
2248
2249 /*
2250 * If the destination name is the same as the voucher name,
2251 * the voucher_entry must already be known. Either that or
2252 * the destination name is MACH_PORT_NULL (i.e. invalid).
2253 */
2254 dest_entry = voucher_entry;
2255 if (dest_entry == IE_NULL) {
2256 goto invalid_dest;
2257 }
2258
2259 /*
2260 * Make sure a future copyin of the reply port will succeed.
2261 * Once we start copying in the dest/voucher pair, we can't
2262 * back out.
2263 */
2264 if (MACH_PORT_VALID(reply_name)) {
2265 assert(reply_type != 0); /* because reply_name not null */
2266
2267 /* It is just WRONG if dest, voucher, and reply are all the same. */
2268 if (voucher_name == reply_name) {
2269 goto invalid_reply;
2270 }
2271 reply_entry = ipc_entry_lookup(space, reply_name);
2272 if (reply_entry == IE_NULL) {
2273 goto invalid_reply;
2274 }
2275 assert(dest_entry != reply_entry); /* names are not equal */
2276 if (!ipc_right_copyin_check(space, reply_name, reply_entry, reply_type)) {
2277 goto invalid_reply;
2278 }
2279 }
2280
2281 /*
2282 * Do the joint copyin of the dest disposition and
2283 * voucher disposition from the one entry/port. We
2284 * already validated that the voucher copyin would
2285 * succeed (above). So, any failure in combining
2286 * the copyins can be blamed on the destination.
2287 */
2288 kr = ipc_right_copyin_two(space, dest_name, dest_entry,
2289 dest_type, voucher_type,
2290 &dest_port, &dest_soright,
2291 &release_port);
2292 if (kr != KERN_SUCCESS) {
2293 assert(kr != KERN_INVALID_CAPABILITY);
2294 goto invalid_dest;
2295 }
2296 voucher_port = (ipc_port_t)dest_port;
2297
2298 /*
2299 * could not have been one of these dispositions,
2300 * validated the port was a true kernel voucher port above,
2301 * AND was successfully able to copyin both dest and voucher.
2302 */
2303 assert(dest_type != MACH_MSG_TYPE_MAKE_SEND);
2304 assert(dest_type != MACH_MSG_TYPE_MAKE_SEND_ONCE);
2305 assert(dest_type != MACH_MSG_TYPE_MOVE_SEND_ONCE);
2306
2307 /*
2308 * Perform the delayed reply right copyin (guaranteed success).
2309 */
2310 if (reply_entry != IE_NULL) {
2311 kr = ipc_right_copyin(space, reply_name, reply_entry,
2312 reply_type, TRUE,
2313 &reply_port, &reply_soright,
2314 &release_port, &assertcnt);
2315 assert(assertcnt == 0);
2316 assert(kr == KERN_SUCCESS);
2317 }
2318
2319 } else {
2320 if (dest_name == reply_name) {
2321 /*
2322 * Destination and reply ports are the same!
2323 * This is very similar to the case where the
2324 * destination and voucher ports were the same
2325 * (except the reply port disposition is not
2326 * previously validated).
2327 */
2328 dest_entry = ipc_entry_lookup(space, dest_name);
2329 if (dest_entry == IE_NULL) {
2330 goto invalid_dest;
2331 }
2332 reply_entry = dest_entry;
2333 assert(reply_type != 0); /* because name not null */
2334
2335 /*
2336 * Do the joint copyin of the dest disposition and
2337 * reply disposition from the one entry/port.
2338 */
2339 kr = ipc_right_copyin_two(space, dest_name, dest_entry,
2340 dest_type, reply_type,
2341 &dest_port, &dest_soright,
2342 &release_port);
2343 if (kr == KERN_INVALID_CAPABILITY) {
2344 goto invalid_reply;
2345 } else if (kr != KERN_SUCCESS) {
2346 goto invalid_dest;
2347 }
2348 reply_port = dest_port;
2349
2350
2351 } else {
2352 /*
2353 * Handle destination and reply independently, as
2354 * they are independent entries (even if the entries
2355 * refer to the same port).
2356 *
2357 * This can be the tough case to make atomic.
2358 *
2359 * The difficult problem is serializing with port death.
2360 * The bad case is when dest_port dies after its copyin,
2361 * reply_port dies before its copyin, and dest_port dies before
2362 * reply_port. Then the copyins operated as if dest_port was
2363 * alive and reply_port was dead, which shouldn't have happened
2364 * because they died in the other order.
2365 *
2366 * Note that it is easy for a user task to tell if
2367 * a copyin happened before or after a port died.
2368 * If a port dies before copyin, a dead-name notification
2369 * is generated and the dead name's urefs are incremented,
2370 * and if the copyin happens first, a port-deleted
2371 * notification is generated.
2372 *
2373 * Even so, avoiding that potentially detectable race is too
2374 * expensive - and no known code cares about it. So, we just
2375 * do the expedient thing and copy them in one after the other.
2376 */
2377
2378 dest_entry = ipc_entry_lookup(space, dest_name);
2379 if (dest_entry == IE_NULL) {
2380 goto invalid_dest;
2381 }
2382 assert(dest_entry != voucher_entry);
2383
2384 /*
2385 * Make sure reply port entry is valid before dest copyin.
2386 */
2387 if (MACH_PORT_VALID(reply_name)) {
2388 if (reply_name == voucher_name) {
2389 goto invalid_reply;
2390 }
2391 reply_entry = ipc_entry_lookup(space, reply_name);
2392 if (reply_entry == IE_NULL) {
2393 goto invalid_reply;
2394 }
2395 assert(dest_entry != reply_entry); /* names are not equal */
2396 assert(reply_type != 0); /* because reply_name not null */
2397
2398 if (!ipc_right_copyin_check(space, reply_name, reply_entry, reply_type)) {
2399 goto invalid_reply;
2400 }
2401 }
2402
2403 /*
2404 * copyin the destination.
2405 */
2406 kr = ipc_right_copyin(space, dest_name, dest_entry,
2407 dest_type, FALSE,
2408 &dest_port, &dest_soright,
2409 &release_port, &assertcnt);
2410 assert(assertcnt == 0);
2411 if (kr != KERN_SUCCESS) {
2412 goto invalid_dest;
2413 }
2414 assert(IO_VALID(dest_port));
2415 assert(!IP_VALID(release_port));
2416
2417 /*
2418 * Copyin the pre-validated reply right.
2419 * It's OK if the reply right has gone dead in the meantime.
2420 */
2421 if (MACH_PORT_VALID(reply_name)) {
2422 kr = ipc_right_copyin(space, reply_name, reply_entry,
2423 reply_type, TRUE,
2424 &reply_port, &reply_soright,
2425 &release_port, &assertcnt);
2426 assert(assertcnt == 0);
2427 assert(kr == KERN_SUCCESS);
2428 } else {
2429 /* convert invalid name to equivalent ipc_object type */
2430 reply_port = (ipc_object_t)CAST_MACH_NAME_TO_PORT(reply_name);
2431 }
2432 }
2433
2434 /*
2435 * Finally can copyin the voucher right now that dest and reply
2436 * are fully copied in (guaranteed success).
2437 */
2438 if (IE_NULL != voucher_entry) {
2439 kr = ipc_right_copyin(space, voucher_name, voucher_entry,
2440 voucher_type, FALSE,
2441 (ipc_object_t *)&voucher_port,
2442 &voucher_soright,
2443 &voucher_release_port,
2444 &assertcnt);
2445 assert(assertcnt == 0);
2446 assert(KERN_SUCCESS == kr);
2447 assert(IP_VALID(voucher_port));
2448 assert(ip_active(voucher_port));
2449 }
2450 }
2451
2452 /*
2453 * The entries might need to be deallocated.
2454 *
2455 * Each entry should be deallocated only once,
2456 * even if it was specified in more than one slot in the header.
2457 * Note that dest can be the same entry as reply or voucher,
2458 * but reply and voucher must be distinct entries.
2459 */
2460 assert(IE_NULL != dest_entry);
2461 if (IE_NULL != reply_entry)
2462 assert(reply_entry != voucher_entry);
2463
2464 if (IE_BITS_TYPE(dest_entry->ie_bits) == MACH_PORT_TYPE_NONE) {
2465 ipc_entry_dealloc(space, dest_name, dest_entry);
2466
2467 if (dest_entry == reply_entry) {
2468 reply_entry = IE_NULL;
2469 }
2470
2471 if (dest_entry == voucher_entry) {
2472 voucher_entry = IE_NULL;
2473 }
2474
2475 dest_entry = IE_NULL;
2476 }
2477 if (IE_NULL != reply_entry &&
2478 IE_BITS_TYPE(reply_entry->ie_bits) == MACH_PORT_TYPE_NONE) {
2479 ipc_entry_dealloc(space, reply_name, reply_entry);
2480 reply_entry = IE_NULL;
2481 }
2482 if (IE_NULL != voucher_entry &&
2483 IE_BITS_TYPE(voucher_entry->ie_bits) == MACH_PORT_TYPE_NONE) {
2484 ipc_entry_dealloc(space, voucher_name, voucher_entry);
2485 voucher_entry = IE_NULL;
2486 }
2487
2488 dest_type = ipc_object_copyin_type(dest_type);
2489 reply_type = ipc_object_copyin_type(reply_type);
2490
2491 /*
2492 * JMM - Without rdar://problem/6275821, this is the last place we can
2493 * re-arm the send-possible notifications. It may trigger unexpectedly
2494 * early (send may NOT have failed), but better than missing. We assure
2495 * we won't miss by forcing MACH_SEND_ALWAYS if we got past arming.
2496 */
2497 if (((*optionp & MACH_SEND_NOTIFY) != 0) &&
2498 dest_type != MACH_MSG_TYPE_PORT_SEND_ONCE &&
2499 dest_entry != IE_NULL && dest_entry->ie_request != IE_REQ_NONE) {
2500 ipc_port_t dport = (ipc_port_t)dest_port;
2501
2502 assert(dport != IP_NULL);
2503 ip_lock(dport);
2504 if (ip_active(dport) && dport->ip_receiver != ipc_space_kernel) {
2505 if (ip_full(dport)) {
2506 #if IMPORTANCE_INHERITANCE
2507 needboost = ipc_port_request_sparm(dport, dest_name,
2508 dest_entry->ie_request,
2509 *optionp,
2510 override);
2511 if (needboost == FALSE)
2512 ip_unlock(dport);
2513 #else
2514 ipc_port_request_sparm(dport, dest_name,
2515 dest_entry->ie_request,
2516 *optionp,
2517 override);
2518 ip_unlock(dport);
2519 #endif /* IMPORTANCE_INHERITANCE */
2520 } else {
2521 *optionp |= MACH_SEND_ALWAYS;
2522 ip_unlock(dport);
2523 }
2524 } else {
2525 ip_unlock(dport);
2526 }
2527 }
2528
2529 is_write_unlock(space);
2530
2531 #if IMPORTANCE_INHERITANCE
2532 /*
2533 * If our request is the first boosting send-possible
2534 * notification this cycle, push the boost down the
2535 * destination port.
2536 */
2537 if (needboost == TRUE) {
2538 ipc_port_t dport = (ipc_port_t)dest_port;
2539
2540 /* dport still locked from above */
2541 if (ipc_port_importance_delta(dport, IPID_OPTION_SENDPOSSIBLE, 1) == FALSE) {
2542 ip_unlock(dport);
2543 }
2544 }
2545 #endif /* IMPORTANCE_INHERITANCE */
2546
2547 if (dest_soright != IP_NULL) {
2548 ipc_notify_port_deleted(dest_soright, dest_name);
2549 }
2550 if (reply_soright != IP_NULL) {
2551 ipc_notify_port_deleted(reply_soright, reply_name);
2552 }
2553 if (voucher_soright != IP_NULL) {
2554 ipc_notify_port_deleted(voucher_soright, voucher_name);
2555 }
2556
2557 /*
2558 * No room to store voucher port in in-kernel msg header,
2559 * so we store it back in the kmsg itself. Extract the
2560 * qos, and apply any override before we enqueue the kmsg.
2561 */
2562 if (IP_VALID(voucher_port)) {
2563
2564 kmsg->ikm_voucher = voucher_port;
2565 voucher_type = MACH_MSG_TYPE_MOVE_SEND;
2566 }
2567
2568 msg->msgh_bits = MACH_MSGH_BITS_SET(dest_type, reply_type, voucher_type, mbits);
2569 msg->msgh_remote_port = (ipc_port_t)dest_port;
2570 msg->msgh_local_port = (ipc_port_t)reply_port;
2571
2572 /* capture the qos value(s) for the kmsg */
2573 ipc_kmsg_set_qos(kmsg, *optionp, override);
2574
2575 if (release_port != IP_NULL)
2576 ip_release(release_port);
2577
2578 if (voucher_release_port != IP_NULL)
2579 ip_release(voucher_release_port);
2580
2581 return MACH_MSG_SUCCESS;
2582
2583 invalid_reply:
2584 is_write_unlock(space);
2585
2586 if (release_port != IP_NULL)
2587 ip_release(release_port);
2588
2589 assert(voucher_port == IP_NULL);
2590 assert(voucher_soright == IP_NULL);
2591
2592 if ((*optionp & MACH_SEND_KERNEL) == 0) {
2593 mach_port_guard_exception(reply_name, 0, 0, kGUARD_EXC_SEND_INVALID_REPLY);
2594 }
2595 return MACH_SEND_INVALID_REPLY;
2596
2597 invalid_dest:
2598 is_write_unlock(space);
2599
2600 if (release_port != IP_NULL)
2601 ip_release(release_port);
2602
2603 if (reply_soright != IP_NULL)
2604 ipc_notify_port_deleted(reply_soright, reply_name);
2605
2606 assert(voucher_port == IP_NULL);
2607 assert(voucher_soright == IP_NULL);
2608
2609 return MACH_SEND_INVALID_DEST;
2610 }
2611
2612 mach_msg_descriptor_t *ipc_kmsg_copyin_port_descriptor(
2613 volatile mach_msg_port_descriptor_t *dsc,
2614 mach_msg_legacy_port_descriptor_t *user_dsc,
2615 ipc_space_t space,
2616 ipc_object_t dest,
2617 ipc_kmsg_t kmsg,
2618 mach_msg_option_t *optionp,
2619 mach_msg_return_t *mr);
2620
2621 void ipc_print_type_name(
2622 int type_name);
2623
2624 mach_msg_descriptor_t *
2625 ipc_kmsg_copyin_port_descriptor(
2626 volatile mach_msg_port_descriptor_t *dsc,
2627 mach_msg_legacy_port_descriptor_t *user_dsc_in,
2628 ipc_space_t space,
2629 ipc_object_t dest,
2630 ipc_kmsg_t kmsg,
2631 mach_msg_option_t *optionp,
2632 mach_msg_return_t *mr)
2633 {
2634 volatile mach_msg_legacy_port_descriptor_t *user_dsc = user_dsc_in;
2635 mach_msg_type_name_t user_disp;
2636 mach_msg_type_name_t result_disp;
2637 mach_port_name_t name;
2638 ipc_object_t object;
2639
2640 user_disp = user_dsc->disposition;
2641 result_disp = ipc_object_copyin_type(user_disp);
2642
2643 name = (mach_port_name_t)user_dsc->name;
2644 if (MACH_PORT_VALID(name)) {
2645
2646 kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object);
2647 if (kr != KERN_SUCCESS) {
2648 if ((*optionp & MACH_SEND_KERNEL) == 0) {
2649 mach_port_guard_exception(name, 0, 0, kGUARD_EXC_SEND_INVALID_RIGHT);
2650 }
2651 *mr = MACH_SEND_INVALID_RIGHT;
2652 return NULL;
2653 }
2654
2655 if ((result_disp == MACH_MSG_TYPE_PORT_RECEIVE) &&
2656 ipc_port_check_circularity((ipc_port_t) object,
2657 (ipc_port_t) dest)) {
2658 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
2659 }
2660 dsc->name = (ipc_port_t) object;
2661 } else {
2662 dsc->name = CAST_MACH_NAME_TO_PORT(name);
2663 }
2664 dsc->disposition = result_disp;
2665 dsc->type = MACH_MSG_PORT_DESCRIPTOR;
2666
2667 dsc->pad_end = 0; // debug, unnecessary
2668
2669 return (mach_msg_descriptor_t *)(user_dsc_in+1);
2670 }
2671
2672 mach_msg_descriptor_t * ipc_kmsg_copyin_ool_descriptor(
2673 mach_msg_ool_descriptor_t *dsc,
2674 mach_msg_descriptor_t *user_dsc,
2675 int is_64bit,
2676 vm_offset_t *paddr,
2677 vm_map_copy_t *copy,
2678 vm_size_t *space_needed,
2679 vm_map_t map,
2680 mach_msg_option_t *optionp,
2681 mach_msg_return_t *mr);
2682
2683 mach_msg_descriptor_t *
2684 ipc_kmsg_copyin_ool_descriptor(
2685 mach_msg_ool_descriptor_t *dsc,
2686 mach_msg_descriptor_t *user_dsc,
2687 int is_64bit,
2688 vm_offset_t *paddr,
2689 vm_map_copy_t *copy,
2690 vm_size_t *space_needed,
2691 vm_map_t map,
2692 __unused mach_msg_option_t *optionp,
2693 mach_msg_return_t *mr)
2694 {
2695 vm_size_t length;
2696 boolean_t dealloc;
2697 mach_msg_copy_options_t copy_options;
2698 mach_vm_offset_t addr;
2699 mach_msg_descriptor_type_t dsc_type;
2700
2701 if (is_64bit) {
2702 mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
2703
2704 addr = (mach_vm_offset_t) user_ool_dsc->address;
2705 length = user_ool_dsc->size;
2706 dealloc = user_ool_dsc->deallocate;
2707 copy_options = user_ool_dsc->copy;
2708 dsc_type = user_ool_dsc->type;
2709
2710 user_dsc = (typeof(user_dsc))(user_ool_dsc+1);
2711 } else {
2712 mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
2713
2714 addr = CAST_USER_ADDR_T(user_ool_dsc->address);
2715 dealloc = user_ool_dsc->deallocate;
2716 copy_options = user_ool_dsc->copy;
2717 dsc_type = user_ool_dsc->type;
2718 length = user_ool_dsc->size;
2719
2720 user_dsc = (typeof(user_dsc))(user_ool_dsc+1);
2721 }
2722
2723 dsc->size = (mach_msg_size_t)length;
2724 dsc->deallocate = dealloc;
2725 dsc->copy = copy_options;
2726 dsc->type = dsc_type;
2727
2728 if (length == 0) {
2729 dsc->address = NULL;
2730 } else if ((length >= MSG_OOL_SIZE_SMALL) &&
2731 (copy_options == MACH_MSG_PHYSICAL_COPY) && !dealloc) {
2732
2733 /*
2734 * If the request is a physical copy and the source
2735 * is not being deallocated, then allocate space
2736 * in the kernel's pageable ipc copy map and copy
2737 * the data in. The semantics guarantee that the
2738 * data will have been physically copied before
2739 * the send operation terminates. Thus if the data
2740 * is not being deallocated, we must be prepared
2741 * to page if the region is sufficiently large.
2742 */
2743 if (copyin(addr, (char *)*paddr, length)) {
2744 *mr = MACH_SEND_INVALID_MEMORY;
2745 return NULL;
2746 }
2747
2748 /*
2749 * The kernel ipc copy map is marked no_zero_fill.
2750 * If the transfer is not a page multiple, we need
2751 * to zero fill the balance.
2752 */
2753 if (!page_aligned(length)) {
2754 (void) memset((void *) (*paddr + length), 0,
2755 round_page(length) - length);
2756 }
2757 if (vm_map_copyin(ipc_kernel_copy_map, (vm_map_address_t)*paddr,
2758 (vm_map_size_t)length, TRUE, copy) != KERN_SUCCESS) {
2759 *mr = MACH_MSG_VM_KERNEL;
2760 return NULL;
2761 }
2762 dsc->address = (void *)*copy;
2763 *paddr += round_page(length);
2764 *space_needed -= round_page(length);
2765 } else {
2766
2767 /*
2768 * Make a vm_map_copy_t of the of the data. If the
2769 * data is small, this will do an optimized physical
2770 * copy. Otherwise, it will do a virtual copy.
2771 *
2772 * NOTE: A virtual copy is OK if the original is being
2773 * deallocted, even if a physical copy was requested.
2774 */
2775 kern_return_t kr = vm_map_copyin(map, addr,
2776 (vm_map_size_t)length, dealloc, copy);
2777 if (kr != KERN_SUCCESS) {
2778 *mr = (kr == KERN_RESOURCE_SHORTAGE) ?
2779 MACH_MSG_VM_KERNEL :
2780 MACH_SEND_INVALID_MEMORY;
2781 return NULL;
2782 }
2783 dsc->address = (void *)*copy;
2784 }
2785 return user_dsc;
2786 }
2787
2788 mach_msg_descriptor_t * ipc_kmsg_copyin_ool_ports_descriptor(
2789 mach_msg_ool_ports_descriptor_t *dsc,
2790 mach_msg_descriptor_t *user_dsc,
2791 int is_64bit,
2792 vm_map_t map,
2793 ipc_space_t space,
2794 ipc_object_t dest,
2795 ipc_kmsg_t kmsg,
2796 mach_msg_option_t *optionp,
2797 mach_msg_return_t *mr);
2798
2799 mach_msg_descriptor_t *
2800 ipc_kmsg_copyin_ool_ports_descriptor(
2801 mach_msg_ool_ports_descriptor_t *dsc,
2802 mach_msg_descriptor_t *user_dsc,
2803 int is_64bit,
2804 vm_map_t map,
2805 ipc_space_t space,
2806 ipc_object_t dest,
2807 ipc_kmsg_t kmsg,
2808 mach_msg_option_t *optionp,
2809 mach_msg_return_t *mr)
2810 {
2811 void *data;
2812 ipc_object_t *objects;
2813 unsigned int i;
2814 mach_vm_offset_t addr;
2815 mach_msg_type_name_t user_disp;
2816 mach_msg_type_name_t result_disp;
2817 mach_msg_type_number_t count;
2818 mach_msg_copy_options_t copy_option;
2819 boolean_t deallocate;
2820 mach_msg_descriptor_type_t type;
2821 vm_size_t ports_length, names_length;
2822
2823 if (is_64bit) {
2824 mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
2825
2826 addr = (mach_vm_offset_t)user_ool_dsc->address;
2827 count = user_ool_dsc->count;
2828 deallocate = user_ool_dsc->deallocate;
2829 copy_option = user_ool_dsc->copy;
2830 user_disp = user_ool_dsc->disposition;
2831 type = user_ool_dsc->type;
2832
2833 user_dsc = (typeof(user_dsc))(user_ool_dsc+1);
2834 } else {
2835 mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
2836
2837 addr = CAST_USER_ADDR_T(user_ool_dsc->address);
2838 count = user_ool_dsc->count;
2839 deallocate = user_ool_dsc->deallocate;
2840 copy_option = user_ool_dsc->copy;
2841 user_disp = user_ool_dsc->disposition;
2842 type = user_ool_dsc->type;
2843
2844 user_dsc = (typeof(user_dsc))(user_ool_dsc+1);
2845 }
2846
2847 dsc->deallocate = deallocate;
2848 dsc->copy = copy_option;
2849 dsc->type = type;
2850 dsc->count = count;
2851 dsc->address = NULL; /* for now */
2852
2853 result_disp = ipc_object_copyin_type(user_disp);
2854 dsc->disposition = result_disp;
2855
2856 /* We always do a 'physical copy', but you have to specify something valid */
2857 if (copy_option != MACH_MSG_PHYSICAL_COPY &&
2858 copy_option != MACH_MSG_VIRTUAL_COPY) {
2859 *mr = MACH_SEND_INVALID_TYPE;
2860 return NULL;
2861 }
2862
2863 /* calculate length of data in bytes, rounding up */
2864
2865 if (os_mul_overflow(count, sizeof(mach_port_t), &ports_length)) {
2866 *mr = MACH_SEND_TOO_LARGE;
2867 return NULL;
2868 }
2869
2870 if (os_mul_overflow(count, sizeof(mach_port_name_t), &names_length)) {
2871 *mr = MACH_SEND_TOO_LARGE;
2872 return NULL;
2873 }
2874
2875 if (ports_length == 0) {
2876 return user_dsc;
2877 }
2878
2879 data = kalloc(ports_length);
2880
2881 if (data == NULL) {
2882 *mr = MACH_SEND_NO_BUFFER;
2883 return NULL;
2884 }
2885
2886 #ifdef __LP64__
2887 mach_port_name_t *names = &((mach_port_name_t *)data)[count];
2888 #else
2889 mach_port_name_t *names = ((mach_port_name_t *)data);
2890 #endif
2891
2892 if (copyinmap(map, addr, names, names_length) != KERN_SUCCESS) {
2893 kfree(data, ports_length);
2894 *mr = MACH_SEND_INVALID_MEMORY;
2895 return NULL;
2896 }
2897
2898 if (deallocate) {
2899 (void) mach_vm_deallocate(map, addr, (mach_vm_size_t)ports_length);
2900 }
2901
2902 objects = (ipc_object_t *) data;
2903 dsc->address = data;
2904
2905 for ( i = 0; i < count; i++) {
2906 mach_port_name_t name = names[i];
2907 ipc_object_t object;
2908
2909 if (!MACH_PORT_VALID(name)) {
2910 objects[i] = (ipc_object_t)CAST_MACH_NAME_TO_PORT(name);
2911 continue;
2912 }
2913
2914 kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object);
2915
2916 if (kr != KERN_SUCCESS) {
2917 unsigned int j;
2918
2919 for(j = 0; j < i; j++) {
2920 object = objects[j];
2921 if (IPC_OBJECT_VALID(object))
2922 ipc_object_destroy(object, result_disp);
2923 }
2924 kfree(data, ports_length);
2925 dsc->address = NULL;
2926 if ((*optionp & MACH_SEND_KERNEL) == 0) {
2927 mach_port_guard_exception(name, 0, 0, kGUARD_EXC_SEND_INVALID_RIGHT);
2928 }
2929 *mr = MACH_SEND_INVALID_RIGHT;
2930 return NULL;
2931 }
2932
2933 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
2934 ipc_port_check_circularity(
2935 (ipc_port_t) object,
2936 (ipc_port_t) dest))
2937 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
2938
2939 objects[i] = object;
2940 }
2941
2942 return user_dsc;
2943 }
2944
2945 /*
2946 * Routine: ipc_kmsg_copyin_body
2947 * Purpose:
2948 * "Copy-in" port rights and out-of-line memory
2949 * in the message body.
2950 *
2951 * In all failure cases, the message is left holding
2952 * no rights or memory. However, the message buffer
2953 * is not deallocated. If successful, the message
2954 * contains a valid destination port.
2955 * Conditions:
2956 * Nothing locked.
2957 * Returns:
2958 * MACH_MSG_SUCCESS Successful copyin.
2959 * MACH_SEND_INVALID_MEMORY Can't grab out-of-line memory.
2960 * MACH_SEND_INVALID_RIGHT Can't copyin port right in body.
2961 * MACH_SEND_INVALID_TYPE Bad type specification.
2962 * MACH_SEND_MSG_TOO_SMALL Body is too small for types/data.
2963 * MACH_SEND_INVALID_RT_OOL_SIZE OOL Buffer too large for RT
2964 * MACH_MSG_INVALID_RT_DESCRIPTOR Dealloc and RT are incompatible
2965 */
2966
2967 mach_msg_return_t
2968 ipc_kmsg_copyin_body(
2969 ipc_kmsg_t kmsg,
2970 ipc_space_t space,
2971 vm_map_t map,
2972 mach_msg_option_t *optionp)
2973 {
2974 ipc_object_t dest;
2975 mach_msg_body_t *body;
2976 mach_msg_descriptor_t *daddr, *naddr;
2977 mach_msg_descriptor_t *user_addr, *kern_addr;
2978 mach_msg_type_number_t dsc_count;
2979 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
2980 boolean_t complex = FALSE;
2981 vm_size_t space_needed = 0;
2982 vm_offset_t paddr = 0;
2983 vm_map_copy_t copy = VM_MAP_COPY_NULL;
2984 mach_msg_type_number_t i;
2985 mach_msg_return_t mr = MACH_MSG_SUCCESS;
2986
2987 vm_size_t descriptor_size = 0;
2988
2989 mach_msg_type_number_t total_ool_port_count = 0;
2990
2991 /*
2992 * Determine if the target is a kernel port.
2993 */
2994 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
2995 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
2996 naddr = (mach_msg_descriptor_t *) (body + 1);
2997
2998 dsc_count = body->msgh_descriptor_count;
2999 if (dsc_count == 0)
3000 return MACH_MSG_SUCCESS;
3001
3002 /*
3003 * Make an initial pass to determine kernal VM space requirements for
3004 * physical copies and possible contraction of the descriptors from
3005 * processes with pointers larger than the kernel's.
3006 */
3007 daddr = NULL;
3008 for (i = 0; i < dsc_count; i++) {
3009 mach_msg_size_t size;
3010 mach_msg_type_number_t ool_port_count = 0;
3011
3012 daddr = naddr;
3013
3014 /* make sure the descriptor fits in the message */
3015 if (is_task_64bit) {
3016 switch (daddr->type.type) {
3017 case MACH_MSG_OOL_DESCRIPTOR:
3018 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3019 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
3020 descriptor_size += 16;
3021 naddr = (typeof(naddr))((vm_offset_t)daddr + 16);
3022 break;
3023 default:
3024 descriptor_size += 12;
3025 naddr = (typeof(naddr))((vm_offset_t)daddr + 12);
3026 break;
3027 }
3028 } else {
3029 descriptor_size += 12;
3030 naddr = (typeof(naddr))((vm_offset_t)daddr + 12);
3031 }
3032
3033 if (naddr > (mach_msg_descriptor_t *)
3034 ((vm_offset_t)kmsg->ikm_header + kmsg->ikm_header->msgh_size)) {
3035 mr = MACH_SEND_MSG_TOO_SMALL;
3036 goto clean_message;
3037 }
3038
3039 switch (daddr->type.type) {
3040 case MACH_MSG_OOL_DESCRIPTOR:
3041 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3042 size = (is_task_64bit) ?
3043 ((mach_msg_ool_descriptor64_t *)daddr)->size :
3044 daddr->out_of_line.size;
3045
3046 if (daddr->out_of_line.copy != MACH_MSG_PHYSICAL_COPY &&
3047 daddr->out_of_line.copy != MACH_MSG_VIRTUAL_COPY) {
3048 /*
3049 * Invalid copy option
3050 */
3051 mr = MACH_SEND_INVALID_TYPE;
3052 goto clean_message;
3053 }
3054
3055 if ((size >= MSG_OOL_SIZE_SMALL) &&
3056 (daddr->out_of_line.copy == MACH_MSG_PHYSICAL_COPY) &&
3057 !(daddr->out_of_line.deallocate)) {
3058
3059 /*
3060 * Out-of-line memory descriptor, accumulate kernel
3061 * memory requirements
3062 */
3063 if (space_needed + round_page(size) <= space_needed) {
3064 /* Overflow dectected */
3065 mr = MACH_MSG_VM_KERNEL;
3066 goto clean_message;
3067 }
3068
3069 space_needed += round_page(size);
3070 if (space_needed > ipc_kmsg_max_vm_space) {
3071 /* Per message kernel memory limit exceeded */
3072 mr = MACH_MSG_VM_KERNEL;
3073 goto clean_message;
3074 }
3075 }
3076 break;
3077 case MACH_MSG_PORT_DESCRIPTOR:
3078 if (os_add_overflow(total_ool_port_count, 1, &total_ool_port_count)) {
3079 /* Overflow detected */
3080 mr = MACH_SEND_TOO_LARGE;
3081 goto clean_message;
3082 }
3083 break;
3084 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
3085 ool_port_count = (is_task_64bit) ?
3086 ((mach_msg_ool_ports_descriptor64_t *)daddr)->count :
3087 daddr->ool_ports.count;
3088
3089 if (os_add_overflow(total_ool_port_count, ool_port_count, &total_ool_port_count)) {
3090 /* Overflow detected */
3091 mr = MACH_SEND_TOO_LARGE;
3092 goto clean_message;
3093 }
3094
3095 if (ool_port_count > (ipc_kmsg_max_vm_space/sizeof(mach_port_t))) {
3096 /* Per message kernel memory limit exceeded */
3097 mr = MACH_SEND_TOO_LARGE;
3098 goto clean_message;
3099 }
3100 break;
3101 }
3102 }
3103
3104 /* Sending more than 16383 rights in one message seems crazy */
3105 if (total_ool_port_count >= (MACH_PORT_UREFS_MAX / 4)) {
3106 mr = MACH_SEND_TOO_LARGE;
3107 goto clean_message;
3108 }
3109
3110 /*
3111 * Allocate space in the pageable kernel ipc copy map for all the
3112 * ool data that is to be physically copied. Map is marked wait for
3113 * space.
3114 */
3115 if (space_needed) {
3116 if (vm_allocate_kernel(ipc_kernel_copy_map, &paddr, space_needed,
3117 VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IPC) != KERN_SUCCESS) {
3118 mr = MACH_MSG_VM_KERNEL;
3119 goto clean_message;
3120 }
3121 }
3122
3123 /* user_addr = just after base as it was copied in */
3124 user_addr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t));
3125
3126 /* Shift the mach_msg_base_t down to make room for dsc_count*16bytes of descriptors */
3127 if(descriptor_size != 16*dsc_count) {
3128 vm_offset_t dsc_adjust = 16*dsc_count - descriptor_size;
3129
3130 memmove((char *)(((vm_offset_t)kmsg->ikm_header) - dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t));
3131 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header - dsc_adjust);
3132
3133 /* Update the message size for the larger in-kernel representation */
3134 kmsg->ikm_header->msgh_size += (mach_msg_size_t)dsc_adjust;
3135 }
3136
3137
3138 /* kern_addr = just after base after it has been (conditionally) moved */
3139 kern_addr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t));
3140
3141 /* handle the OOL regions and port descriptors. */
3142 for(i=0;i<dsc_count;i++) {
3143 switch (user_addr->type.type) {
3144 case MACH_MSG_PORT_DESCRIPTOR:
3145 user_addr = ipc_kmsg_copyin_port_descriptor((mach_msg_port_descriptor_t *)kern_addr,
3146 (mach_msg_legacy_port_descriptor_t *)user_addr, space, dest, kmsg, optionp, &mr);
3147 kern_addr++;
3148 complex = TRUE;
3149 break;
3150 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3151 case MACH_MSG_OOL_DESCRIPTOR:
3152 user_addr = ipc_kmsg_copyin_ool_descriptor((mach_msg_ool_descriptor_t *)kern_addr,
3153 user_addr, is_task_64bit, &paddr, &copy, &space_needed, map, optionp, &mr);
3154 kern_addr++;
3155 complex = TRUE;
3156 break;
3157 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
3158 user_addr = ipc_kmsg_copyin_ool_ports_descriptor((mach_msg_ool_ports_descriptor_t *)kern_addr,
3159 user_addr, is_task_64bit, map, space, dest, kmsg, optionp, &mr);
3160 kern_addr++;
3161 complex = TRUE;
3162 break;
3163 default:
3164 /* Invalid descriptor */
3165 mr = MACH_SEND_INVALID_TYPE;
3166 break;
3167 }
3168
3169 if (MACH_MSG_SUCCESS != mr) {
3170 /* clean from start of message descriptors to i */
3171 ipc_kmsg_clean_partial(kmsg, i,
3172 (mach_msg_descriptor_t *)((mach_msg_base_t *)kmsg->ikm_header + 1),
3173 paddr, space_needed);
3174 goto out;
3175 }
3176 } /* End of loop */
3177
3178 if (!complex) {
3179 kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_COMPLEX;
3180 }
3181 out:
3182 return mr;
3183
3184 clean_message:
3185 /* no descriptors have been copied in yet */
3186 ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0);
3187 return mr;
3188 }
3189
3190
3191 /*
3192 * Routine: ipc_kmsg_copyin
3193 * Purpose:
3194 * "Copy-in" port rights and out-of-line memory
3195 * in the message.
3196 *
3197 * In all failure cases, the message is left holding
3198 * no rights or memory. However, the message buffer
3199 * is not deallocated. If successful, the message
3200 * contains a valid destination port.
3201 * Conditions:
3202 * Nothing locked.
3203 * Returns:
3204 * MACH_MSG_SUCCESS Successful copyin.
3205 * MACH_SEND_INVALID_HEADER
3206 * Illegal value in the message header bits.
3207 * MACH_SEND_INVALID_DEST Can't copyin destination port.
3208 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
3209 * MACH_SEND_INVALID_MEMORY Can't grab out-of-line memory.
3210 * MACH_SEND_INVALID_RIGHT Can't copyin port right in body.
3211 * MACH_SEND_INVALID_TYPE Bad type specification.
3212 * MACH_SEND_MSG_TOO_SMALL Body is too small for types/data.
3213 */
3214
3215 mach_msg_return_t
3216 ipc_kmsg_copyin(
3217 ipc_kmsg_t kmsg,
3218 ipc_space_t space,
3219 vm_map_t map,
3220 mach_msg_priority_t override,
3221 mach_msg_option_t *optionp)
3222 {
3223 mach_msg_return_t mr;
3224
3225 kmsg->ikm_header->msgh_bits &= MACH_MSGH_BITS_USER;
3226
3227 mr = ipc_kmsg_copyin_header(kmsg, space, override, optionp);
3228
3229 if (mr != MACH_MSG_SUCCESS)
3230 return mr;
3231
3232 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_MSG_SEND) | DBG_FUNC_NONE,
3233 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3234 (uintptr_t)kmsg->ikm_header->msgh_bits,
3235 (uintptr_t)kmsg->ikm_header->msgh_id,
3236 VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(kmsg->ikm_voucher)),
3237 0);
3238
3239 DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_copyin header:\n%.8x\n%.8x\n%p\n%p\n%p\n%.8x\n",
3240 kmsg->ikm_header->msgh_size,
3241 kmsg->ikm_header->msgh_bits,
3242 kmsg->ikm_header->msgh_remote_port,
3243 kmsg->ikm_header->msgh_local_port,
3244 kmsg->ikm_voucher,
3245 kmsg->ikm_header->msgh_id);
3246
3247 if ((kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) == 0)
3248 return MACH_MSG_SUCCESS;
3249
3250 mr = ipc_kmsg_copyin_body( kmsg, space, map, optionp);
3251
3252 /* unreachable if !DEBUG */
3253 __unreachable_ok_push
3254 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK))
3255 {
3256 kprintf("body:\n");
3257 uint32_t i;
3258 for(i=0;i*4 < (kmsg->ikm_header->msgh_size - sizeof(mach_msg_header_t));i++)
3259 {
3260 kprintf("%.4x\n",((uint32_t *)(kmsg->ikm_header + 1))[i]);
3261 }
3262 }
3263 __unreachable_ok_pop
3264
3265 return mr;
3266 }
3267
3268 /*
3269 * Routine: ipc_kmsg_copyin_from_kernel
3270 * Purpose:
3271 * "Copy-in" port rights and out-of-line memory
3272 * in a message sent from the kernel.
3273 *
3274 * Because the message comes from the kernel,
3275 * the implementation assumes there are no errors
3276 * or peculiarities in the message.
3277 * Conditions:
3278 * Nothing locked.
3279 */
3280
3281 mach_msg_return_t
3282 ipc_kmsg_copyin_from_kernel(
3283 ipc_kmsg_t kmsg)
3284 {
3285 mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits;
3286 mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits);
3287 mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits);
3288 ipc_object_t remote = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
3289 ipc_object_t local = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
3290
3291 /* translate the destination and reply ports */
3292 if (!IO_VALID(remote))
3293 return MACH_SEND_INVALID_DEST;
3294
3295 ipc_object_copyin_from_kernel(remote, rname);
3296 if (IO_VALID(local))
3297 ipc_object_copyin_from_kernel(local, lname);
3298
3299 /*
3300 * The common case is a complex message with no reply port,
3301 * because that is what the memory_object interface uses.
3302 */
3303
3304 if (bits == (MACH_MSGH_BITS_COMPLEX |
3305 MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) {
3306 bits = (MACH_MSGH_BITS_COMPLEX |
3307 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0));
3308
3309 kmsg->ikm_header->msgh_bits = bits;
3310 } else {
3311 bits = (MACH_MSGH_BITS_OTHER(bits) |
3312 MACH_MSGH_BITS(ipc_object_copyin_type(rname),
3313 ipc_object_copyin_type(lname)));
3314
3315 kmsg->ikm_header->msgh_bits = bits;
3316 if ((bits & MACH_MSGH_BITS_COMPLEX) == 0)
3317 return MACH_MSG_SUCCESS;
3318 }
3319 {
3320 mach_msg_descriptor_t *saddr;
3321 mach_msg_body_t *body;
3322 mach_msg_type_number_t i, count;
3323
3324 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
3325 saddr = (mach_msg_descriptor_t *) (body + 1);
3326 count = body->msgh_descriptor_count;
3327
3328 for (i = 0; i < count; i++, saddr++) {
3329
3330 switch (saddr->type.type) {
3331
3332 case MACH_MSG_PORT_DESCRIPTOR: {
3333 mach_msg_type_name_t name;
3334 ipc_object_t object;
3335 mach_msg_port_descriptor_t *dsc;
3336
3337 dsc = &saddr->port;
3338
3339 /* this is really the type SEND, SEND_ONCE, etc. */
3340 name = dsc->disposition;
3341 object = (ipc_object_t) dsc->name;
3342 dsc->disposition = ipc_object_copyin_type(name);
3343
3344 if (!IO_VALID(object)) {
3345 break;
3346 }
3347
3348 ipc_object_copyin_from_kernel(object, name);
3349
3350 /* CDY avoid circularity when the destination is also */
3351 /* the kernel. This check should be changed into an */
3352 /* assert when the new kobject model is in place since*/
3353 /* ports will not be used in kernel to kernel chats */
3354
3355 if (((ipc_port_t)remote)->ip_receiver != ipc_space_kernel) {
3356 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3357 ipc_port_check_circularity((ipc_port_t) object,
3358 (ipc_port_t) remote)) {
3359 kmsg->ikm_header->msgh_bits |=
3360 MACH_MSGH_BITS_CIRCULAR;
3361 }
3362 }
3363 break;
3364 }
3365 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3366 case MACH_MSG_OOL_DESCRIPTOR: {
3367 /*
3368 * The sender should supply ready-made memory, i.e.
3369 * a vm_map_copy_t, so we don't need to do anything.
3370 */
3371 break;
3372 }
3373 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
3374 ipc_object_t *objects;
3375 unsigned int j;
3376 mach_msg_type_name_t name;
3377 mach_msg_ool_ports_descriptor_t *dsc;
3378
3379 dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports;
3380
3381 /* this is really the type SEND, SEND_ONCE, etc. */
3382 name = dsc->disposition;
3383 dsc->disposition = ipc_object_copyin_type(name);
3384
3385 objects = (ipc_object_t *) dsc->address;
3386
3387 for ( j = 0; j < dsc->count; j++) {
3388 ipc_object_t object = objects[j];
3389
3390 if (!IO_VALID(object))
3391 continue;
3392
3393 ipc_object_copyin_from_kernel(object, name);
3394
3395 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3396 ipc_port_check_circularity(
3397 (ipc_port_t) object,
3398 (ipc_port_t) remote))
3399 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
3400 }
3401 break;
3402 }
3403 default: {
3404 #if MACH_ASSERT
3405 panic("ipc_kmsg_copyin_from_kernel: bad descriptor");
3406 #endif /* MACH_ASSERT */
3407 }
3408 }
3409 }
3410 }
3411 return MACH_MSG_SUCCESS;
3412 }
3413
3414 #if IKM_SUPPORT_LEGACY
3415 mach_msg_return_t
3416 ipc_kmsg_copyin_from_kernel_legacy(
3417 ipc_kmsg_t kmsg)
3418 {
3419 mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits;
3420 mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits);
3421 mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits);
3422 ipc_object_t remote = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
3423 ipc_object_t local = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
3424
3425 /* translate the destination and reply ports */
3426 if (!IO_VALID(remote))
3427 return MACH_SEND_INVALID_DEST;
3428
3429 ipc_object_copyin_from_kernel(remote, rname);
3430 if (IO_VALID(local))
3431 ipc_object_copyin_from_kernel(local, lname);
3432
3433 /*
3434 * The common case is a complex message with no reply port,
3435 * because that is what the memory_object interface uses.
3436 */
3437
3438 if (bits == (MACH_MSGH_BITS_COMPLEX |
3439 MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) {
3440 bits = (MACH_MSGH_BITS_COMPLEX |
3441 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0));
3442
3443 kmsg->ikm_header->msgh_bits = bits;
3444 } else {
3445 bits = (MACH_MSGH_BITS_OTHER(bits) |
3446 MACH_MSGH_BITS(ipc_object_copyin_type(rname),
3447 ipc_object_copyin_type(lname)));
3448
3449 kmsg->ikm_header->msgh_bits = bits;
3450 if ((bits & MACH_MSGH_BITS_COMPLEX) == 0)
3451 return MACH_MSG_SUCCESS;
3452 }
3453 {
3454 mach_msg_legacy_descriptor_t *saddr;
3455 mach_msg_descriptor_t *daddr;
3456 mach_msg_body_t *body;
3457 mach_msg_type_number_t i, count;
3458
3459 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
3460 saddr = (typeof(saddr)) (body + 1);
3461 count = body->msgh_descriptor_count;
3462
3463 if(count) {
3464 vm_offset_t dsc_adjust = 4*count;
3465 memmove((char *)(((vm_offset_t)kmsg->ikm_header) - dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t));
3466 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header - dsc_adjust);
3467 /* Update the message size for the larger in-kernel representation */
3468 kmsg->ikm_header->msgh_size += dsc_adjust;
3469 }
3470 daddr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t));
3471
3472 for (i = 0; i < count; i++, saddr++, daddr++) {
3473 switch (saddr->type.type) {
3474
3475 case MACH_MSG_PORT_DESCRIPTOR: {
3476 mach_msg_type_name_t name;
3477 ipc_object_t object;
3478 mach_msg_legacy_port_descriptor_t *dsc;
3479 mach_msg_port_descriptor_t *dest_dsc;
3480
3481 dsc = (typeof(dsc))&saddr->port;
3482 dest_dsc = &daddr->port;
3483
3484 /* this is really the type SEND, SEND_ONCE, etc. */
3485 name = dsc->disposition;
3486 object = (ipc_object_t) CAST_MACH_NAME_TO_PORT(dsc->name);
3487 dest_dsc->disposition = ipc_object_copyin_type(name);
3488 dest_dsc->name = (mach_port_t)object;
3489 dest_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
3490
3491 if (!IO_VALID(object)) {
3492 break;
3493 }
3494
3495 ipc_object_copyin_from_kernel(object, name);
3496
3497 /* CDY avoid circularity when the destination is also */
3498 /* the kernel. This check should be changed into an */
3499 /* assert when the new kobject model is in place since*/
3500 /* ports will not be used in kernel to kernel chats */
3501
3502 if (((ipc_port_t)remote)->ip_receiver != ipc_space_kernel) {
3503 if ((dest_dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3504 ipc_port_check_circularity((ipc_port_t) object,
3505 (ipc_port_t) remote)) {
3506 kmsg->ikm_header->msgh_bits |=
3507 MACH_MSGH_BITS_CIRCULAR;
3508 }
3509 }
3510 break;
3511 }
3512 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3513 case MACH_MSG_OOL_DESCRIPTOR: {
3514 /* The sender should supply ready-made memory, i.e. a vm_map_copy_t
3515 * so we don't need to do anything special. */
3516
3517 mach_msg_ool_descriptor32_t *source_dsc = &saddr->out_of_line32;
3518 mach_msg_ool_descriptor_t *dest_dsc = (typeof(dest_dsc))&daddr->out_of_line;
3519
3520 vm_offset_t address = source_dsc->address;
3521 vm_size_t size = source_dsc->size;
3522 boolean_t deallocate = source_dsc->deallocate;
3523 mach_msg_copy_options_t copy = source_dsc->copy;
3524 mach_msg_descriptor_type_t type = source_dsc->type;
3525
3526 dest_dsc->address = (void *)address;
3527 dest_dsc->size = size;
3528 dest_dsc->deallocate = deallocate;
3529 dest_dsc->copy = copy;
3530 dest_dsc->type = type;
3531 break;
3532 }
3533 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
3534 ipc_object_t *objects;
3535 unsigned int j;
3536 mach_msg_type_name_t name;
3537 mach_msg_ool_ports_descriptor_t *dest_dsc;
3538
3539 mach_msg_ool_ports_descriptor32_t *source_dsc = &saddr->ool_ports32;
3540 dest_dsc = (typeof(dest_dsc))&daddr->ool_ports;
3541
3542 boolean_t deallocate = source_dsc->deallocate;
3543 mach_msg_copy_options_t copy = source_dsc->copy;
3544 mach_msg_size_t port_count = source_dsc->count;
3545 mach_msg_type_name_t disposition = source_dsc->disposition;
3546
3547 /* this is really the type SEND, SEND_ONCE, etc. */
3548 name = disposition;
3549 disposition = ipc_object_copyin_type(name);
3550
3551 objects = (ipc_object_t *) (uintptr_t)source_dsc->address;
3552
3553 for ( j = 0; j < port_count; j++) {
3554 ipc_object_t object = objects[j];
3555
3556 if (!IO_VALID(object))
3557 continue;
3558
3559 ipc_object_copyin_from_kernel(object, name);
3560
3561 if ((disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3562 ipc_port_check_circularity(
3563 (ipc_port_t) object,
3564 (ipc_port_t) remote))
3565 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
3566 }
3567
3568 dest_dsc->address = objects;
3569 dest_dsc->deallocate = deallocate;
3570 dest_dsc->copy = copy;
3571 dest_dsc->disposition = disposition;
3572 dest_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
3573 dest_dsc->count = port_count;
3574 break;
3575 }
3576 default: {
3577 #if MACH_ASSERT
3578 panic("ipc_kmsg_copyin_from_kernel: bad descriptor");
3579 #endif /* MACH_ASSERT */
3580 }
3581 }
3582 }
3583 }
3584 return MACH_MSG_SUCCESS;
3585 }
3586 #endif /* IKM_SUPPORT_LEGACY */
3587
3588 /*
3589 * Routine: ipc_kmsg_copyout_header
3590 * Purpose:
3591 * "Copy-out" port rights in the header of a message.
3592 * Operates atomically; if it doesn't succeed the
3593 * message header and the space are left untouched.
3594 * If it does succeed the remote/local port fields
3595 * contain port names instead of object pointers,
3596 * and the bits field is updated.
3597 * Conditions:
3598 * Nothing locked.
3599 * Returns:
3600 * MACH_MSG_SUCCESS Copied out port rights.
3601 * MACH_RCV_INVALID_NOTIFY
3602 * Notify is non-null and doesn't name a receive right.
3603 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
3604 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE
3605 * The space is dead.
3606 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE
3607 * No room in space for another name.
3608 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_KERNEL
3609 * Couldn't allocate memory for the reply port.
3610 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_KERNEL
3611 * Couldn't allocate memory for the dead-name request.
3612 */
3613
3614 mach_msg_return_t
3615 ipc_kmsg_copyout_header(
3616 ipc_kmsg_t kmsg,
3617 ipc_space_t space,
3618 mach_msg_option_t option)
3619 {
3620 mach_msg_header_t *msg = kmsg->ikm_header;
3621 mach_msg_bits_t mbits = msg->msgh_bits;
3622 ipc_port_t dest = (ipc_port_t) msg->msgh_remote_port;
3623
3624 assert(IP_VALID(dest));
3625
3626 /*
3627 * While we still hold a reference on the received-from port,
3628 * process all send-possible notfications we received along with
3629 * the message.
3630 */
3631 ipc_port_spnotify(dest);
3632
3633 {
3634 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
3635 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
3636 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
3637 ipc_port_t reply = msg->msgh_local_port;
3638 ipc_port_t release_reply_port = IP_NULL;
3639 mach_port_name_t dest_name, reply_name;
3640
3641 ipc_port_t voucher = kmsg->ikm_voucher;
3642 ipc_port_t release_voucher_port = IP_NULL;
3643 mach_port_name_t voucher_name;
3644
3645 uint32_t entries_held = 0;
3646 boolean_t need_write_lock = FALSE;
3647 kern_return_t kr;
3648
3649 /*
3650 * Reserve any potentially needed entries in the target space.
3651 * We'll free any unused before unlocking the space.
3652 */
3653 if (IP_VALID(reply)) {
3654 entries_held++;
3655 need_write_lock = TRUE;
3656 }
3657 if (IP_VALID(voucher)) {
3658 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
3659
3660 if ((option & MACH_RCV_VOUCHER) != 0)
3661 entries_held++;
3662 need_write_lock = TRUE;
3663 }
3664
3665 if (need_write_lock) {
3666
3667 is_write_lock(space);
3668
3669 while(entries_held) {
3670 if (!is_active(space)) {
3671 is_write_unlock(space);
3672 return (MACH_RCV_HEADER_ERROR|
3673 MACH_MSG_IPC_SPACE);
3674 }
3675
3676 kr = ipc_entries_hold(space, entries_held);
3677 if (KERN_SUCCESS == kr)
3678 break;
3679
3680 kr = ipc_entry_grow_table(space, ITS_SIZE_NONE);
3681 if (KERN_SUCCESS != kr)
3682 return(MACH_RCV_HEADER_ERROR|
3683 MACH_MSG_IPC_SPACE);
3684 /* space was unlocked and relocked - retry */
3685 }
3686
3687 /* Handle reply port. */
3688 if (IP_VALID(reply)) {
3689 ipc_entry_t entry;
3690
3691 /* Is there already an entry we can use? */
3692 if ((reply_type != MACH_MSG_TYPE_PORT_SEND_ONCE) &&
3693 ipc_right_reverse(space, (ipc_object_t) reply, &reply_name, &entry)) {
3694 /* reply port is locked and active */
3695 assert(entry->ie_bits & MACH_PORT_TYPE_SEND_RECEIVE);
3696 } else {
3697 ip_lock(reply);
3698 if (!ip_active(reply)) {
3699 ip_unlock(reply);
3700
3701 release_reply_port = reply;
3702 reply = IP_DEAD;
3703 reply_name = MACH_PORT_DEAD;
3704 goto done_with_reply;
3705 }
3706
3707 /* claim a held entry for the reply port */
3708 assert(entries_held > 0);
3709 entries_held--;
3710 ipc_entry_claim(space, &reply_name, &entry);
3711 assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE);
3712 assert(entry->ie_object == IO_NULL);
3713 entry->ie_object = (ipc_object_t) reply;
3714 }
3715
3716 /* space and reply port are locked and active */
3717 ip_reference(reply); /* hold onto the reply port */
3718
3719 kr = ipc_right_copyout(space, reply_name, entry,
3720 reply_type, TRUE, (ipc_object_t) reply);
3721 assert(kr == KERN_SUCCESS);
3722 /* reply port is unlocked */
3723 } else
3724 reply_name = CAST_MACH_PORT_TO_NAME(reply);
3725
3726 done_with_reply:
3727
3728 /* Handle voucher port. */
3729 if (voucher_type != MACH_MSGH_BITS_ZERO) {
3730 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
3731
3732 if (!IP_VALID(voucher)) {
3733 if ((option & MACH_RCV_VOUCHER) == 0) {
3734 voucher_type = MACH_MSGH_BITS_ZERO;
3735 }
3736 voucher_name = MACH_PORT_NULL;
3737 goto done_with_voucher;
3738 }
3739
3740 /* clear voucher from its hiding place back in the kmsg */
3741 kmsg->ikm_voucher = IP_NULL;
3742
3743 if ((option & MACH_RCV_VOUCHER) != 0) {
3744 ipc_entry_t entry;
3745
3746 if (ipc_right_reverse(space, (ipc_object_t) voucher,
3747 &voucher_name, &entry)) {
3748 /* voucher port locked */
3749 assert(entry->ie_bits & MACH_PORT_TYPE_SEND);
3750 } else {
3751 assert(entries_held > 0);
3752 entries_held--;
3753 ipc_entry_claim(space, &voucher_name, &entry);
3754 assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE);
3755 assert(entry->ie_object == IO_NULL);
3756 entry->ie_object = (ipc_object_t) voucher;
3757 ip_lock(voucher);
3758 }
3759 /* space is locked and active */
3760
3761 assert(ip_active(voucher));
3762 assert(ip_kotype(voucher) == IKOT_VOUCHER);
3763 kr = ipc_right_copyout(space, voucher_name, entry,
3764 MACH_MSG_TYPE_MOVE_SEND, TRUE,
3765 (ipc_object_t) voucher);
3766 /* voucher port is unlocked */
3767 } else {
3768 voucher_type = MACH_MSGH_BITS_ZERO;
3769 release_voucher_port = voucher;
3770 voucher_name = MACH_PORT_NULL;
3771 }
3772 } else {
3773 voucher_name = msg->msgh_voucher_port;
3774 }
3775
3776 done_with_voucher:
3777
3778 ip_lock(dest);
3779 is_write_unlock(space);
3780
3781 } else {
3782 /*
3783 * No reply or voucher port! This is an easy case.
3784 * We only need to have the space locked
3785 * when locking the destination.
3786 */
3787
3788 is_read_lock(space);
3789 if (!is_active(space)) {
3790 is_read_unlock(space);
3791 return MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE;
3792 }
3793
3794 ip_lock(dest);
3795 is_read_unlock(space);
3796
3797 reply_name = CAST_MACH_PORT_TO_NAME(reply);
3798
3799 if (voucher_type != MACH_MSGH_BITS_ZERO) {
3800 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
3801 if ((option & MACH_RCV_VOUCHER) == 0) {
3802 voucher_type = MACH_MSGH_BITS_ZERO;
3803 }
3804 voucher_name = MACH_PORT_NULL;
3805 } else {
3806 voucher_name = msg->msgh_voucher_port;
3807 }
3808 }
3809
3810 /*
3811 * At this point, the space is unlocked and the destination
3812 * port is locked. (Lock taken while space was locked.)
3813 * reply_name is taken care of; we still need dest_name.
3814 * We still hold a ref for reply (if it is valid).
3815 *
3816 * If the space holds receive rights for the destination,
3817 * we return its name for the right. Otherwise the task
3818 * managed to destroy or give away the receive right between
3819 * receiving the message and this copyout. If the destination
3820 * is dead, return MACH_PORT_DEAD, and if the receive right
3821 * exists somewhere else (another space, in transit)
3822 * return MACH_PORT_NULL.
3823 *
3824 * Making this copyout operation atomic with the previous
3825 * copyout of the reply port is a bit tricky. If there was
3826 * no real reply port (it wasn't IP_VALID) then this isn't
3827 * an issue. If the reply port was dead at copyout time,
3828 * then we are OK, because if dest is dead we serialize
3829 * after the death of both ports and if dest is alive
3830 * we serialize after reply died but before dest's (later) death.
3831 * So assume reply was alive when we copied it out. If dest
3832 * is alive, then we are OK because we serialize before
3833 * the ports' deaths. So assume dest is dead when we look at it.
3834 * If reply dies/died after dest, then we are OK because
3835 * we serialize after dest died but before reply dies.
3836 * So the hard case is when reply is alive at copyout,
3837 * dest is dead at copyout, and reply died before dest died.
3838 * In this case pretend that dest is still alive, so
3839 * we serialize while both ports are alive.
3840 *
3841 * Because the space lock is held across the copyout of reply
3842 * and locking dest, the receive right for dest can't move
3843 * in or out of the space while the copyouts happen, so
3844 * that isn't an atomicity problem. In the last hard case
3845 * above, this implies that when dest is dead that the
3846 * space couldn't have had receive rights for dest at
3847 * the time reply was copied-out, so when we pretend
3848 * that dest is still alive, we can return MACH_PORT_NULL.
3849 *
3850 * If dest == reply, then we have to make it look like
3851 * either both copyouts happened before the port died,
3852 * or both happened after the port died. This special
3853 * case works naturally if the timestamp comparison
3854 * is done correctly.
3855 */
3856
3857 if (ip_active(dest)) {
3858 ipc_object_copyout_dest(space, (ipc_object_t) dest,
3859 dest_type, &dest_name);
3860 /* dest is unlocked */
3861
3862 } else {
3863 ipc_port_timestamp_t timestamp;
3864
3865 timestamp = dest->ip_timestamp;
3866 ip_unlock(dest);
3867 ip_release(dest);
3868
3869 if (IP_VALID(reply)) {
3870 ip_lock(reply);
3871 if (ip_active(reply) ||
3872 IP_TIMESTAMP_ORDER(timestamp,
3873 reply->ip_timestamp))
3874 dest_name = MACH_PORT_DEAD;
3875 else
3876 dest_name = MACH_PORT_NULL;
3877 ip_unlock(reply);
3878 } else
3879 dest_name = MACH_PORT_DEAD;
3880 }
3881
3882 if (IP_VALID(reply))
3883 ip_release(reply);
3884
3885 if (IP_VALID(release_reply_port)) {
3886 if (reply_type == MACH_MSG_TYPE_PORT_SEND_ONCE)
3887 ipc_port_release_sonce(release_reply_port);
3888 else
3889 ipc_port_release_send(release_reply_port);
3890 }
3891
3892 if (IP_VALID(release_voucher_port))
3893 ipc_port_release_send(release_voucher_port);
3894
3895
3896 if ((option & MACH_RCV_VOUCHER) != 0) {
3897 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV) | DBG_FUNC_NONE,
3898 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3899 (uintptr_t)kmsg->ikm_header->msgh_bits,
3900 (uintptr_t)kmsg->ikm_header->msgh_id,
3901 VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(voucher)),
3902 0);
3903 } else {
3904 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV_VOUCHER_REFUSED) | DBG_FUNC_NONE,
3905 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3906 (uintptr_t)kmsg->ikm_header->msgh_bits,
3907 (uintptr_t)kmsg->ikm_header->msgh_id,
3908 VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(voucher)),
3909 0);
3910 }
3911
3912 msg->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type,
3913 voucher_type, mbits);
3914 msg->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
3915 msg->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
3916 msg->msgh_voucher_port = voucher_name;
3917 }
3918
3919 return MACH_MSG_SUCCESS;
3920 }
3921
3922 /*
3923 * Routine: ipc_kmsg_copyout_object
3924 * Purpose:
3925 * Copy-out a port right. Always returns a name,
3926 * even for unsuccessful return codes. Always
3927 * consumes the supplied object.
3928 * Conditions:
3929 * Nothing locked.
3930 * Returns:
3931 * MACH_MSG_SUCCESS The space acquired the right
3932 * (name is valid) or the object is dead (MACH_PORT_DEAD).
3933 * MACH_MSG_IPC_SPACE No room in space for the right,
3934 * or the space is dead. (Name is MACH_PORT_NULL.)
3935 * MACH_MSG_IPC_KERNEL Kernel resource shortage.
3936 * (Name is MACH_PORT_NULL.)
3937 */
3938
3939 mach_msg_return_t
3940 ipc_kmsg_copyout_object(
3941 ipc_space_t space,
3942 ipc_object_t object,
3943 mach_msg_type_name_t msgt_name,
3944 mach_port_name_t *namep)
3945 {
3946 kern_return_t kr;
3947
3948 if (!IO_VALID(object)) {
3949 *namep = CAST_MACH_PORT_TO_NAME(object);
3950 return MACH_MSG_SUCCESS;
3951 }
3952
3953 kr = ipc_object_copyout(space, object, msgt_name, TRUE, namep);
3954 if (kr != KERN_SUCCESS) {
3955 ipc_object_destroy(object, msgt_name);
3956
3957 if (kr == KERN_INVALID_CAPABILITY)
3958 *namep = MACH_PORT_DEAD;
3959 else {
3960 *namep = MACH_PORT_NULL;
3961
3962 if (kr == KERN_RESOURCE_SHORTAGE)
3963 return MACH_MSG_IPC_KERNEL;
3964 else
3965 return MACH_MSG_IPC_SPACE;
3966 }
3967 }
3968
3969 return MACH_MSG_SUCCESS;
3970 }
3971
3972 mach_msg_descriptor_t *
3973 ipc_kmsg_copyout_port_descriptor(mach_msg_descriptor_t *dsc,
3974 mach_msg_descriptor_t *user_dsc,
3975 ipc_space_t space,
3976 kern_return_t *mr);
3977 mach_msg_descriptor_t *
3978 ipc_kmsg_copyout_port_descriptor(mach_msg_descriptor_t *dsc,
3979 mach_msg_descriptor_t *dest_dsc,
3980 ipc_space_t space,
3981 kern_return_t *mr)
3982 {
3983 mach_port_t port;
3984 mach_port_name_t name;
3985 mach_msg_type_name_t disp;
3986
3987
3988 /* Copyout port right carried in the message */
3989 port = dsc->port.name;
3990 disp = dsc->port.disposition;
3991 *mr |= ipc_kmsg_copyout_object(space,
3992 (ipc_object_t)port,
3993 disp,
3994 &name);
3995
3996 if(current_task() == kernel_task)
3997 {
3998 mach_msg_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc;
3999 user_dsc--; // point to the start of this port descriptor
4000 bzero((void *)user_dsc, sizeof(*user_dsc));
4001 user_dsc->name = CAST_MACH_NAME_TO_PORT(name);
4002 user_dsc->disposition = disp;
4003 user_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
4004 dest_dsc = (typeof(dest_dsc))user_dsc;
4005 } else {
4006 mach_msg_legacy_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc;
4007 user_dsc--; // point to the start of this port descriptor
4008 bzero((void *)user_dsc, sizeof(*user_dsc));
4009 user_dsc->name = CAST_MACH_PORT_TO_NAME(name);
4010 user_dsc->disposition = disp;
4011 user_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
4012 dest_dsc = (typeof(dest_dsc))user_dsc;
4013 }
4014
4015 return (mach_msg_descriptor_t *)dest_dsc;
4016 }
4017
4018 mach_msg_descriptor_t *
4019 ipc_kmsg_copyout_ool_descriptor(mach_msg_ool_descriptor_t *dsc, mach_msg_descriptor_t *user_dsc, int is_64bit, vm_map_t map, mach_msg_return_t *mr);
4020 mach_msg_descriptor_t *
4021 ipc_kmsg_copyout_ool_descriptor(mach_msg_ool_descriptor_t *dsc, mach_msg_descriptor_t *user_dsc, int is_64bit, vm_map_t map, mach_msg_return_t *mr)
4022 {
4023 vm_map_copy_t copy;
4024 vm_map_address_t rcv_addr;
4025 mach_msg_copy_options_t copy_options;
4026 vm_map_size_t size;
4027 mach_msg_descriptor_type_t dsc_type;
4028
4029 //SKIP_PORT_DESCRIPTORS(saddr, sdsc_count);
4030
4031 copy = (vm_map_copy_t)dsc->address;
4032 size = (vm_map_size_t)dsc->size;
4033 copy_options = dsc->copy;
4034 assert(copy_options != MACH_MSG_KALLOC_COPY_T);
4035 dsc_type = dsc->type;
4036
4037 if (copy != VM_MAP_COPY_NULL) {
4038 kern_return_t kr;
4039
4040 rcv_addr = 0;
4041 if (vm_map_copy_validate_size(map, copy, &size) == FALSE)
4042 panic("Inconsistent OOL/copyout size on %p: expected %d, got %lld @%p",
4043 dsc, dsc->size, (unsigned long long)copy->size, copy);
4044 kr = vm_map_copyout_size(map, &rcv_addr, copy, size);
4045 if (kr != KERN_SUCCESS) {
4046 if (kr == KERN_RESOURCE_SHORTAGE)
4047 *mr |= MACH_MSG_VM_KERNEL;
4048 else
4049 *mr |= MACH_MSG_VM_SPACE;
4050 vm_map_copy_discard(copy);
4051 rcv_addr = 0;
4052 size = 0;
4053 }
4054 } else {
4055 rcv_addr = 0;
4056 size = 0;
4057 }
4058
4059 /*
4060 * Now update the descriptor as the user would see it.
4061 * This may require expanding the descriptor to the user
4062 * visible size. There is already space allocated for
4063 * this in what naddr points to.
4064 */
4065 if(current_task() == kernel_task)
4066 {
4067 mach_msg_ool_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4068 user_ool_dsc--;
4069 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4070
4071 user_ool_dsc->address = (void *)(uintptr_t)rcv_addr;
4072 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4073 TRUE : FALSE;
4074 user_ool_dsc->copy = copy_options;
4075 user_ool_dsc->type = dsc_type;
4076 user_ool_dsc->size = (mach_msg_size_t)size;
4077
4078 user_dsc = (typeof(user_dsc))user_ool_dsc;
4079 } else if (is_64bit) {
4080 mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4081 user_ool_dsc--;
4082 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4083
4084 user_ool_dsc->address = rcv_addr;
4085 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4086 TRUE : FALSE;
4087 user_ool_dsc->copy = copy_options;
4088 user_ool_dsc->type = dsc_type;
4089 user_ool_dsc->size = (mach_msg_size_t)size;
4090
4091 user_dsc = (typeof(user_dsc))user_ool_dsc;
4092 } else {
4093 mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4094 user_ool_dsc--;
4095 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4096
4097 user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr);
4098 user_ool_dsc->size = (mach_msg_size_t)size;
4099 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4100 TRUE : FALSE;
4101 user_ool_dsc->copy = copy_options;
4102 user_ool_dsc->type = dsc_type;
4103
4104 user_dsc = (typeof(user_dsc))user_ool_dsc;
4105 }
4106 return user_dsc;
4107 }
4108
4109 mach_msg_descriptor_t *
4110 ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc,
4111 mach_msg_descriptor_t *user_dsc,
4112 int is_64bit,
4113 vm_map_t map,
4114 ipc_space_t space,
4115 ipc_kmsg_t kmsg,
4116 mach_msg_return_t *mr);
4117 mach_msg_descriptor_t *
4118 ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc,
4119 mach_msg_descriptor_t *user_dsc,
4120 int is_64bit,
4121 vm_map_t map,
4122 ipc_space_t space,
4123 ipc_kmsg_t kmsg,
4124 mach_msg_return_t *mr)
4125 {
4126 mach_vm_offset_t rcv_addr = 0;
4127 mach_msg_type_name_t disp;
4128 mach_msg_type_number_t count, i;
4129 vm_size_t ports_length, names_length;
4130
4131 mach_msg_copy_options_t copy_options = MACH_MSG_VIRTUAL_COPY;
4132
4133 //SKIP_PORT_DESCRIPTORS(saddr, sdsc_count);
4134
4135 count = dsc->count;
4136 disp = dsc->disposition;
4137 ports_length = count * sizeof(mach_port_t);
4138 names_length = count * sizeof(mach_port_name_t);
4139
4140 if (ports_length != 0 && dsc->address != 0) {
4141
4142 /*
4143 * Check to see if there is an overwrite descriptor
4144 * specified in the scatter list for this ool data.
4145 * The descriptor has already been verified.
4146 */
4147 #if 0
4148 if (saddr != MACH_MSG_DESCRIPTOR_NULL) {
4149 if (differs) {
4150 OTHER_OOL_DESCRIPTOR *scatter_dsc;
4151
4152 scatter_dsc = (OTHER_OOL_DESCRIPTOR *)saddr;
4153 rcv_addr = (mach_vm_offset_t) scatter_dsc->address;
4154 copy_options = scatter_dsc->copy;
4155 } else {
4156 mach_msg_ool_descriptor_t *scatter_dsc;
4157
4158 scatter_dsc = &saddr->out_of_line;
4159 rcv_addr = CAST_USER_ADDR_T(scatter_dsc->address);
4160 copy_options = scatter_dsc->copy;
4161 }
4162 INCREMENT_SCATTER(saddr, sdsc_count, differs);
4163 }
4164 #endif
4165
4166 if (copy_options == MACH_MSG_VIRTUAL_COPY) {
4167 /*
4168 * Dynamically allocate the region
4169 */
4170 vm_tag_t tag;
4171 if (vm_kernel_map_is_kernel(map)) tag = VM_KERN_MEMORY_IPC;
4172 else tag = VM_MEMORY_MACH_MSG;
4173
4174 kern_return_t kr;
4175 if ((kr = mach_vm_allocate_kernel(map, &rcv_addr,
4176 (mach_vm_size_t)names_length,
4177 VM_FLAGS_ANYWHERE, tag)) != KERN_SUCCESS) {
4178 ipc_kmsg_clean_body(kmsg, 1, (mach_msg_descriptor_t *)dsc);
4179 rcv_addr = 0;
4180
4181 if (kr == KERN_RESOURCE_SHORTAGE){
4182 *mr |= MACH_MSG_VM_KERNEL;
4183 } else {
4184 *mr |= MACH_MSG_VM_SPACE;
4185 }
4186 }
4187 }
4188
4189 /*
4190 * Handle the port rights and copy out the names
4191 * for those rights out to user-space.
4192 */
4193 if (rcv_addr != 0) {
4194 mach_port_t *objects = (mach_port_t *) dsc->address;
4195 mach_port_name_t *names = (mach_port_name_t *) dsc->address;
4196
4197 /* copyout port rights carried in the message */
4198
4199 for ( i = 0; i < count ; i++) {
4200 ipc_object_t object = (ipc_object_t)objects[i];
4201
4202 *mr |= ipc_kmsg_copyout_object(space, object,
4203 disp, &names[i]);
4204 }
4205
4206 /* copyout to memory allocated above */
4207 void *data = dsc->address;
4208 if (copyoutmap(map, data, rcv_addr, names_length) != KERN_SUCCESS)
4209 *mr |= MACH_MSG_VM_SPACE;
4210 kfree(data, ports_length);
4211 }
4212 } else {
4213 rcv_addr = 0;
4214 }
4215
4216 /*
4217 * Now update the descriptor based on the information
4218 * calculated above.
4219 */
4220 if(current_task() == kernel_task) {
4221 mach_msg_ool_ports_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4222 user_ool_dsc--;
4223 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4224
4225 user_ool_dsc->address = (void *)(uintptr_t)rcv_addr;
4226 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4227 TRUE : FALSE;
4228 user_ool_dsc->copy = copy_options;
4229 user_ool_dsc->disposition = disp;
4230 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
4231 user_ool_dsc->count = count;
4232
4233 user_dsc = (typeof(user_dsc))user_ool_dsc;
4234 } if (is_64bit) {
4235 mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4236 user_ool_dsc--;
4237 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4238
4239 user_ool_dsc->address = rcv_addr;
4240 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4241 TRUE : FALSE;
4242 user_ool_dsc->copy = copy_options;
4243 user_ool_dsc->disposition = disp;
4244 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
4245 user_ool_dsc->count = count;
4246
4247 user_dsc = (typeof(user_dsc))user_ool_dsc;
4248 } else {
4249 mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4250 user_ool_dsc--;
4251 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4252
4253 user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr);
4254 user_ool_dsc->count = count;
4255 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4256 TRUE : FALSE;
4257 user_ool_dsc->copy = copy_options;
4258 user_ool_dsc->disposition = disp;
4259 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
4260
4261 user_dsc = (typeof(user_dsc))user_ool_dsc;
4262 }
4263 return user_dsc;
4264 }
4265
4266 /*
4267 * Routine: ipc_kmsg_copyout_body
4268 * Purpose:
4269 * "Copy-out" port rights and out-of-line memory
4270 * in the body of a message.
4271 *
4272 * The error codes are a combination of special bits.
4273 * The copyout proceeds despite errors.
4274 * Conditions:
4275 * Nothing locked.
4276 * Returns:
4277 * MACH_MSG_SUCCESS Successful copyout.
4278 * MACH_MSG_IPC_SPACE No room for port right in name space.
4279 * MACH_MSG_VM_SPACE No room for memory in address space.
4280 * MACH_MSG_IPC_KERNEL Resource shortage handling port right.
4281 * MACH_MSG_VM_KERNEL Resource shortage handling memory.
4282 * MACH_MSG_INVALID_RT_DESCRIPTOR Descriptor incompatible with RT
4283 */
4284
4285 mach_msg_return_t
4286 ipc_kmsg_copyout_body(
4287 ipc_kmsg_t kmsg,
4288 ipc_space_t space,
4289 vm_map_t map,
4290 mach_msg_body_t *slist)
4291 {
4292 mach_msg_body_t *body;
4293 mach_msg_descriptor_t *kern_dsc, *user_dsc;
4294 mach_msg_descriptor_t *saddr;
4295 mach_msg_type_number_t dsc_count, sdsc_count;
4296 int i;
4297 mach_msg_return_t mr = MACH_MSG_SUCCESS;
4298 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
4299
4300 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
4301 dsc_count = body->msgh_descriptor_count;
4302 kern_dsc = (mach_msg_descriptor_t *) (body + 1);
4303 /* Point user_dsc just after the end of all the descriptors */
4304 user_dsc = &kern_dsc[dsc_count];
4305
4306 /* Do scatter list setup */
4307 if (slist != MACH_MSG_BODY_NULL) {
4308 panic("Scatter lists disabled");
4309 saddr = (mach_msg_descriptor_t *) (slist + 1);
4310 sdsc_count = slist->msgh_descriptor_count;
4311 }
4312 else {
4313 saddr = MACH_MSG_DESCRIPTOR_NULL;
4314 sdsc_count = 0;
4315 }
4316
4317 /* Now process the descriptors */
4318 for (i = dsc_count-1; i >= 0; i--) {
4319 switch (kern_dsc[i].type.type) {
4320
4321 case MACH_MSG_PORT_DESCRIPTOR:
4322 user_dsc = ipc_kmsg_copyout_port_descriptor(&kern_dsc[i], user_dsc, space, &mr);
4323 break;
4324 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
4325 case MACH_MSG_OOL_DESCRIPTOR :
4326 user_dsc = ipc_kmsg_copyout_ool_descriptor(
4327 (mach_msg_ool_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, map, &mr);
4328 break;
4329 case MACH_MSG_OOL_PORTS_DESCRIPTOR :
4330 user_dsc = ipc_kmsg_copyout_ool_ports_descriptor(
4331 (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, map, space, kmsg, &mr);
4332 break;
4333 default : {
4334 panic("untyped IPC copyout body: invalid message descriptor");
4335 }
4336 }
4337 }
4338
4339 if(user_dsc != kern_dsc) {
4340 vm_offset_t dsc_adjust = (vm_offset_t)user_dsc - (vm_offset_t)kern_dsc;
4341 memmove((char *)((vm_offset_t)kmsg->ikm_header + dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t));
4342 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header + dsc_adjust);
4343 /* Update the message size for the smaller user representation */
4344 kmsg->ikm_header->msgh_size -= (mach_msg_size_t)dsc_adjust;
4345 }
4346
4347 return mr;
4348 }
4349
4350 /*
4351 * Routine: ipc_kmsg_copyout_size
4352 * Purpose:
4353 * Compute the size of the message as copied out to the given
4354 * map. If the destination map's pointers are a different size
4355 * than the kernel's, we have to allow for expansion/
4356 * contraction of the descriptors as appropriate.
4357 * Conditions:
4358 * Nothing locked.
4359 * Returns:
4360 * size of the message as it would be received.
4361 */
4362
4363 mach_msg_size_t
4364 ipc_kmsg_copyout_size(
4365 ipc_kmsg_t kmsg,
4366 vm_map_t map)
4367 {
4368 mach_msg_size_t send_size;
4369
4370 send_size = kmsg->ikm_header->msgh_size;
4371
4372 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
4373
4374 #if defined(__LP64__)
4375 send_size -= LEGACY_HEADER_SIZE_DELTA;
4376 #endif
4377
4378 if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
4379
4380 mach_msg_body_t *body;
4381 mach_msg_descriptor_t *saddr, *eaddr;
4382
4383 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
4384 saddr = (mach_msg_descriptor_t *) (body + 1);
4385 eaddr = saddr + body->msgh_descriptor_count;
4386
4387 for ( ; saddr < eaddr; saddr++ ) {
4388 switch (saddr->type.type) {
4389 case MACH_MSG_OOL_DESCRIPTOR:
4390 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
4391 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
4392 if(!is_task_64bit)
4393 send_size -= DESC_SIZE_ADJUSTMENT;
4394 break;
4395 case MACH_MSG_PORT_DESCRIPTOR:
4396 send_size -= DESC_SIZE_ADJUSTMENT;
4397 break;
4398 default:
4399 break;
4400 }
4401 }
4402 }
4403 return send_size;
4404 }
4405
4406 /*
4407 * Routine: ipc_kmsg_copyout
4408 * Purpose:
4409 * "Copy-out" port rights and out-of-line memory
4410 * in the message.
4411 * Conditions:
4412 * Nothing locked.
4413 * Returns:
4414 * MACH_MSG_SUCCESS Copied out all rights and memory.
4415 * MACH_RCV_HEADER_ERROR + special bits
4416 * Rights and memory in the message are intact.
4417 * MACH_RCV_BODY_ERROR + special bits
4418 * The message header was successfully copied out.
4419 * As much of the body was handled as possible.
4420 */
4421
4422 mach_msg_return_t
4423 ipc_kmsg_copyout(
4424 ipc_kmsg_t kmsg,
4425 ipc_space_t space,
4426 vm_map_t map,
4427 mach_msg_body_t *slist,
4428 mach_msg_option_t option)
4429 {
4430 mach_msg_return_t mr;
4431
4432 mr = ipc_kmsg_copyout_header(kmsg, space, option);
4433 if (mr != MACH_MSG_SUCCESS) {
4434 return mr;
4435 }
4436
4437 if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
4438 mr = ipc_kmsg_copyout_body(kmsg, space, map, slist);
4439
4440 if (mr != MACH_MSG_SUCCESS)
4441 mr |= MACH_RCV_BODY_ERROR;
4442 }
4443
4444 return mr;
4445 }
4446
4447 /*
4448 * Routine: ipc_kmsg_copyout_pseudo
4449 * Purpose:
4450 * Does a pseudo-copyout of the message.
4451 * This is like a regular copyout, except
4452 * that the ports in the header are handled
4453 * as if they are in the body. They aren't reversed.
4454 *
4455 * The error codes are a combination of special bits.
4456 * The copyout proceeds despite errors.
4457 * Conditions:
4458 * Nothing locked.
4459 * Returns:
4460 * MACH_MSG_SUCCESS Successful copyout.
4461 * MACH_MSG_IPC_SPACE No room for port right in name space.
4462 * MACH_MSG_VM_SPACE No room for memory in address space.
4463 * MACH_MSG_IPC_KERNEL Resource shortage handling port right.
4464 * MACH_MSG_VM_KERNEL Resource shortage handling memory.
4465 */
4466
4467 mach_msg_return_t
4468 ipc_kmsg_copyout_pseudo(
4469 ipc_kmsg_t kmsg,
4470 ipc_space_t space,
4471 vm_map_t map,
4472 mach_msg_body_t *slist)
4473 {
4474 mach_msg_bits_t mbits = kmsg->ikm_header->msgh_bits;
4475 ipc_object_t dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4476 ipc_object_t reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
4477 ipc_object_t voucher = (ipc_object_t) kmsg->ikm_voucher;
4478 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
4479 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
4480 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
4481 mach_port_name_t voucher_name = kmsg->ikm_header->msgh_voucher_port;
4482 mach_port_name_t dest_name, reply_name;
4483 mach_msg_return_t mr;
4484
4485 /* Set ith_knote to ITH_KNOTE_PSEUDO */
4486 current_thread()->ith_knote = ITH_KNOTE_PSEUDO;
4487
4488 assert(IO_VALID(dest));
4489
4490 #if 0
4491 /*
4492 * If we did this here, it looks like we wouldn't need the undo logic
4493 * at the end of ipc_kmsg_send() in the error cases. Not sure which
4494 * would be more elegant to keep.
4495 */
4496 ipc_importance_clean(kmsg);
4497 #else
4498 /* just assert it is already clean */
4499 ipc_importance_assert_clean(kmsg);
4500 #endif
4501
4502 mr = (ipc_kmsg_copyout_object(space, dest, dest_type, &dest_name) |
4503 ipc_kmsg_copyout_object(space, reply, reply_type, &reply_name));
4504
4505 kmsg->ikm_header->msgh_bits = mbits & MACH_MSGH_BITS_USER;
4506 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(dest_name);
4507 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(reply_name);
4508
4509 if (IO_VALID(voucher)) {
4510 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
4511
4512 kmsg->ikm_voucher = IP_NULL;
4513 mr |= ipc_kmsg_copyout_object(space, voucher, voucher_type, &voucher_name);
4514 kmsg->ikm_header->msgh_voucher_port = voucher_name;
4515 }
4516
4517 if (mbits & MACH_MSGH_BITS_COMPLEX) {
4518 mr |= ipc_kmsg_copyout_body(kmsg, space, map, slist);
4519 }
4520
4521 return mr;
4522 }
4523
4524 /*
4525 * Routine: ipc_kmsg_copyout_dest
4526 * Purpose:
4527 * Copies out the destination port in the message.
4528 * Destroys all other rights and memory in the message.
4529 * Conditions:
4530 * Nothing locked.
4531 */
4532
4533 void
4534 ipc_kmsg_copyout_dest(
4535 ipc_kmsg_t kmsg,
4536 ipc_space_t space)
4537 {
4538 mach_msg_bits_t mbits;
4539 ipc_object_t dest;
4540 ipc_object_t reply;
4541 ipc_object_t voucher;
4542 mach_msg_type_name_t dest_type;
4543 mach_msg_type_name_t reply_type;
4544 mach_msg_type_name_t voucher_type;
4545 mach_port_name_t dest_name, reply_name, voucher_name;
4546
4547 mbits = kmsg->ikm_header->msgh_bits;
4548 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4549 reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
4550 voucher = (ipc_object_t) kmsg->ikm_voucher;
4551 voucher_name = kmsg->ikm_header->msgh_voucher_port;
4552 dest_type = MACH_MSGH_BITS_REMOTE(mbits);
4553 reply_type = MACH_MSGH_BITS_LOCAL(mbits);
4554 voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
4555
4556 assert(IO_VALID(dest));
4557
4558 ipc_importance_assert_clean(kmsg);
4559
4560 io_lock(dest);
4561 if (io_active(dest)) {
4562 ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
4563 /* dest is unlocked */
4564 } else {
4565 io_unlock(dest);
4566 io_release(dest);
4567 dest_name = MACH_PORT_DEAD;
4568 }
4569
4570 if (IO_VALID(reply)) {
4571 ipc_object_destroy(reply, reply_type);
4572 reply_name = MACH_PORT_NULL;
4573 } else
4574 reply_name = CAST_MACH_PORT_TO_NAME(reply);
4575
4576 if (IO_VALID(voucher)) {
4577 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
4578
4579 kmsg->ikm_voucher = IP_NULL;
4580 ipc_object_destroy((ipc_object_t)voucher, voucher_type);
4581 voucher_name = MACH_PORT_NULL;
4582 }
4583
4584 kmsg->ikm_header->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type,
4585 voucher_type, mbits);
4586 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
4587 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
4588 kmsg->ikm_header->msgh_voucher_port = voucher_name;
4589
4590 if (mbits & MACH_MSGH_BITS_COMPLEX) {
4591 mach_msg_body_t *body;
4592
4593 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
4594 ipc_kmsg_clean_body(kmsg, body->msgh_descriptor_count,
4595 (mach_msg_descriptor_t *)(body + 1));
4596 }
4597 }
4598
4599 /*
4600 * Routine: ipc_kmsg_copyout_to_kernel
4601 * Purpose:
4602 * Copies out the destination and reply ports in the message.
4603 * Leaves all other rights and memory in the message alone.
4604 * Conditions:
4605 * Nothing locked.
4606 *
4607 * Derived from ipc_kmsg_copyout_dest.
4608 * Use by mach_msg_rpc_from_kernel (which used to use copyout_dest).
4609 * We really do want to save rights and memory.
4610 */
4611
4612 void
4613 ipc_kmsg_copyout_to_kernel(
4614 ipc_kmsg_t kmsg,
4615 ipc_space_t space)
4616 {
4617 ipc_object_t dest;
4618 mach_port_t reply;
4619 mach_msg_type_name_t dest_type;
4620 mach_msg_type_name_t reply_type;
4621 mach_port_name_t dest_name;
4622
4623 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4624 reply = kmsg->ikm_header->msgh_local_port;
4625 dest_type = MACH_MSGH_BITS_REMOTE(kmsg->ikm_header->msgh_bits);
4626 reply_type = MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits);
4627
4628 assert(IO_VALID(dest));
4629
4630 io_lock(dest);
4631 if (io_active(dest)) {
4632 ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
4633 /* dest is unlocked */
4634 } else {
4635 io_unlock(dest);
4636 io_release(dest);
4637 dest_name = MACH_PORT_DEAD;
4638 }
4639
4640 kmsg->ikm_header->msgh_bits =
4641 (MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) |
4642 MACH_MSGH_BITS(reply_type, dest_type));
4643 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
4644 kmsg->ikm_header->msgh_remote_port = reply;
4645 }
4646
4647 #if IKM_SUPPORT_LEGACY
4648 void
4649 ipc_kmsg_copyout_to_kernel_legacy(
4650 ipc_kmsg_t kmsg,
4651 ipc_space_t space)
4652 {
4653 ipc_object_t dest;
4654 ipc_object_t reply;
4655 mach_msg_type_name_t dest_type;
4656 mach_msg_type_name_t reply_type;
4657 mach_port_name_t dest_name, reply_name;
4658
4659 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4660 reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
4661 dest_type = MACH_MSGH_BITS_REMOTE(kmsg->ikm_header->msgh_bits);
4662 reply_type = MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits);
4663
4664 assert(IO_VALID(dest));
4665
4666 io_lock(dest);
4667 if (io_active(dest)) {
4668 ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
4669 /* dest is unlocked */
4670 } else {
4671 io_unlock(dest);
4672 io_release(dest);
4673 dest_name = MACH_PORT_DEAD;
4674 }
4675
4676 reply_name = CAST_MACH_PORT_TO_NAME(reply);
4677
4678 kmsg->ikm_header->msgh_bits =
4679 (MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) |
4680 MACH_MSGH_BITS(reply_type, dest_type));
4681 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
4682 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
4683
4684 mach_msg_descriptor_t *saddr;
4685 mach_msg_legacy_descriptor_t *daddr;
4686 mach_msg_type_number_t i, count = ((mach_msg_base_t *)kmsg->ikm_header)->body.msgh_descriptor_count;
4687 saddr = (mach_msg_descriptor_t *) (((mach_msg_base_t *)kmsg->ikm_header) + 1);
4688 saddr = &saddr[count-1];
4689 daddr = (mach_msg_legacy_descriptor_t *)&saddr[count];
4690 daddr--;
4691
4692 vm_offset_t dsc_adjust = 0;
4693
4694 for (i = 0; i < count; i++, saddr--, daddr--) {
4695 switch (saddr->type.type) {
4696 case MACH_MSG_PORT_DESCRIPTOR: {
4697 mach_msg_port_descriptor_t *dsc = &saddr->port;
4698 mach_msg_legacy_port_descriptor_t *dest_dsc = &daddr->port;
4699
4700 mach_port_t name = dsc->name;
4701 mach_msg_type_name_t disposition = dsc->disposition;
4702
4703 dest_dsc->name = CAST_MACH_PORT_TO_NAME(name);
4704 dest_dsc->disposition = disposition;
4705 dest_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
4706 break;
4707 }
4708 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
4709 case MACH_MSG_OOL_DESCRIPTOR: {
4710 /* The sender should supply ready-made memory, i.e. a vm_map_copy_t
4711 * so we don't need to do anything special. */
4712
4713 mach_msg_ool_descriptor_t *source_dsc = (typeof(source_dsc))&saddr->out_of_line;
4714
4715 mach_msg_ool_descriptor32_t *dest_dsc = &daddr->out_of_line32;
4716
4717 vm_offset_t address = (vm_offset_t)source_dsc->address;
4718 vm_size_t size = source_dsc->size;
4719 boolean_t deallocate = source_dsc->deallocate;
4720 mach_msg_copy_options_t copy = source_dsc->copy;
4721 mach_msg_descriptor_type_t type = source_dsc->type;
4722
4723 dest_dsc->address = address;
4724 dest_dsc->size = size;
4725 dest_dsc->deallocate = deallocate;
4726 dest_dsc->copy = copy;
4727 dest_dsc->type = type;
4728 break;
4729 }
4730 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
4731 mach_msg_ool_ports_descriptor_t *source_dsc = (typeof(source_dsc))&saddr->ool_ports;
4732
4733 mach_msg_ool_ports_descriptor32_t *dest_dsc = &daddr->ool_ports32;
4734
4735 vm_offset_t address = (vm_offset_t)source_dsc->address;
4736 vm_size_t port_count = source_dsc->count;
4737 boolean_t deallocate = source_dsc->deallocate;
4738 mach_msg_copy_options_t copy = source_dsc->copy;
4739 mach_msg_descriptor_type_t type = source_dsc->type;
4740
4741 dest_dsc->address = address;
4742 dest_dsc->count = port_count;
4743 dest_dsc->deallocate = deallocate;
4744 dest_dsc->copy = copy;
4745 dest_dsc->type = type;
4746 break;
4747 }
4748 default: {
4749 #if MACH_ASSERT
4750 panic("ipc_kmsg_copyin_from_kernel: bad descriptor");
4751 #endif /* MACH_ASSERT */
4752 }
4753 }
4754 }
4755
4756 if(count) {
4757 dsc_adjust = 4*count;
4758 memmove((char *)((vm_offset_t)kmsg->ikm_header + dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t));
4759 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header + dsc_adjust);
4760 /* Update the message size for the smaller user representation */
4761 kmsg->ikm_header->msgh_size -= dsc_adjust;
4762 }
4763 }
4764 #endif /* IKM_SUPPORT_LEGACY */
4765
4766 #ifdef __arm64__
4767 /*
4768 * Just sets those parts of the trailer that aren't set up at allocation time.
4769 */
4770 static void
4771 ipc_kmsg_munge_trailer(mach_msg_max_trailer_t *in, void *_out, boolean_t is64bit)
4772 {
4773 if (is64bit) {
4774 mach_msg_max_trailer64_t *out = (mach_msg_max_trailer64_t*)_out;
4775 out->msgh_seqno = in->msgh_seqno;
4776 out->msgh_context = in->msgh_context;
4777 out->msgh_trailer_size = in->msgh_trailer_size;
4778 out->msgh_ad = in->msgh_ad;
4779 } else {
4780 mach_msg_max_trailer32_t *out = (mach_msg_max_trailer32_t*)_out;
4781 out->msgh_seqno = in->msgh_seqno;
4782 out->msgh_context = (mach_port_context32_t)in->msgh_context;
4783 out->msgh_trailer_size = in->msgh_trailer_size;
4784 out->msgh_ad = in->msgh_ad;
4785 }
4786 }
4787 #endif /* __arm64__ */
4788
4789 mach_msg_trailer_size_t
4790 ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused,
4791 mach_msg_option_t option, thread_t thread,
4792 mach_port_seqno_t seqno, boolean_t minimal_trailer,
4793 mach_vm_offset_t context)
4794 {
4795 mach_msg_max_trailer_t *trailer;
4796
4797 #ifdef __arm64__
4798 mach_msg_max_trailer_t tmp_trailer; /* This accommodates U64, and we'll munge */
4799 void *real_trailer_out = (void*)(mach_msg_max_trailer_t *)
4800 ((vm_offset_t)kmsg->ikm_header +
4801 round_msg(kmsg->ikm_header->msgh_size));
4802
4803 /*
4804 * Populate scratch with initial values set up at message allocation time.
4805 * After, we reinterpret the space in the message as the right type
4806 * of trailer for the address space in question.
4807 */
4808 bcopy(real_trailer_out, &tmp_trailer, MAX_TRAILER_SIZE);
4809 trailer = &tmp_trailer;
4810 #else /* __arm64__ */
4811 (void)thread;
4812 trailer = (mach_msg_max_trailer_t *)
4813 ((vm_offset_t)kmsg->ikm_header +
4814 round_msg(kmsg->ikm_header->msgh_size));
4815 #endif /* __arm64__ */
4816
4817 if (!(option & MACH_RCV_TRAILER_MASK)) {
4818 return trailer->msgh_trailer_size;
4819 }
4820
4821 trailer->msgh_seqno = seqno;
4822 trailer->msgh_context = context;
4823 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(thread_is_64bit_addr(thread), option);
4824
4825 if (minimal_trailer) {
4826 goto done;
4827 }
4828
4829 if (MACH_RCV_TRAILER_ELEMENTS(option) >=
4830 MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_AV)){
4831 trailer->msgh_ad = 0;
4832 }
4833
4834 /*
4835 * The ipc_kmsg_t holds a reference to the label of a label
4836 * handle, not the port. We must get a reference to the port
4837 * and a send right to copyout to the receiver.
4838 */
4839
4840 if (option & MACH_RCV_TRAILER_ELEMENTS (MACH_RCV_TRAILER_LABELS)) {
4841 trailer->msgh_labels.sender = 0;
4842 }
4843
4844 done:
4845 #ifdef __arm64__
4846 ipc_kmsg_munge_trailer(trailer, real_trailer_out, thread_is_64bit_addr(thread));
4847 #endif /* __arm64__ */
4848
4849 return trailer->msgh_trailer_size;
4850 }
mirror of XNU