]> git.saurik.com Git - apple/xnu.git/blob - osfmk/ipc/ipc_kmsg.c
projects / apple / xnu.git / blob
? search:


ef7e61a4c4de89635428425e5fbe0bece9087a0e
[apple/xnu.git] / osfmk / ipc / ipc_kmsg.c
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 * NOTICE: This file was modified by McAfee Research in 2004 to introduce
58 * support for mandatory and extensible security protections. This notice
59 * is included in support of clause 2.2 (b) of the Apple Public License,
60 * Version 2.0.
61 * Copyright (c) 2005 SPARTA, Inc.
62 */
63 /*
64 */
65 /*
66 * File: ipc/ipc_kmsg.c
67 * Author: Rich Draves
68 * Date: 1989
69 *
70 * Operations on kernel messages.
71 */
72
73
74 #include <mach/mach_types.h>
75 #include <mach/boolean.h>
76 #include <mach/kern_return.h>
77 #include <mach/message.h>
78 #include <mach/port.h>
79 #include <mach/vm_map.h>
80 #include <mach/mach_vm.h>
81 #include <mach/vm_statistics.h>
82
83 #include <kern/kern_types.h>
84 #include <kern/assert.h>
85 #include <kern/debug.h>
86 #include <kern/ipc_kobject.h>
87 #include <kern/kalloc.h>
88 #include <kern/zalloc.h>
89 #include <kern/processor.h>
90 #include <kern/thread.h>
91 #include <kern/sched_prim.h>
92 #include <kern/misc_protos.h>
93 #include <kern/counters.h>
94 #include <kern/cpu_data.h>
95 #include <kern/policy_internal.h>
96
97 #include <machine/machlimits.h>
98
99 #include <vm/vm_map.h>
100 #include <vm/vm_object.h>
101 #include <vm/vm_kern.h>
102
103 #include <ipc/port.h>
104 #include <ipc/ipc_types.h>
105 #include <ipc/ipc_entry.h>
106 #include <ipc/ipc_kmsg.h>
107 #include <ipc/ipc_notify.h>
108 #include <ipc/ipc_object.h>
109 #include <ipc/ipc_space.h>
110 #include <ipc/ipc_port.h>
111 #include <ipc/ipc_right.h>
112 #include <ipc/ipc_hash.h>
113 #include <ipc/ipc_table.h>
114 #include <ipc/ipc_importance.h>
115 #if MACH_FLIPC
116 #include <kern/mach_node.h>
117 #include <ipc/flipc.h>
118 #endif
119
120 #include <os/overflow.h>
121
122 #include <security/mac_mach_internal.h>
123
124 #include <device/device_server.h>
125
126 #include <string.h>
127
128 #ifdef ppc
129 #include <ppc/Firmware.h>
130 #include <ppc/low_trace.h>
131 #endif
132
133 #if DEBUG
134 #define DEBUG_MSGS_K64 1
135 #endif
136
137 #include <sys/kdebug.h>
138 #include <libkern/OSAtomic.h>
139
140 #pragma pack(4)
141
142 typedef struct
143 {
144 mach_msg_bits_t msgh_bits;
145 mach_msg_size_t msgh_size;
146 mach_port_name_t msgh_remote_port;
147 mach_port_name_t msgh_local_port;
148 mach_port_name_t msgh_voucher_port;
149 mach_msg_id_t msgh_id;
150 } mach_msg_legacy_header_t;
151
152 typedef struct
153 {
154 mach_msg_legacy_header_t header;
155 mach_msg_body_t body;
156 } mach_msg_legacy_base_t;
157
158 typedef struct
159 {
160 mach_port_name_t name;
161 mach_msg_size_t pad1;
162 uint32_t pad2 : 16;
163 mach_msg_type_name_t disposition : 8;
164 mach_msg_descriptor_type_t type : 8;
165 } mach_msg_legacy_port_descriptor_t;
166
167
168 typedef union
169 {
170 mach_msg_legacy_port_descriptor_t port;
171 mach_msg_ool_descriptor32_t out_of_line32;
172 mach_msg_ool_ports_descriptor32_t ool_ports32;
173 mach_msg_type_descriptor_t type;
174 } mach_msg_legacy_descriptor_t;
175
176 #pragma pack()
177
178 #define LEGACY_HEADER_SIZE_DELTA ((mach_msg_size_t)(sizeof(mach_msg_header_t) - sizeof(mach_msg_legacy_header_t)))
179
180 // END LP64 fixes
181
182
183 #if DEBUG_MSGS_K64
184 extern void ipc_pset_print64(
185 ipc_pset_t pset);
186
187 extern void ipc_kmsg_print64(
188 ipc_kmsg_t kmsg,
189 const char *str);
190
191 extern void ipc_msg_print64(
192 mach_msg_header_t *msgh);
193
194 extern ipc_port_t ipc_name_to_data64(
195 task_t task,
196 mach_port_name_t name);
197
198 /*
199 * Forward declarations
200 */
201 void ipc_msg_print_untyped64(
202 mach_msg_body_t *body);
203
204 const char * ipc_type_name64(
205 int type_name,
206 boolean_t received);
207
208 void ipc_print_type_name64(
209 int type_name);
210
211 const char *
212 msgh_bit_decode64(
213 mach_msg_bits_t bit);
214
215 const char *
216 mm_copy_options_string64(
217 mach_msg_copy_options_t option);
218
219 void db_print_msg_uid64(mach_msg_header_t *);
220
221 static void
222 ipc_msg_body_print64(void *body, int size)
223 {
224 uint32_t *word = (uint32_t *) body;
225 uint32_t *end = (uint32_t *)(((uintptr_t) body) + size
226 - sizeof(mach_msg_header_t));
227 int i;
228
229 kprintf(" body(%p-%p):\n %p: ", body, end, word);
230 for (;;) {
231 for (i = 0; i < 8; i++, word++) {
232 if (word >= end) {
233 kprintf("\n");
234 return;
235 }
236 kprintf("%08x ", *word);
237 }
238 kprintf("\n %p: ", word);
239 }
240 }
241
242
243 const char *
244 ipc_type_name64(
245 int type_name,
246 boolean_t received)
247 {
248 switch (type_name) {
249 case MACH_MSG_TYPE_PORT_NAME:
250 return "port_name";
251
252 case MACH_MSG_TYPE_MOVE_RECEIVE:
253 if (received) {
254 return "port_receive";
255 } else {
256 return "move_receive";
257 }
258
259 case MACH_MSG_TYPE_MOVE_SEND:
260 if (received) {
261 return "port_send";
262 } else {
263 return "move_send";
264 }
265
266 case MACH_MSG_TYPE_MOVE_SEND_ONCE:
267 if (received) {
268 return "port_send_once";
269 } else {
270 return "move_send_once";
271 }
272
273 case MACH_MSG_TYPE_COPY_SEND:
274 return "copy_send";
275
276 case MACH_MSG_TYPE_MAKE_SEND:
277 return "make_send";
278
279 case MACH_MSG_TYPE_MAKE_SEND_ONCE:
280 return "make_send_once";
281
282 default:
283 return (char *) 0;
284 }
285 }
286
287 void
288 ipc_print_type_name64(
289 int type_name)
290 {
291 const char *name = ipc_type_name64(type_name, TRUE);
292 if (name) {
293 kprintf("%s", name);
294 } else {
295 kprintf("type%d", type_name);
296 }
297 }
298
299 /*
300 * ipc_kmsg_print64 [ debug ]
301 */
302 void
303 ipc_kmsg_print64(
304 ipc_kmsg_t kmsg,
305 const char *str)
306 {
307 kprintf("%s kmsg=%p:\n", str, kmsg);
308 kprintf(" next=%p, prev=%p, size=%d",
309 kmsg->ikm_next,
310 kmsg->ikm_prev,
311 kmsg->ikm_size);
312 kprintf("\n");
313 ipc_msg_print64(kmsg->ikm_header);
314 }
315
316 const char *
317 msgh_bit_decode64(
318 mach_msg_bits_t bit)
319 {
320 switch (bit) {
321 case MACH_MSGH_BITS_COMPLEX: return "complex";
322 case MACH_MSGH_BITS_CIRCULAR: return "circular";
323 default: return (char *) 0;
324 }
325 }
326
327 /*
328 * ipc_msg_print64 [ debug ]
329 */
330 void
331 ipc_msg_print64(
332 mach_msg_header_t *msgh)
333 {
334 mach_msg_bits_t mbits;
335 unsigned int bit, i;
336 const char *bit_name;
337 int needs_comma;
338
339 mbits = msgh->msgh_bits;
340 kprintf(" msgh_bits=0x%x: l=0x%x,r=0x%x\n",
341 mbits,
342 MACH_MSGH_BITS_LOCAL(msgh->msgh_bits),
343 MACH_MSGH_BITS_REMOTE(msgh->msgh_bits));
344
345 mbits = MACH_MSGH_BITS_OTHER(mbits) & MACH_MSGH_BITS_USED;
346 kprintf(" decoded bits: ");
347 needs_comma = 0;
348 for (i = 0, bit = 1; i < sizeof(mbits) * 8; ++i, bit <<= 1) {
349 if ((mbits & bit) == 0)
350 continue;
351 bit_name = msgh_bit_decode64((mach_msg_bits_t)bit);
352 if (bit_name)
353 kprintf("%s%s", needs_comma ? "," : "", bit_name);
354 else
355 kprintf("%sunknown(0x%x),", needs_comma ? "," : "", bit);
356 ++needs_comma;
357 }
358 if (msgh->msgh_bits & ~MACH_MSGH_BITS_USED) {
359 kprintf("%sunused=0x%x,", needs_comma ? "," : "",
360 msgh->msgh_bits & ~MACH_MSGH_BITS_USED);
361 }
362 kprintf("\n");
363
364 needs_comma = 1;
365 if (msgh->msgh_remote_port) {
366 kprintf(" remote=%p(", msgh->msgh_remote_port);
367 ipc_print_type_name64(MACH_MSGH_BITS_REMOTE(msgh->msgh_bits));
368 kprintf(")");
369 } else {
370 kprintf(" remote=null");
371 }
372
373 if (msgh->msgh_local_port) {
374 kprintf("%slocal=%p(", needs_comma ? "," : "",
375 msgh->msgh_local_port);
376 ipc_print_type_name64(MACH_MSGH_BITS_LOCAL(msgh->msgh_bits));
377 kprintf(")\n");
378 } else {
379 kprintf("local=null\n");
380 }
381
382 kprintf(" msgh_id=%d, size=%d\n",
383 msgh->msgh_id,
384 msgh->msgh_size);
385
386 if (mbits & MACH_MSGH_BITS_COMPLEX) {
387 ipc_msg_print_untyped64((mach_msg_body_t *) (msgh + 1));
388 }
389
390 ipc_msg_body_print64((void *)(msgh + 1), msgh->msgh_size);
391 }
392
393
394 const char *
395 mm_copy_options_string64(
396 mach_msg_copy_options_t option)
397 {
398 const char *name;
399
400 switch (option) {
401 case MACH_MSG_PHYSICAL_COPY:
402 name = "PHYSICAL";
403 break;
404 case MACH_MSG_VIRTUAL_COPY:
405 name = "VIRTUAL";
406 break;
407 case MACH_MSG_OVERWRITE:
408 name = "OVERWRITE(DEPRECATED)";
409 break;
410 case MACH_MSG_ALLOCATE:
411 name = "ALLOCATE";
412 break;
413 case MACH_MSG_KALLOC_COPY_T:
414 name = "KALLOC_COPY_T";
415 break;
416 default:
417 name = "unknown";
418 break;
419 }
420 return name;
421 }
422
423 void
424 ipc_msg_print_untyped64(
425 mach_msg_body_t *body)
426 {
427 mach_msg_descriptor_t *saddr, *send;
428 mach_msg_descriptor_type_t type;
429
430 kprintf(" %d descriptors: \n", body->msgh_descriptor_count);
431
432 saddr = (mach_msg_descriptor_t *) (body + 1);
433 send = saddr + body->msgh_descriptor_count;
434
435 for ( ; saddr < send; saddr++ ) {
436
437 type = saddr->type.type;
438
439 switch (type) {
440
441 case MACH_MSG_PORT_DESCRIPTOR: {
442 mach_msg_port_descriptor_t *dsc;
443
444 dsc = &saddr->port;
445 kprintf(" PORT name = %p disp = ", dsc->name);
446 ipc_print_type_name64(dsc->disposition);
447 kprintf("\n");
448 break;
449 }
450 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
451 case MACH_MSG_OOL_DESCRIPTOR: {
452 mach_msg_ool_descriptor_t *dsc;
453
454 dsc = (mach_msg_ool_descriptor_t *) &saddr->out_of_line;
455 kprintf(" OOL%s addr = %p size = 0x%x copy = %s %s\n",
456 type == MACH_MSG_OOL_DESCRIPTOR ? "" : " VOLATILE",
457 dsc->address, dsc->size,
458 mm_copy_options_string64(dsc->copy),
459 dsc->deallocate ? "DEALLOC" : "");
460 break;
461 }
462 case MACH_MSG_OOL_PORTS_DESCRIPTOR : {
463 mach_msg_ool_ports_descriptor_t *dsc;
464
465 dsc = (mach_msg_ool_ports_descriptor_t *) &saddr->ool_ports;
466
467 kprintf(" OOL_PORTS addr = %p count = 0x%x ",
468 dsc->address, dsc->count);
469 kprintf("disp = ");
470 ipc_print_type_name64(dsc->disposition);
471 kprintf(" copy = %s %s\n",
472 mm_copy_options_string64(dsc->copy),
473 dsc->deallocate ? "DEALLOC" : "");
474 break;
475 }
476
477 default: {
478 kprintf(" UNKNOWN DESCRIPTOR 0x%x\n", type);
479 break;
480 }
481 }
482 }
483 }
484
485 #define DEBUG_IPC_KMSG_PRINT(kmsg,string) \
486 __unreachable_ok_push \
487 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \
488 ipc_kmsg_print64(kmsg, string); \
489 } \
490 __unreachable_ok_pop
491
492 #define DEBUG_IPC_MSG_BODY_PRINT(body,size) \
493 __unreachable_ok_push \
494 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \
495 ipc_msg_body_print64(body,size);\
496 } \
497 __unreachable_ok_pop
498 #else /* !DEBUG_MSGS_K64 */
499 #define DEBUG_IPC_KMSG_PRINT(kmsg,string)
500 #define DEBUG_IPC_MSG_BODY_PRINT(body,size)
501 #endif /* !DEBUG_MSGS_K64 */
502
503 extern vm_map_t ipc_kernel_copy_map;
504 extern vm_size_t ipc_kmsg_max_space;
505 extern vm_size_t ipc_kmsg_max_vm_space;
506 extern vm_size_t ipc_kmsg_max_body_space;
507 extern vm_size_t msg_ool_size_small;
508
509 #define MSG_OOL_SIZE_SMALL msg_ool_size_small
510
511 #if defined(__LP64__)
512 #define MAP_SIZE_DIFFERS(map) (map->max_offset < MACH_VM_MAX_ADDRESS)
513 #define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor32_t
514 #define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor32_t
515 #else
516 #define MAP_SIZE_DIFFERS(map) (map->max_offset > VM_MAX_ADDRESS)
517 #define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor64_t
518 #define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor64_t
519 #endif
520
521 #define DESC_SIZE_ADJUSTMENT ((mach_msg_size_t)(sizeof(mach_msg_ool_descriptor64_t) - \
522 sizeof(mach_msg_ool_descriptor32_t)))
523
524 /* scatter list macros */
525
526 #define SKIP_PORT_DESCRIPTORS(s, c) \
527 MACRO_BEGIN \
528 if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \
529 while ((c) > 0) { \
530 if ((s)->type.type != MACH_MSG_PORT_DESCRIPTOR) \
531 break; \
532 (s)++; (c)--; \
533 } \
534 if (c == 0) \
535 (s) = MACH_MSG_DESCRIPTOR_NULL; \
536 } \
537 MACRO_END
538
539 #define INCREMENT_SCATTER(s, c, d) \
540 MACRO_BEGIN \
541 if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \
542 s = (d) ? (mach_msg_descriptor_t *) \
543 ((OTHER_OOL_DESCRIPTOR *)(s) + 1) : \
544 (s + 1); \
545 (c)--; \
546 } \
547 MACRO_END
548
549 #define KMSG_TRACE_FLAG_TRACED 0x000001
550 #define KMSG_TRACE_FLAG_COMPLEX 0x000002
551 #define KMSG_TRACE_FLAG_OOLMEM 0x000004
552 #define KMSG_TRACE_FLAG_VCPY 0x000008
553 #define KMSG_TRACE_FLAG_PCPY 0x000010
554 #define KMSG_TRACE_FLAG_SND64 0x000020
555 #define KMSG_TRACE_FLAG_RAISEIMP 0x000040
556 #define KMSG_TRACE_FLAG_APP_SRC 0x000080
557 #define KMSG_TRACE_FLAG_APP_DST 0x000100
558 #define KMSG_TRACE_FLAG_DAEMON_SRC 0x000200
559 #define KMSG_TRACE_FLAG_DAEMON_DST 0x000400
560 #define KMSG_TRACE_FLAG_DST_NDFLTQ 0x000800
561 #define KMSG_TRACE_FLAG_SRC_NDFLTQ 0x001000
562 #define KMSG_TRACE_FLAG_DST_SONCE 0x002000
563 #define KMSG_TRACE_FLAG_SRC_SONCE 0x004000
564 #define KMSG_TRACE_FLAG_CHECKIN 0x008000
565 #define KMSG_TRACE_FLAG_ONEWAY 0x010000
566 #define KMSG_TRACE_FLAG_IOKIT 0x020000
567 #define KMSG_TRACE_FLAG_SNDRCV 0x040000
568 #define KMSG_TRACE_FLAG_DSTQFULL 0x080000
569 #define KMSG_TRACE_FLAG_VOUCHER 0x100000
570 #define KMSG_TRACE_FLAG_TIMER 0x200000
571 #define KMSG_TRACE_FLAG_SEMA 0x400000
572 #define KMSG_TRACE_FLAG_DTMPOWNER 0x800000
573
574 #define KMSG_TRACE_FLAGS_MASK 0xffffff
575 #define KMSG_TRACE_FLAGS_SHIFT 8
576
577 #define KMSG_TRACE_PORTS_MASK 0xff
578 #define KMSG_TRACE_PORTS_SHIFT 0
579
580 #if (KDEBUG_LEVEL >= KDEBUG_LEVEL_STANDARD)
581 extern boolean_t kdebug_debugid_enabled(uint32_t debugid);
582
583 void ipc_kmsg_trace_send(ipc_kmsg_t kmsg,
584 mach_msg_option_t option)
585 {
586 task_t send_task = TASK_NULL;
587 ipc_port_t dst_port, src_port;
588 boolean_t is_task_64bit;
589 mach_msg_header_t *msg;
590 mach_msg_trailer_t *trailer;
591
592 int kotype = 0;
593 uint32_t msg_size = 0;
594 uint32_t msg_flags = KMSG_TRACE_FLAG_TRACED;
595 uint32_t num_ports = 0;
596 uint32_t send_pid, dst_pid;
597
598 /*
599 * check to see not only if ktracing is enabled, but if we will
600 * _actually_ emit the KMSG_INFO tracepoint. This saves us a
601 * significant amount of processing (and a port lock hold) in
602 * the non-tracing case.
603 */
604 if (__probable((kdebug_enable & KDEBUG_TRACE) == 0))
605 return;
606 if (!kdebug_debugid_enabled(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO)))
607 return;
608
609 msg = kmsg->ikm_header;
610
611 dst_port = (ipc_port_t)(msg->msgh_remote_port);
612 if (!IPC_PORT_VALID(dst_port))
613 return;
614
615 /*
616 * Message properties / options
617 */
618 if ((option & (MACH_SEND_MSG|MACH_RCV_MSG)) == (MACH_SEND_MSG|MACH_RCV_MSG))
619 msg_flags |= KMSG_TRACE_FLAG_SNDRCV;
620
621 if (msg->msgh_id >= is_iokit_subsystem.start &&
622 msg->msgh_id < is_iokit_subsystem.end + 100)
623 msg_flags |= KMSG_TRACE_FLAG_IOKIT;
624 /* magic XPC checkin message id (XPC_MESSAGE_ID_CHECKIN) from libxpc */
625 else if (msg->msgh_id == 0x77303074u /* w00t */)
626 msg_flags |= KMSG_TRACE_FLAG_CHECKIN;
627
628 if (msg->msgh_bits & MACH_MSGH_BITS_RAISEIMP)
629 msg_flags |= KMSG_TRACE_FLAG_RAISEIMP;
630
631 if (unsafe_convert_port_to_voucher(kmsg->ikm_voucher))
632 msg_flags |= KMSG_TRACE_FLAG_VOUCHER;
633
634 /*
635 * Sending task / port
636 */
637 send_task = current_task();
638 send_pid = task_pid(send_task);
639
640 if (send_pid != 0) {
641 if (task_is_daemon(send_task))
642 msg_flags |= KMSG_TRACE_FLAG_DAEMON_SRC;
643 else if (task_is_app(send_task))
644 msg_flags |= KMSG_TRACE_FLAG_APP_SRC;
645 }
646
647 is_task_64bit = (send_task->map->max_offset > VM_MAX_ADDRESS);
648 if (is_task_64bit)
649 msg_flags |= KMSG_TRACE_FLAG_SND64;
650
651 src_port = (ipc_port_t)(msg->msgh_local_port);
652 if (src_port) {
653 if (src_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT)
654 msg_flags |= KMSG_TRACE_FLAG_SRC_NDFLTQ;
655 switch (MACH_MSGH_BITS_LOCAL(msg->msgh_bits)) {
656 case MACH_MSG_TYPE_MOVE_SEND_ONCE:
657 msg_flags |= KMSG_TRACE_FLAG_SRC_SONCE;
658 break;
659 default:
660 break;
661 }
662 } else {
663 msg_flags |= KMSG_TRACE_FLAG_ONEWAY;
664 }
665
666
667 /*
668 * Destination task / port
669 */
670 ip_lock(dst_port);
671 if (!ip_active(dst_port)) {
672 /* dst port is being torn down */
673 dst_pid = (uint32_t)0xfffffff0;
674 } else if (dst_port->ip_tempowner) {
675 msg_flags |= KMSG_TRACE_FLAG_DTMPOWNER;
676 if (IIT_NULL != dst_port->ip_imp_task)
677 dst_pid = task_pid(dst_port->ip_imp_task->iit_task);
678 else
679 dst_pid = (uint32_t)0xfffffff1;
680 } else if (dst_port->ip_receiver_name == MACH_PORT_NULL) {
681 /* dst_port is otherwise in-transit */
682 dst_pid = (uint32_t)0xfffffff2;
683 } else {
684 if (dst_port->ip_receiver == ipc_space_kernel) {
685 dst_pid = 0;
686 } else {
687 ipc_space_t dst_space;
688 dst_space = dst_port->ip_receiver;
689 if (dst_space && is_active(dst_space)) {
690 dst_pid = task_pid(dst_space->is_task);
691 if (task_is_daemon(dst_space->is_task))
692 msg_flags |= KMSG_TRACE_FLAG_DAEMON_DST;
693 else if (task_is_app(dst_space->is_task))
694 msg_flags |= KMSG_TRACE_FLAG_APP_DST;
695 } else {
696 /* receiving task is being torn down */
697 dst_pid = (uint32_t)0xfffffff3;
698 }
699 }
700 }
701
702 if (dst_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT)
703 msg_flags |= KMSG_TRACE_FLAG_DST_NDFLTQ;
704 if (imq_full(&dst_port->ip_messages))
705 msg_flags |= KMSG_TRACE_FLAG_DSTQFULL;
706
707 kotype = ip_kotype(dst_port);
708
709 ip_unlock(dst_port);
710
711 switch (kotype) {
712 case IKOT_SEMAPHORE:
713 msg_flags |= KMSG_TRACE_FLAG_SEMA;
714 break;
715 case IKOT_TIMER:
716 case IKOT_CLOCK:
717 msg_flags |= KMSG_TRACE_FLAG_TIMER;
718 break;
719 case IKOT_MASTER_DEVICE:
720 case IKOT_IOKIT_CONNECT:
721 case IKOT_IOKIT_OBJECT:
722 case IKOT_IOKIT_SPARE:
723 msg_flags |= KMSG_TRACE_FLAG_IOKIT;
724 break;
725 default:
726 break;
727 }
728
729 switch(MACH_MSGH_BITS_REMOTE(msg->msgh_bits)) {
730 case MACH_MSG_TYPE_PORT_SEND_ONCE:
731 msg_flags |= KMSG_TRACE_FLAG_DST_SONCE;
732 break;
733 default:
734 break;
735 }
736
737
738 /*
739 * Message size / content
740 */
741 msg_size = msg->msgh_size - sizeof(mach_msg_header_t);
742
743 if (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
744 mach_msg_body_t *msg_body;
745 mach_msg_descriptor_t *kern_dsc;
746 int dsc_count;
747
748 msg_flags |= KMSG_TRACE_FLAG_COMPLEX;
749
750 msg_body = (mach_msg_body_t *)(kmsg->ikm_header + 1);
751 dsc_count = (int)msg_body->msgh_descriptor_count;
752 kern_dsc = (mach_msg_descriptor_t *)(msg_body + 1);
753
754 /* this is gross: see ipc_kmsg_copyin_body()... */
755 if (!is_task_64bit)
756 msg_size -= (dsc_count * 12);
757
758 for (int i = 0; i < dsc_count; i++) {
759 switch (kern_dsc[i].type.type) {
760 case MACH_MSG_PORT_DESCRIPTOR:
761 num_ports++;
762 if (is_task_64bit)
763 msg_size -= 12;
764 break;
765 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
766 case MACH_MSG_OOL_DESCRIPTOR: {
767 mach_msg_ool_descriptor_t *dsc;
768 dsc = (mach_msg_ool_descriptor_t *)&kern_dsc[i];
769 msg_flags |= KMSG_TRACE_FLAG_OOLMEM;
770 msg_size += dsc->size;
771 if ((dsc->size >= MSG_OOL_SIZE_SMALL) &&
772 (dsc->copy == MACH_MSG_PHYSICAL_COPY) &&
773 !dsc->deallocate)
774 msg_flags |= KMSG_TRACE_FLAG_PCPY;
775 else if (dsc->size <= MSG_OOL_SIZE_SMALL)
776 msg_flags |= KMSG_TRACE_FLAG_PCPY;
777 else
778 msg_flags |= KMSG_TRACE_FLAG_VCPY;
779 if (is_task_64bit)
780 msg_size -= 16;
781 } break;
782 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
783 mach_msg_ool_ports_descriptor_t *dsc;
784 dsc = (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i];
785 num_ports += dsc->count;
786 if (is_task_64bit)
787 msg_size -= 16;
788 } break;
789 default:
790 break;
791 }
792 }
793 }
794
795 /*
796 * Trailer contents
797 */
798 trailer = (mach_msg_trailer_t *)((vm_offset_t)msg +
799 (vm_offset_t)msg->msgh_size);
800 if (trailer->msgh_trailer_size <= sizeof(mach_msg_security_trailer_t)) {
801 extern security_token_t KERNEL_SECURITY_TOKEN;
802 mach_msg_security_trailer_t *strailer;
803 strailer = (mach_msg_security_trailer_t *)trailer;
804 /*
805 * verify the sender PID: replies from the kernel often look
806 * like self-talk because the sending port is not reset.
807 */
808 if (memcmp(&strailer->msgh_sender,
809 &KERNEL_SECURITY_TOKEN,
810 sizeof(KERNEL_SECURITY_TOKEN)) == 0) {
811 send_pid = 0;
812 msg_flags &= ~(KMSG_TRACE_FLAG_APP_SRC | KMSG_TRACE_FLAG_DAEMON_SRC);
813 }
814 }
815
816 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END,
817 (uintptr_t)send_pid,
818 (uintptr_t)dst_pid,
819 (uintptr_t)msg_size,
820 (uintptr_t)(
821 ((msg_flags & KMSG_TRACE_FLAGS_MASK) << KMSG_TRACE_FLAGS_SHIFT) |
822 ((num_ports & KMSG_TRACE_PORTS_MASK) << KMSG_TRACE_PORTS_SHIFT)
823 )
824 );
825 }
826 #endif
827
828 /* zone for cached ipc_kmsg_t structures */
829 zone_t ipc_kmsg_zone;
830
831 /*
832 * Forward declarations
833 */
834
835 void ipc_kmsg_clean(
836 ipc_kmsg_t kmsg);
837
838 void ipc_kmsg_clean_body(
839 ipc_kmsg_t kmsg,
840 mach_msg_type_number_t number,
841 mach_msg_descriptor_t *desc);
842
843 void ipc_kmsg_clean_partial(
844 ipc_kmsg_t kmsg,
845 mach_msg_type_number_t number,
846 mach_msg_descriptor_t *desc,
847 vm_offset_t paddr,
848 vm_size_t length);
849
850 mach_msg_return_t ipc_kmsg_copyin_body(
851 ipc_kmsg_t kmsg,
852 ipc_space_t space,
853 vm_map_t map);
854
855 /*
856 * We keep a per-processor cache of kernel message buffers.
857 * The cache saves the overhead/locking of using kalloc/kfree.
858 * The per-processor cache seems to miss less than a per-thread cache,
859 * and it also uses less memory. Access to the cache doesn't
860 * require locking.
861 */
862
863 /*
864 * Routine: ipc_kmsg_alloc
865 * Purpose:
866 * Allocate a kernel message structure. If we can get one from
867 * the cache, that is best. Otherwise, allocate a new one.
868 * Conditions:
869 * Nothing locked.
870 */
871 ipc_kmsg_t
872 ipc_kmsg_alloc(
873 mach_msg_size_t msg_and_trailer_size)
874 {
875 mach_msg_size_t max_expanded_size;
876 ipc_kmsg_t kmsg;
877
878 /*
879 * LP64support -
880 * Pad the allocation in case we need to expand the
881 * message descrptors for user spaces with pointers larger than
882 * the kernel's own, or vice versa. We don't know how many descriptors
883 * there are yet, so just assume the whole body could be
884 * descriptors (if there could be any at all).
885 *
886 * The expansion space is left in front of the header,
887 * because it is easier to pull the header and descriptors
888 * forward as we process them than it is to push all the
889 * data backwards.
890 */
891 mach_msg_size_t size = msg_and_trailer_size - MAX_TRAILER_SIZE;
892
893 /* compare against implementation upper limit for the body */
894 if (size > ipc_kmsg_max_body_space)
895 return IKM_NULL;
896
897 if (size > sizeof(mach_msg_base_t)) {
898 mach_msg_size_t max_desc = (mach_msg_size_t)(((size - sizeof(mach_msg_base_t)) /
899 sizeof(mach_msg_ool_descriptor32_t)) *
900 DESC_SIZE_ADJUSTMENT);
901
902 /* make sure expansion won't cause wrap */
903 if (msg_and_trailer_size > MACH_MSG_SIZE_MAX - max_desc)
904 return IKM_NULL;
905
906 max_expanded_size = msg_and_trailer_size + max_desc;
907 } else
908 max_expanded_size = msg_and_trailer_size;
909
910 if (max_expanded_size < IKM_SAVED_MSG_SIZE)
911 max_expanded_size = IKM_SAVED_MSG_SIZE; /* round up for ikm_cache */
912
913 if (max_expanded_size == IKM_SAVED_MSG_SIZE) {
914 struct ikm_cache *cache;
915 unsigned int i;
916
917 disable_preemption();
918 cache = &PROCESSOR_DATA(current_processor(), ikm_cache);
919 if ((i = cache->avail) > 0) {
920 assert(i <= IKM_STASH);
921 kmsg = cache->entries[--i];
922 cache->avail = i;
923 enable_preemption();
924 ikm_check_init(kmsg, max_expanded_size);
925 ikm_set_header(kmsg, msg_and_trailer_size);
926 return (kmsg);
927 }
928 enable_preemption();
929 kmsg = (ipc_kmsg_t)zalloc(ipc_kmsg_zone);
930 } else {
931 kmsg = (ipc_kmsg_t)kalloc(ikm_plus_overhead(max_expanded_size));
932 }
933
934 if (kmsg != IKM_NULL) {
935 ikm_init(kmsg, max_expanded_size);
936 ikm_set_header(kmsg, msg_and_trailer_size);
937 }
938
939 return(kmsg);
940 }
941
942 /*
943 * Routine: ipc_kmsg_free
944 * Purpose:
945 * Free a kernel message buffer. If the kms is preallocated
946 * to a port, just "put it back (marked unused)." We have to
947 * do this with the port locked. The port may have its hold
948 * on our message released. In that case, we have to just
949 * revert the message to a traditional one and free it normally.
950 * Conditions:
951 * Nothing locked.
952 */
953
954 void
955 ipc_kmsg_free(
956 ipc_kmsg_t kmsg)
957 {
958 mach_msg_size_t size = kmsg->ikm_size;
959 ipc_port_t port;
960
961 assert(!IP_VALID(kmsg->ikm_voucher));
962
963 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_FREE) | DBG_FUNC_NONE,
964 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
965 0, 0, 0, 0);
966
967 /*
968 * Check to see if the message is bound to the port. If so,
969 * mark it not in use. If the port isn't already dead, then
970 * leave the message associated with it. Otherwise, free it.
971 */
972 port = ikm_prealloc_inuse_port(kmsg);
973 if (port != IP_NULL) {
974 ip_lock(port);
975 ikm_prealloc_clear_inuse(kmsg, port);
976 if (ip_active(port) && (port->ip_premsg == kmsg)) {
977 assert(IP_PREALLOC(port));
978 ip_unlock(port);
979 ip_release(port);
980 return;
981 }
982 ip_unlock(port);
983 ip_release(port); /* May be last reference */
984 }
985
986 /*
987 * Peek and see if it has to go back in the cache.
988 */
989 if (kmsg->ikm_size == IKM_SAVED_MSG_SIZE) {
990 struct ikm_cache *cache;
991 unsigned int i;
992
993 disable_preemption();
994 cache = &PROCESSOR_DATA(current_processor(), ikm_cache);
995 if ((i = cache->avail) < IKM_STASH) {
996 cache->entries[i] = kmsg;
997 cache->avail = i + 1;
998 enable_preemption();
999 return;
1000 }
1001 enable_preemption();
1002 zfree(ipc_kmsg_zone, kmsg);
1003 return;
1004 }
1005 kfree(kmsg, ikm_plus_overhead(size));
1006 }
1007
1008
1009 /*
1010 * Routine: ipc_kmsg_enqueue
1011 * Purpose:
1012 * Enqueue a kmsg.
1013 */
1014
1015 void
1016 ipc_kmsg_enqueue(
1017 ipc_kmsg_queue_t queue,
1018 ipc_kmsg_t kmsg)
1019 {
1020 ipc_kmsg_t first = queue->ikmq_base;
1021 ipc_kmsg_t last;
1022
1023 if (first == IKM_NULL) {
1024 queue->ikmq_base = kmsg;
1025 kmsg->ikm_next = kmsg;
1026 kmsg->ikm_prev = kmsg;
1027 } else {
1028 last = first->ikm_prev;
1029 kmsg->ikm_next = first;
1030 kmsg->ikm_prev = last;
1031 first->ikm_prev = kmsg;
1032 last->ikm_next = kmsg;
1033 }
1034 }
1035
1036 /*
1037 * Routine: ipc_kmsg_enqueue_qos
1038 * Purpose:
1039 * Enqueue a kmsg, propagating qos
1040 * overrides towards the head of the queue.
1041 *
1042 * Returns:
1043 * whether the head of the queue had
1044 * it's override-qos adjusted because
1045 * of this insertion.
1046 */
1047
1048 boolean_t
1049 ipc_kmsg_enqueue_qos(
1050 ipc_kmsg_queue_t queue,
1051 ipc_kmsg_t kmsg)
1052 {
1053 ipc_kmsg_t first = queue->ikmq_base;
1054 ipc_kmsg_t prev;
1055 mach_msg_priority_t override;
1056
1057 if (first == IKM_NULL) {
1058 /* insert a first message */
1059 queue->ikmq_base = kmsg;
1060 kmsg->ikm_next = kmsg;
1061 kmsg->ikm_prev = kmsg;
1062 return TRUE;
1063 }
1064
1065 /* insert at the tail */
1066 prev = first->ikm_prev;
1067 kmsg->ikm_next = first;
1068 kmsg->ikm_prev = prev;
1069 first->ikm_prev = kmsg;
1070 prev->ikm_next = kmsg;
1071
1072 /* apply QoS overrides towards the head */
1073 override = kmsg->ikm_qos_override;
1074 while (prev != kmsg &&
1075 override > prev->ikm_qos_override) {
1076 prev->ikm_qos_override = override;
1077 prev = prev->ikm_prev;
1078 }
1079
1080 /* did we adjust everything? */
1081 return (prev == kmsg);
1082 }
1083
1084 /*
1085 * Routine: ipc_kmsg_override_qos
1086 * Purpose:
1087 * Update the override for a given kmsg already
1088 * enqueued, propagating qos override adjustments
1089 * towards the head of the queue.
1090 *
1091 * Returns:
1092 * whether the head of the queue had
1093 * it's override-qos adjusted because
1094 * of this insertion.
1095 */
1096
1097 boolean_t
1098 ipc_kmsg_override_qos(
1099 ipc_kmsg_queue_t queue,
1100 ipc_kmsg_t kmsg,
1101 mach_msg_priority_t override)
1102 {
1103 ipc_kmsg_t first = queue->ikmq_base;
1104 ipc_kmsg_t cur = kmsg;
1105
1106 /* apply QoS overrides towards the head */
1107 while (override > cur->ikm_qos_override) {
1108 cur->ikm_qos_override = override;
1109 if (cur == first)
1110 return TRUE;
1111 cur = cur->ikm_next;
1112 }
1113 return FALSE;
1114 }
1115
1116 /*
1117 * Routine: ipc_kmsg_dequeue
1118 * Purpose:
1119 * Dequeue and return a kmsg.
1120 */
1121
1122 ipc_kmsg_t
1123 ipc_kmsg_dequeue(
1124 ipc_kmsg_queue_t queue)
1125 {
1126 ipc_kmsg_t first;
1127
1128 first = ipc_kmsg_queue_first(queue);
1129
1130 if (first != IKM_NULL)
1131 ipc_kmsg_rmqueue(queue, first);
1132
1133 return first;
1134 }
1135
1136 /*
1137 * Routine: ipc_kmsg_rmqueue
1138 * Purpose:
1139 * Pull a kmsg out of a queue.
1140 */
1141
1142 void
1143 ipc_kmsg_rmqueue(
1144 ipc_kmsg_queue_t queue,
1145 ipc_kmsg_t kmsg)
1146 {
1147 ipc_kmsg_t next, prev;
1148
1149 assert(queue->ikmq_base != IKM_NULL);
1150
1151 next = kmsg->ikm_next;
1152 prev = kmsg->ikm_prev;
1153
1154 if (next == kmsg) {
1155 assert(prev == kmsg);
1156 assert(queue->ikmq_base == kmsg);
1157
1158 queue->ikmq_base = IKM_NULL;
1159 } else {
1160 if (queue->ikmq_base == kmsg)
1161 queue->ikmq_base = next;
1162
1163 next->ikm_prev = prev;
1164 prev->ikm_next = next;
1165 }
1166 /* XXX Temporary debug logic */
1167 assert((kmsg->ikm_next = IKM_BOGUS) == IKM_BOGUS);
1168 assert((kmsg->ikm_prev = IKM_BOGUS) == IKM_BOGUS);
1169 }
1170
1171 /*
1172 * Routine: ipc_kmsg_queue_next
1173 * Purpose:
1174 * Return the kmsg following the given kmsg.
1175 * (Or IKM_NULL if it is the last one in the queue.)
1176 */
1177
1178 ipc_kmsg_t
1179 ipc_kmsg_queue_next(
1180 ipc_kmsg_queue_t queue,
1181 ipc_kmsg_t kmsg)
1182 {
1183 ipc_kmsg_t next;
1184
1185 assert(queue->ikmq_base != IKM_NULL);
1186
1187 next = kmsg->ikm_next;
1188 if (queue->ikmq_base == next)
1189 next = IKM_NULL;
1190
1191 return next;
1192 }
1193
1194 /*
1195 * Routine: ipc_kmsg_destroy
1196 * Purpose:
1197 * Destroys a kernel message. Releases all rights,
1198 * references, and memory held by the message.
1199 * Frees the message.
1200 * Conditions:
1201 * No locks held.
1202 */
1203
1204 void
1205 ipc_kmsg_destroy(
1206 ipc_kmsg_t kmsg)
1207 {
1208 /*
1209 * Destroying a message can cause more messages to be destroyed.
1210 * Curtail recursion by putting messages on the deferred
1211 * destruction queue. If this was the first message on the
1212 * queue, this instance must process the full queue.
1213 */
1214 if (ipc_kmsg_delayed_destroy(kmsg))
1215 ipc_kmsg_reap_delayed();
1216 }
1217
1218 /*
1219 * Routine: ipc_kmsg_delayed_destroy
1220 * Purpose:
1221 * Enqueues a kernel message for deferred destruction.
1222 * Returns:
1223 * Boolean indicator that the caller is responsible to reap
1224 * deferred messages.
1225 */
1226
1227 boolean_t ipc_kmsg_delayed_destroy(
1228 ipc_kmsg_t kmsg)
1229 {
1230 ipc_kmsg_queue_t queue = &(current_thread()->ith_messages);
1231 boolean_t first = ipc_kmsg_queue_empty(queue);
1232
1233 ipc_kmsg_enqueue(queue, kmsg);
1234 return first;
1235 }
1236
1237 /*
1238 * Routine: ipc_kmsg_destroy_queue
1239 * Purpose:
1240 * Destroys messages from the per-thread
1241 * deferred reaping queue.
1242 * Conditions:
1243 * No locks held.
1244 */
1245
1246 void
1247 ipc_kmsg_reap_delayed(void)
1248 {
1249 ipc_kmsg_queue_t queue = &(current_thread()->ith_messages);
1250 ipc_kmsg_t kmsg;
1251
1252 /*
1253 * must leave kmsg in queue while cleaning it to assure
1254 * no nested calls recurse into here.
1255 */
1256 while ((kmsg = ipc_kmsg_queue_first(queue)) != IKM_NULL) {
1257 ipc_kmsg_clean(kmsg);
1258 ipc_kmsg_rmqueue(queue, kmsg);
1259 ipc_kmsg_free(kmsg);
1260 }
1261 }
1262
1263 /*
1264 * Routine: ipc_kmsg_clean_body
1265 * Purpose:
1266 * Cleans the body of a kernel message.
1267 * Releases all rights, references, and memory.
1268 *
1269 * Conditions:
1270 * No locks held.
1271 */
1272 static unsigned int _ipc_kmsg_clean_invalid_desc = 0;
1273 void
1274 ipc_kmsg_clean_body(
1275 __unused ipc_kmsg_t kmsg,
1276 mach_msg_type_number_t number,
1277 mach_msg_descriptor_t *saddr)
1278 {
1279 mach_msg_type_number_t i;
1280
1281 if ( number == 0 )
1282 return;
1283
1284 for (i = 0 ; i < number; i++, saddr++ ) {
1285
1286 switch (saddr->type.type) {
1287
1288 case MACH_MSG_PORT_DESCRIPTOR: {
1289 mach_msg_port_descriptor_t *dsc;
1290
1291 dsc = &saddr->port;
1292
1293 /*
1294 * Destroy port rights carried in the message
1295 */
1296 if (!IO_VALID((ipc_object_t) dsc->name))
1297 continue;
1298 ipc_object_destroy((ipc_object_t) dsc->name, dsc->disposition);
1299 break;
1300 }
1301 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
1302 case MACH_MSG_OOL_DESCRIPTOR : {
1303 mach_msg_ool_descriptor_t *dsc;
1304
1305 dsc = (mach_msg_ool_descriptor_t *)&saddr->out_of_line;
1306
1307 /*
1308 * Destroy memory carried in the message
1309 */
1310 if (dsc->size == 0) {
1311 assert(dsc->address == (void *) 0);
1312 } else {
1313 vm_map_copy_discard((vm_map_copy_t) dsc->address);
1314 }
1315 break;
1316 }
1317 case MACH_MSG_OOL_PORTS_DESCRIPTOR : {
1318 ipc_object_t *objects;
1319 mach_msg_type_number_t j;
1320 mach_msg_ool_ports_descriptor_t *dsc;
1321
1322 dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports;
1323 objects = (ipc_object_t *) dsc->address;
1324
1325 if (dsc->count == 0) {
1326 break;
1327 }
1328
1329 assert(objects != (ipc_object_t *) 0);
1330
1331 /* destroy port rights carried in the message */
1332
1333 for (j = 0; j < dsc->count; j++) {
1334 ipc_object_t object = objects[j];
1335
1336 if (!IO_VALID(object))
1337 continue;
1338
1339 ipc_object_destroy(object, dsc->disposition);
1340 }
1341
1342 /* destroy memory carried in the message */
1343
1344 assert(dsc->count != 0);
1345
1346 kfree(dsc->address,
1347 (vm_size_t) dsc->count * sizeof(mach_port_t));
1348 break;
1349 }
1350 default : {
1351 _ipc_kmsg_clean_invalid_desc++; /* don't understand this type of descriptor */
1352 }
1353 }
1354 }
1355 }
1356
1357 /*
1358 * Routine: ipc_kmsg_clean_partial
1359 * Purpose:
1360 * Cleans a partially-acquired kernel message.
1361 * number is the index of the type descriptor
1362 * in the body of the message that contained the error.
1363 * If dolast, the memory and port rights in this last
1364 * type spec are also cleaned. In that case, number
1365 * specifies the number of port rights to clean.
1366 * Conditions:
1367 * Nothing locked.
1368 */
1369
1370 void
1371 ipc_kmsg_clean_partial(
1372 ipc_kmsg_t kmsg,
1373 mach_msg_type_number_t number,
1374 mach_msg_descriptor_t *desc,
1375 vm_offset_t paddr,
1376 vm_size_t length)
1377 {
1378 ipc_object_t object;
1379 mach_msg_bits_t mbits = kmsg->ikm_header->msgh_bits;
1380
1381 /* deal with importance chain while we still have dest and voucher references */
1382 ipc_importance_clean(kmsg);
1383
1384 object = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
1385 assert(IO_VALID(object));
1386 ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits));
1387
1388 object = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
1389 if (IO_VALID(object))
1390 ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits));
1391
1392 object = (ipc_object_t) kmsg->ikm_voucher;
1393 if (IO_VALID(object)) {
1394 assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND);
1395 ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND);
1396 kmsg->ikm_voucher = IP_NULL;
1397 }
1398
1399 if (paddr) {
1400 (void) vm_deallocate(ipc_kernel_copy_map, paddr, length);
1401 }
1402
1403 ipc_kmsg_clean_body(kmsg, number, desc);
1404 }
1405
1406 /*
1407 * Routine: ipc_kmsg_clean
1408 * Purpose:
1409 * Cleans a kernel message. Releases all rights,
1410 * references, and memory held by the message.
1411 * Conditions:
1412 * No locks held.
1413 */
1414
1415 void
1416 ipc_kmsg_clean(
1417 ipc_kmsg_t kmsg)
1418 {
1419 ipc_object_t object;
1420 mach_msg_bits_t mbits;
1421
1422 /* deal with importance chain while we still have dest and voucher references */
1423 ipc_importance_clean(kmsg);
1424
1425 mbits = kmsg->ikm_header->msgh_bits;
1426 object = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
1427 if (IO_VALID(object))
1428 ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits));
1429
1430 object = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
1431 if (IO_VALID(object))
1432 ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits));
1433
1434 object = (ipc_object_t) kmsg->ikm_voucher;
1435 if (IO_VALID(object)) {
1436 assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND);
1437 ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND);
1438 kmsg->ikm_voucher = IP_NULL;
1439 }
1440
1441 if (mbits & MACH_MSGH_BITS_COMPLEX) {
1442 mach_msg_body_t *body;
1443
1444 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
1445 ipc_kmsg_clean_body(kmsg, body->msgh_descriptor_count,
1446 (mach_msg_descriptor_t *)(body + 1));
1447 }
1448 }
1449
1450 /*
1451 * Routine: ipc_kmsg_set_prealloc
1452 * Purpose:
1453 * Assign a kmsg as a preallocated message buffer to a port.
1454 * Conditions:
1455 * port locked.
1456 */
1457
1458 void
1459 ipc_kmsg_set_prealloc(
1460 ipc_kmsg_t kmsg,
1461 ipc_port_t port)
1462 {
1463 assert(kmsg->ikm_prealloc == IP_NULL);
1464
1465 kmsg->ikm_prealloc = IP_NULL;
1466 IP_SET_PREALLOC(port, kmsg);
1467 }
1468
1469 /*
1470 * Routine: ipc_kmsg_clear_prealloc
1471 * Purpose:
1472 * Release the Assignment of a preallocated message buffer from a port.
1473 * Conditions:
1474 * port locked.
1475 */
1476 void
1477 ipc_kmsg_clear_prealloc(
1478 ipc_kmsg_t kmsg,
1479 ipc_port_t port)
1480 {
1481 assert(kmsg->ikm_prealloc == port);
1482
1483 kmsg->ikm_prealloc = IP_NULL;
1484 IP_CLEAR_PREALLOC(port, kmsg);
1485 }
1486
1487 /*
1488 * Routine: ipc_kmsg_prealloc
1489 * Purpose:
1490 * Wraper to ipc_kmsg_alloc() to account for
1491 * header expansion requirements.
1492 */
1493 ipc_kmsg_t
1494 ipc_kmsg_prealloc(mach_msg_size_t size)
1495 {
1496 #if defined(__LP64__)
1497 if (size > MACH_MSG_SIZE_MAX - LEGACY_HEADER_SIZE_DELTA)
1498 return IKM_NULL;
1499
1500 size += LEGACY_HEADER_SIZE_DELTA;
1501 #endif
1502 return ipc_kmsg_alloc(size);
1503 }
1504
1505
1506 /*
1507 * Routine: ipc_kmsg_get
1508 * Purpose:
1509 * Allocates a kernel message buffer.
1510 * Copies a user message to the message buffer.
1511 * Conditions:
1512 * Nothing locked.
1513 * Returns:
1514 * MACH_MSG_SUCCESS Acquired a message buffer.
1515 * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
1516 * MACH_SEND_MSG_TOO_SMALL Message size not long-word multiple.
1517 * MACH_SEND_TOO_LARGE Message too large to ever be sent.
1518 * MACH_SEND_NO_BUFFER Couldn't allocate a message buffer.
1519 * MACH_SEND_INVALID_DATA Couldn't copy message data.
1520 */
1521
1522 mach_msg_return_t
1523 ipc_kmsg_get(
1524 mach_vm_address_t msg_addr,
1525 mach_msg_size_t size,
1526 ipc_kmsg_t *kmsgp)
1527 {
1528 mach_msg_size_t msg_and_trailer_size;
1529 ipc_kmsg_t kmsg;
1530 mach_msg_max_trailer_t *trailer;
1531 mach_msg_legacy_base_t legacy_base;
1532 mach_msg_size_t len_copied;
1533 legacy_base.body.msgh_descriptor_count = 0;
1534
1535 if ((size < sizeof(mach_msg_legacy_header_t)) || (size & 3))
1536 return MACH_SEND_MSG_TOO_SMALL;
1537
1538 if (size > ipc_kmsg_max_body_space)
1539 return MACH_SEND_TOO_LARGE;
1540
1541 if(size == sizeof(mach_msg_legacy_header_t))
1542 len_copied = sizeof(mach_msg_legacy_header_t);
1543 else
1544 len_copied = sizeof(mach_msg_legacy_base_t);
1545
1546 if (copyinmsg(msg_addr, (char *)&legacy_base, len_copied))
1547 return MACH_SEND_INVALID_DATA;
1548
1549 msg_addr += sizeof(legacy_base.header);
1550 #if defined(__LP64__)
1551 size += LEGACY_HEADER_SIZE_DELTA;
1552 #endif
1553 /* unreachable if !DEBUG */
1554 __unreachable_ok_push
1555 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) {
1556 unsigned int j;
1557 for (j=0; j<sizeof(legacy_base.header); j++) {
1558 kprintf("%02x\n", ((unsigned char*)&legacy_base.header)[j]);
1559 }
1560 }
1561 __unreachable_ok_pop
1562
1563 msg_and_trailer_size = size + MAX_TRAILER_SIZE;
1564 kmsg = ipc_kmsg_alloc(msg_and_trailer_size);
1565 if (kmsg == IKM_NULL)
1566 return MACH_SEND_NO_BUFFER;
1567
1568 kmsg->ikm_header->msgh_size = size;
1569 kmsg->ikm_header->msgh_bits = legacy_base.header.msgh_bits;
1570 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(legacy_base.header.msgh_remote_port);
1571 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(legacy_base.header.msgh_local_port);
1572 kmsg->ikm_header->msgh_voucher_port = legacy_base.header.msgh_voucher_port;
1573 kmsg->ikm_header->msgh_id = legacy_base.header.msgh_id;
1574
1575 DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_get header:\n"
1576 " size: 0x%.8x\n"
1577 " bits: 0x%.8x\n"
1578 " remote_port: %p\n"
1579 " local_port: %p\n"
1580 " voucher_port: 0x%.8x\n"
1581 " id: %.8d\n",
1582 kmsg->ikm_header->msgh_size,
1583 kmsg->ikm_header->msgh_bits,
1584 kmsg->ikm_header->msgh_remote_port,
1585 kmsg->ikm_header->msgh_local_port,
1586 kmsg->ikm_header->msgh_voucher_port,
1587 kmsg->ikm_header->msgh_id);
1588
1589 if (copyinmsg(msg_addr, (char *)(kmsg->ikm_header + 1), size - (mach_msg_size_t)sizeof(mach_msg_header_t))) {
1590 ipc_kmsg_free(kmsg);
1591 return MACH_SEND_INVALID_DATA;
1592 }
1593
1594 /* unreachable if !DEBUG */
1595 __unreachable_ok_push
1596 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK))
1597 {
1598 kprintf("body: size: %lu\n", (size - sizeof(mach_msg_header_t)));
1599 uint32_t i;
1600 for(i=0;i*4 < (size - sizeof(mach_msg_header_t));i++)
1601 {
1602 kprintf("%.4x\n",((uint32_t *)(kmsg->ikm_header + 1))[i]);
1603 }
1604 }
1605 __unreachable_ok_pop
1606 DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_get()");
1607
1608 /*
1609 * I reserve for the trailer the largest space (MAX_TRAILER_SIZE)
1610 * However, the internal size field of the trailer (msgh_trailer_size)
1611 * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to optimize
1612 * the cases where no implicit data is requested.
1613 */
1614 trailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + size);
1615 trailer->msgh_sender = current_thread()->task->sec_token;
1616 trailer->msgh_audit = current_thread()->task->audit_token;
1617 trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
1618 trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
1619
1620 #ifdef ppc
1621 if(trcWork.traceMask) dbgTrace(0x1100, (unsigned int)kmsg->ikm_header->msgh_id,
1622 (unsigned int)kmsg->ikm_header->msgh_remote_port,
1623 (unsigned int)kmsg->ikm_header->msgh_local_port, 0);
1624 #endif
1625
1626 trailer->msgh_labels.sender = 0;
1627 *kmsgp = kmsg;
1628 return MACH_MSG_SUCCESS;
1629 }
1630
1631 /*
1632 * Routine: ipc_kmsg_get_from_kernel
1633 * Purpose:
1634 * First checks for a preallocated message
1635 * reserved for kernel clients. If not found -
1636 * allocates a new kernel message buffer.
1637 * Copies a kernel message to the message buffer.
1638 * Only resource errors are allowed.
1639 * Conditions:
1640 * Nothing locked.
1641 * Ports in header are ipc_port_t.
1642 * Returns:
1643 * MACH_MSG_SUCCESS Acquired a message buffer.
1644 * MACH_SEND_NO_BUFFER Couldn't allocate a message buffer.
1645 */
1646
1647 mach_msg_return_t
1648 ipc_kmsg_get_from_kernel(
1649 mach_msg_header_t *msg,
1650 mach_msg_size_t size,
1651 ipc_kmsg_t *kmsgp)
1652 {
1653 ipc_kmsg_t kmsg;
1654 mach_msg_size_t msg_and_trailer_size;
1655 mach_msg_max_trailer_t *trailer;
1656 ipc_port_t dest_port;
1657
1658 assert(size >= sizeof(mach_msg_header_t));
1659 assert((size & 3) == 0);
1660
1661 dest_port = (ipc_port_t)msg->msgh_remote_port;
1662
1663 msg_and_trailer_size = size + MAX_TRAILER_SIZE;
1664
1665 /*
1666 * See if the port has a pre-allocated kmsg for kernel
1667 * clients. These are set up for those kernel clients
1668 * which cannot afford to wait.
1669 */
1670 if (IP_VALID(dest_port) && IP_PREALLOC(dest_port)) {
1671 mach_msg_size_t max_desc = 0;
1672
1673 ip_lock(dest_port);
1674 if (!ip_active(dest_port)) {
1675 ip_unlock(dest_port);
1676 return MACH_SEND_NO_BUFFER;
1677 }
1678 assert(IP_PREALLOC(dest_port));
1679 kmsg = dest_port->ip_premsg;
1680 if (ikm_prealloc_inuse(kmsg)) {
1681 ip_unlock(dest_port);
1682 return MACH_SEND_NO_BUFFER;
1683 }
1684 #if !defined(__LP64__)
1685 if (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
1686 assert(size > sizeof(mach_msg_base_t));
1687 max_desc = ((mach_msg_base_t *)msg)->body.msgh_descriptor_count *
1688 DESC_SIZE_ADJUSTMENT;
1689 }
1690 #endif
1691 if (msg_and_trailer_size > kmsg->ikm_size - max_desc) {
1692 ip_unlock(dest_port);
1693 return MACH_SEND_TOO_LARGE;
1694 }
1695 ikm_prealloc_set_inuse(kmsg, dest_port);
1696 ikm_set_header(kmsg, msg_and_trailer_size);
1697 ip_unlock(dest_port);
1698 }
1699 else
1700 {
1701 kmsg = ipc_kmsg_alloc(msg_and_trailer_size);
1702 if (kmsg == IKM_NULL)
1703 return MACH_SEND_NO_BUFFER;
1704 }
1705
1706 (void) memcpy((void *) kmsg->ikm_header, (const void *) msg, size);
1707
1708 kmsg->ikm_header->msgh_size = size;
1709
1710 /*
1711 * I reserve for the trailer the largest space (MAX_TRAILER_SIZE)
1712 * However, the internal size field of the trailer (msgh_trailer_size)
1713 * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to
1714 * optimize the cases where no implicit data is requested.
1715 */
1716 trailer = (mach_msg_max_trailer_t *)
1717 ((vm_offset_t)kmsg->ikm_header + size);
1718 trailer->msgh_sender = KERNEL_SECURITY_TOKEN;
1719 trailer->msgh_audit = KERNEL_AUDIT_TOKEN;
1720 trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
1721 trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
1722
1723 trailer->msgh_labels.sender = 0;
1724
1725 *kmsgp = kmsg;
1726 return MACH_MSG_SUCCESS;
1727 }
1728
1729 /*
1730 * Routine: ipc_kmsg_send
1731 * Purpose:
1732 * Send a message. The message holds a reference
1733 * for the destination port in the msgh_remote_port field.
1734 *
1735 * If unsuccessful, the caller still has possession of
1736 * the message and must do something with it. If successful,
1737 * the message is queued, given to a receiver, destroyed,
1738 * or handled directly by the kernel via mach_msg.
1739 * Conditions:
1740 * Nothing locked.
1741 * Returns:
1742 * MACH_MSG_SUCCESS The message was accepted.
1743 * MACH_SEND_TIMED_OUT Caller still has message.
1744 * MACH_SEND_INTERRUPTED Caller still has message.
1745 * MACH_SEND_INVALID_DEST Caller still has message.
1746 */
1747
1748
1749 mach_msg_return_t
1750 ipc_kmsg_send(
1751 ipc_kmsg_t kmsg,
1752 mach_msg_option_t option,
1753 mach_msg_timeout_t send_timeout)
1754 {
1755 ipc_port_t port;
1756 thread_t th = current_thread();
1757 mach_msg_return_t error = MACH_MSG_SUCCESS;
1758 boolean_t kernel_reply = FALSE;
1759
1760 /* Check if honor qlimit flag is set on thread. */
1761 if ((th->options & TH_OPT_HONOR_QLIMIT) == TH_OPT_HONOR_QLIMIT) {
1762 /* Remove the MACH_SEND_ALWAYS flag to honor queue limit. */
1763 option &= (~MACH_SEND_ALWAYS);
1764 /* Add the timeout flag since the message queue might be full. */
1765 option |= MACH_SEND_TIMEOUT;
1766 th->options &= (~TH_OPT_HONOR_QLIMIT);
1767 }
1768
1769 #if IMPORTANCE_INHERITANCE
1770 boolean_t did_importance = FALSE;
1771 #if IMPORTANCE_DEBUG
1772 mach_msg_id_t imp_msgh_id = -1;
1773 int sender_pid = -1;
1774 #endif /* IMPORTANCE_DEBUG */
1775 #endif /* IMPORTANCE_INHERITANCE */
1776
1777 /* don't allow the creation of a circular loop */
1778 if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_CIRCULAR) {
1779 ipc_kmsg_destroy(kmsg);
1780 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_MSGH_BITS_CIRCULAR);
1781 return MACH_MSG_SUCCESS;
1782 }
1783
1784 ipc_voucher_send_preprocessing(kmsg);
1785
1786 port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port;
1787 assert(IP_VALID(port));
1788 ip_lock(port);
1789
1790 #if IMPORTANCE_INHERITANCE
1791 retry:
1792 #endif /* IMPORTANCE_INHERITANCE */
1793 /*
1794 * Can't deliver to a dead port.
1795 * However, we can pretend it got sent
1796 * and was then immediately destroyed.
1797 */
1798 if (!ip_active(port)) {
1799 ip_unlock(port);
1800 #if MACH_FLIPC
1801 if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport))
1802 flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
1803 #endif
1804 ip_release(port); /* JMM - Future: release right, not just ref */
1805 kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
1806 ipc_kmsg_destroy(kmsg);
1807 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST);
1808 return MACH_MSG_SUCCESS;
1809 }
1810
1811 if (port->ip_receiver == ipc_space_kernel) {
1812
1813 /*
1814 * We can check ip_receiver == ipc_space_kernel
1815 * before checking that the port is active because
1816 * ipc_port_dealloc_kernel clears ip_receiver
1817 * before destroying a kernel port.
1818 */
1819 assert(ip_active(port));
1820 port->ip_messages.imq_seqno++;
1821 ip_unlock(port);
1822
1823 current_task()->messages_sent++;
1824
1825 /*
1826 * Call the server routine, and get the reply message to send.
1827 */
1828 kmsg = ipc_kobject_server(kmsg, option);
1829 if (kmsg == IKM_NULL)
1830 return MACH_MSG_SUCCESS;
1831
1832 /* restart the KMSG_INFO tracing for the reply message */
1833 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_START);
1834 port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port;
1835 assert(IP_VALID(port));
1836 ip_lock(port);
1837 /* fall thru with reply - same options */
1838 kernel_reply = TRUE;
1839 if (!ip_active(port))
1840 error = MACH_SEND_INVALID_DEST;
1841 }
1842
1843 #if IMPORTANCE_INHERITANCE
1844 /*
1845 * Need to see if this message needs importance donation and/or
1846 * propagation. That routine can drop the port lock temporarily.
1847 * If it does we'll have to revalidate the destination.
1848 */
1849 if (did_importance == FALSE) {
1850 did_importance = TRUE;
1851 if (ipc_importance_send(kmsg, option))
1852 goto retry;
1853 }
1854 #endif /* IMPORTANCE_INHERITANCE */
1855
1856 if (error != MACH_MSG_SUCCESS) {
1857 ip_unlock(port);
1858 } else {
1859 /*
1860 * We have a valid message and a valid reference on the port.
1861 * we can unlock the port and call mqueue_send() on its message
1862 * queue. Lock message queue while port is locked.
1863 */
1864 imq_lock(&port->ip_messages);
1865 ip_unlock(port);
1866
1867 error = ipc_mqueue_send(&port->ip_messages, kmsg, option,
1868 send_timeout);
1869 }
1870
1871 #if IMPORTANCE_INHERITANCE
1872 if (did_importance == TRUE) {
1873 __unused int importance_cleared = 0;
1874 switch (error) {
1875 case MACH_SEND_TIMED_OUT:
1876 case MACH_SEND_NO_BUFFER:
1877 case MACH_SEND_INTERRUPTED:
1878 case MACH_SEND_INVALID_DEST:
1879 /*
1880 * We still have the kmsg and its
1881 * reference on the port. But we
1882 * have to back out the importance
1883 * boost.
1884 *
1885 * The port could have changed hands,
1886 * be inflight to another destination,
1887 * etc... But in those cases our
1888 * back-out will find the new owner
1889 * (and all the operations that
1890 * transferred the right should have
1891 * applied their own boost adjustments
1892 * to the old owner(s)).
1893 */
1894 importance_cleared = 1;
1895 ipc_importance_clean(kmsg);
1896 break;
1897
1898 case MACH_MSG_SUCCESS:
1899 default:
1900 break;
1901 }
1902 #if IMPORTANCE_DEBUG
1903 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_END,
1904 task_pid(current_task()), sender_pid, imp_msgh_id, importance_cleared, 0);
1905 #endif /* IMPORTANCE_DEBUG */
1906 }
1907 #endif /* IMPORTANCE_INHERITANCE */
1908
1909 /*
1910 * If the port has been destroyed while we wait, treat the message
1911 * as a successful delivery (like we do for an inactive port).
1912 */
1913 if (error == MACH_SEND_INVALID_DEST) {
1914 #if MACH_FLIPC
1915 if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport))
1916 flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
1917 #endif
1918 ip_release(port); /* JMM - Future: release right, not just ref */
1919 kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
1920 ipc_kmsg_destroy(kmsg);
1921 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST);
1922 return MACH_MSG_SUCCESS;
1923 }
1924
1925 if (error != MACH_MSG_SUCCESS && kernel_reply) {
1926 /*
1927 * Kernel reply messages that fail can't be allowed to
1928 * pseudo-receive on error conditions. We need to just treat
1929 * the message as a successful delivery.
1930 */
1931 #if MACH_FLIPC
1932 if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport))
1933 flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
1934 #endif
1935 ip_release(port); /* JMM - Future: release right, not just ref */
1936 kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
1937 ipc_kmsg_destroy(kmsg);
1938 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, error);
1939 return MACH_MSG_SUCCESS;
1940 }
1941 return error;
1942 }
1943
1944 /*
1945 * Routine: ipc_kmsg_put
1946 * Purpose:
1947 * Copies a message buffer to a user message.
1948 * Copies only the specified number of bytes.
1949 * Frees the message buffer.
1950 * Conditions:
1951 * Nothing locked. The message buffer must have clean
1952 * header fields.
1953 * Returns:
1954 * MACH_MSG_SUCCESS Copied data out of message buffer.
1955 * MACH_RCV_INVALID_DATA Couldn't copy to user message.
1956 */
1957
1958 mach_msg_return_t
1959 ipc_kmsg_put(
1960 ipc_kmsg_t kmsg,
1961 mach_msg_option_t option,
1962 mach_vm_address_t rcv_addr,
1963 mach_msg_size_t rcv_size,
1964 mach_msg_size_t trailer_size,
1965 mach_msg_size_t *sizep)
1966 {
1967 mach_msg_size_t size = kmsg->ikm_header->msgh_size + trailer_size;
1968 mach_msg_return_t mr;
1969
1970 DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_put()");
1971
1972
1973 DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_put header:\n"
1974 " size: 0x%.8x\n"
1975 " bits: 0x%.8x\n"
1976 " remote_port: %p\n"
1977 " local_port: %p\n"
1978 " voucher_port: 0x%.8x\n"
1979 " id: %.8d\n",
1980 kmsg->ikm_header->msgh_size,
1981 kmsg->ikm_header->msgh_bits,
1982 kmsg->ikm_header->msgh_remote_port,
1983 kmsg->ikm_header->msgh_local_port,
1984 kmsg->ikm_header->msgh_voucher_port,
1985 kmsg->ikm_header->msgh_id);
1986
1987 #if defined(__LP64__)
1988 if (current_task() != kernel_task) { /* don't if receiver expects fully-cooked in-kernel msg; ux_exception */
1989 mach_msg_legacy_header_t *legacy_header =
1990 (mach_msg_legacy_header_t *)((vm_offset_t)(kmsg->ikm_header) + LEGACY_HEADER_SIZE_DELTA);
1991
1992 mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits;
1993 mach_msg_size_t msg_size = kmsg->ikm_header->msgh_size;
1994 mach_port_name_t remote_port = CAST_MACH_PORT_TO_NAME(kmsg->ikm_header->msgh_remote_port);
1995 mach_port_name_t local_port = CAST_MACH_PORT_TO_NAME(kmsg->ikm_header->msgh_local_port);
1996 mach_port_name_t voucher_port = kmsg->ikm_header->msgh_voucher_port;
1997 mach_msg_id_t id = kmsg->ikm_header->msgh_id;
1998
1999 legacy_header->msgh_id = id;
2000 legacy_header->msgh_local_port = local_port;
2001 legacy_header->msgh_remote_port = remote_port;
2002 legacy_header->msgh_voucher_port = voucher_port;
2003 legacy_header->msgh_size = msg_size - LEGACY_HEADER_SIZE_DELTA;
2004 legacy_header->msgh_bits = bits;
2005
2006 size -= LEGACY_HEADER_SIZE_DELTA;
2007 kmsg->ikm_header = (mach_msg_header_t *)legacy_header;
2008 }
2009 #endif
2010
2011 /* unreachable if !DEBUG */
2012 __unreachable_ok_push
2013 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) {
2014 kprintf("ipc_kmsg_put header+body: %d\n", (size));
2015 uint32_t i;
2016 for(i=0;i*4 < size;i++)
2017 {
2018 kprintf("%.4x\n",((uint32_t *)kmsg->ikm_header)[i]);
2019 }
2020 kprintf("type: %d\n", ((mach_msg_type_descriptor_t *)(((mach_msg_base_t *)kmsg->ikm_header)+1))->type);
2021 }
2022 __unreachable_ok_pop
2023
2024 /* Re-Compute target address if using stack-style delivery */
2025 if (option & MACH_RCV_STACK) {
2026 rcv_addr += rcv_size - size;
2027 }
2028
2029 if (copyoutmsg((const char *) kmsg->ikm_header, rcv_addr, size)) {
2030 mr = MACH_RCV_INVALID_DATA;
2031 size = 0;
2032 } else
2033 mr = MACH_MSG_SUCCESS;
2034
2035 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_LINK) | DBG_FUNC_NONE,
2036 (rcv_addr >= VM_MIN_KERNEL_AND_KEXT_ADDRESS ||
2037 rcv_addr + size >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) ? (uintptr_t)0 : (uintptr_t)rcv_addr,
2038 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
2039 1 /* this is on the receive/copyout path */,
2040 0,
2041 0);
2042 ipc_kmsg_free(kmsg);
2043
2044 if (sizep)
2045 *sizep = size;
2046 return mr;
2047 }
2048
2049 /*
2050 * Routine: ipc_kmsg_put_to_kernel
2051 * Purpose:
2052 * Copies a message buffer to a kernel message.
2053 * Frees the message buffer.
2054 * No errors allowed.
2055 * Conditions:
2056 * Nothing locked.
2057 */
2058
2059 void
2060 ipc_kmsg_put_to_kernel(
2061 mach_msg_header_t *msg,
2062 ipc_kmsg_t kmsg,
2063 mach_msg_size_t size)
2064 {
2065 (void) memcpy((void *) msg, (const void *) kmsg->ikm_header, size);
2066
2067 ipc_kmsg_free(kmsg);
2068 }
2069
2070 unsigned long pthread_priority_canonicalize(unsigned long priority, boolean_t propagation);
2071
2072 static void
2073 ipc_kmsg_set_qos(
2074 ipc_kmsg_t kmsg,
2075 mach_msg_option_t options,
2076 mach_msg_priority_t override)
2077 {
2078 kern_return_t kr;
2079
2080 kr = ipc_get_pthpriority_from_kmsg_voucher(kmsg, &kmsg->ikm_qos);
2081 if (kr != KERN_SUCCESS) {
2082 kmsg->ikm_qos = MACH_MSG_PRIORITY_UNSPECIFIED;
2083 }
2084 kmsg->ikm_qos_override = kmsg->ikm_qos;
2085
2086 if (options & MACH_SEND_OVERRIDE) {
2087 unsigned long canonical;
2088 mach_msg_priority_t canon;
2089
2090 canonical = pthread_priority_canonicalize(override, TRUE);
2091 canon = (mach_msg_priority_t)canonical;
2092 if (canon > kmsg->ikm_qos)
2093 kmsg->ikm_qos_override = canon;
2094 }
2095 }
2096
2097 /*
2098 * Routine: ipc_kmsg_copyin_header
2099 * Purpose:
2100 * "Copy-in" port rights in the header of a message.
2101 * Operates atomically; if it doesn't succeed the
2102 * message header and the space are left untouched.
2103 * If it does succeed the remote/local port fields
2104 * contain object pointers instead of port names,
2105 * and the bits field is updated. The destination port
2106 * will be a valid port pointer.
2107 *
2108 * Conditions:
2109 * Nothing locked.
2110 * Returns:
2111 * MACH_MSG_SUCCESS Successful copyin.
2112 * MACH_SEND_INVALID_HEADER
2113 * Illegal value in the message header bits.
2114 * MACH_SEND_INVALID_DEST The space is dead.
2115 * MACH_SEND_INVALID_DEST Can't copyin destination port.
2116 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
2117 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
2118 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
2119 */
2120
2121 mach_msg_return_t
2122 ipc_kmsg_copyin_header(
2123 ipc_kmsg_t kmsg,
2124 ipc_space_t space,
2125 mach_msg_priority_t override,
2126 mach_msg_option_t *optionp)
2127 {
2128 mach_msg_header_t *msg = kmsg->ikm_header;
2129 mach_msg_bits_t mbits = msg->msgh_bits & MACH_MSGH_BITS_USER;
2130 mach_port_name_t dest_name = CAST_MACH_PORT_TO_NAME(msg->msgh_remote_port);
2131 mach_port_name_t reply_name = CAST_MACH_PORT_TO_NAME(msg->msgh_local_port);
2132 mach_port_name_t voucher_name = MACH_PORT_NULL;
2133 kern_return_t kr;
2134
2135 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
2136 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
2137 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
2138 ipc_object_t dest_port = IO_NULL;
2139 ipc_object_t reply_port = IO_NULL;
2140 ipc_port_t dest_soright = IP_NULL;
2141 ipc_port_t reply_soright = IP_NULL;
2142 ipc_port_t voucher_soright = IP_NULL;
2143 ipc_port_t release_port = IP_NULL;
2144 ipc_port_t voucher_port = IP_NULL;
2145 ipc_port_t voucher_release_port = IP_NULL;
2146 ipc_entry_t dest_entry = IE_NULL;
2147 ipc_entry_t reply_entry = IE_NULL;
2148 ipc_entry_t voucher_entry = IE_NULL;
2149
2150 int assertcnt = 0;
2151 #if IMPORTANCE_INHERITANCE
2152 boolean_t needboost = FALSE;
2153 #endif /* IMPORTANCE_INHERITANCE */
2154
2155 if ((mbits != msg->msgh_bits) ||
2156 (!MACH_MSG_TYPE_PORT_ANY_SEND(dest_type)) ||
2157 ((reply_type == 0) ?
2158 (reply_name != MACH_PORT_NULL) :
2159 !MACH_MSG_TYPE_PORT_ANY_SEND(reply_type)))
2160 return MACH_SEND_INVALID_HEADER;
2161
2162 if (!MACH_PORT_VALID(dest_name))
2163 return MACH_SEND_INVALID_DEST;
2164
2165 is_write_lock(space);
2166 if (!is_active(space)) {
2167 is_write_unlock(space);
2168 return MACH_SEND_INVALID_DEST;
2169 }
2170 /* space locked and active */
2171
2172 /*
2173 * If there is a voucher specified, make sure the disposition is
2174 * valid and the entry actually refers to a voucher port. Don't
2175 * actually copy in until we validate destination and reply.
2176 */
2177 if (voucher_type != MACH_MSGH_BITS_ZERO) {
2178
2179 voucher_name = msg->msgh_voucher_port;
2180
2181 if (voucher_name == MACH_PORT_DEAD ||
2182 (voucher_type != MACH_MSG_TYPE_MOVE_SEND &&
2183 voucher_type != MACH_MSG_TYPE_COPY_SEND)) {
2184 is_write_unlock(space);
2185 return MACH_SEND_INVALID_VOUCHER;
2186 }
2187
2188 if (voucher_name != MACH_PORT_NULL) {
2189 voucher_entry = ipc_entry_lookup(space, voucher_name);
2190 if (voucher_entry == IE_NULL ||
2191 (voucher_entry->ie_bits & MACH_PORT_TYPE_SEND) == 0 ||
2192 io_kotype(voucher_entry->ie_object) != IKOT_VOUCHER) {
2193 is_write_unlock(space);
2194 return MACH_SEND_INVALID_VOUCHER;
2195 }
2196 } else {
2197 voucher_type = MACH_MSG_TYPE_MOVE_SEND;
2198 }
2199 }
2200
2201 /*
2202 * Handle combinations of validating destination and reply; along
2203 * with copying in destination, reply, and voucher in an atomic way.
2204 */
2205
2206 if (dest_name == voucher_name) {
2207
2208 /*
2209 * If the destination name is the same as the voucher name,
2210 * the voucher_entry must already be known. Either that or
2211 * the destination name is MACH_PORT_NULL (i.e. invalid).
2212 */
2213 dest_entry = voucher_entry;
2214 if (dest_entry == IE_NULL) {
2215 goto invalid_dest;
2216 }
2217
2218 /*
2219 * Make sure a future copyin of the reply port will succeed.
2220 * Once we start copying in the dest/voucher pair, we can't
2221 * back out.
2222 */
2223 if (MACH_PORT_VALID(reply_name)) {
2224 assert(reply_type != 0); /* because reply_name not null */
2225
2226 /* It is just WRONG if dest, voucher, and reply are all the same. */
2227 if (voucher_name == reply_name) {
2228 goto invalid_reply;
2229 }
2230 reply_entry = ipc_entry_lookup(space, reply_name);
2231 if (reply_entry == IE_NULL) {
2232 goto invalid_reply;
2233 }
2234 assert(dest_entry != reply_entry); /* names are not equal */
2235 if (!ipc_right_copyin_check(space, reply_name, reply_entry, reply_type)) {
2236 goto invalid_reply;
2237 }
2238 }
2239
2240 /*
2241 * Do the joint copyin of the dest disposition and
2242 * voucher disposition from the one entry/port. We
2243 * already validated that the voucher copyin would
2244 * succeed (above). So, any failure in combining
2245 * the copyins can be blamed on the destination.
2246 */
2247 kr = ipc_right_copyin_two(space, dest_name, dest_entry,
2248 dest_type, voucher_type,
2249 &dest_port, &dest_soright,
2250 &release_port);
2251 if (kr != KERN_SUCCESS) {
2252 assert(kr != KERN_INVALID_CAPABILITY);
2253 goto invalid_dest;
2254 }
2255 voucher_port = (ipc_port_t)dest_port;
2256
2257 /*
2258 * could not have been one of these dispositions,
2259 * validated the port was a true kernel voucher port above,
2260 * AND was successfully able to copyin both dest and voucher.
2261 */
2262 assert(dest_type != MACH_MSG_TYPE_MAKE_SEND);
2263 assert(dest_type != MACH_MSG_TYPE_MAKE_SEND_ONCE);
2264 assert(dest_type != MACH_MSG_TYPE_MOVE_SEND_ONCE);
2265
2266 /*
2267 * Perform the delayed reply right copyin (guaranteed success).
2268 */
2269 if (reply_entry != IE_NULL) {
2270 kr = ipc_right_copyin(space, reply_name, reply_entry,
2271 reply_type, TRUE,
2272 &reply_port, &reply_soright,
2273 &release_port, &assertcnt);
2274 assert(assertcnt == 0);
2275 assert(kr == KERN_SUCCESS);
2276 }
2277
2278 } else {
2279 if (dest_name == reply_name) {
2280 /*
2281 * Destination and reply ports are the same!
2282 * This is very similar to the case where the
2283 * destination and voucher ports were the same
2284 * (except the reply port disposition is not
2285 * previously validated).
2286 */
2287 dest_entry = ipc_entry_lookup(space, dest_name);
2288 if (dest_entry == IE_NULL) {
2289 goto invalid_dest;
2290 }
2291 reply_entry = dest_entry;
2292 assert(reply_type != 0); /* because name not null */
2293
2294 /*
2295 * Do the joint copyin of the dest disposition and
2296 * reply disposition from the one entry/port.
2297 */
2298 kr = ipc_right_copyin_two(space, dest_name, dest_entry,
2299 dest_type, reply_type,
2300 &dest_port, &dest_soright,
2301 &release_port);
2302 if (kr == KERN_INVALID_CAPABILITY) {
2303 goto invalid_reply;
2304 } else if (kr != KERN_SUCCESS) {
2305 goto invalid_dest;
2306 }
2307 reply_port = dest_port;
2308
2309
2310 } else {
2311 /*
2312 * Handle destination and reply independently, as
2313 * they are independent entries (even if the entries
2314 * refer to the same port).
2315 *
2316 * This can be the tough case to make atomic.
2317 *
2318 * The difficult problem is serializing with port death.
2319 * The bad case is when dest_port dies after its copyin,
2320 * reply_port dies before its copyin, and dest_port dies before
2321 * reply_port. Then the copyins operated as if dest_port was
2322 * alive and reply_port was dead, which shouldn't have happened
2323 * because they died in the other order.
2324 *
2325 * Note that it is easy for a user task to tell if
2326 * a copyin happened before or after a port died.
2327 * If a port dies before copyin, a dead-name notification
2328 * is generated and the dead name's urefs are incremented,
2329 * and if the copyin happens first, a port-deleted
2330 * notification is generated.
2331 *
2332 * Even so, avoiding that potentially detectable race is too
2333 * expensive - and no known code cares about it. So, we just
2334 * do the expedient thing and copy them in one after the other.
2335 */
2336
2337 dest_entry = ipc_entry_lookup(space, dest_name);
2338 if (dest_entry == IE_NULL) {
2339 goto invalid_dest;
2340 }
2341 assert(dest_entry != voucher_entry);
2342
2343 /*
2344 * Make sure reply port entry is valid before dest copyin.
2345 */
2346 if (MACH_PORT_VALID(reply_name)) {
2347 if (reply_name == voucher_name) {
2348 goto invalid_reply;
2349 }
2350 reply_entry = ipc_entry_lookup(space, reply_name);
2351 if (reply_entry == IE_NULL) {
2352 goto invalid_reply;
2353 }
2354 assert(dest_entry != reply_entry); /* names are not equal */
2355 assert(reply_type != 0); /* because reply_name not null */
2356
2357 if (!ipc_right_copyin_check(space, reply_name, reply_entry, reply_type)) {
2358 goto invalid_reply;
2359 }
2360 }
2361
2362 /*
2363 * copyin the destination.
2364 */
2365 kr = ipc_right_copyin(space, dest_name, dest_entry,
2366 dest_type, FALSE,
2367 &dest_port, &dest_soright,
2368 &release_port, &assertcnt);
2369 assert(assertcnt == 0);
2370 if (kr != KERN_SUCCESS) {
2371 goto invalid_dest;
2372 }
2373 assert(IO_VALID(dest_port));
2374 assert(!IP_VALID(release_port));
2375
2376 /*
2377 * Copyin the pre-validated reply right.
2378 * It's OK if the reply right has gone dead in the meantime.
2379 */
2380 if (MACH_PORT_VALID(reply_name)) {
2381 kr = ipc_right_copyin(space, reply_name, reply_entry,
2382 reply_type, TRUE,
2383 &reply_port, &reply_soright,
2384 &release_port, &assertcnt);
2385 assert(assertcnt == 0);
2386 assert(kr == KERN_SUCCESS);
2387 } else {
2388 /* convert invalid name to equivalent ipc_object type */
2389 reply_port = (ipc_object_t)CAST_MACH_NAME_TO_PORT(reply_name);
2390 }
2391 }
2392
2393 /*
2394 * Finally can copyin the voucher right now that dest and reply
2395 * are fully copied in (guaranteed success).
2396 */
2397 if (IE_NULL != voucher_entry) {
2398 kr = ipc_right_copyin(space, voucher_name, voucher_entry,
2399 voucher_type, FALSE,
2400 (ipc_object_t *)&voucher_port,
2401 &voucher_soright,
2402 &voucher_release_port,
2403 &assertcnt);
2404 assert(assertcnt == 0);
2405 assert(KERN_SUCCESS == kr);
2406 assert(IP_VALID(voucher_port));
2407 assert(ip_active(voucher_port));
2408 }
2409 }
2410
2411 /*
2412 * The entries might need to be deallocated.
2413 *
2414 * Each entry should be deallocated only once,
2415 * even if it was specified in more than one slot in the header.
2416 * Note that dest can be the same entry as reply or voucher,
2417 * but reply and voucher must be distinct entries.
2418 */
2419 assert(IE_NULL != dest_entry);
2420 if (IE_NULL != reply_entry)
2421 assert(reply_entry != voucher_entry);
2422
2423 if (IE_BITS_TYPE(dest_entry->ie_bits) == MACH_PORT_TYPE_NONE) {
2424 ipc_entry_dealloc(space, dest_name, dest_entry);
2425
2426 if (dest_entry == reply_entry) {
2427 reply_entry = IE_NULL;
2428 }
2429
2430 if (dest_entry == voucher_entry) {
2431 voucher_entry = IE_NULL;
2432 }
2433
2434 dest_entry = IE_NULL;
2435 }
2436 if (IE_NULL != reply_entry &&
2437 IE_BITS_TYPE(reply_entry->ie_bits) == MACH_PORT_TYPE_NONE) {
2438 ipc_entry_dealloc(space, reply_name, reply_entry);
2439 reply_entry = IE_NULL;
2440 }
2441 if (IE_NULL != voucher_entry &&
2442 IE_BITS_TYPE(voucher_entry->ie_bits) == MACH_PORT_TYPE_NONE) {
2443 ipc_entry_dealloc(space, voucher_name, voucher_entry);
2444 voucher_entry = IE_NULL;
2445 }
2446
2447 dest_type = ipc_object_copyin_type(dest_type);
2448 reply_type = ipc_object_copyin_type(reply_type);
2449
2450 /*
2451 * JMM - Without rdar://problem/6275821, this is the last place we can
2452 * re-arm the send-possible notifications. It may trigger unexpectedly
2453 * early (send may NOT have failed), but better than missing. We assure
2454 * we won't miss by forcing MACH_SEND_ALWAYS if we got past arming.
2455 */
2456 if (((*optionp & MACH_SEND_NOTIFY) != 0) &&
2457 dest_type != MACH_MSG_TYPE_PORT_SEND_ONCE &&
2458 dest_entry != IE_NULL && dest_entry->ie_request != IE_REQ_NONE) {
2459 ipc_port_t dport = (ipc_port_t)dest_port;
2460
2461 assert(dport != IP_NULL);
2462 ip_lock(dport);
2463 if (ip_active(dport) && dport->ip_receiver != ipc_space_kernel) {
2464 if (ip_full(dport)) {
2465 #if IMPORTANCE_INHERITANCE
2466 needboost = ipc_port_request_sparm(dport, dest_name,
2467 dest_entry->ie_request,
2468 *optionp,
2469 override);
2470 if (needboost == FALSE)
2471 ip_unlock(dport);
2472 #else
2473 ipc_port_request_sparm(dport, dest_name,
2474 dest_entry->ie_request,
2475 *optionp,
2476 override);
2477 ip_unlock(dport);
2478 #endif /* IMPORTANCE_INHERITANCE */
2479 } else {
2480 *optionp |= MACH_SEND_ALWAYS;
2481 ip_unlock(dport);
2482 }
2483 } else {
2484 ip_unlock(dport);
2485 }
2486 }
2487
2488 is_write_unlock(space);
2489
2490 #if IMPORTANCE_INHERITANCE
2491 /*
2492 * If our request is the first boosting send-possible
2493 * notification this cycle, push the boost down the
2494 * destination port.
2495 */
2496 if (needboost == TRUE) {
2497 ipc_port_t dport = (ipc_port_t)dest_port;
2498
2499 /* dport still locked from above */
2500 if (ipc_port_importance_delta(dport, IPID_OPTION_SENDPOSSIBLE, 1) == FALSE) {
2501 ip_unlock(dport);
2502 }
2503 }
2504 #endif /* IMPORTANCE_INHERITANCE */
2505
2506 if (dest_soright != IP_NULL) {
2507 ipc_notify_port_deleted(dest_soright, dest_name);
2508 }
2509 if (reply_soright != IP_NULL) {
2510 ipc_notify_port_deleted(reply_soright, reply_name);
2511 }
2512 if (voucher_soright != IP_NULL) {
2513 ipc_notify_port_deleted(voucher_soright, voucher_name);
2514 }
2515
2516 /*
2517 * No room to store voucher port in in-kernel msg header,
2518 * so we store it back in the kmsg itself. Extract the
2519 * qos, and apply any override before we enqueue the kmsg.
2520 */
2521 if (IP_VALID(voucher_port)) {
2522
2523 kmsg->ikm_voucher = voucher_port;
2524 voucher_type = MACH_MSG_TYPE_MOVE_SEND;
2525 }
2526
2527 /* capture the qos value(s) for the kmsg */
2528 ipc_kmsg_set_qos(kmsg, *optionp, override);
2529
2530 msg->msgh_bits = MACH_MSGH_BITS_SET(dest_type, reply_type, voucher_type, mbits);
2531 msg->msgh_remote_port = (ipc_port_t)dest_port;
2532 msg->msgh_local_port = (ipc_port_t)reply_port;
2533
2534 if (release_port != IP_NULL)
2535 ip_release(release_port);
2536
2537 if (voucher_release_port != IP_NULL)
2538 ip_release(voucher_release_port);
2539
2540 return MACH_MSG_SUCCESS;
2541
2542 invalid_reply:
2543 is_write_unlock(space);
2544
2545 if (release_port != IP_NULL)
2546 ip_release(release_port);
2547
2548 assert(voucher_port == IP_NULL);
2549 assert(voucher_soright == IP_NULL);
2550
2551 return MACH_SEND_INVALID_REPLY;
2552
2553 invalid_dest:
2554 is_write_unlock(space);
2555
2556 if (release_port != IP_NULL)
2557 ip_release(release_port);
2558
2559 if (reply_soright != IP_NULL)
2560 ipc_notify_port_deleted(reply_soright, reply_name);
2561
2562 assert(voucher_port == IP_NULL);
2563 assert(voucher_soright == IP_NULL);
2564
2565 return MACH_SEND_INVALID_DEST;
2566 }
2567
2568 mach_msg_descriptor_t *ipc_kmsg_copyin_port_descriptor(
2569 volatile mach_msg_port_descriptor_t *dsc,
2570 mach_msg_legacy_port_descriptor_t *user_dsc,
2571 ipc_space_t space,
2572 ipc_object_t dest,
2573 ipc_kmsg_t kmsg,
2574 mach_msg_return_t *mr);
2575
2576 void ipc_print_type_name(
2577 int type_name);
2578 mach_msg_descriptor_t *
2579 ipc_kmsg_copyin_port_descriptor(
2580 volatile mach_msg_port_descriptor_t *dsc,
2581 mach_msg_legacy_port_descriptor_t *user_dsc_in,
2582 ipc_space_t space,
2583 ipc_object_t dest,
2584 ipc_kmsg_t kmsg,
2585 mach_msg_return_t *mr)
2586 {
2587 volatile mach_msg_legacy_port_descriptor_t *user_dsc = user_dsc_in;
2588 mach_msg_type_name_t user_disp;
2589 mach_msg_type_name_t result_disp;
2590 mach_port_name_t name;
2591 ipc_object_t object;
2592
2593 user_disp = user_dsc->disposition;
2594 result_disp = ipc_object_copyin_type(user_disp);
2595
2596 name = (mach_port_name_t)user_dsc->name;
2597 if (MACH_PORT_VALID(name)) {
2598
2599 kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object);
2600 if (kr != KERN_SUCCESS) {
2601 *mr = MACH_SEND_INVALID_RIGHT;
2602 return NULL;
2603 }
2604
2605 if ((result_disp == MACH_MSG_TYPE_PORT_RECEIVE) &&
2606 ipc_port_check_circularity((ipc_port_t) object,
2607 (ipc_port_t) dest)) {
2608 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
2609 }
2610 dsc->name = (ipc_port_t) object;
2611 } else {
2612 dsc->name = CAST_MACH_NAME_TO_PORT(name);
2613 }
2614 dsc->disposition = result_disp;
2615 dsc->type = MACH_MSG_PORT_DESCRIPTOR;
2616
2617 dsc->pad_end = 0; // debug, unnecessary
2618
2619 return (mach_msg_descriptor_t *)(user_dsc_in+1);
2620 }
2621
2622 mach_msg_descriptor_t * ipc_kmsg_copyin_ool_descriptor(
2623 mach_msg_ool_descriptor_t *dsc,
2624 mach_msg_descriptor_t *user_dsc,
2625 int is_64bit,
2626 vm_offset_t *paddr,
2627 vm_map_copy_t *copy,
2628 vm_size_t *space_needed,
2629 vm_map_t map,
2630 mach_msg_return_t *mr);
2631 mach_msg_descriptor_t *
2632 ipc_kmsg_copyin_ool_descriptor(
2633 mach_msg_ool_descriptor_t *dsc,
2634 mach_msg_descriptor_t *user_dsc,
2635 int is_64bit,
2636 vm_offset_t *paddr,
2637 vm_map_copy_t *copy,
2638 vm_size_t *space_needed,
2639 vm_map_t map,
2640 mach_msg_return_t *mr)
2641 {
2642 vm_size_t length;
2643 boolean_t dealloc;
2644 mach_msg_copy_options_t copy_options;
2645 mach_vm_offset_t addr;
2646 mach_msg_descriptor_type_t dsc_type;
2647
2648 if (is_64bit) {
2649 mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
2650
2651 addr = (mach_vm_offset_t) user_ool_dsc->address;
2652 length = user_ool_dsc->size;
2653 dealloc = user_ool_dsc->deallocate;
2654 copy_options = user_ool_dsc->copy;
2655 dsc_type = user_ool_dsc->type;
2656
2657 user_dsc = (typeof(user_dsc))(user_ool_dsc+1);
2658 } else {
2659 mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
2660
2661 addr = CAST_USER_ADDR_T(user_ool_dsc->address);
2662 dealloc = user_ool_dsc->deallocate;
2663 copy_options = user_ool_dsc->copy;
2664 dsc_type = user_ool_dsc->type;
2665 length = user_ool_dsc->size;
2666
2667 user_dsc = (typeof(user_dsc))(user_ool_dsc+1);
2668 }
2669
2670 dsc->size = (mach_msg_size_t)length;
2671 dsc->deallocate = dealloc;
2672 dsc->copy = copy_options;
2673 dsc->type = dsc_type;
2674
2675 if (length == 0) {
2676 dsc->address = NULL;
2677 } else if ((length >= MSG_OOL_SIZE_SMALL) &&
2678 (copy_options == MACH_MSG_PHYSICAL_COPY) && !dealloc) {
2679
2680 /*
2681 * If the request is a physical copy and the source
2682 * is not being deallocated, then allocate space
2683 * in the kernel's pageable ipc copy map and copy
2684 * the data in. The semantics guarantee that the
2685 * data will have been physically copied before
2686 * the send operation terminates. Thus if the data
2687 * is not being deallocated, we must be prepared
2688 * to page if the region is sufficiently large.
2689 */
2690 if (copyin(addr, (char *)*paddr, length)) {
2691 *mr = MACH_SEND_INVALID_MEMORY;
2692 return NULL;
2693 }
2694
2695 /*
2696 * The kernel ipc copy map is marked no_zero_fill.
2697 * If the transfer is not a page multiple, we need
2698 * to zero fill the balance.
2699 */
2700 if (!page_aligned(length)) {
2701 (void) memset((void *) (*paddr + length), 0,
2702 round_page(length) - length);
2703 }
2704 if (vm_map_copyin(ipc_kernel_copy_map, (vm_map_address_t)*paddr,
2705 (vm_map_size_t)length, TRUE, copy) != KERN_SUCCESS) {
2706 *mr = MACH_MSG_VM_KERNEL;
2707 return NULL;
2708 }
2709 dsc->address = (void *)*copy;
2710 *paddr += round_page(length);
2711 *space_needed -= round_page(length);
2712 } else {
2713
2714 /*
2715 * Make a vm_map_copy_t of the of the data. If the
2716 * data is small, this will do an optimized physical
2717 * copy. Otherwise, it will do a virtual copy.
2718 *
2719 * NOTE: A virtual copy is OK if the original is being
2720 * deallocted, even if a physical copy was requested.
2721 */
2722 kern_return_t kr = vm_map_copyin(map, addr,
2723 (vm_map_size_t)length, dealloc, copy);
2724 if (kr != KERN_SUCCESS) {
2725 *mr = (kr == KERN_RESOURCE_SHORTAGE) ?
2726 MACH_MSG_VM_KERNEL :
2727 MACH_SEND_INVALID_MEMORY;
2728 return NULL;
2729 }
2730 dsc->address = (void *)*copy;
2731 }
2732 return user_dsc;
2733 }
2734
2735 mach_msg_descriptor_t * ipc_kmsg_copyin_ool_ports_descriptor(
2736 mach_msg_ool_ports_descriptor_t *dsc,
2737 mach_msg_descriptor_t *user_dsc,
2738 int is_64bit,
2739 vm_map_t map,
2740 ipc_space_t space,
2741 ipc_object_t dest,
2742 ipc_kmsg_t kmsg,
2743 mach_msg_return_t *mr);
2744 mach_msg_descriptor_t *
2745 ipc_kmsg_copyin_ool_ports_descriptor(
2746 mach_msg_ool_ports_descriptor_t *dsc,
2747 mach_msg_descriptor_t *user_dsc,
2748 int is_64bit,
2749 vm_map_t map,
2750 ipc_space_t space,
2751 ipc_object_t dest,
2752 ipc_kmsg_t kmsg,
2753 mach_msg_return_t *mr)
2754 {
2755 void *data;
2756 ipc_object_t *objects;
2757 unsigned int i;
2758 mach_vm_offset_t addr;
2759 mach_msg_type_name_t user_disp;
2760 mach_msg_type_name_t result_disp;
2761 mach_msg_type_number_t count;
2762 mach_msg_copy_options_t copy_option;
2763 boolean_t deallocate;
2764 mach_msg_descriptor_type_t type;
2765 vm_size_t ports_length, names_length;
2766
2767 if (is_64bit) {
2768 mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
2769
2770 addr = (mach_vm_offset_t)user_ool_dsc->address;
2771 count = user_ool_dsc->count;
2772 deallocate = user_ool_dsc->deallocate;
2773 copy_option = user_ool_dsc->copy;
2774 user_disp = user_ool_dsc->disposition;
2775 type = user_ool_dsc->type;
2776
2777 user_dsc = (typeof(user_dsc))(user_ool_dsc+1);
2778 } else {
2779 mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
2780
2781 addr = CAST_USER_ADDR_T(user_ool_dsc->address);
2782 count = user_ool_dsc->count;
2783 deallocate = user_ool_dsc->deallocate;
2784 copy_option = user_ool_dsc->copy;
2785 user_disp = user_ool_dsc->disposition;
2786 type = user_ool_dsc->type;
2787
2788 user_dsc = (typeof(user_dsc))(user_ool_dsc+1);
2789 }
2790
2791 dsc->deallocate = deallocate;
2792 dsc->copy = copy_option;
2793 dsc->type = type;
2794 dsc->count = count;
2795 dsc->address = NULL; /* for now */
2796
2797 result_disp = ipc_object_copyin_type(user_disp);
2798 dsc->disposition = result_disp;
2799
2800 /* We always do a 'physical copy', but you have to specify something valid */
2801 if (copy_option != MACH_MSG_PHYSICAL_COPY &&
2802 copy_option != MACH_MSG_VIRTUAL_COPY) {
2803 *mr = MACH_SEND_INVALID_TYPE;
2804 return NULL;
2805 }
2806
2807 /* calculate length of data in bytes, rounding up */
2808
2809 if (os_mul_overflow(count, sizeof(mach_port_t), &ports_length)) {
2810 *mr = MACH_SEND_TOO_LARGE;
2811 return NULL;
2812 }
2813
2814 if (os_mul_overflow(count, sizeof(mach_port_name_t), &names_length)) {
2815 *mr = MACH_SEND_TOO_LARGE;
2816 return NULL;
2817 }
2818
2819 if (ports_length == 0) {
2820 return user_dsc;
2821 }
2822
2823 data = kalloc(ports_length);
2824
2825 if (data == NULL) {
2826 *mr = MACH_SEND_NO_BUFFER;
2827 return NULL;
2828 }
2829
2830 #ifdef __LP64__
2831 mach_port_name_t *names = &((mach_port_name_t *)data)[count];
2832 #else
2833 mach_port_name_t *names = ((mach_port_name_t *)data);
2834 #endif
2835
2836 if (copyinmap(map, addr, names, names_length) != KERN_SUCCESS) {
2837 kfree(data, ports_length);
2838 *mr = MACH_SEND_INVALID_MEMORY;
2839 return NULL;
2840 }
2841
2842 if (deallocate) {
2843 (void) mach_vm_deallocate(map, addr, (mach_vm_size_t)ports_length);
2844 }
2845
2846 objects = (ipc_object_t *) data;
2847 dsc->address = data;
2848
2849 for ( i = 0; i < count; i++) {
2850 mach_port_name_t name = names[i];
2851 ipc_object_t object;
2852
2853 if (!MACH_PORT_VALID(name)) {
2854 objects[i] = (ipc_object_t)CAST_MACH_NAME_TO_PORT(name);
2855 continue;
2856 }
2857
2858 kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object);
2859
2860 if (kr != KERN_SUCCESS) {
2861 unsigned int j;
2862
2863 for(j = 0; j < i; j++) {
2864 object = objects[j];
2865 if (IPC_OBJECT_VALID(object))
2866 ipc_object_destroy(object, result_disp);
2867 }
2868 kfree(data, ports_length);
2869 dsc->address = NULL;
2870 *mr = MACH_SEND_INVALID_RIGHT;
2871 return NULL;
2872 }
2873
2874 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
2875 ipc_port_check_circularity(
2876 (ipc_port_t) object,
2877 (ipc_port_t) dest))
2878 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
2879
2880 objects[i] = object;
2881 }
2882
2883 return user_dsc;
2884 }
2885
2886 /*
2887 * Routine: ipc_kmsg_copyin_body
2888 * Purpose:
2889 * "Copy-in" port rights and out-of-line memory
2890 * in the message body.
2891 *
2892 * In all failure cases, the message is left holding
2893 * no rights or memory. However, the message buffer
2894 * is not deallocated. If successful, the message
2895 * contains a valid destination port.
2896 * Conditions:
2897 * Nothing locked.
2898 * Returns:
2899 * MACH_MSG_SUCCESS Successful copyin.
2900 * MACH_SEND_INVALID_MEMORY Can't grab out-of-line memory.
2901 * MACH_SEND_INVALID_RIGHT Can't copyin port right in body.
2902 * MACH_SEND_INVALID_TYPE Bad type specification.
2903 * MACH_SEND_MSG_TOO_SMALL Body is too small for types/data.
2904 * MACH_SEND_INVALID_RT_OOL_SIZE OOL Buffer too large for RT
2905 * MACH_MSG_INVALID_RT_DESCRIPTOR Dealloc and RT are incompatible
2906 */
2907
2908 mach_msg_return_t
2909 ipc_kmsg_copyin_body(
2910 ipc_kmsg_t kmsg,
2911 ipc_space_t space,
2912 vm_map_t map)
2913 {
2914 ipc_object_t dest;
2915 mach_msg_body_t *body;
2916 mach_msg_descriptor_t *daddr, *naddr;
2917 mach_msg_descriptor_t *user_addr, *kern_addr;
2918 mach_msg_type_number_t dsc_count;
2919 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
2920 boolean_t complex = FALSE;
2921 vm_size_t space_needed = 0;
2922 vm_offset_t paddr = 0;
2923 vm_map_copy_t copy = VM_MAP_COPY_NULL;
2924 mach_msg_type_number_t i;
2925 mach_msg_return_t mr = MACH_MSG_SUCCESS;
2926
2927 vm_size_t descriptor_size = 0;
2928
2929 mach_msg_type_number_t total_ool_port_count = 0;
2930
2931 /*
2932 * Determine if the target is a kernel port.
2933 */
2934 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
2935 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
2936 naddr = (mach_msg_descriptor_t *) (body + 1);
2937
2938 dsc_count = body->msgh_descriptor_count;
2939 if (dsc_count == 0)
2940 return MACH_MSG_SUCCESS;
2941
2942 /*
2943 * Make an initial pass to determine kernal VM space requirements for
2944 * physical copies and possible contraction of the descriptors from
2945 * processes with pointers larger than the kernel's.
2946 */
2947 daddr = NULL;
2948 for (i = 0; i < dsc_count; i++) {
2949 mach_msg_size_t size;
2950 mach_msg_type_number_t ool_port_count = 0;
2951
2952 daddr = naddr;
2953
2954 /* make sure the descriptor fits in the message */
2955 if (is_task_64bit) {
2956 switch (daddr->type.type) {
2957 case MACH_MSG_OOL_DESCRIPTOR:
2958 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
2959 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
2960 descriptor_size += 16;
2961 naddr = (typeof(naddr))((vm_offset_t)daddr + 16);
2962 break;
2963 default:
2964 descriptor_size += 12;
2965 naddr = (typeof(naddr))((vm_offset_t)daddr + 12);
2966 break;
2967 }
2968 } else {
2969 descriptor_size += 12;
2970 naddr = (typeof(naddr))((vm_offset_t)daddr + 12);
2971 }
2972
2973 if (naddr > (mach_msg_descriptor_t *)
2974 ((vm_offset_t)kmsg->ikm_header + kmsg->ikm_header->msgh_size)) {
2975 mr = MACH_SEND_MSG_TOO_SMALL;
2976 goto clean_message;
2977 }
2978
2979 switch (daddr->type.type) {
2980 case MACH_MSG_OOL_DESCRIPTOR:
2981 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
2982 size = (is_task_64bit) ?
2983 ((mach_msg_ool_descriptor64_t *)daddr)->size :
2984 daddr->out_of_line.size;
2985
2986 if (daddr->out_of_line.copy != MACH_MSG_PHYSICAL_COPY &&
2987 daddr->out_of_line.copy != MACH_MSG_VIRTUAL_COPY) {
2988 /*
2989 * Invalid copy option
2990 */
2991 mr = MACH_SEND_INVALID_TYPE;
2992 goto clean_message;
2993 }
2994
2995 if ((size >= MSG_OOL_SIZE_SMALL) &&
2996 (daddr->out_of_line.copy == MACH_MSG_PHYSICAL_COPY) &&
2997 !(daddr->out_of_line.deallocate)) {
2998
2999 /*
3000 * Out-of-line memory descriptor, accumulate kernel
3001 * memory requirements
3002 */
3003 if (space_needed + round_page(size) <= space_needed) {
3004 /* Overflow dectected */
3005 mr = MACH_MSG_VM_KERNEL;
3006 goto clean_message;
3007 }
3008
3009 space_needed += round_page(size);
3010 if (space_needed > ipc_kmsg_max_vm_space) {
3011 /* Per message kernel memory limit exceeded */
3012 mr = MACH_MSG_VM_KERNEL;
3013 goto clean_message;
3014 }
3015 }
3016 break;
3017 case MACH_MSG_PORT_DESCRIPTOR:
3018 if (os_add_overflow(total_ool_port_count, 1, &total_ool_port_count)) {
3019 /* Overflow detected */
3020 mr = MACH_SEND_TOO_LARGE;
3021 goto clean_message;
3022 }
3023 break;
3024 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
3025 ool_port_count = (is_task_64bit) ?
3026 ((mach_msg_ool_ports_descriptor64_t *)daddr)->count :
3027 daddr->ool_ports.count;
3028
3029 if (os_add_overflow(total_ool_port_count, ool_port_count, &total_ool_port_count)) {
3030 /* Overflow detected */
3031 mr = MACH_SEND_TOO_LARGE;
3032 goto clean_message;
3033 }
3034
3035 if (ool_port_count > (ipc_kmsg_max_vm_space/sizeof(mach_port_t))) {
3036 /* Per message kernel memory limit exceeded */
3037 mr = MACH_SEND_TOO_LARGE;
3038 goto clean_message;
3039 }
3040 break;
3041 }
3042 }
3043
3044 /* Sending more than 16383 rights in one message seems crazy */
3045 if (total_ool_port_count >= (MACH_PORT_UREFS_MAX / 4)) {
3046 mr = MACH_SEND_TOO_LARGE;
3047 goto clean_message;
3048 }
3049
3050 /*
3051 * Allocate space in the pageable kernel ipc copy map for all the
3052 * ool data that is to be physically copied. Map is marked wait for
3053 * space.
3054 */
3055 if (space_needed) {
3056 if (vm_allocate(ipc_kernel_copy_map, &paddr, space_needed,
3057 VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IPC)) != KERN_SUCCESS) {
3058 mr = MACH_MSG_VM_KERNEL;
3059 goto clean_message;
3060 }
3061 }
3062
3063 /* user_addr = just after base as it was copied in */
3064 user_addr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t));
3065
3066 /* Shift the mach_msg_base_t down to make room for dsc_count*16bytes of descriptors */
3067 if(descriptor_size != 16*dsc_count) {
3068 vm_offset_t dsc_adjust = 16*dsc_count - descriptor_size;
3069
3070 memmove((char *)(((vm_offset_t)kmsg->ikm_header) - dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t));
3071 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header - dsc_adjust);
3072
3073 /* Update the message size for the larger in-kernel representation */
3074 kmsg->ikm_header->msgh_size += (mach_msg_size_t)dsc_adjust;
3075 }
3076
3077
3078 /* kern_addr = just after base after it has been (conditionally) moved */
3079 kern_addr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t));
3080
3081 /* handle the OOL regions and port descriptors. */
3082 for(i=0;i<dsc_count;i++) {
3083 switch (user_addr->type.type) {
3084 case MACH_MSG_PORT_DESCRIPTOR:
3085 user_addr = ipc_kmsg_copyin_port_descriptor((mach_msg_port_descriptor_t *)kern_addr,
3086 (mach_msg_legacy_port_descriptor_t *)user_addr, space, dest, kmsg, &mr);
3087 kern_addr++;
3088 complex = TRUE;
3089 break;
3090 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3091 case MACH_MSG_OOL_DESCRIPTOR:
3092 user_addr = ipc_kmsg_copyin_ool_descriptor((mach_msg_ool_descriptor_t *)kern_addr,
3093 user_addr, is_task_64bit, &paddr, &copy, &space_needed, map, &mr);
3094 kern_addr++;
3095 complex = TRUE;
3096 break;
3097 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
3098 user_addr = ipc_kmsg_copyin_ool_ports_descriptor((mach_msg_ool_ports_descriptor_t *)kern_addr,
3099 user_addr, is_task_64bit, map, space, dest, kmsg, &mr);
3100 kern_addr++;
3101 complex = TRUE;
3102 break;
3103 default:
3104 /* Invalid descriptor */
3105 mr = MACH_SEND_INVALID_TYPE;
3106 break;
3107 }
3108
3109 if (MACH_MSG_SUCCESS != mr) {
3110 /* clean from start of message descriptors to i */
3111 ipc_kmsg_clean_partial(kmsg, i,
3112 (mach_msg_descriptor_t *)((mach_msg_base_t *)kmsg->ikm_header + 1),
3113 paddr, space_needed);
3114 goto out;
3115 }
3116 } /* End of loop */
3117
3118 if (!complex) {
3119 kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_COMPLEX;
3120 }
3121 out:
3122 return mr;
3123
3124 clean_message:
3125 /* no descriptors have been copied in yet */
3126 ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0);
3127 return mr;
3128 }
3129
3130
3131 /*
3132 * Routine: ipc_kmsg_copyin
3133 * Purpose:
3134 * "Copy-in" port rights and out-of-line memory
3135 * in the message.
3136 *
3137 * In all failure cases, the message is left holding
3138 * no rights or memory. However, the message buffer
3139 * is not deallocated. If successful, the message
3140 * contains a valid destination port.
3141 * Conditions:
3142 * Nothing locked.
3143 * Returns:
3144 * MACH_MSG_SUCCESS Successful copyin.
3145 * MACH_SEND_INVALID_HEADER
3146 * Illegal value in the message header bits.
3147 * MACH_SEND_INVALID_DEST Can't copyin destination port.
3148 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
3149 * MACH_SEND_INVALID_MEMORY Can't grab out-of-line memory.
3150 * MACH_SEND_INVALID_RIGHT Can't copyin port right in body.
3151 * MACH_SEND_INVALID_TYPE Bad type specification.
3152 * MACH_SEND_MSG_TOO_SMALL Body is too small for types/data.
3153 */
3154
3155 mach_msg_return_t
3156 ipc_kmsg_copyin(
3157 ipc_kmsg_t kmsg,
3158 ipc_space_t space,
3159 vm_map_t map,
3160 mach_msg_priority_t override,
3161 mach_msg_option_t *optionp)
3162 {
3163 mach_msg_return_t mr;
3164
3165 kmsg->ikm_header->msgh_bits &= MACH_MSGH_BITS_USER;
3166
3167 mr = ipc_kmsg_copyin_header(kmsg, space, override, optionp);
3168
3169 if (mr != MACH_MSG_SUCCESS)
3170 return mr;
3171
3172 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_MSG_SEND) | DBG_FUNC_NONE,
3173 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3174 (uintptr_t)kmsg->ikm_header->msgh_bits,
3175 (uintptr_t)kmsg->ikm_header->msgh_id,
3176 VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(kmsg->ikm_voucher)),
3177 0);
3178
3179 DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_copyin header:\n%.8x\n%.8x\n%p\n%p\n%p\n%.8x\n",
3180 kmsg->ikm_header->msgh_size,
3181 kmsg->ikm_header->msgh_bits,
3182 kmsg->ikm_header->msgh_remote_port,
3183 kmsg->ikm_header->msgh_local_port,
3184 kmsg->ikm_voucher,
3185 kmsg->ikm_header->msgh_id);
3186
3187 if ((kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) == 0)
3188 return MACH_MSG_SUCCESS;
3189
3190 mr = ipc_kmsg_copyin_body( kmsg, space, map);
3191
3192 /* unreachable if !DEBUG */
3193 __unreachable_ok_push
3194 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK))
3195 {
3196 kprintf("body:\n");
3197 uint32_t i;
3198 for(i=0;i*4 < (kmsg->ikm_header->msgh_size - sizeof(mach_msg_header_t));i++)
3199 {
3200 kprintf("%.4x\n",((uint32_t *)(kmsg->ikm_header + 1))[i]);
3201 }
3202 }
3203 __unreachable_ok_pop
3204
3205 return mr;
3206 }
3207
3208 /*
3209 * Routine: ipc_kmsg_copyin_from_kernel
3210 * Purpose:
3211 * "Copy-in" port rights and out-of-line memory
3212 * in a message sent from the kernel.
3213 *
3214 * Because the message comes from the kernel,
3215 * the implementation assumes there are no errors
3216 * or peculiarities in the message.
3217 * Conditions:
3218 * Nothing locked.
3219 */
3220
3221 mach_msg_return_t
3222 ipc_kmsg_copyin_from_kernel(
3223 ipc_kmsg_t kmsg)
3224 {
3225 mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits;
3226 mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits);
3227 mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits);
3228 ipc_object_t remote = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
3229 ipc_object_t local = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
3230
3231 /* translate the destination and reply ports */
3232 if (!IO_VALID(remote))
3233 return MACH_SEND_INVALID_DEST;
3234
3235 ipc_object_copyin_from_kernel(remote, rname);
3236 if (IO_VALID(local))
3237 ipc_object_copyin_from_kernel(local, lname);
3238
3239 /*
3240 * The common case is a complex message with no reply port,
3241 * because that is what the memory_object interface uses.
3242 */
3243
3244 if (bits == (MACH_MSGH_BITS_COMPLEX |
3245 MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) {
3246 bits = (MACH_MSGH_BITS_COMPLEX |
3247 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0));
3248
3249 kmsg->ikm_header->msgh_bits = bits;
3250 } else {
3251 bits = (MACH_MSGH_BITS_OTHER(bits) |
3252 MACH_MSGH_BITS(ipc_object_copyin_type(rname),
3253 ipc_object_copyin_type(lname)));
3254
3255 kmsg->ikm_header->msgh_bits = bits;
3256 if ((bits & MACH_MSGH_BITS_COMPLEX) == 0)
3257 return MACH_MSG_SUCCESS;
3258 }
3259 {
3260 mach_msg_descriptor_t *saddr;
3261 mach_msg_body_t *body;
3262 mach_msg_type_number_t i, count;
3263
3264 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
3265 saddr = (mach_msg_descriptor_t *) (body + 1);
3266 count = body->msgh_descriptor_count;
3267
3268 for (i = 0; i < count; i++, saddr++) {
3269
3270 switch (saddr->type.type) {
3271
3272 case MACH_MSG_PORT_DESCRIPTOR: {
3273 mach_msg_type_name_t name;
3274 ipc_object_t object;
3275 mach_msg_port_descriptor_t *dsc;
3276
3277 dsc = &saddr->port;
3278
3279 /* this is really the type SEND, SEND_ONCE, etc. */
3280 name = dsc->disposition;
3281 object = (ipc_object_t) dsc->name;
3282 dsc->disposition = ipc_object_copyin_type(name);
3283
3284 if (!IO_VALID(object)) {
3285 break;
3286 }
3287
3288 ipc_object_copyin_from_kernel(object, name);
3289
3290 /* CDY avoid circularity when the destination is also */
3291 /* the kernel. This check should be changed into an */
3292 /* assert when the new kobject model is in place since*/
3293 /* ports will not be used in kernel to kernel chats */
3294
3295 if (((ipc_port_t)remote)->ip_receiver != ipc_space_kernel) {
3296 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3297 ipc_port_check_circularity((ipc_port_t) object,
3298 (ipc_port_t) remote)) {
3299 kmsg->ikm_header->msgh_bits |=
3300 MACH_MSGH_BITS_CIRCULAR;
3301 }
3302 }
3303 break;
3304 }
3305 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3306 case MACH_MSG_OOL_DESCRIPTOR: {
3307 /*
3308 * The sender should supply ready-made memory, i.e.
3309 * a vm_map_copy_t, so we don't need to do anything.
3310 */
3311 break;
3312 }
3313 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
3314 ipc_object_t *objects;
3315 unsigned int j;
3316 mach_msg_type_name_t name;
3317 mach_msg_ool_ports_descriptor_t *dsc;
3318
3319 dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports;
3320
3321 /* this is really the type SEND, SEND_ONCE, etc. */
3322 name = dsc->disposition;
3323 dsc->disposition = ipc_object_copyin_type(name);
3324
3325 objects = (ipc_object_t *) dsc->address;
3326
3327 for ( j = 0; j < dsc->count; j++) {
3328 ipc_object_t object = objects[j];
3329
3330 if (!IO_VALID(object))
3331 continue;
3332
3333 ipc_object_copyin_from_kernel(object, name);
3334
3335 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3336 ipc_port_check_circularity(
3337 (ipc_port_t) object,
3338 (ipc_port_t) remote))
3339 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
3340 }
3341 break;
3342 }
3343 default: {
3344 #if MACH_ASSERT
3345 panic("ipc_kmsg_copyin_from_kernel: bad descriptor");
3346 #endif /* MACH_ASSERT */
3347 }
3348 }
3349 }
3350 }
3351 return MACH_MSG_SUCCESS;
3352 }
3353
3354 #if IKM_SUPPORT_LEGACY
3355 mach_msg_return_t
3356 ipc_kmsg_copyin_from_kernel_legacy(
3357 ipc_kmsg_t kmsg)
3358 {
3359 mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits;
3360 mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits);
3361 mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits);
3362 ipc_object_t remote = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
3363 ipc_object_t local = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
3364
3365 /* translate the destination and reply ports */
3366 if (!IO_VALID(remote))
3367 return MACH_SEND_INVALID_DEST;
3368
3369 ipc_object_copyin_from_kernel(remote, rname);
3370 if (IO_VALID(local))
3371 ipc_object_copyin_from_kernel(local, lname);
3372
3373 /*
3374 * The common case is a complex message with no reply port,
3375 * because that is what the memory_object interface uses.
3376 */
3377
3378 if (bits == (MACH_MSGH_BITS_COMPLEX |
3379 MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) {
3380 bits = (MACH_MSGH_BITS_COMPLEX |
3381 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0));
3382
3383 kmsg->ikm_header->msgh_bits = bits;
3384 } else {
3385 bits = (MACH_MSGH_BITS_OTHER(bits) |
3386 MACH_MSGH_BITS(ipc_object_copyin_type(rname),
3387 ipc_object_copyin_type(lname)));
3388
3389 kmsg->ikm_header->msgh_bits = bits;
3390 if ((bits & MACH_MSGH_BITS_COMPLEX) == 0)
3391 return MACH_MSG_SUCCESS;
3392 }
3393 {
3394 mach_msg_legacy_descriptor_t *saddr;
3395 mach_msg_descriptor_t *daddr;
3396 mach_msg_body_t *body;
3397 mach_msg_type_number_t i, count;
3398
3399 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
3400 saddr = (typeof(saddr)) (body + 1);
3401 count = body->msgh_descriptor_count;
3402
3403 if(count) {
3404 vm_offset_t dsc_adjust = 4*count;
3405 memmove((char *)(((vm_offset_t)kmsg->ikm_header) - dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t));
3406 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header - dsc_adjust);
3407 /* Update the message size for the larger in-kernel representation */
3408 kmsg->ikm_header->msgh_size += dsc_adjust;
3409 }
3410 daddr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t));
3411
3412 for (i = 0; i < count; i++, saddr++, daddr++) {
3413 switch (saddr->type.type) {
3414
3415 case MACH_MSG_PORT_DESCRIPTOR: {
3416 mach_msg_type_name_t name;
3417 ipc_object_t object;
3418 mach_msg_legacy_port_descriptor_t *dsc;
3419 mach_msg_port_descriptor_t *dest_dsc;
3420
3421 dsc = (typeof(dsc))&saddr->port;
3422 dest_dsc = &daddr->port;
3423
3424 /* this is really the type SEND, SEND_ONCE, etc. */
3425 name = dsc->disposition;
3426 object = (ipc_object_t) CAST_MACH_NAME_TO_PORT(dsc->name);
3427 dest_dsc->disposition = ipc_object_copyin_type(name);
3428 dest_dsc->name = (mach_port_t)object;
3429 dest_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
3430
3431 if (!IO_VALID(object)) {
3432 break;
3433 }
3434
3435 ipc_object_copyin_from_kernel(object, name);
3436
3437 /* CDY avoid circularity when the destination is also */
3438 /* the kernel. This check should be changed into an */
3439 /* assert when the new kobject model is in place since*/
3440 /* ports will not be used in kernel to kernel chats */
3441
3442 if (((ipc_port_t)remote)->ip_receiver != ipc_space_kernel) {
3443 if ((dest_dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3444 ipc_port_check_circularity((ipc_port_t) object,
3445 (ipc_port_t) remote)) {
3446 kmsg->ikm_header->msgh_bits |=
3447 MACH_MSGH_BITS_CIRCULAR;
3448 }
3449 }
3450 break;
3451 }
3452 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3453 case MACH_MSG_OOL_DESCRIPTOR: {
3454 /* The sender should supply ready-made memory, i.e. a vm_map_copy_t
3455 * so we don't need to do anything special. */
3456
3457 mach_msg_ool_descriptor32_t *source_dsc = &saddr->out_of_line32;
3458 mach_msg_ool_descriptor_t *dest_dsc = (typeof(dest_dsc))&daddr->out_of_line;
3459
3460 vm_offset_t address = source_dsc->address;
3461 vm_size_t size = source_dsc->size;
3462 boolean_t deallocate = source_dsc->deallocate;
3463 mach_msg_copy_options_t copy = source_dsc->copy;
3464 mach_msg_descriptor_type_t type = source_dsc->type;
3465
3466 dest_dsc->address = (void *)address;
3467 dest_dsc->size = size;
3468 dest_dsc->deallocate = deallocate;
3469 dest_dsc->copy = copy;
3470 dest_dsc->type = type;
3471 break;
3472 }
3473 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
3474 ipc_object_t *objects;
3475 unsigned int j;
3476 mach_msg_type_name_t name;
3477 mach_msg_ool_ports_descriptor_t *dest_dsc;
3478
3479 mach_msg_ool_ports_descriptor32_t *source_dsc = &saddr->ool_ports32;
3480 dest_dsc = (typeof(dest_dsc))&daddr->ool_ports;
3481
3482 boolean_t deallocate = source_dsc->deallocate;
3483 mach_msg_copy_options_t copy = source_dsc->copy;
3484 mach_msg_size_t port_count = source_dsc->count;
3485 mach_msg_type_name_t disposition = source_dsc->disposition;
3486
3487 /* this is really the type SEND, SEND_ONCE, etc. */
3488 name = disposition;
3489 disposition = ipc_object_copyin_type(name);
3490
3491 objects = (ipc_object_t *) (uintptr_t)source_dsc->address;
3492
3493 for ( j = 0; j < port_count; j++) {
3494 ipc_object_t object = objects[j];
3495
3496 if (!IO_VALID(object))
3497 continue;
3498
3499 ipc_object_copyin_from_kernel(object, name);
3500
3501 if ((disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3502 ipc_port_check_circularity(
3503 (ipc_port_t) object,
3504 (ipc_port_t) remote))
3505 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
3506 }
3507
3508 dest_dsc->address = objects;
3509 dest_dsc->deallocate = deallocate;
3510 dest_dsc->copy = copy;
3511 dest_dsc->disposition = disposition;
3512 dest_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
3513 dest_dsc->count = port_count;
3514 break;
3515 }
3516 default: {
3517 #if MACH_ASSERT
3518 panic("ipc_kmsg_copyin_from_kernel: bad descriptor");
3519 #endif /* MACH_ASSERT */
3520 }
3521 }
3522 }
3523 }
3524 return MACH_MSG_SUCCESS;
3525 }
3526 #endif /* IKM_SUPPORT_LEGACY */
3527
3528 /*
3529 * Routine: ipc_kmsg_copyout_header
3530 * Purpose:
3531 * "Copy-out" port rights in the header of a message.
3532 * Operates atomically; if it doesn't succeed the
3533 * message header and the space are left untouched.
3534 * If it does succeed the remote/local port fields
3535 * contain port names instead of object pointers,
3536 * and the bits field is updated.
3537 * Conditions:
3538 * Nothing locked.
3539 * Returns:
3540 * MACH_MSG_SUCCESS Copied out port rights.
3541 * MACH_RCV_INVALID_NOTIFY
3542 * Notify is non-null and doesn't name a receive right.
3543 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
3544 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE
3545 * The space is dead.
3546 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE
3547 * No room in space for another name.
3548 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_KERNEL
3549 * Couldn't allocate memory for the reply port.
3550 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_KERNEL
3551 * Couldn't allocate memory for the dead-name request.
3552 */
3553
3554 mach_msg_return_t
3555 ipc_kmsg_copyout_header(
3556 ipc_kmsg_t kmsg,
3557 ipc_space_t space,
3558 mach_msg_option_t option)
3559 {
3560 mach_msg_header_t *msg = kmsg->ikm_header;
3561 mach_msg_bits_t mbits = msg->msgh_bits;
3562 ipc_port_t dest = (ipc_port_t) msg->msgh_remote_port;
3563
3564 assert(IP_VALID(dest));
3565
3566 /*
3567 * While we still hold a reference on the received-from port,
3568 * process all send-possible notfications we received along with
3569 * the message.
3570 */
3571 ipc_port_spnotify(dest);
3572
3573 {
3574 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
3575 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
3576 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
3577 ipc_port_t reply = msg->msgh_local_port;
3578 ipc_port_t release_reply_port = IP_NULL;
3579 mach_port_name_t dest_name, reply_name;
3580
3581 ipc_port_t voucher = kmsg->ikm_voucher;
3582 ipc_port_t release_voucher_port = IP_NULL;
3583 mach_port_name_t voucher_name;
3584
3585 uint32_t entries_held = 0;
3586 boolean_t need_write_lock = FALSE;
3587 kern_return_t kr;
3588
3589 /*
3590 * Reserve any potentially needed entries in the target space.
3591 * We'll free any unused before unlocking the space.
3592 */
3593 if (IP_VALID(reply)) {
3594 entries_held++;
3595 need_write_lock = TRUE;
3596 }
3597 if (IP_VALID(voucher)) {
3598 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
3599
3600 if ((option & MACH_RCV_VOUCHER) != 0)
3601 entries_held++;
3602 need_write_lock = TRUE;
3603 }
3604
3605 if (need_write_lock) {
3606
3607 is_write_lock(space);
3608
3609 while(entries_held) {
3610 if (!is_active(space)) {
3611 is_write_unlock(space);
3612 return (MACH_RCV_HEADER_ERROR|
3613 MACH_MSG_IPC_SPACE);
3614 }
3615
3616 kr = ipc_entries_hold(space, entries_held);
3617 if (KERN_SUCCESS == kr)
3618 break;
3619
3620 kr = ipc_entry_grow_table(space, ITS_SIZE_NONE);
3621 if (KERN_SUCCESS != kr)
3622 return(MACH_RCV_HEADER_ERROR|
3623 MACH_MSG_IPC_SPACE);
3624 /* space was unlocked and relocked - retry */
3625 }
3626
3627 /* Handle reply port. */
3628 if (IP_VALID(reply)) {
3629 ipc_entry_t entry;
3630
3631 /* Is there already an entry we can use? */
3632 if ((reply_type != MACH_MSG_TYPE_PORT_SEND_ONCE) &&
3633 ipc_right_reverse(space, (ipc_object_t) reply, &reply_name, &entry)) {
3634 /* reply port is locked and active */
3635 assert(entry->ie_bits & MACH_PORT_TYPE_SEND_RECEIVE);
3636 } else {
3637 ip_lock(reply);
3638 if (!ip_active(reply)) {
3639 ip_unlock(reply);
3640
3641 release_reply_port = reply;
3642 reply = IP_DEAD;
3643 reply_name = MACH_PORT_DEAD;
3644 goto done_with_reply;
3645 }
3646
3647 /* claim a held entry for the reply port */
3648 assert(entries_held > 0);
3649 entries_held--;
3650 ipc_entry_claim(space, &reply_name, &entry);
3651 assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE);
3652 assert(entry->ie_object == IO_NULL);
3653 entry->ie_object = (ipc_object_t) reply;
3654 }
3655
3656 /* space and reply port are locked and active */
3657 ip_reference(reply); /* hold onto the reply port */
3658
3659 kr = ipc_right_copyout(space, reply_name, entry,
3660 reply_type, TRUE, (ipc_object_t) reply);
3661 assert(kr == KERN_SUCCESS);
3662 /* reply port is unlocked */
3663 } else
3664 reply_name = CAST_MACH_PORT_TO_NAME(reply);
3665
3666 done_with_reply:
3667
3668 /* Handle voucher port. */
3669 if (voucher_type != MACH_MSGH_BITS_ZERO) {
3670 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
3671
3672 if (!IP_VALID(voucher)) {
3673 if ((option & MACH_RCV_VOUCHER) == 0) {
3674 voucher_type = MACH_MSGH_BITS_ZERO;
3675 }
3676 voucher_name = MACH_PORT_NULL;
3677 goto done_with_voucher;
3678 }
3679
3680 /* clear voucher from its hiding place back in the kmsg */
3681 kmsg->ikm_voucher = IP_NULL;
3682
3683 if ((option & MACH_RCV_VOUCHER) != 0) {
3684 ipc_entry_t entry;
3685
3686 if (ipc_right_reverse(space, (ipc_object_t) voucher,
3687 &voucher_name, &entry)) {
3688 /* voucher port locked */
3689 assert(entry->ie_bits & MACH_PORT_TYPE_SEND);
3690 } else {
3691 assert(entries_held > 0);
3692 entries_held--;
3693 ipc_entry_claim(space, &voucher_name, &entry);
3694 assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE);
3695 assert(entry->ie_object == IO_NULL);
3696 entry->ie_object = (ipc_object_t) voucher;
3697 ip_lock(voucher);
3698 }
3699 /* space is locked and active */
3700
3701 assert(ip_active(voucher));
3702 assert(ip_kotype(voucher) == IKOT_VOUCHER);
3703 kr = ipc_right_copyout(space, voucher_name, entry,
3704 MACH_MSG_TYPE_MOVE_SEND, TRUE,
3705 (ipc_object_t) voucher);
3706 /* voucher port is unlocked */
3707 } else {
3708 voucher_type = MACH_MSGH_BITS_ZERO;
3709 release_voucher_port = voucher;
3710 voucher_name = MACH_PORT_NULL;
3711 }
3712 } else {
3713 voucher_name = msg->msgh_voucher_port;
3714 }
3715
3716 done_with_voucher:
3717
3718 ip_lock(dest);
3719 is_write_unlock(space);
3720
3721 } else {
3722 /*
3723 * No reply or voucher port! This is an easy case.
3724 * We only need to have the space locked
3725 * when locking the destination.
3726 */
3727
3728 is_read_lock(space);
3729 if (!is_active(space)) {
3730 is_read_unlock(space);
3731 return MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE;
3732 }
3733
3734 ip_lock(dest);
3735 is_read_unlock(space);
3736
3737 reply_name = CAST_MACH_PORT_TO_NAME(reply);
3738
3739 if (voucher_type != MACH_MSGH_BITS_ZERO) {
3740 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
3741 if ((option & MACH_RCV_VOUCHER) == 0) {
3742 voucher_type = MACH_MSGH_BITS_ZERO;
3743 }
3744 voucher_name = MACH_PORT_NULL;
3745 } else {
3746 voucher_name = msg->msgh_voucher_port;
3747 }
3748 }
3749
3750 /*
3751 * At this point, the space is unlocked and the destination
3752 * port is locked. (Lock taken while space was locked.)
3753 * reply_name is taken care of; we still need dest_name.
3754 * We still hold a ref for reply (if it is valid).
3755 *
3756 * If the space holds receive rights for the destination,
3757 * we return its name for the right. Otherwise the task
3758 * managed to destroy or give away the receive right between
3759 * receiving the message and this copyout. If the destination
3760 * is dead, return MACH_PORT_DEAD, and if the receive right
3761 * exists somewhere else (another space, in transit)
3762 * return MACH_PORT_NULL.
3763 *
3764 * Making this copyout operation atomic with the previous
3765 * copyout of the reply port is a bit tricky. If there was
3766 * no real reply port (it wasn't IP_VALID) then this isn't
3767 * an issue. If the reply port was dead at copyout time,
3768 * then we are OK, because if dest is dead we serialize
3769 * after the death of both ports and if dest is alive
3770 * we serialize after reply died but before dest's (later) death.
3771 * So assume reply was alive when we copied it out. If dest
3772 * is alive, then we are OK because we serialize before
3773 * the ports' deaths. So assume dest is dead when we look at it.
3774 * If reply dies/died after dest, then we are OK because
3775 * we serialize after dest died but before reply dies.
3776 * So the hard case is when reply is alive at copyout,
3777 * dest is dead at copyout, and reply died before dest died.
3778 * In this case pretend that dest is still alive, so
3779 * we serialize while both ports are alive.
3780 *
3781 * Because the space lock is held across the copyout of reply
3782 * and locking dest, the receive right for dest can't move
3783 * in or out of the space while the copyouts happen, so
3784 * that isn't an atomicity problem. In the last hard case
3785 * above, this implies that when dest is dead that the
3786 * space couldn't have had receive rights for dest at
3787 * the time reply was copied-out, so when we pretend
3788 * that dest is still alive, we can return MACH_PORT_NULL.
3789 *
3790 * If dest == reply, then we have to make it look like
3791 * either both copyouts happened before the port died,
3792 * or both happened after the port died. This special
3793 * case works naturally if the timestamp comparison
3794 * is done correctly.
3795 */
3796
3797 if (ip_active(dest)) {
3798 ipc_object_copyout_dest(space, (ipc_object_t) dest,
3799 dest_type, &dest_name);
3800 /* dest is unlocked */
3801
3802 } else {
3803 ipc_port_timestamp_t timestamp;
3804
3805 timestamp = dest->ip_timestamp;
3806 ip_unlock(dest);
3807 ip_release(dest);
3808
3809 if (IP_VALID(reply)) {
3810 ip_lock(reply);
3811 if (ip_active(reply) ||
3812 IP_TIMESTAMP_ORDER(timestamp,
3813 reply->ip_timestamp))
3814 dest_name = MACH_PORT_DEAD;
3815 else
3816 dest_name = MACH_PORT_NULL;
3817 ip_unlock(reply);
3818 } else
3819 dest_name = MACH_PORT_DEAD;
3820 }
3821
3822 if (IP_VALID(reply))
3823 ip_release(reply);
3824
3825 if (IP_VALID(release_reply_port)) {
3826 if (reply_type == MACH_MSG_TYPE_PORT_SEND_ONCE)
3827 ipc_port_release_sonce(release_reply_port);
3828 else
3829 ipc_port_release_send(release_reply_port);
3830 }
3831
3832 if (IP_VALID(release_voucher_port))
3833 ipc_port_release_send(release_voucher_port);
3834
3835
3836 if ((option & MACH_RCV_VOUCHER) != 0) {
3837 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV) | DBG_FUNC_NONE,
3838 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3839 (uintptr_t)kmsg->ikm_header->msgh_bits,
3840 (uintptr_t)kmsg->ikm_header->msgh_id,
3841 VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(voucher)),
3842 0);
3843 } else {
3844 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV_VOUCHER_REFUSED) | DBG_FUNC_NONE,
3845 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3846 (uintptr_t)kmsg->ikm_header->msgh_bits,
3847 (uintptr_t)kmsg->ikm_header->msgh_id,
3848 VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(voucher)),
3849 0);
3850 }
3851
3852 msg->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type,
3853 voucher_type, mbits);
3854 msg->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
3855 msg->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
3856 msg->msgh_voucher_port = voucher_name;
3857 }
3858
3859 return MACH_MSG_SUCCESS;
3860 }
3861
3862 /*
3863 * Routine: ipc_kmsg_copyout_object
3864 * Purpose:
3865 * Copy-out a port right. Always returns a name,
3866 * even for unsuccessful return codes. Always
3867 * consumes the supplied object.
3868 * Conditions:
3869 * Nothing locked.
3870 * Returns:
3871 * MACH_MSG_SUCCESS The space acquired the right
3872 * (name is valid) or the object is dead (MACH_PORT_DEAD).
3873 * MACH_MSG_IPC_SPACE No room in space for the right,
3874 * or the space is dead. (Name is MACH_PORT_NULL.)
3875 * MACH_MSG_IPC_KERNEL Kernel resource shortage.
3876 * (Name is MACH_PORT_NULL.)
3877 */
3878
3879 mach_msg_return_t
3880 ipc_kmsg_copyout_object(
3881 ipc_space_t space,
3882 ipc_object_t object,
3883 mach_msg_type_name_t msgt_name,
3884 mach_port_name_t *namep)
3885 {
3886 kern_return_t kr;
3887
3888 if (!IO_VALID(object)) {
3889 *namep = CAST_MACH_PORT_TO_NAME(object);
3890 return MACH_MSG_SUCCESS;
3891 }
3892
3893 kr = ipc_object_copyout(space, object, msgt_name, TRUE, namep);
3894 if (kr != KERN_SUCCESS) {
3895 ipc_object_destroy(object, msgt_name);
3896
3897 if (kr == KERN_INVALID_CAPABILITY)
3898 *namep = MACH_PORT_DEAD;
3899 else {
3900 *namep = MACH_PORT_NULL;
3901
3902 if (kr == KERN_RESOURCE_SHORTAGE)
3903 return MACH_MSG_IPC_KERNEL;
3904 else
3905 return MACH_MSG_IPC_SPACE;
3906 }
3907 }
3908
3909 return MACH_MSG_SUCCESS;
3910 }
3911
3912 mach_msg_descriptor_t *
3913 ipc_kmsg_copyout_port_descriptor(mach_msg_descriptor_t *dsc,
3914 mach_msg_descriptor_t *user_dsc,
3915 ipc_space_t space,
3916 kern_return_t *mr);
3917 mach_msg_descriptor_t *
3918 ipc_kmsg_copyout_port_descriptor(mach_msg_descriptor_t *dsc,
3919 mach_msg_descriptor_t *dest_dsc,
3920 ipc_space_t space,
3921 kern_return_t *mr)
3922 {
3923 mach_port_t port;
3924 mach_port_name_t name;
3925 mach_msg_type_name_t disp;
3926
3927
3928 /* Copyout port right carried in the message */
3929 port = dsc->port.name;
3930 disp = dsc->port.disposition;
3931 *mr |= ipc_kmsg_copyout_object(space,
3932 (ipc_object_t)port,
3933 disp,
3934 &name);
3935
3936 if(current_task() == kernel_task)
3937 {
3938 mach_msg_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc;
3939 user_dsc--; // point to the start of this port descriptor
3940 bzero((void *)user_dsc, sizeof(*user_dsc));
3941 user_dsc->name = CAST_MACH_NAME_TO_PORT(name);
3942 user_dsc->disposition = disp;
3943 user_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
3944 dest_dsc = (typeof(dest_dsc))user_dsc;
3945 } else {
3946 mach_msg_legacy_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc;
3947 user_dsc--; // point to the start of this port descriptor
3948 bzero((void *)user_dsc, sizeof(*user_dsc));
3949 user_dsc->name = CAST_MACH_PORT_TO_NAME(name);
3950 user_dsc->disposition = disp;
3951 user_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
3952 dest_dsc = (typeof(dest_dsc))user_dsc;
3953 }
3954
3955 return (mach_msg_descriptor_t *)dest_dsc;
3956 }
3957
3958 mach_msg_descriptor_t *
3959 ipc_kmsg_copyout_ool_descriptor(mach_msg_ool_descriptor_t *dsc, mach_msg_descriptor_t *user_dsc, int is_64bit, vm_map_t map, mach_msg_return_t *mr);
3960 mach_msg_descriptor_t *
3961 ipc_kmsg_copyout_ool_descriptor(mach_msg_ool_descriptor_t *dsc, mach_msg_descriptor_t *user_dsc, int is_64bit, vm_map_t map, mach_msg_return_t *mr)
3962 {
3963 vm_map_copy_t copy;
3964 vm_map_address_t rcv_addr;
3965 mach_msg_copy_options_t copy_options;
3966 vm_map_size_t size;
3967 mach_msg_descriptor_type_t dsc_type;
3968
3969 //SKIP_PORT_DESCRIPTORS(saddr, sdsc_count);
3970
3971 copy = (vm_map_copy_t)dsc->address;
3972 size = (vm_map_size_t)dsc->size;
3973 copy_options = dsc->copy;
3974 assert(copy_options != MACH_MSG_KALLOC_COPY_T);
3975 dsc_type = dsc->type;
3976
3977 if (copy != VM_MAP_COPY_NULL) {
3978 kern_return_t kr;
3979
3980 rcv_addr = 0;
3981 if (vm_map_copy_validate_size(map, copy, &size) == FALSE)
3982 panic("Inconsistent OOL/copyout size on %p: expected %d, got %lld @%p",
3983 dsc, dsc->size, (unsigned long long)copy->size, copy);
3984 kr = vm_map_copyout_size(map, &rcv_addr, copy, size);
3985 if (kr != KERN_SUCCESS) {
3986 if (kr == KERN_RESOURCE_SHORTAGE)
3987 *mr |= MACH_MSG_VM_KERNEL;
3988 else
3989 *mr |= MACH_MSG_VM_SPACE;
3990 vm_map_copy_discard(copy);
3991 rcv_addr = 0;
3992 size = 0;
3993 }
3994 } else {
3995 rcv_addr = 0;
3996 size = 0;
3997 }
3998
3999 /*
4000 * Now update the descriptor as the user would see it.
4001 * This may require expanding the descriptor to the user
4002 * visible size. There is already space allocated for
4003 * this in what naddr points to.
4004 */
4005 if(current_task() == kernel_task)
4006 {
4007 mach_msg_ool_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4008 user_ool_dsc--;
4009 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4010
4011 user_ool_dsc->address = (void *)(uintptr_t)rcv_addr;
4012 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4013 TRUE : FALSE;
4014 user_ool_dsc->copy = copy_options;
4015 user_ool_dsc->type = dsc_type;
4016 user_ool_dsc->size = (mach_msg_size_t)size;
4017
4018 user_dsc = (typeof(user_dsc))user_ool_dsc;
4019 } else if (is_64bit) {
4020 mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4021 user_ool_dsc--;
4022 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4023
4024 user_ool_dsc->address = rcv_addr;
4025 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4026 TRUE : FALSE;
4027 user_ool_dsc->copy = copy_options;
4028 user_ool_dsc->type = dsc_type;
4029 user_ool_dsc->size = (mach_msg_size_t)size;
4030
4031 user_dsc = (typeof(user_dsc))user_ool_dsc;
4032 } else {
4033 mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4034 user_ool_dsc--;
4035 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4036
4037 user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr);
4038 user_ool_dsc->size = (mach_msg_size_t)size;
4039 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4040 TRUE : FALSE;
4041 user_ool_dsc->copy = copy_options;
4042 user_ool_dsc->type = dsc_type;
4043
4044 user_dsc = (typeof(user_dsc))user_ool_dsc;
4045 }
4046 return user_dsc;
4047 }
4048
4049 mach_msg_descriptor_t *
4050 ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc,
4051 mach_msg_descriptor_t *user_dsc,
4052 int is_64bit,
4053 vm_map_t map,
4054 ipc_space_t space,
4055 ipc_kmsg_t kmsg,
4056 mach_msg_return_t *mr);
4057 mach_msg_descriptor_t *
4058 ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc,
4059 mach_msg_descriptor_t *user_dsc,
4060 int is_64bit,
4061 vm_map_t map,
4062 ipc_space_t space,
4063 ipc_kmsg_t kmsg,
4064 mach_msg_return_t *mr)
4065 {
4066 mach_vm_offset_t rcv_addr = 0;
4067 mach_msg_type_name_t disp;
4068 mach_msg_type_number_t count, i;
4069 vm_size_t ports_length, names_length;
4070
4071 mach_msg_copy_options_t copy_options = MACH_MSG_VIRTUAL_COPY;
4072
4073 //SKIP_PORT_DESCRIPTORS(saddr, sdsc_count);
4074
4075 count = dsc->count;
4076 disp = dsc->disposition;
4077 ports_length = count * sizeof(mach_port_t);
4078 names_length = count * sizeof(mach_port_name_t);
4079
4080 if (ports_length != 0 && dsc->address != 0) {
4081
4082 /*
4083 * Check to see if there is an overwrite descriptor
4084 * specified in the scatter list for this ool data.
4085 * The descriptor has already been verified.
4086 */
4087 #if 0
4088 if (saddr != MACH_MSG_DESCRIPTOR_NULL) {
4089 if (differs) {
4090 OTHER_OOL_DESCRIPTOR *scatter_dsc;
4091
4092 scatter_dsc = (OTHER_OOL_DESCRIPTOR *)saddr;
4093 rcv_addr = (mach_vm_offset_t) scatter_dsc->address;
4094 copy_options = scatter_dsc->copy;
4095 } else {
4096 mach_msg_ool_descriptor_t *scatter_dsc;
4097
4098 scatter_dsc = &saddr->out_of_line;
4099 rcv_addr = CAST_USER_ADDR_T(scatter_dsc->address);
4100 copy_options = scatter_dsc->copy;
4101 }
4102 INCREMENT_SCATTER(saddr, sdsc_count, differs);
4103 }
4104 #endif
4105
4106 if (copy_options == MACH_MSG_VIRTUAL_COPY) {
4107 /*
4108 * Dynamically allocate the region
4109 */
4110 int anywhere = VM_FLAGS_ANYWHERE;
4111 if (vm_kernel_map_is_kernel(map)) anywhere |= VM_MAKE_TAG(VM_KERN_MEMORY_IPC);
4112 else anywhere |= VM_MAKE_TAG(VM_MEMORY_MACH_MSG);
4113
4114 kern_return_t kr;
4115 if ((kr = mach_vm_allocate(map, &rcv_addr,
4116 (mach_vm_size_t)names_length,
4117 anywhere)) != KERN_SUCCESS) {
4118 ipc_kmsg_clean_body(kmsg, 1, (mach_msg_descriptor_t *)dsc);
4119 rcv_addr = 0;
4120
4121 if (kr == KERN_RESOURCE_SHORTAGE){
4122 *mr |= MACH_MSG_VM_KERNEL;
4123 } else {
4124 *mr |= MACH_MSG_VM_SPACE;
4125 }
4126 }
4127 }
4128
4129 /*
4130 * Handle the port rights and copy out the names
4131 * for those rights out to user-space.
4132 */
4133 if (rcv_addr != 0) {
4134 mach_port_t *objects = (mach_port_t *) dsc->address;
4135 mach_port_name_t *names = (mach_port_name_t *) dsc->address;
4136
4137 /* copyout port rights carried in the message */
4138
4139 for ( i = 0; i < count ; i++) {
4140 ipc_object_t object = (ipc_object_t)objects[i];
4141
4142 *mr |= ipc_kmsg_copyout_object(space, object,
4143 disp, &names[i]);
4144 }
4145
4146 /* copyout to memory allocated above */
4147 void *data = dsc->address;
4148 if (copyoutmap(map, data, rcv_addr, names_length) != KERN_SUCCESS)
4149 *mr |= MACH_MSG_VM_SPACE;
4150 kfree(data, ports_length);
4151 }
4152 } else {
4153 rcv_addr = 0;
4154 }
4155
4156 /*
4157 * Now update the descriptor based on the information
4158 * calculated above.
4159 */
4160 if(current_task() == kernel_task) {
4161 mach_msg_ool_ports_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4162 user_ool_dsc--;
4163 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4164
4165 user_ool_dsc->address = (void *)(uintptr_t)rcv_addr;
4166 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4167 TRUE : FALSE;
4168 user_ool_dsc->copy = copy_options;
4169 user_ool_dsc->disposition = disp;
4170 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
4171 user_ool_dsc->count = count;
4172
4173 user_dsc = (typeof(user_dsc))user_ool_dsc;
4174 } if (is_64bit) {
4175 mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4176 user_ool_dsc--;
4177 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4178
4179 user_ool_dsc->address = rcv_addr;
4180 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4181 TRUE : FALSE;
4182 user_ool_dsc->copy = copy_options;
4183 user_ool_dsc->disposition = disp;
4184 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
4185 user_ool_dsc->count = count;
4186
4187 user_dsc = (typeof(user_dsc))user_ool_dsc;
4188 } else {
4189 mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4190 user_ool_dsc--;
4191 bzero((void *)user_ool_dsc, sizeof(*user_ool_dsc));
4192
4193 user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr);
4194 user_ool_dsc->count = count;
4195 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4196 TRUE : FALSE;
4197 user_ool_dsc->copy = copy_options;
4198 user_ool_dsc->disposition = disp;
4199 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
4200
4201 user_dsc = (typeof(user_dsc))user_ool_dsc;
4202 }
4203 return user_dsc;
4204 }
4205
4206 /*
4207 * Routine: ipc_kmsg_copyout_body
4208 * Purpose:
4209 * "Copy-out" port rights and out-of-line memory
4210 * in the body of a message.
4211 *
4212 * The error codes are a combination of special bits.
4213 * The copyout proceeds despite errors.
4214 * Conditions:
4215 * Nothing locked.
4216 * Returns:
4217 * MACH_MSG_SUCCESS Successful copyout.
4218 * MACH_MSG_IPC_SPACE No room for port right in name space.
4219 * MACH_MSG_VM_SPACE No room for memory in address space.
4220 * MACH_MSG_IPC_KERNEL Resource shortage handling port right.
4221 * MACH_MSG_VM_KERNEL Resource shortage handling memory.
4222 * MACH_MSG_INVALID_RT_DESCRIPTOR Descriptor incompatible with RT
4223 */
4224
4225 mach_msg_return_t
4226 ipc_kmsg_copyout_body(
4227 ipc_kmsg_t kmsg,
4228 ipc_space_t space,
4229 vm_map_t map,
4230 mach_msg_body_t *slist)
4231 {
4232 mach_msg_body_t *body;
4233 mach_msg_descriptor_t *kern_dsc, *user_dsc;
4234 mach_msg_descriptor_t *saddr;
4235 mach_msg_type_number_t dsc_count, sdsc_count;
4236 int i;
4237 mach_msg_return_t mr = MACH_MSG_SUCCESS;
4238 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
4239
4240 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
4241 dsc_count = body->msgh_descriptor_count;
4242 kern_dsc = (mach_msg_descriptor_t *) (body + 1);
4243 /* Point user_dsc just after the end of all the descriptors */
4244 user_dsc = &kern_dsc[dsc_count];
4245
4246 /* Do scatter list setup */
4247 if (slist != MACH_MSG_BODY_NULL) {
4248 panic("Scatter lists disabled");
4249 saddr = (mach_msg_descriptor_t *) (slist + 1);
4250 sdsc_count = slist->msgh_descriptor_count;
4251 }
4252 else {
4253 saddr = MACH_MSG_DESCRIPTOR_NULL;
4254 sdsc_count = 0;
4255 }
4256
4257 /* Now process the descriptors */
4258 for (i = dsc_count-1; i >= 0; i--) {
4259 switch (kern_dsc[i].type.type) {
4260
4261 case MACH_MSG_PORT_DESCRIPTOR:
4262 user_dsc = ipc_kmsg_copyout_port_descriptor(&kern_dsc[i], user_dsc, space, &mr);
4263 break;
4264 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
4265 case MACH_MSG_OOL_DESCRIPTOR :
4266 user_dsc = ipc_kmsg_copyout_ool_descriptor(
4267 (mach_msg_ool_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, map, &mr);
4268 break;
4269 case MACH_MSG_OOL_PORTS_DESCRIPTOR :
4270 user_dsc = ipc_kmsg_copyout_ool_ports_descriptor(
4271 (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, map, space, kmsg, &mr);
4272 break;
4273 default : {
4274 panic("untyped IPC copyout body: invalid message descriptor");
4275 }
4276 }
4277 }
4278
4279 if(user_dsc != kern_dsc) {
4280 vm_offset_t dsc_adjust = (vm_offset_t)user_dsc - (vm_offset_t)kern_dsc;
4281 memmove((char *)((vm_offset_t)kmsg->ikm_header + dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t));
4282 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header + dsc_adjust);
4283 /* Update the message size for the smaller user representation */
4284 kmsg->ikm_header->msgh_size -= (mach_msg_size_t)dsc_adjust;
4285 }
4286
4287 return mr;
4288 }
4289
4290 /*
4291 * Routine: ipc_kmsg_copyout_size
4292 * Purpose:
4293 * Compute the size of the message as copied out to the given
4294 * map. If the destination map's pointers are a different size
4295 * than the kernel's, we have to allow for expansion/
4296 * contraction of the descriptors as appropriate.
4297 * Conditions:
4298 * Nothing locked.
4299 * Returns:
4300 * size of the message as it would be received.
4301 */
4302
4303 mach_msg_size_t
4304 ipc_kmsg_copyout_size(
4305 ipc_kmsg_t kmsg,
4306 vm_map_t map)
4307 {
4308 mach_msg_size_t send_size;
4309
4310 send_size = kmsg->ikm_header->msgh_size;
4311
4312 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
4313
4314 #if defined(__LP64__)
4315 send_size -= LEGACY_HEADER_SIZE_DELTA;
4316 #endif
4317
4318 if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
4319
4320 mach_msg_body_t *body;
4321 mach_msg_descriptor_t *saddr, *eaddr;
4322
4323 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
4324 saddr = (mach_msg_descriptor_t *) (body + 1);
4325 eaddr = saddr + body->msgh_descriptor_count;
4326
4327 for ( ; saddr < eaddr; saddr++ ) {
4328 switch (saddr->type.type) {
4329 case MACH_MSG_OOL_DESCRIPTOR:
4330 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
4331 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
4332 if(!is_task_64bit)
4333 send_size -= DESC_SIZE_ADJUSTMENT;
4334 break;
4335 case MACH_MSG_PORT_DESCRIPTOR:
4336 send_size -= DESC_SIZE_ADJUSTMENT;
4337 break;
4338 default:
4339 break;
4340 }
4341 }
4342 }
4343 return send_size;
4344 }
4345
4346 /*
4347 * Routine: ipc_kmsg_copyout
4348 * Purpose:
4349 * "Copy-out" port rights and out-of-line memory
4350 * in the message.
4351 * Conditions:
4352 * Nothing locked.
4353 * Returns:
4354 * MACH_MSG_SUCCESS Copied out all rights and memory.
4355 * MACH_RCV_HEADER_ERROR + special bits
4356 * Rights and memory in the message are intact.
4357 * MACH_RCV_BODY_ERROR + special bits
4358 * The message header was successfully copied out.
4359 * As much of the body was handled as possible.
4360 */
4361
4362 mach_msg_return_t
4363 ipc_kmsg_copyout(
4364 ipc_kmsg_t kmsg,
4365 ipc_space_t space,
4366 vm_map_t map,
4367 mach_msg_body_t *slist,
4368 mach_msg_option_t option)
4369 {
4370 mach_msg_return_t mr;
4371
4372 mr = ipc_kmsg_copyout_header(kmsg, space, option);
4373 if (mr != MACH_MSG_SUCCESS) {
4374 return mr;
4375 }
4376
4377 if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
4378 mr = ipc_kmsg_copyout_body(kmsg, space, map, slist);
4379
4380 if (mr != MACH_MSG_SUCCESS)
4381 mr |= MACH_RCV_BODY_ERROR;
4382 }
4383
4384 return mr;
4385 }
4386
4387 /*
4388 * Routine: ipc_kmsg_copyout_pseudo
4389 * Purpose:
4390 * Does a pseudo-copyout of the message.
4391 * This is like a regular copyout, except
4392 * that the ports in the header are handled
4393 * as if they are in the body. They aren't reversed.
4394 *
4395 * The error codes are a combination of special bits.
4396 * The copyout proceeds despite errors.
4397 * Conditions:
4398 * Nothing locked.
4399 * Returns:
4400 * MACH_MSG_SUCCESS Successful copyout.
4401 * MACH_MSG_IPC_SPACE No room for port right in name space.
4402 * MACH_MSG_VM_SPACE No room for memory in address space.
4403 * MACH_MSG_IPC_KERNEL Resource shortage handling port right.
4404 * MACH_MSG_VM_KERNEL Resource shortage handling memory.
4405 */
4406
4407 mach_msg_return_t
4408 ipc_kmsg_copyout_pseudo(
4409 ipc_kmsg_t kmsg,
4410 ipc_space_t space,
4411 vm_map_t map,
4412 mach_msg_body_t *slist)
4413 {
4414 mach_msg_bits_t mbits = kmsg->ikm_header->msgh_bits;
4415 ipc_object_t dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4416 ipc_object_t reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
4417 ipc_object_t voucher = (ipc_object_t) kmsg->ikm_voucher;
4418 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
4419 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
4420 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
4421 mach_port_name_t voucher_name = kmsg->ikm_header->msgh_voucher_port;
4422 mach_port_name_t dest_name, reply_name;
4423 mach_msg_return_t mr;
4424
4425 assert(IO_VALID(dest));
4426
4427 #if 0
4428 /*
4429 * If we did this here, it looks like we wouldn't need the undo logic
4430 * at the end of ipc_kmsg_send() in the error cases. Not sure which
4431 * would be more elegant to keep.
4432 */
4433 ipc_importance_clean(kmsg);
4434 #else
4435 /* just assert it is already clean */
4436 ipc_importance_assert_clean(kmsg);
4437 #endif
4438
4439 mr = (ipc_kmsg_copyout_object(space, dest, dest_type, &dest_name) |
4440 ipc_kmsg_copyout_object(space, reply, reply_type, &reply_name));
4441
4442 kmsg->ikm_header->msgh_bits = mbits & MACH_MSGH_BITS_USER;
4443 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(dest_name);
4444 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(reply_name);
4445
4446 if (IO_VALID(voucher)) {
4447 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
4448
4449 kmsg->ikm_voucher = IP_NULL;
4450 mr |= ipc_kmsg_copyout_object(space, voucher, voucher_type, &voucher_name);
4451 kmsg->ikm_header->msgh_voucher_port = voucher_name;
4452 }
4453
4454 if (mbits & MACH_MSGH_BITS_COMPLEX) {
4455 mr |= ipc_kmsg_copyout_body(kmsg, space, map, slist);
4456 }
4457
4458 return mr;
4459 }
4460
4461 /*
4462 * Routine: ipc_kmsg_copyout_dest
4463 * Purpose:
4464 * Copies out the destination port in the message.
4465 * Destroys all other rights and memory in the message.
4466 * Conditions:
4467 * Nothing locked.
4468 */
4469
4470 void
4471 ipc_kmsg_copyout_dest(
4472 ipc_kmsg_t kmsg,
4473 ipc_space_t space)
4474 {
4475 mach_msg_bits_t mbits;
4476 ipc_object_t dest;
4477 ipc_object_t reply;
4478 ipc_object_t voucher;
4479 mach_msg_type_name_t dest_type;
4480 mach_msg_type_name_t reply_type;
4481 mach_msg_type_name_t voucher_type;
4482 mach_port_name_t dest_name, reply_name, voucher_name;
4483
4484 mbits = kmsg->ikm_header->msgh_bits;
4485 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4486 reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
4487 voucher = (ipc_object_t) kmsg->ikm_voucher;
4488 voucher_name = kmsg->ikm_header->msgh_voucher_port;
4489 dest_type = MACH_MSGH_BITS_REMOTE(mbits);
4490 reply_type = MACH_MSGH_BITS_LOCAL(mbits);
4491 voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
4492
4493 assert(IO_VALID(dest));
4494
4495 ipc_importance_assert_clean(kmsg);
4496
4497 io_lock(dest);
4498 if (io_active(dest)) {
4499 ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
4500 /* dest is unlocked */
4501 } else {
4502 io_unlock(dest);
4503 io_release(dest);
4504 dest_name = MACH_PORT_DEAD;
4505 }
4506
4507 if (IO_VALID(reply)) {
4508 ipc_object_destroy(reply, reply_type);
4509 reply_name = MACH_PORT_NULL;
4510 } else
4511 reply_name = CAST_MACH_PORT_TO_NAME(reply);
4512
4513 if (IO_VALID(voucher)) {
4514 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
4515
4516 kmsg->ikm_voucher = IP_NULL;
4517 ipc_object_destroy((ipc_object_t)voucher, voucher_type);
4518 voucher_name = MACH_PORT_NULL;
4519 }
4520
4521 kmsg->ikm_header->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type,
4522 voucher_type, mbits);
4523 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
4524 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
4525 kmsg->ikm_header->msgh_voucher_port = voucher_name;
4526
4527 if (mbits & MACH_MSGH_BITS_COMPLEX) {
4528 mach_msg_body_t *body;
4529
4530 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
4531 ipc_kmsg_clean_body(kmsg, body->msgh_descriptor_count,
4532 (mach_msg_descriptor_t *)(body + 1));
4533 }
4534 }
4535
4536 /*
4537 * Routine: ipc_kmsg_copyout_to_kernel
4538 * Purpose:
4539 * Copies out the destination and reply ports in the message.
4540 * Leaves all other rights and memory in the message alone.
4541 * Conditions:
4542 * Nothing locked.
4543 *
4544 * Derived from ipc_kmsg_copyout_dest.
4545 * Use by mach_msg_rpc_from_kernel (which used to use copyout_dest).
4546 * We really do want to save rights and memory.
4547 */
4548
4549 void
4550 ipc_kmsg_copyout_to_kernel(
4551 ipc_kmsg_t kmsg,
4552 ipc_space_t space)
4553 {
4554 ipc_object_t dest;
4555 ipc_object_t reply;
4556 mach_msg_type_name_t dest_type;
4557 mach_msg_type_name_t reply_type;
4558 mach_port_name_t dest_name, reply_name;
4559
4560 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4561 reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
4562 dest_type = MACH_MSGH_BITS_REMOTE(kmsg->ikm_header->msgh_bits);
4563 reply_type = MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits);
4564
4565 assert(IO_VALID(dest));
4566
4567 io_lock(dest);
4568 if (io_active(dest)) {
4569 ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
4570 /* dest is unlocked */
4571 } else {
4572 io_unlock(dest);
4573 io_release(dest);
4574 dest_name = MACH_PORT_DEAD;
4575 }
4576
4577 reply_name = CAST_MACH_PORT_TO_NAME(reply);
4578
4579 kmsg->ikm_header->msgh_bits =
4580 (MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) |
4581 MACH_MSGH_BITS(reply_type, dest_type));
4582 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
4583 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
4584 }
4585
4586 #if IKM_SUPPORT_LEGACY
4587 void
4588 ipc_kmsg_copyout_to_kernel_legacy(
4589 ipc_kmsg_t kmsg,
4590 ipc_space_t space)
4591 {
4592 ipc_object_t dest;
4593 ipc_object_t reply;
4594 mach_msg_type_name_t dest_type;
4595 mach_msg_type_name_t reply_type;
4596 mach_port_name_t dest_name, reply_name;
4597
4598 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4599 reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
4600 dest_type = MACH_MSGH_BITS_REMOTE(kmsg->ikm_header->msgh_bits);
4601 reply_type = MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits);
4602
4603 assert(IO_VALID(dest));
4604
4605 io_lock(dest);
4606 if (io_active(dest)) {
4607 ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
4608 /* dest is unlocked */
4609 } else {
4610 io_unlock(dest);
4611 io_release(dest);
4612 dest_name = MACH_PORT_DEAD;
4613 }
4614
4615 reply_name = CAST_MACH_PORT_TO_NAME(reply);
4616
4617 kmsg->ikm_header->msgh_bits =
4618 (MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) |
4619 MACH_MSGH_BITS(reply_type, dest_type));
4620 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
4621 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
4622
4623 mach_msg_descriptor_t *saddr;
4624 mach_msg_legacy_descriptor_t *daddr;
4625 mach_msg_type_number_t i, count = ((mach_msg_base_t *)kmsg->ikm_header)->body.msgh_descriptor_count;
4626 saddr = (mach_msg_descriptor_t *) (((mach_msg_base_t *)kmsg->ikm_header) + 1);
4627 saddr = &saddr[count-1];
4628 daddr = (mach_msg_legacy_descriptor_t *)&saddr[count];
4629 daddr--;
4630
4631 vm_offset_t dsc_adjust = 0;
4632
4633 for (i = 0; i < count; i++, saddr--, daddr--) {
4634 switch (saddr->type.type) {
4635 case MACH_MSG_PORT_DESCRIPTOR: {
4636 mach_msg_port_descriptor_t *dsc = &saddr->port;
4637 mach_msg_legacy_port_descriptor_t *dest_dsc = &daddr->port;
4638
4639 mach_port_t name = dsc->name;
4640 mach_msg_type_name_t disposition = dsc->disposition;
4641
4642 dest_dsc->name = CAST_MACH_PORT_TO_NAME(name);
4643 dest_dsc->disposition = disposition;
4644 dest_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
4645 break;
4646 }
4647 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
4648 case MACH_MSG_OOL_DESCRIPTOR: {
4649 /* The sender should supply ready-made memory, i.e. a vm_map_copy_t
4650 * so we don't need to do anything special. */
4651
4652 mach_msg_ool_descriptor_t *source_dsc = (typeof(source_dsc))&saddr->out_of_line;
4653
4654 mach_msg_ool_descriptor32_t *dest_dsc = &daddr->out_of_line32;
4655
4656 vm_offset_t address = (vm_offset_t)source_dsc->address;
4657 vm_size_t size = source_dsc->size;
4658 boolean_t deallocate = source_dsc->deallocate;
4659 mach_msg_copy_options_t copy = source_dsc->copy;
4660 mach_msg_descriptor_type_t type = source_dsc->type;
4661
4662 dest_dsc->address = address;
4663 dest_dsc->size = size;
4664 dest_dsc->deallocate = deallocate;
4665 dest_dsc->copy = copy;
4666 dest_dsc->type = type;
4667 break;
4668 }
4669 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
4670 mach_msg_ool_ports_descriptor_t *source_dsc = (typeof(source_dsc))&saddr->ool_ports;
4671
4672 mach_msg_ool_ports_descriptor32_t *dest_dsc = &daddr->ool_ports32;
4673
4674 vm_offset_t address = (vm_offset_t)source_dsc->address;
4675 vm_size_t port_count = source_dsc->count;
4676 boolean_t deallocate = source_dsc->deallocate;
4677 mach_msg_copy_options_t copy = source_dsc->copy;
4678 mach_msg_descriptor_type_t type = source_dsc->type;
4679
4680 dest_dsc->address = address;
4681 dest_dsc->count = port_count;
4682 dest_dsc->deallocate = deallocate;
4683 dest_dsc->copy = copy;
4684 dest_dsc->type = type;
4685 break;
4686 }
4687 default: {
4688 #if MACH_ASSERT
4689 panic("ipc_kmsg_copyin_from_kernel: bad descriptor");
4690 #endif /* MACH_ASSERT */
4691 }
4692 }
4693 }
4694
4695 if(count) {
4696 dsc_adjust = 4*count;
4697 memmove((char *)((vm_offset_t)kmsg->ikm_header + dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t));
4698 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header + dsc_adjust);
4699 /* Update the message size for the smaller user representation */
4700 kmsg->ikm_header->msgh_size -= dsc_adjust;
4701 }
4702 }
4703 #endif /* IKM_SUPPORT_LEGACY */
4704
4705
4706 mach_msg_trailer_size_t
4707 ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused,
4708 mach_msg_option_t option, thread_t thread,
4709 mach_port_seqno_t seqno, boolean_t minimal_trailer,
4710 mach_vm_offset_t context)
4711 {
4712 mach_msg_max_trailer_t *trailer;
4713
4714 (void)thread;
4715 trailer = (mach_msg_max_trailer_t *)
4716 ((vm_offset_t)kmsg->ikm_header +
4717 round_msg(kmsg->ikm_header->msgh_size));
4718
4719 if (!(option & MACH_RCV_TRAILER_MASK)) {
4720 return trailer->msgh_trailer_size;
4721 }
4722
4723 trailer->msgh_seqno = seqno;
4724 trailer->msgh_context = context;
4725 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(thread_is_64bit(thread), option);
4726
4727 if (minimal_trailer) {
4728 goto done;
4729 }
4730
4731 if (MACH_RCV_TRAILER_ELEMENTS(option) >=
4732 MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_AV)){
4733 trailer->msgh_ad = 0;
4734 }
4735
4736 /*
4737 * The ipc_kmsg_t holds a reference to the label of a label
4738 * handle, not the port. We must get a reference to the port
4739 * and a send right to copyout to the receiver.
4740 */
4741
4742 if (option & MACH_RCV_TRAILER_ELEMENTS (MACH_RCV_TRAILER_LABELS)) {
4743 trailer->msgh_labels.sender = 0;
4744 }
4745
4746 done:
4747
4748 return trailer->msgh_trailer_size;
4749 }
mirror of XNU