]> git.saurik.com Git - apple/xnu.git/blame - osfmk/ipc/ipc_kmsg.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-3789.21.4.tar.gz
[apple/xnu.git] / osfmk / ipc / ipc_kmsg.c
CommitLineData
1c79356b 1/*
2d21ac55 2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
1c79356b 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
1c79356b 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b
A		 27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
2d21ac55
A		 56/*
57 * NOTICE: This file was modified by McAfee Research in 2004 to introduce
58 * support for mandatory and extensible security protections. This notice
59 * is included in support of clause 2.2 (b) of the Apple Public License,
60 * Version 2.0.
61 * Copyright (c) 2005 SPARTA, Inc.
62 */
1c79356b
A		 63/*
64 */
65/*
66 * File: ipc/ipc_kmsg.c
67 * Author: Rich Draves
68 * Date: 1989
69 *
70 * Operations on kernel messages.
71 */
72
1c79356b 73
91447636 74#include <mach/mach_types.h>
1c79356b
A		 75#include <mach/boolean.h>
76#include <mach/kern_return.h>
77#include <mach/message.h>
78#include <mach/port.h>
91447636
A		 79#include <mach/vm_map.h>
80#include <mach/mach_vm.h>
1c79356b 81#include <mach/vm_statistics.h>
91447636
A		 82
83#include <kern/kern_types.h>
1c79356b 84#include <kern/assert.h>
b0d623f7 85#include <kern/debug.h>
91447636 86#include <kern/ipc_kobject.h>
1c79356b 87#include <kern/kalloc.h>
91447636
A		 88#include <kern/zalloc.h>
89#include <kern/processor.h>
1c79356b
A		 90#include <kern/thread.h>
91#include <kern/sched_prim.h>
1c79356b
A		 92#include <kern/misc_protos.h>
93#include <kern/counters.h>
91447636 94#include <kern/cpu_data.h>
39037602 95#include <kern/policy_internal.h>
91447636 96
b0d623f7
A		 97#include <machine/machlimits.h>
98
1c79356b
A		 99#include <vm/vm_map.h>
100#include <vm/vm_object.h>
101#include <vm/vm_kern.h>
91447636 102
1c79356b 103#include <ipc/port.h>
91447636 104#include <ipc/ipc_types.h>
1c79356b
A		 105#include <ipc/ipc_entry.h>
106#include <ipc/ipc_kmsg.h>
107#include <ipc/ipc_notify.h>
108#include <ipc/ipc_object.h>
109#include <ipc/ipc_space.h>
110#include <ipc/ipc_port.h>
111#include <ipc/ipc_right.h>
112#include <ipc/ipc_hash.h>
113#include <ipc/ipc_table.h>
fe8ab488 114#include <ipc/ipc_importance.h>
39037602
A		 115#if MACH_FLIPC
116#include <kern/mach_node.h>
117#include <ipc/flipc.h>
118#endif
1c79356b 119
2d21ac55
A		 120#include <security/mac_mach_internal.h>
121
39037602
A		 122#include <device/device_server.h>
123
1c79356b
A		 124#include <string.h>
125
55e303ae
A		 126#ifdef ppc
127#include <ppc/Firmware.h>
128#include <ppc/low_trace.h>
129#endif
130
b0d623f7
A		 131#if DEBUG
132#define DEBUG_MSGS_K64 1
133#endif
134
39236c6e
A		 135#include <sys/kdebug.h>
136#include <libkern/OSAtomic.h>
137
b0d623f7
A		 138#pragma pack(4)
139
140typedef struct
141{
142 mach_msg_bits_t msgh_bits;
143 mach_msg_size_t msgh_size;
fe8ab488
A		 144 mach_port_name_t msgh_remote_port;
145 mach_port_name_t msgh_local_port;
146 mach_port_name_t msgh_voucher_port;
b0d623f7
A		 147 mach_msg_id_t msgh_id;
148} mach_msg_legacy_header_t;
149
150typedef struct
151{
152 mach_msg_legacy_header_t header;
153 mach_msg_body_t body;
154} mach_msg_legacy_base_t;
155
156typedef struct
157{
158 mach_port_name_t name;
159 mach_msg_size_t pad1;
160 uint32_t pad2 : 16;
161 mach_msg_type_name_t disposition : 8;
162 mach_msg_descriptor_type_t type : 8;
163} mach_msg_legacy_port_descriptor_t;
164
165
166typedef union
167{
168 mach_msg_legacy_port_descriptor_t port;
169 mach_msg_ool_descriptor32_t out_of_line32;
170 mach_msg_ool_ports_descriptor32_t ool_ports32;
171 mach_msg_type_descriptor_t type;
172} mach_msg_legacy_descriptor_t;
173
174#pragma pack()
175
176#define LEGACY_HEADER_SIZE_DELTA ((mach_msg_size_t)(sizeof(mach_msg_header_t) - sizeof(mach_msg_legacy_header_t)))
fe8ab488 177
b0d623f7
A		 178// END LP64 fixes
179
180
181#if DEBUG_MSGS_K64
182extern void ipc_pset_print64(
183 ipc_pset_t pset);
184
185extern void ipc_kmsg_print64(
186 ipc_kmsg_t kmsg,
187 const char *str);
188
189extern void ipc_msg_print64(
190 mach_msg_header_t *msgh);
191
192extern ipc_port_t ipc_name_to_data64(
193 task_t task,
194 mach_port_name_t name);
195
196/*
197 * Forward declarations
198 */
199void ipc_msg_print_untyped64(
200 mach_msg_body_t *body);
201
202const char * ipc_type_name64(
203 int type_name,
204 boolean_t received);
205
206void ipc_print_type_name64(
207 int type_name);
208
209const char *
210msgh_bit_decode64(
211 mach_msg_bits_t bit);
212
213const char *
214mm_copy_options_string64(
215 mach_msg_copy_options_t option);
216
217void db_print_msg_uid64(mach_msg_header_t *);
218
219static void
220ipc_msg_body_print64(void *body, int size)
221{
222 uint32_t *word = (uint32_t *) body;
223 uint32_t *end = (uint32_t *)(((uintptr_t) body) + size
224 - sizeof(mach_msg_header_t));
225 int i;
226
227 kprintf(" body(%p-%p):\n %p: ", body, end, word);
228 for (;;) {
229 for (i = 0; i < 8; i++, word++) {
230 if (word >= end) {
231 kprintf("\n");
232 return;
233 }
234 kprintf("%08x ", *word);
235 }
236 kprintf("\n %p: ", word);
237 }
238}
239
240
241const char *
242ipc_type_name64(
243 int type_name,
244 boolean_t received)
245{
246 switch (type_name) {
247 case MACH_MSG_TYPE_PORT_NAME:
248 return "port_name";
249
250 case MACH_MSG_TYPE_MOVE_RECEIVE:
251 if (received) {
252 return "port_receive";
253 } else {
254 return "move_receive";
255 }
256
257 case MACH_MSG_TYPE_MOVE_SEND:
258 if (received) {
259 return "port_send";
260 } else {
261 return "move_send";
262 }
263
264 case MACH_MSG_TYPE_MOVE_SEND_ONCE:
265 if (received) {
266 return "port_send_once";
267 } else {
268 return "move_send_once";
269 }
270
271 case MACH_MSG_TYPE_COPY_SEND:
272 return "copy_send";
273
274 case MACH_MSG_TYPE_MAKE_SEND:
275 return "make_send";
276
277 case MACH_MSG_TYPE_MAKE_SEND_ONCE:
278 return "make_send_once";
279
280 default:
281 return (char *) 0;
282 }
283}
284
285void
286ipc_print_type_name64(
287 int type_name)
288{
289 const char *name = ipc_type_name64(type_name, TRUE);
290 if (name) {
291 kprintf("%s", name);
292 } else {
293 kprintf("type%d", type_name);
294 }
295}
296
297/*
298 * ipc_kmsg_print64 [ debug ]
299 */
300void
301ipc_kmsg_print64(
302 ipc_kmsg_t kmsg,
303 const char *str)
304{
305 kprintf("%s kmsg=%p:\n", str, kmsg);
306 kprintf(" next=%p, prev=%p, size=%d",
307 kmsg->ikm_next,
308 kmsg->ikm_prev,
309 kmsg->ikm_size);
310 kprintf("\n");
311 ipc_msg_print64(kmsg->ikm_header);
312}
313
314const char *
315msgh_bit_decode64(
316 mach_msg_bits_t bit)
317{
318 switch (bit) {
319 case MACH_MSGH_BITS_COMPLEX: return "complex";
320 case MACH_MSGH_BITS_CIRCULAR: return "circular";
321 default: return (char *) 0;
322 }
323}
324
325/*
326 * ipc_msg_print64 [ debug ]
327 */
328void
329ipc_msg_print64(
330 mach_msg_header_t *msgh)
331{
332 mach_msg_bits_t mbits;
333 unsigned int bit, i;
334 const char *bit_name;
335 int needs_comma;
336
337 mbits = msgh->msgh_bits;
338 kprintf(" msgh_bits=0x%x: l=0x%x,r=0x%x\n",
339 mbits,
340 MACH_MSGH_BITS_LOCAL(msgh->msgh_bits),
341 MACH_MSGH_BITS_REMOTE(msgh->msgh_bits));
342
343 mbits = MACH_MSGH_BITS_OTHER(mbits) & MACH_MSGH_BITS_USED;
344 kprintf(" decoded bits: ");
345 needs_comma = 0;
346 for (i = 0, bit = 1; i < sizeof(mbits) * 8; ++i, bit <<= 1) {
347 if ((mbits & bit) == 0)
348 continue;
349 bit_name = msgh_bit_decode64((mach_msg_bits_t)bit);
350 if (bit_name)
351 kprintf("%s%s", needs_comma ? "," : "", bit_name);
352 else
353 kprintf("%sunknown(0x%x),", needs_comma ? "," : "", bit);
354 ++needs_comma;
355 }
356 if (msgh->msgh_bits & ~MACH_MSGH_BITS_USED) {
357 kprintf("%sunused=0x%x,", needs_comma ? "," : "",
358 msgh->msgh_bits & ~MACH_MSGH_BITS_USED);
359 }
360 kprintf("\n");
361
362 needs_comma = 1;
363 if (msgh->msgh_remote_port) {
364 kprintf(" remote=%p(", msgh->msgh_remote_port);
365 ipc_print_type_name64(MACH_MSGH_BITS_REMOTE(msgh->msgh_bits));
366 kprintf(")");
367 } else {
368 kprintf(" remote=null");
369 }
370
371 if (msgh->msgh_local_port) {
372 kprintf("%slocal=%p(", needs_comma ? "," : "",
373 msgh->msgh_local_port);
374 ipc_print_type_name64(MACH_MSGH_BITS_LOCAL(msgh->msgh_bits));
375 kprintf(")\n");
376 } else {
377 kprintf("local=null\n");
378 }
379
380 kprintf(" msgh_id=%d, size=%d\n",
381 msgh->msgh_id,
382 msgh->msgh_size);
383
384 if (mbits & MACH_MSGH_BITS_COMPLEX) {
385 ipc_msg_print_untyped64((mach_msg_body_t *) (msgh + 1));
386 }
387
388 ipc_msg_body_print64((void *)(msgh + 1), msgh->msgh_size);
389}
390
391
392const char *
393mm_copy_options_string64(
394 mach_msg_copy_options_t option)
395{
396 const char *name;
397
398 switch (option) {
399 case MACH_MSG_PHYSICAL_COPY:
400 name = "PHYSICAL";
401 break;
402 case MACH_MSG_VIRTUAL_COPY:
403 name = "VIRTUAL";
404 break;
405 case MACH_MSG_OVERWRITE:
3e170ce0 406 name = "OVERWRITE(DEPRECATED)";
b0d623f7
A		 407 break;
408 case MACH_MSG_ALLOCATE:
409 name = "ALLOCATE";
410 break;
411 case MACH_MSG_KALLOC_COPY_T:
412 name = "KALLOC_COPY_T";
413 break;
414 default:
415 name = "unknown";
416 break;
417 }
418 return name;
419}
420
421void
422ipc_msg_print_untyped64(
423 mach_msg_body_t *body)
424{
425 mach_msg_descriptor_t *saddr, *send;
426 mach_msg_descriptor_type_t type;
427
428 kprintf(" %d descriptors: \n", body->msgh_descriptor_count);
429
430 saddr = (mach_msg_descriptor_t *) (body + 1);
431 send = saddr + body->msgh_descriptor_count;
432
433 for ( ; saddr < send; saddr++ ) {
434
435 type = saddr->type.type;
436
437 switch (type) {
438
439 case MACH_MSG_PORT_DESCRIPTOR: {
440 mach_msg_port_descriptor_t *dsc;
441
442 dsc = &saddr->port;
443 kprintf(" PORT name = %p disp = ", dsc->name);
444 ipc_print_type_name64(dsc->disposition);
445 kprintf("\n");
446 break;
447 }
448 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
449 case MACH_MSG_OOL_DESCRIPTOR: {
450 mach_msg_ool_descriptor_t *dsc;
451
452 dsc = (mach_msg_ool_descriptor_t *) &saddr->out_of_line;
453 kprintf(" OOL%s addr = %p size = 0x%x copy = %s %s\n",
454 type == MACH_MSG_OOL_DESCRIPTOR ? "" : " VOLATILE",
455 dsc->address, dsc->size,
456 mm_copy_options_string64(dsc->copy),
457 dsc->deallocate ? "DEALLOC" : "");
458 break;
459 }
460 case MACH_MSG_OOL_PORTS_DESCRIPTOR : {
461 mach_msg_ool_ports_descriptor_t *dsc;
462
463 dsc = (mach_msg_ool_ports_descriptor_t *) &saddr->ool_ports;
464
465 kprintf(" OOL_PORTS addr = %p count = 0x%x ",
466 dsc->address, dsc->count);
467 kprintf("disp = ");
468 ipc_print_type_name64(dsc->disposition);
469 kprintf(" copy = %s %s\n",
470 mm_copy_options_string64(dsc->copy),
471 dsc->deallocate ? "DEALLOC" : "");
472 break;
473 }
474
475 default: {
476 kprintf(" UNKNOWN DESCRIPTOR 0x%x\n", type);
477 break;
478 }
479 }
480 }
481}
482
483#define DEBUG_IPC_KMSG_PRINT(kmsg,string) \
fe8ab488 484 __unreachable_ok_push \
b0d623f7
A		 485 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \
486 ipc_kmsg_print64(kmsg, string); \
fe8ab488
A		 487 } \
488 __unreachable_ok_pop
489
b0d623f7 490#define DEBUG_IPC_MSG_BODY_PRINT(body,size) \
fe8ab488 491 __unreachable_ok_push \
b0d623f7
A		 492 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \
493 ipc_msg_body_print64(body,size);\
fe8ab488
A		 494 } \
495 __unreachable_ok_pop
b0d623f7
A		 496#else /* !DEBUG_MSGS_K64 */
497#define DEBUG_IPC_KMSG_PRINT(kmsg,string)
498#define DEBUG_IPC_MSG_BODY_PRINT(body,size)
499#endif /* !DEBUG_MSGS_K64 */
55e303ae 500
1c79356b 501extern vm_map_t ipc_kernel_copy_map;
316670eb 502extern vm_size_t ipc_kmsg_max_space;
1c79356b 503extern vm_size_t ipc_kmsg_max_vm_space;
316670eb 504extern vm_size_t ipc_kmsg_max_body_space;
1c79356b
A		 505extern vm_size_t msg_ool_size_small;
506
507#define MSG_OOL_SIZE_SMALL msg_ool_size_small
508
91447636
A		 509#if defined(__LP64__)
510#define MAP_SIZE_DIFFERS(map) (map->max_offset < MACH_VM_MAX_ADDRESS)
511#define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor32_t
512#define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor32_t
513#else
514#define MAP_SIZE_DIFFERS(map) (map->max_offset > VM_MAX_ADDRESS)
515#define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor64_t
516#define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor64_t
517#endif
518
b0d623f7
A		 519#define DESC_SIZE_ADJUSTMENT ((mach_msg_size_t)(sizeof(mach_msg_ool_descriptor64_t) - \
520 sizeof(mach_msg_ool_descriptor32_t)))
91447636
A		 521
522/* scatter list macros */
523
524#define SKIP_PORT_DESCRIPTORS(s, c) \
525MACRO_BEGIN \
526 if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \
527 while ((c) > 0) { \
528 if ((s)->type.type != MACH_MSG_PORT_DESCRIPTOR) \
529 break; \
530 (s)++; (c)--; \
531 } \
532 if (c == 0) \
533 (s) = MACH_MSG_DESCRIPTOR_NULL; \
534 } \
535MACRO_END
536
537#define INCREMENT_SCATTER(s, c, d) \
538MACRO_BEGIN \
539 if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \
540 s = (d) ? (mach_msg_descriptor_t *) \
541 ((OTHER_OOL_DESCRIPTOR *)(s) + 1) : \
542 (s + 1); \
543 (c)--; \
544 } \
545MACRO_END
546
39037602
A		 547#define KMSG_TRACE_FLAG_TRACED 0x000001
548#define KMSG_TRACE_FLAG_COMPLEX 0x000002
549#define KMSG_TRACE_FLAG_OOLMEM 0x000004
550#define KMSG_TRACE_FLAG_VCPY 0x000008
551#define KMSG_TRACE_FLAG_PCPY 0x000010
552#define KMSG_TRACE_FLAG_SND64 0x000020
553#define KMSG_TRACE_FLAG_RAISEIMP 0x000040
554#define KMSG_TRACE_FLAG_APP_SRC 0x000080
555#define KMSG_TRACE_FLAG_APP_DST 0x000100
556#define KMSG_TRACE_FLAG_DAEMON_SRC 0x000200
557#define KMSG_TRACE_FLAG_DAEMON_DST 0x000400
558#define KMSG_TRACE_FLAG_DST_NDFLTQ 0x000800
559#define KMSG_TRACE_FLAG_SRC_NDFLTQ 0x001000
560#define KMSG_TRACE_FLAG_DST_SONCE 0x002000
561#define KMSG_TRACE_FLAG_SRC_SONCE 0x004000
562#define KMSG_TRACE_FLAG_CHECKIN 0x008000
563#define KMSG_TRACE_FLAG_ONEWAY 0x010000
564#define KMSG_TRACE_FLAG_IOKIT 0x020000
565#define KMSG_TRACE_FLAG_SNDRCV 0x040000
566#define KMSG_TRACE_FLAG_DSTQFULL 0x080000
567#define KMSG_TRACE_FLAG_VOUCHER 0x100000
568#define KMSG_TRACE_FLAG_TIMER 0x200000
569#define KMSG_TRACE_FLAG_SEMA 0x400000
570#define KMSG_TRACE_FLAG_DTMPOWNER 0x800000
571
572#define KMSG_TRACE_FLAGS_MASK 0xffffff
573#define KMSG_TRACE_FLAGS_SHIFT 8
574
575#define KMSG_TRACE_PORTS_MASK 0xff
576#define KMSG_TRACE_PORTS_SHIFT 0
577
578#if (KDEBUG_LEVEL >= KDEBUG_LEVEL_STANDARD)
579extern boolean_t kdebug_debugid_enabled(uint32_t debugid);
580
581void ipc_kmsg_trace_send(ipc_kmsg_t kmsg,
582 mach_msg_option_t option)
583{
584 task_t send_task = TASK_NULL;
585 ipc_port_t dst_port, src_port;
586 boolean_t is_task_64bit;
587 mach_msg_header_t *msg;
588 mach_msg_trailer_t *trailer;
589
590 int kotype = 0;
591 uint32_t msg_size = 0;
592 uint32_t msg_flags = KMSG_TRACE_FLAG_TRACED;
593 uint32_t num_ports = 0;
594 uint32_t send_pid, dst_pid;
595
596 /*
597 * check to see not only if ktracing is enabled, but if we will
598 * _actually_ emit the KMSG_INFO tracepoint. This saves us a
599 * significant amount of processing (and a port lock hold) in
600 * the non-tracing case.
601 */
602 if (__probable((kdebug_enable & KDEBUG_TRACE) == 0))
603 return;
604 if (!kdebug_debugid_enabled(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO)))
605 return;
606
607 msg = kmsg->ikm_header;
608
609 dst_port = (ipc_port_t)(msg->msgh_remote_port);
610 if (!IPC_PORT_VALID(dst_port))
611 return;
612
613 /*
614 * Message properties / options
615 */
616 if ((option & (MACH_SEND_MSG|MACH_RCV_MSG)) == (MACH_SEND_MSG|MACH_RCV_MSG))
617 msg_flags |= KMSG_TRACE_FLAG_SNDRCV;
618
619 if (msg->msgh_id >= is_iokit_subsystem.start &&
620 msg->msgh_id < is_iokit_subsystem.end + 100)
621 msg_flags |= KMSG_TRACE_FLAG_IOKIT;
622 /* magic XPC checkin message id (XPC_MESSAGE_ID_CHECKIN) from libxpc */
623 else if (msg->msgh_id == 0x77303074u /* w00t */)
624 msg_flags |= KMSG_TRACE_FLAG_CHECKIN;
625
626 if (msg->msgh_bits & MACH_MSGH_BITS_RAISEIMP)
627 msg_flags |= KMSG_TRACE_FLAG_RAISEIMP;
628
629 if (unsafe_convert_port_to_voucher(kmsg->ikm_voucher))
630 msg_flags |= KMSG_TRACE_FLAG_VOUCHER;
631
632 /*
633 * Sending task / port
634 */
635 send_task = current_task();
636 send_pid = task_pid(send_task);
637
638 if (send_pid != 0) {
639 if (task_is_daemon(send_task))
640 msg_flags |= KMSG_TRACE_FLAG_DAEMON_SRC;
641 else if (task_is_app(send_task))
642 msg_flags |= KMSG_TRACE_FLAG_APP_SRC;
643 }
644
645 is_task_64bit = (send_task->map->max_offset > VM_MAX_ADDRESS);
646 if (is_task_64bit)
647 msg_flags |= KMSG_TRACE_FLAG_SND64;
648
649 src_port = (ipc_port_t)(msg->msgh_local_port);
650 if (src_port) {
651 if (src_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT)
652 msg_flags |= KMSG_TRACE_FLAG_SRC_NDFLTQ;
653 switch (MACH_MSGH_BITS_LOCAL(msg->msgh_bits)) {
654 case MACH_MSG_TYPE_MOVE_SEND_ONCE:
655 msg_flags |= KMSG_TRACE_FLAG_SRC_SONCE;
656 break;
657 default:
658 break;
659 }
660 } else {
661 msg_flags |= KMSG_TRACE_FLAG_ONEWAY;
662 }
663
664
665 /*
666 * Destination task / port
667 */
668 ip_lock(dst_port);
669 if (!ip_active(dst_port)) {
670 /* dst port is being torn down */
671 dst_pid = (uint32_t)0xfffffff0;
672 } else if (dst_port->ip_tempowner) {
673 msg_flags |= KMSG_TRACE_FLAG_DTMPOWNER;
674 if (IIT_NULL != dst_port->ip_imp_task)
675 dst_pid = task_pid(dst_port->ip_imp_task->iit_task);
676 else
677 dst_pid = (uint32_t)0xfffffff1;
678 } else if (dst_port->ip_receiver_name == MACH_PORT_NULL) {
679 /* dst_port is otherwise in-transit */
680 dst_pid = (uint32_t)0xfffffff2;
681 } else {
682 if (dst_port->ip_receiver == ipc_space_kernel) {
683 dst_pid = 0;
684 } else {
685 ipc_space_t dst_space;
686 dst_space = dst_port->ip_receiver;
687 if (dst_space && is_active(dst_space)) {
688 dst_pid = task_pid(dst_space->is_task);
689 if (task_is_daemon(dst_space->is_task))
690 msg_flags |= KMSG_TRACE_FLAG_DAEMON_DST;
691 else if (task_is_app(dst_space->is_task))
692 msg_flags |= KMSG_TRACE_FLAG_APP_DST;
693 } else {
694 /* receiving task is being torn down */
695 dst_pid = (uint32_t)0xfffffff3;
696 }
697 }
698 }
699
700 if (dst_port->ip_messages.imq_qlimit != MACH_PORT_QLIMIT_DEFAULT)
701 msg_flags |= KMSG_TRACE_FLAG_DST_NDFLTQ;
702 if (imq_full(&dst_port->ip_messages))
703 msg_flags |= KMSG_TRACE_FLAG_DSTQFULL;
704
705 kotype = ip_kotype(dst_port);
706
707 ip_unlock(dst_port);
708
709 switch (kotype) {
710 case IKOT_SEMAPHORE:
711 msg_flags |= KMSG_TRACE_FLAG_SEMA;
712 break;
713 case IKOT_TIMER:
714 case IKOT_CLOCK:
715 msg_flags |= KMSG_TRACE_FLAG_TIMER;
716 break;
717 case IKOT_MASTER_DEVICE:
718 case IKOT_IOKIT_CONNECT:
719 case IKOT_IOKIT_OBJECT:
720 case IKOT_IOKIT_SPARE:
721 msg_flags |= KMSG_TRACE_FLAG_IOKIT;
722 break;
723 default:
724 break;
725 }
726
727 switch(MACH_MSGH_BITS_REMOTE(msg->msgh_bits)) {
728 case MACH_MSG_TYPE_PORT_SEND_ONCE:
729 msg_flags |= KMSG_TRACE_FLAG_DST_SONCE;
730 break;
731 default:
732 break;
733 }
734
735
736 /*
737 * Message size / content
738 */
739 msg_size = msg->msgh_size - sizeof(mach_msg_header_t);
740
741 if (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
742 mach_msg_body_t *msg_body;
743 mach_msg_descriptor_t *kern_dsc;
744 int dsc_count;
745
746 msg_flags |= KMSG_TRACE_FLAG_COMPLEX;
747
748 msg_body = (mach_msg_body_t *)(kmsg->ikm_header + 1);
749 dsc_count = (int)msg_body->msgh_descriptor_count;
750 kern_dsc = (mach_msg_descriptor_t *)(msg_body + 1);
751
752 /* this is gross: see ipc_kmsg_copyin_body()... */
753 if (!is_task_64bit)
754 msg_size -= (dsc_count * 12);
755
756 for (int i = 0; i < dsc_count; i++) {
757 switch (kern_dsc[i].type.type) {
758 case MACH_MSG_PORT_DESCRIPTOR:
759 num_ports++;
760 if (is_task_64bit)
761 msg_size -= 12;
762 break;
763 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
764 case MACH_MSG_OOL_DESCRIPTOR: {
765 mach_msg_ool_descriptor_t *dsc;
766 dsc = (mach_msg_ool_descriptor_t *)&kern_dsc[i];
767 msg_flags |= KMSG_TRACE_FLAG_OOLMEM;
768 msg_size += dsc->size;
769 if ((dsc->size >= MSG_OOL_SIZE_SMALL) &&
770 (dsc->copy == MACH_MSG_PHYSICAL_COPY) &&
771 !dsc->deallocate)
772 msg_flags |= KMSG_TRACE_FLAG_PCPY;
773 else if (dsc->size <= MSG_OOL_SIZE_SMALL)
774 msg_flags |= KMSG_TRACE_FLAG_PCPY;
775 else
776 msg_flags |= KMSG_TRACE_FLAG_VCPY;
777 if (is_task_64bit)
778 msg_size -= 16;
779 } break;
780 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
781 mach_msg_ool_ports_descriptor_t *dsc;
782 dsc = (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i];
783 num_ports += dsc->count;
784 if (is_task_64bit)
785 msg_size -= 16;
786 } break;
787 default:
788 break;
789 }
790 }
791 }
792
793 /*
794 * Trailer contents
795 */
796 trailer = (mach_msg_trailer_t *)((vm_offset_t)msg +
797 (vm_offset_t)msg->msgh_size);
798 if (trailer->msgh_trailer_size <= sizeof(mach_msg_security_trailer_t)) {
799 extern security_token_t KERNEL_SECURITY_TOKEN;
800 mach_msg_security_trailer_t *strailer;
801 strailer = (mach_msg_security_trailer_t *)trailer;
802 /*
803 * verify the sender PID: replies from the kernel often look
804 * like self-talk because the sending port is not reset.
805 */
806 if (memcmp(&strailer->msgh_sender,
807 &KERNEL_SECURITY_TOKEN,
808 sizeof(KERNEL_SECURITY_TOKEN)) == 0) {
809 send_pid = 0;
810 msg_flags &= ~(KMSG_TRACE_FLAG_APP_SRC | KMSG_TRACE_FLAG_DAEMON_SRC);
811 }
812 }
813
814 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END,
815 (uintptr_t)send_pid,
816 (uintptr_t)dst_pid,
817 (uintptr_t)msg_size,
818 (uintptr_t)(
819 ((msg_flags & KMSG_TRACE_FLAGS_MASK) << KMSG_TRACE_FLAGS_SHIFT) |
820 ((num_ports & KMSG_TRACE_PORTS_MASK) << KMSG_TRACE_PORTS_SHIFT)
821 )
822 );
823}
824#endif
825
91447636
A		 826/* zone for cached ipc_kmsg_t structures */
827zone_t ipc_kmsg_zone;
1c79356b
A		 828
829/*
830 * Forward declarations
831 */
832
833void ipc_kmsg_clean(
834 ipc_kmsg_t kmsg);
835
836void ipc_kmsg_clean_body(
837 ipc_kmsg_t kmsg,
91447636
A		 838 mach_msg_type_number_t number,
839 mach_msg_descriptor_t *desc);
1c79356b
A		 840
841void ipc_kmsg_clean_partial(
842 ipc_kmsg_t kmsg,
843 mach_msg_type_number_t number,
91447636 844 mach_msg_descriptor_t *desc,
1c79356b
A		 845 vm_offset_t paddr,
846 vm_size_t length);
847
1c79356b
A		 848mach_msg_return_t ipc_kmsg_copyin_body(
849 ipc_kmsg_t kmsg,
850 ipc_space_t space,
851 vm_map_t map);
852
1c79356b
A		 853/*
854 * We keep a per-processor cache of kernel message buffers.
855 * The cache saves the overhead/locking of using kalloc/kfree.
856 * The per-processor cache seems to miss less than a per-thread cache,
857 * and it also uses less memory. Access to the cache doesn't
858 * require locking.
859 */
1c79356b
A		 860
861/*
862 * Routine: ipc_kmsg_alloc
863 * Purpose:
864 * Allocate a kernel message structure. If we can get one from
865 * the cache, that is best. Otherwise, allocate a new one.
866 * Conditions:
867 * Nothing locked.
868 */
869ipc_kmsg_t
870ipc_kmsg_alloc(
871 mach_msg_size_t msg_and_trailer_size)
872{
91447636 873 mach_msg_size_t max_expanded_size;
1c79356b
A		 874 ipc_kmsg_t kmsg;
875
91447636
A		 876 /*
877 * LP64support -
878 * Pad the allocation in case we need to expand the
879 * message descrptors for user spaces with pointers larger than
b0d623f7 880 * the kernel's own, or vice versa. We don't know how many descriptors
91447636
A		 881 * there are yet, so just assume the whole body could be
882 * descriptors (if there could be any at all).
883 *
884 * The expansion space is left in front of the header,
885 * because it is easier to pull the header and descriptors
886 * forward as we process them than it is to push all the
887 * data backwards.
888 */
8ad349bb 889 mach_msg_size_t size = msg_and_trailer_size - MAX_TRAILER_SIZE;
316670eb
A		 890
891 /* compare against implementation upper limit for the body */
892 if (size > ipc_kmsg_max_body_space)
893 return IKM_NULL;
894
8ad349bb 895 if (size > sizeof(mach_msg_base_t)) {
b0d623f7
A		 896 mach_msg_size_t max_desc = (mach_msg_size_t)(((size - sizeof(mach_msg_base_t)) /
897 sizeof(mach_msg_ool_descriptor32_t)) *
898 DESC_SIZE_ADJUSTMENT);
316670eb
A		 899
900 /* make sure expansion won't cause wrap */
b7266188 901 if (msg_and_trailer_size > MACH_MSG_SIZE_MAX - max_desc)
8ad349bb 902 return IKM_NULL;
b0d623f7 903
8ad349bb
A		 904 max_expanded_size = msg_and_trailer_size + max_desc;
905 } else
fe8ab488 906 max_expanded_size = msg_and_trailer_size;
91447636 907
316670eb 908 if (max_expanded_size < IKM_SAVED_MSG_SIZE)
8ad349bb 909 max_expanded_size = IKM_SAVED_MSG_SIZE; /* round up for ikm_cache */
91447636
A		 910
911 if (max_expanded_size == IKM_SAVED_MSG_SIZE) {
912 struct ikm_cache *cache;
913 unsigned int i;
1c79356b
A		 914
915 disable_preemption();
91447636
A		 916 cache = &PROCESSOR_DATA(current_processor(), ikm_cache);
917 if ((i = cache->avail) > 0) {
1c79356b 918 assert(i <= IKM_STASH);
91447636
A		 919 kmsg = cache->entries[--i];
920 cache->avail = i;
1c79356b 921 enable_preemption();
b7266188
A		 922 ikm_check_init(kmsg, max_expanded_size);
923 ikm_set_header(kmsg, msg_and_trailer_size);
1c79356b
A		 924 return (kmsg);
925 }
926 enable_preemption();
91447636
A		 927 kmsg = (ipc_kmsg_t)zalloc(ipc_kmsg_zone);
928 } else {
929 kmsg = (ipc_kmsg_t)kalloc(ikm_plus_overhead(max_expanded_size));
1c79356b
A		 930 }
931
1c79356b 932 if (kmsg != IKM_NULL) {
91447636 933 ikm_init(kmsg, max_expanded_size);
b7266188 934 ikm_set_header(kmsg, msg_and_trailer_size);
1c79356b 935 }
2d21ac55 936
1c79356b
A		 937 return(kmsg);
938}
939
940/*
941 * Routine: ipc_kmsg_free
942 * Purpose:
943 * Free a kernel message buffer. If the kms is preallocated
944 * to a port, just "put it back (marked unused)." We have to
945 * do this with the port locked. The port may have its hold
946 * on our message released. In that case, we have to just
947 * revert the message to a traditional one and free it normally.
948 * Conditions:
949 * Nothing locked.
950 */
951
952void
953ipc_kmsg_free(
954 ipc_kmsg_t kmsg)
955{
956 mach_msg_size_t size = kmsg->ikm_size;
957 ipc_port_t port;
958
fe8ab488
A		 959 assert(!IP_VALID(kmsg->ikm_voucher));
960
961 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_FREE) | DBG_FUNC_NONE,
962 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
963 0, 0, 0, 0);
2d21ac55 964
1c79356b
A		 965 /*
966 * Check to see if the message is bound to the port. If so,
967 * mark it not in use. If the port isn't already dead, then
91447636 968 * leave the message associated with it. Otherwise, free it.
1c79356b
A		 969 */
970 port = ikm_prealloc_inuse_port(kmsg);
971 if (port != IP_NULL) {
972 ip_lock(port);
973 ikm_prealloc_clear_inuse(kmsg, port);
974 if (ip_active(port) && (port->ip_premsg == kmsg)) {
975 assert(IP_PREALLOC(port));
976 ip_unlock(port);
316670eb 977 ip_release(port);
1c79356b
A		 978 return;
979 }
316670eb
A		 980 ip_unlock(port);
981 ip_release(port); /* May be last reference */
1c79356b
A		 982 }
983
984 /*
985 * Peek and see if it has to go back in the cache.
986 */
91447636
A		 987 if (kmsg->ikm_size == IKM_SAVED_MSG_SIZE) {
988 struct ikm_cache *cache;
989 unsigned int i;
1c79356b
A		 990
991 disable_preemption();
91447636
A		 992 cache = &PROCESSOR_DATA(current_processor(), ikm_cache);
993 if ((i = cache->avail) < IKM_STASH) {
994 cache->entries[i] = kmsg;
995 cache->avail = i + 1;
1c79356b
A		 996 enable_preemption();
997 return;
998 }
999 enable_preemption();
91447636
A		 1000 zfree(ipc_kmsg_zone, kmsg);
1001 return;
1c79356b 1002 }
91447636 1003 kfree(kmsg, ikm_plus_overhead(size));
1c79356b
A		 1004}
1005
1006
1007/*
1008 * Routine: ipc_kmsg_enqueue
1009 * Purpose:
1010 * Enqueue a kmsg.
1011 */
1012
1013void
1014ipc_kmsg_enqueue(
1015 ipc_kmsg_queue_t queue,
1016 ipc_kmsg_t kmsg)
1017{
39037602
A		 1018 ipc_kmsg_t first = queue->ikmq_base;
1019 ipc_kmsg_t last;
1020
1021 if (first == IKM_NULL) {
1022 queue->ikmq_base = kmsg;
1023 kmsg->ikm_next = kmsg;
1024 kmsg->ikm_prev = kmsg;
1025 } else {
1026 last = first->ikm_prev;
1027 kmsg->ikm_next = first;
1028 kmsg->ikm_prev = last;
1029 first->ikm_prev = kmsg;
1030 last->ikm_next = kmsg;
1031 }
1032}
1033
1034/*
1035 * Routine: ipc_kmsg_enqueue_qos
1036 * Purpose:
1037 * Enqueue a kmsg, propagating qos
1038 * overrides towards the head of the queue.
1039 *
1040 * Returns:
1041 * whether the head of the queue had
1042 * it's override-qos adjusted because
1043 * of this insertion.
1044 */
1045
1046boolean_t
1047ipc_kmsg_enqueue_qos(
1048 ipc_kmsg_queue_t queue,
1049 ipc_kmsg_t kmsg)
1050{
1051 ipc_kmsg_t first = queue->ikmq_base;
1052 ipc_kmsg_t prev;
1053 mach_msg_priority_t override;
1054
1055 if (first == IKM_NULL) {
1056 /* insert a first message */
1057 queue->ikmq_base = kmsg;
1058 kmsg->ikm_next = kmsg;
1059 kmsg->ikm_prev = kmsg;
1060 return TRUE;
1061 }
1062
1063 /* insert at the tail */
1064 prev = first->ikm_prev;
1065 kmsg->ikm_next = first;
1066 kmsg->ikm_prev = prev;
1067 first->ikm_prev = kmsg;
1068 prev->ikm_next = kmsg;
1069
1070 /* apply QoS overrides towards the head */
1071 override = kmsg->ikm_qos_override;
1072 while (prev != kmsg &&
1073 override > prev->ikm_qos_override) {
1074 prev->ikm_qos_override = override;
1075 prev = prev->ikm_prev;
1076 }
1077
1078 /* did we adjust everything? */
1079 return (prev == kmsg);
1080}
1081
1082/*
1083 * Routine: ipc_kmsg_override_qos
1084 * Purpose:
1085 * Update the override for a given kmsg already
1086 * enqueued, propagating qos override adjustments
1087 * towards the head of the queue.
1088 *
1089 * Returns:
1090 * whether the head of the queue had
1091 * it's override-qos adjusted because
1092 * of this insertion.
1093 */
1094
1095boolean_t
1096ipc_kmsg_override_qos(
1097 ipc_kmsg_queue_t queue,
1098 ipc_kmsg_t kmsg,
1099 mach_msg_priority_t override)
1100{
1101 ipc_kmsg_t first = queue->ikmq_base;
1102 ipc_kmsg_t cur = kmsg;
1103
1104 /* apply QoS overrides towards the head */
1105 while (override > cur->ikm_qos_override) {
1106 cur->ikm_qos_override = override;
1107 if (cur == first)
1108 return TRUE;
1109 cur = cur->ikm_next;
1110 }
1111 return FALSE;
1c79356b
A		 1112}
1113
1114/*
1115 * Routine: ipc_kmsg_dequeue
1116 * Purpose:
1117 * Dequeue and return a kmsg.
1118 */
1119
1120ipc_kmsg_t
1121ipc_kmsg_dequeue(
1122 ipc_kmsg_queue_t queue)
1123{
1124 ipc_kmsg_t first;
1125
1126 first = ipc_kmsg_queue_first(queue);
1127
1128 if (first != IKM_NULL)
39037602 1129 ipc_kmsg_rmqueue(queue, first);
1c79356b
A		 1130
1131 return first;
1132}
1133
1134/*
1135 * Routine: ipc_kmsg_rmqueue
1136 * Purpose:
1137 * Pull a kmsg out of a queue.
1138 */
1139
1140void
1141ipc_kmsg_rmqueue(
1142 ipc_kmsg_queue_t queue,
1143 ipc_kmsg_t kmsg)
1144{
1145 ipc_kmsg_t next, prev;
1146
1147 assert(queue->ikmq_base != IKM_NULL);
1148
1149 next = kmsg->ikm_next;
1150 prev = kmsg->ikm_prev;
1151
1152 if (next == kmsg) {
1153 assert(prev == kmsg);
1154 assert(queue->ikmq_base == kmsg);
1155
1156 queue->ikmq_base = IKM_NULL;
1157 } else {
1158 if (queue->ikmq_base == kmsg)
1159 queue->ikmq_base = next;
1160
1161 next->ikm_prev = prev;
1162 prev->ikm_next = next;
1163 }
1164 /* XXX Temporary debug logic */
2d21ac55
A		 1165 assert((kmsg->ikm_next = IKM_BOGUS) == IKM_BOGUS);
1166 assert((kmsg->ikm_prev = IKM_BOGUS) == IKM_BOGUS);
1c79356b
A		 1167}
1168
1169/*
1170 * Routine: ipc_kmsg_queue_next
1171 * Purpose:
1172 * Return the kmsg following the given kmsg.
1173 * (Or IKM_NULL if it is the last one in the queue.)
1174 */
1175
1176ipc_kmsg_t
1177ipc_kmsg_queue_next(
1178 ipc_kmsg_queue_t queue,
1179 ipc_kmsg_t kmsg)
1180{
1181 ipc_kmsg_t next;
1182
1183 assert(queue->ikmq_base != IKM_NULL);
1184
1185 next = kmsg->ikm_next;
1186 if (queue->ikmq_base == next)
1187 next = IKM_NULL;
1188
1189 return next;
1190}
1191
1192/*
1193 * Routine: ipc_kmsg_destroy
1194 * Purpose:
1195 * Destroys a kernel message. Releases all rights,
1196 * references, and memory held by the message.
1197 * Frees the message.
1198 * Conditions:
1199 * No locks held.
1200 */
1201
1202void
1203ipc_kmsg_destroy(
1204 ipc_kmsg_t kmsg)
1205{
1c79356b 1206 /*
6d2010ae
A		 1207 * Destroying a message can cause more messages to be destroyed.
1208 * Curtail recursion by putting messages on the deferred
1209 * destruction queue. If this was the first message on the
1210 * queue, this instance must process the full queue.
1c79356b 1211 */
6d2010ae
A		 1212 if (ipc_kmsg_delayed_destroy(kmsg))
1213 ipc_kmsg_reap_delayed();
1214}
1c79356b 1215
6d2010ae
A		 1216/*
1217 * Routine: ipc_kmsg_delayed_destroy
1218 * Purpose:
1219 * Enqueues a kernel message for deferred destruction.
1220 * Returns:
1221 * Boolean indicator that the caller is responsible to reap
1222 * deferred messages.
1223 */
1c79356b 1224
6d2010ae
A		 1225boolean_t ipc_kmsg_delayed_destroy(
1226 ipc_kmsg_t kmsg)
1227{
1228 ipc_kmsg_queue_t queue = &(current_thread()->ith_messages);
1229 boolean_t first = ipc_kmsg_queue_empty(queue);
1c79356b 1230
6d2010ae
A		 1231 ipc_kmsg_enqueue(queue, kmsg);
1232 return first;
1c79356b
A		 1233}
1234
1235/*
6d2010ae 1236 * Routine: ipc_kmsg_destroy_queue
1c79356b 1237 * Purpose:
6d2010ae
A		 1238 * Destroys messages from the per-thread
1239 * deferred reaping queue.
1c79356b
A		 1240 * Conditions:
1241 * No locks held.
1242 */
6d2010ae 1243
91447636 1244void
6d2010ae 1245ipc_kmsg_reap_delayed(void)
1c79356b 1246{
6d2010ae
A		 1247 ipc_kmsg_queue_t queue = &(current_thread()->ith_messages);
1248 ipc_kmsg_t kmsg;
1c79356b 1249
6d2010ae
A		 1250 /*
1251 * must leave kmsg in queue while cleaning it to assure
1252 * no nested calls recurse into here.
1253 */
1254 while ((kmsg = ipc_kmsg_queue_first(queue)) != IKM_NULL) {
1255 ipc_kmsg_clean(kmsg);
1256 ipc_kmsg_rmqueue(queue, kmsg);
1257 ipc_kmsg_free(kmsg);
1258 }
1c79356b
A		 1259}
1260
1261/*
1262 * Routine: ipc_kmsg_clean_body
1263 * Purpose:
1264 * Cleans the body of a kernel message.
1265 * Releases all rights, references, and memory.
1266 *
1267 * Conditions:
1268 * No locks held.
1269 */
6d2010ae 1270static unsigned int _ipc_kmsg_clean_invalid_desc = 0;
1c79356b
A		 1271void
1272ipc_kmsg_clean_body(
91447636
A		 1273 __unused ipc_kmsg_t kmsg,
1274 mach_msg_type_number_t number,
1275 mach_msg_descriptor_t *saddr)
1c79356b 1276{
b0d623f7 1277 mach_msg_type_number_t i;
1c79356b
A		 1278
1279 if ( number == 0 )
1280 return;
1281
b0d623f7 1282 for (i = 0 ; i < number; i++, saddr++ ) {
1c79356b
A		 1283
1284 switch (saddr->type.type) {
1285
1286 case MACH_MSG_PORT_DESCRIPTOR: {
1287 mach_msg_port_descriptor_t *dsc;
1288
1289 dsc = &saddr->port;
1290
1291 /*
1292 * Destroy port rights carried in the message
1293 */
1294 if (!IO_VALID((ipc_object_t) dsc->name))
1295 continue;
1296 ipc_object_destroy((ipc_object_t) dsc->name, dsc->disposition);
1297 break;
1298 }
1299 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
1300 case MACH_MSG_OOL_DESCRIPTOR : {
1301 mach_msg_ool_descriptor_t *dsc;
1302
b0d623f7 1303 dsc = (mach_msg_ool_descriptor_t *)&saddr->out_of_line;
1c79356b
A		 1304
1305 /*
1306 * Destroy memory carried in the message
1307 */
1308 if (dsc->size == 0) {
1309 assert(dsc->address == (void *) 0);
1310 } else {
1311 vm_map_copy_discard((vm_map_copy_t) dsc->address);
1312 }
1313 break;
1314 }
1315 case MACH_MSG_OOL_PORTS_DESCRIPTOR : {
1316 ipc_object_t *objects;
1317 mach_msg_type_number_t j;
1318 mach_msg_ool_ports_descriptor_t *dsc;
1319
b0d623f7 1320 dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports;
1c79356b
A		 1321 objects = (ipc_object_t *) dsc->address;
1322
1323 if (dsc->count == 0) {
1324 break;
1325 }
1326
1327 assert(objects != (ipc_object_t *) 0);
1328
1329 /* destroy port rights carried in the message */
1330
1331 for (j = 0; j < dsc->count; j++) {
1332 ipc_object_t object = objects[j];
1333
1334 if (!IO_VALID(object))
1335 continue;
1336
1337 ipc_object_destroy(object, dsc->disposition);
1338 }
1339
1340 /* destroy memory carried in the message */
1341
1342 assert(dsc->count != 0);
1343
91447636 1344 kfree(dsc->address,
b0d623f7 1345 (vm_size_t) dsc->count * sizeof(mach_port_t));
1c79356b
A		 1346 break;
1347 }
1348 default : {
6d2010ae 1349 _ipc_kmsg_clean_invalid_desc++; /* don't understand this type of descriptor */
1c79356b
A		 1350 }
1351 }
1352 }
1353}
1354
1355/*
1356 * Routine: ipc_kmsg_clean_partial
1357 * Purpose:
1358 * Cleans a partially-acquired kernel message.
1359 * number is the index of the type descriptor
1360 * in the body of the message that contained the error.
1361 * If dolast, the memory and port rights in this last
1362 * type spec are also cleaned. In that case, number
1363 * specifies the number of port rights to clean.
1364 * Conditions:
1365 * Nothing locked.
1366 */
1367
1368void
1369ipc_kmsg_clean_partial(
1370 ipc_kmsg_t kmsg,
1371 mach_msg_type_number_t number,
91447636 1372 mach_msg_descriptor_t *desc,
1c79356b
A		 1373 vm_offset_t paddr,
1374 vm_size_t length)
1375{
1376 ipc_object_t object;
91447636 1377 mach_msg_bits_t mbits = kmsg->ikm_header->msgh_bits;
1c79356b 1378
fe8ab488
A		 1379 /* deal with importance chain while we still have dest and voucher references */
1380 ipc_importance_clean(kmsg);
1381
91447636 1382 object = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
1c79356b 1383 assert(IO_VALID(object));
6d2010ae 1384 ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits));
1c79356b 1385
91447636 1386 object = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
1c79356b
A		 1387 if (IO_VALID(object))
1388 ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits));
1389
fe8ab488
A		 1390 object = (ipc_object_t) kmsg->ikm_voucher;
1391 if (IO_VALID(object)) {
1392 assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND);
1393 ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND);
1394 kmsg->ikm_voucher = IP_NULL;
1395 }
1396
1c79356b
A		 1397 if (paddr) {
1398 (void) vm_deallocate(ipc_kernel_copy_map, paddr, length);
1399 }
1400
91447636 1401 ipc_kmsg_clean_body(kmsg, number, desc);
1c79356b
A		 1402}
1403
1404/*
1405 * Routine: ipc_kmsg_clean
1406 * Purpose:
1407 * Cleans a kernel message. Releases all rights,
1408 * references, and memory held by the message.
1409 * Conditions:
1410 * No locks held.
1411 */
1412
1413void
1414ipc_kmsg_clean(
1415 ipc_kmsg_t kmsg)
1416{
1417 ipc_object_t object;
1418 mach_msg_bits_t mbits;
1419
fe8ab488
A		 1420 /* deal with importance chain while we still have dest and voucher references */
1421 ipc_importance_clean(kmsg);
1422
91447636
A		 1423 mbits = kmsg->ikm_header->msgh_bits;
1424 object = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
1c79356b 1425 if (IO_VALID(object))
6d2010ae 1426 ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits));
1c79356b 1427
91447636 1428 object = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
1c79356b
A		 1429 if (IO_VALID(object))
1430 ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits));
1431
fe8ab488
A		 1432 object = (ipc_object_t) kmsg->ikm_voucher;
1433 if (IO_VALID(object)) {
1434 assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND);
1435 ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND);
1436 kmsg->ikm_voucher = IP_NULL;
1437 }
1438
1c79356b
A		 1439 if (mbits & MACH_MSGH_BITS_COMPLEX) {
1440 mach_msg_body_t *body;
1441
91447636
A		 1442 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
1443 ipc_kmsg_clean_body(kmsg, body->msgh_descriptor_count,
1444 (mach_msg_descriptor_t *)(body + 1));
1c79356b
A		 1445 }
1446}
1447
1448/*
1449 * Routine: ipc_kmsg_set_prealloc
1450 * Purpose:
1451 * Assign a kmsg as a preallocated message buffer to a port.
1452 * Conditions:
1453 * port locked.
1454 */
1455
1456void
1457ipc_kmsg_set_prealloc(
1458 ipc_kmsg_t kmsg,
1459 ipc_port_t port)
1460{
1461 assert(kmsg->ikm_prealloc == IP_NULL);
1462
1463 kmsg->ikm_prealloc = IP_NULL;
1464 IP_SET_PREALLOC(port, kmsg);
1465}
1466
1467/*
1468 * Routine: ipc_kmsg_clear_prealloc
1469 * Purpose:
1470 * Release the Assignment of a preallocated message buffer from a port.
1471 * Conditions:
1472 * port locked.
1473 */
1474void
1475ipc_kmsg_clear_prealloc(
1476 ipc_kmsg_t kmsg,
1477 ipc_port_t port)
1478{
1479 assert(kmsg->ikm_prealloc == port);
1480
1481 kmsg->ikm_prealloc = IP_NULL;
1482 IP_CLEAR_PREALLOC(port, kmsg);
1483}
1484
b7266188
A		 1485/*
1486 * Routine: ipc_kmsg_prealloc
1487 * Purpose:
1488 * Wraper to ipc_kmsg_alloc() to account for
1489 * header expansion requirements.
1490 */
1491ipc_kmsg_t
1492ipc_kmsg_prealloc(mach_msg_size_t size)
1493{
1494#if defined(__LP64__)
1495 if (size > MACH_MSG_SIZE_MAX - LEGACY_HEADER_SIZE_DELTA)
1496 return IKM_NULL;
1497
1498 size += LEGACY_HEADER_SIZE_DELTA;
1499#endif
1500 return ipc_kmsg_alloc(size);
1501}
91447636
A		 1502
1503
1c79356b
A		 1504/*
1505 * Routine: ipc_kmsg_get
1506 * Purpose:
1507 * Allocates a kernel message buffer.
1508 * Copies a user message to the message buffer.
1509 * Conditions:
1510 * Nothing locked.
1511 * Returns:
1512 * MACH_MSG_SUCCESS Acquired a message buffer.
1513 * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
1514 * MACH_SEND_MSG_TOO_SMALL Message size not long-word multiple.
316670eb 1515 * MACH_SEND_TOO_LARGE Message too large to ever be sent.
1c79356b
A		 1516 * MACH_SEND_NO_BUFFER Couldn't allocate a message buffer.
1517 * MACH_SEND_INVALID_DATA Couldn't copy message data.
1518 */
1519
1520mach_msg_return_t
1521ipc_kmsg_get(
91447636
A		 1522 mach_vm_address_t msg_addr,
1523 mach_msg_size_t size,
1c79356b
A		 1524 ipc_kmsg_t *kmsgp)
1525{
1526 mach_msg_size_t msg_and_trailer_size;
1527 ipc_kmsg_t kmsg;
55e303ae 1528 mach_msg_max_trailer_t *trailer;
b0d623f7
A		 1529 mach_msg_legacy_base_t legacy_base;
1530 mach_msg_size_t len_copied;
1531 legacy_base.body.msgh_descriptor_count = 0;
1c79356b 1532
b0d623f7 1533 if ((size < sizeof(mach_msg_legacy_header_t)) || (size & 3))
1c79356b
A		 1534 return MACH_SEND_MSG_TOO_SMALL;
1535
316670eb 1536 if (size > ipc_kmsg_max_body_space)
8ad349bb
A		 1537 return MACH_SEND_TOO_LARGE;
1538
b0d623f7
A		 1539 if(size == sizeof(mach_msg_legacy_header_t))
1540 len_copied = sizeof(mach_msg_legacy_header_t);
1541 else
1542 len_copied = sizeof(mach_msg_legacy_base_t);
1c79356b 1543
b0d623f7
A		 1544 if (copyinmsg(msg_addr, (char *)&legacy_base, len_copied))
1545 return MACH_SEND_INVALID_DATA;
1546
1547 msg_addr += sizeof(legacy_base.header);
1548#if defined(__LP64__)
1549 size += LEGACY_HEADER_SIZE_DELTA;
1550#endif
fe8ab488
A		 1551 /* unreachable if !DEBUG */
1552 __unreachable_ok_push
b0d623f7
A		 1553 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) {
1554 unsigned int j;
1555 for (j=0; j<sizeof(legacy_base.header); j++) {
1556 kprintf("%02x\n", ((unsigned char*)&legacy_base.header)[j]);
1557 }
1558 }
fe8ab488 1559 __unreachable_ok_pop
1c79356b 1560
b0d623f7
A		 1561 msg_and_trailer_size = size + MAX_TRAILER_SIZE;
1562 kmsg = ipc_kmsg_alloc(msg_and_trailer_size);
1c79356b
A		 1563 if (kmsg == IKM_NULL)
1564 return MACH_SEND_NO_BUFFER;
1565
b0d623f7
A		 1566 kmsg->ikm_header->msgh_size = size;
1567 kmsg->ikm_header->msgh_bits = legacy_base.header.msgh_bits;
1568 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(legacy_base.header.msgh_remote_port);
1569 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(legacy_base.header.msgh_local_port);
fe8ab488 1570 kmsg->ikm_header->msgh_voucher_port = legacy_base.header.msgh_voucher_port;
b0d623f7
A		 1571 kmsg->ikm_header->msgh_id = legacy_base.header.msgh_id;
1572
1573 DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_get header:\n"
1574 " size: 0x%.8x\n"
1575 " bits: 0x%.8x\n"
1576 " remote_port: %p\n"
1577 " local_port: %p\n"
fe8ab488 1578 " voucher_port: 0x%.8x\n"
b0d623f7
A		 1579 " id: %.8d\n",
1580 kmsg->ikm_header->msgh_size,
1581 kmsg->ikm_header->msgh_bits,
1582 kmsg->ikm_header->msgh_remote_port,
1583 kmsg->ikm_header->msgh_local_port,
fe8ab488 1584 kmsg->ikm_header->msgh_voucher_port,
b0d623f7
A		 1585 kmsg->ikm_header->msgh_id);
1586
1587 if (copyinmsg(msg_addr, (char *)(kmsg->ikm_header + 1), size - (mach_msg_size_t)sizeof(mach_msg_header_t))) {
1c79356b
A		 1588 ipc_kmsg_free(kmsg);
1589 return MACH_SEND_INVALID_DATA;
1590 }
1591
fe8ab488
A		 1592 /* unreachable if !DEBUG */
1593 __unreachable_ok_push
b0d623f7
A		 1594 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK))
1595 {
1596 kprintf("body: size: %lu\n", (size - sizeof(mach_msg_header_t)));
1597 uint32_t i;
1598 for(i=0;i*4 < (size - sizeof(mach_msg_header_t));i++)
1599 {
1600 kprintf("%.4x\n",((uint32_t *)(kmsg->ikm_header + 1))[i]);
1601 }
1602 }
fe8ab488 1603 __unreachable_ok_pop
b0d623f7 1604 DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_get()");
1c79356b
A		 1605
1606 /*
1607 * I reserve for the trailer the largest space (MAX_TRAILER_SIZE)
1608 * However, the internal size field of the trailer (msgh_trailer_size)
1609 * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to optimize
1610 * the cases where no implicit data is requested.
1611 */
91447636
A		 1612 trailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + size);
1613 trailer->msgh_sender = current_thread()->task->sec_token;
1614 trailer->msgh_audit = current_thread()->task->audit_token;
1c79356b
A		 1615 trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
1616 trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
b0d623f7 1617
55e303ae 1618#ifdef ppc
b36670ce 1619 if(trcWork.traceMask) dbgTrace(0x1100, (unsigned int)kmsg->ikm_header->msgh_id,
91447636
A		 1620 (unsigned int)kmsg->ikm_header->msgh_remote_port,
1621 (unsigned int)kmsg->ikm_header->msgh_local_port, 0);
55e303ae 1622#endif
2d21ac55 1623
b0d623f7 1624 trailer->msgh_labels.sender = 0;
1c79356b
A		 1625 *kmsgp = kmsg;
1626 return MACH_MSG_SUCCESS;
1627}
1628
1629/*
1630 * Routine: ipc_kmsg_get_from_kernel
1631 * Purpose:
91447636
A		 1632 * First checks for a preallocated message
1633 * reserved for kernel clients. If not found -
1634 * allocates a new kernel message buffer.
1c79356b
A		 1635 * Copies a kernel message to the message buffer.
1636 * Only resource errors are allowed.
1637 * Conditions:
1638 * Nothing locked.
1639 * Ports in header are ipc_port_t.
1640 * Returns:
1641 * MACH_MSG_SUCCESS Acquired a message buffer.
1642 * MACH_SEND_NO_BUFFER Couldn't allocate a message buffer.
1643 */
1644
1645mach_msg_return_t
1646ipc_kmsg_get_from_kernel(
1647 mach_msg_header_t *msg,
91447636 1648 mach_msg_size_t size,
1c79356b
A		 1649 ipc_kmsg_t *kmsgp)
1650{
1651 ipc_kmsg_t kmsg;
1652 mach_msg_size_t msg_and_trailer_size;
55e303ae 1653 mach_msg_max_trailer_t *trailer;
1c79356b
A		 1654 ipc_port_t dest_port;
1655
1656 assert(size >= sizeof(mach_msg_header_t));
316670eb 1657 assert((size & 3) == 0);
1c79356b 1658
1c79356b
A		 1659 dest_port = (ipc_port_t)msg->msgh_remote_port;
1660
1661 msg_and_trailer_size = size + MAX_TRAILER_SIZE;
1662
1663 /*
1664 * See if the port has a pre-allocated kmsg for kernel
1665 * clients. These are set up for those kernel clients
1666 * which cannot afford to wait.
1667 */
6d2010ae 1668 if (IP_VALID(dest_port) && IP_PREALLOC(dest_port)) {
b7266188
A		 1669 mach_msg_size_t max_desc = 0;
1670
1c79356b
A		 1671 ip_lock(dest_port);
1672 if (!ip_active(dest_port)) {
1673 ip_unlock(dest_port);
1674 return MACH_SEND_NO_BUFFER;
1675 }
1676 assert(IP_PREALLOC(dest_port));
1677 kmsg = dest_port->ip_premsg;
1c79356b
A		 1678 if (ikm_prealloc_inuse(kmsg)) {
1679 ip_unlock(dest_port);
1680 return MACH_SEND_NO_BUFFER;
1681 }
b7266188
A		 1682#if !defined(__LP64__)
1683 if (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
1684 assert(size > sizeof(mach_msg_base_t));
1685 max_desc = ((mach_msg_base_t *)msg)->body.msgh_descriptor_count *
1686 DESC_SIZE_ADJUSTMENT;
1687 }
1688#endif
1689 if (msg_and_trailer_size > kmsg->ikm_size - max_desc) {
1690 ip_unlock(dest_port);
1691 return MACH_SEND_TOO_LARGE;
1692 }
1c79356b 1693 ikm_prealloc_set_inuse(kmsg, dest_port);
b7266188 1694 ikm_set_header(kmsg, msg_and_trailer_size);
1c79356b 1695 ip_unlock(dest_port);
b0d623f7
A		 1696 }
1697 else
b0d623f7 1698 {
1c79356b
A		 1699 kmsg = ipc_kmsg_alloc(msg_and_trailer_size);
1700 if (kmsg == IKM_NULL)
1701 return MACH_SEND_NO_BUFFER;
1702 }
1703
91447636 1704 (void) memcpy((void *) kmsg->ikm_header, (const void *) msg, size);
1c79356b 1705
91447636 1706 kmsg->ikm_header->msgh_size = size;
1c79356b
A		 1707
1708 /*
1709 * I reserve for the trailer the largest space (MAX_TRAILER_SIZE)
1710 * However, the internal size field of the trailer (msgh_trailer_size)
1711 * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to
1712 * optimize the cases where no implicit data is requested.
1713 */
55e303ae 1714 trailer = (mach_msg_max_trailer_t *)
91447636 1715 ((vm_offset_t)kmsg->ikm_header + size);
1c79356b 1716 trailer->msgh_sender = KERNEL_SECURITY_TOKEN;
55e303ae 1717 trailer->msgh_audit = KERNEL_AUDIT_TOKEN;
1c79356b
A		 1718 trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
1719 trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
1720
2d21ac55
A		 1721 trailer->msgh_labels.sender = 0;
1722
1c79356b
A		 1723 *kmsgp = kmsg;
1724 return MACH_MSG_SUCCESS;
1725}
1726
1727/*
1728 * Routine: ipc_kmsg_send
1729 * Purpose:
1730 * Send a message. The message holds a reference
1731 * for the destination port in the msgh_remote_port field.
1732 *
1733 * If unsuccessful, the caller still has possession of
1734 * the message and must do something with it. If successful,
1735 * the message is queued, given to a receiver, destroyed,
1736 * or handled directly by the kernel via mach_msg.
1737 * Conditions:
1738 * Nothing locked.
1739 * Returns:
1740 * MACH_MSG_SUCCESS The message was accepted.
1741 * MACH_SEND_TIMED_OUT Caller still has message.
1742 * MACH_SEND_INTERRUPTED Caller still has message.
6d2010ae 1743 * MACH_SEND_INVALID_DEST Caller still has message.
1c79356b 1744 */
39236c6e
A		 1745
1746
1c79356b
A		 1747mach_msg_return_t
1748ipc_kmsg_send(
1749 ipc_kmsg_t kmsg,
1750 mach_msg_option_t option,
91447636 1751 mach_msg_timeout_t send_timeout)
1c79356b 1752{
1c79356b 1753 ipc_port_t port;
3e170ce0 1754 thread_t th = current_thread();
b0d623f7 1755 mach_msg_return_t error = MACH_MSG_SUCCESS;
4bd07ac2 1756 boolean_t kernel_reply = FALSE;
91447636 1757
3e170ce0
A		 1758 /* Check if honor qlimit flag is set on thread. */
1759 if ((th->options & TH_OPT_HONOR_QLIMIT) == TH_OPT_HONOR_QLIMIT) {
1760 /* Remove the MACH_SEND_ALWAYS flag to honor queue limit. */
1761 option &= (~MACH_SEND_ALWAYS);
1762 /* Add the timeout flag since the message queue might be full. */
1763 option |= MACH_SEND_TIMEOUT;
1764 th->options &= (~TH_OPT_HONOR_QLIMIT);
1765 }
1766
39236c6e
A		 1767#if IMPORTANCE_INHERITANCE
1768 boolean_t did_importance = FALSE;
1769#if IMPORTANCE_DEBUG
1770 mach_msg_id_t imp_msgh_id = -1;
1771 int sender_pid = -1;
1772#endif /* IMPORTANCE_DEBUG */
1773#endif /* IMPORTANCE_INHERITANCE */
1774
1775 /* don't allow the creation of a circular loop */
1776 if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_CIRCULAR) {
1777 ipc_kmsg_destroy(kmsg);
39037602 1778 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_MSGH_BITS_CIRCULAR);
39236c6e
A		 1779 return MACH_MSG_SUCCESS;
1780 }
1781
490019cf
A		 1782 ipc_voucher_send_preprocessing(kmsg);
1783
91447636 1784 port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port;
1c79356b 1785 assert(IP_VALID(port));
1c79356b
A		 1786 ip_lock(port);
1787
fe8ab488
A		 1788#if IMPORTANCE_INHERITANCE
1789retry:
1790#endif /* IMPORTANCE_INHERITANCE */
1791 /*
1792 * Can't deliver to a dead port.
1793 * However, we can pretend it got sent
1794 * and was then immediately destroyed.
1795 */
1796 if (!ip_active(port)) {
1797 ip_unlock(port);
39037602
A		 1798#if MACH_FLIPC
1799 if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport))
1800 flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
1801#endif
fe8ab488
A		 1802 ip_release(port); /* JMM - Future: release right, not just ref */
1803 kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
1804 ipc_kmsg_destroy(kmsg);
39037602 1805 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST);
fe8ab488
A		 1806 return MACH_MSG_SUCCESS;
1807 }
1808
1c79356b
A		 1809 if (port->ip_receiver == ipc_space_kernel) {
1810
1811 /*
1812 * We can check ip_receiver == ipc_space_kernel
1813 * before checking that the port is active because
1814 * ipc_port_dealloc_kernel clears ip_receiver
1815 * before destroying a kernel port.
1816 */
1817 assert(ip_active(port));
1818 port->ip_messages.imq_seqno++;
1819 ip_unlock(port);
1820
1821 current_task()->messages_sent++;
1822
1823 /*
1824 * Call the server routine, and get the reply message to send.
1825 */
39037602 1826 kmsg = ipc_kobject_server(kmsg, option);
1c79356b
A		 1827 if (kmsg == IKM_NULL)
1828 return MACH_MSG_SUCCESS;
1829
39037602
A		 1830 /* restart the KMSG_INFO tracing for the reply message */
1831 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_START);
91447636 1832 port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port;
1c79356b
A		 1833 assert(IP_VALID(port));
1834 ip_lock(port);
1835 /* fall thru with reply - same options */
4bd07ac2 1836 kernel_reply = TRUE;
39037602
A		 1837 if (!ip_active(port))
1838 error = MACH_SEND_INVALID_DEST;
1c79356b
A		 1839 }
1840
39236c6e
A		 1841#if IMPORTANCE_INHERITANCE
1842 /*
1843 * Need to see if this message needs importance donation and/or
fe8ab488
A		 1844 * propagation. That routine can drop the port lock temporarily.
1845 * If it does we'll have to revalidate the destination.
39236c6e 1846 */
fe8ab488 1847 if (did_importance == FALSE) {
39236c6e 1848 did_importance = TRUE;
fe8ab488
A		 1849 if (ipc_importance_send(kmsg, option))
1850 goto retry;
1c79356b 1851 }
39236c6e 1852#endif /* IMPORTANCE_INHERITANCE */
1c79356b 1853
39037602
A		 1854 if (error != MACH_MSG_SUCCESS) {
1855 ip_unlock(port);
1856 } else {
1857 /*
1858 * We have a valid message and a valid reference on the port.
1859 * we can unlock the port and call mqueue_send() on its message
1860 * queue. Lock message queue while port is locked.
1861 */
1862 imq_lock(&port->ip_messages);
1863 ip_unlock(port);
39236c6e 1864
39037602
A		 1865 error = ipc_mqueue_send(&port->ip_messages, kmsg, option,
1866 send_timeout);
1867 }
b0d623f7 1868
39236c6e
A		 1869#if IMPORTANCE_INHERITANCE
1870 if (did_importance == TRUE) {
1871 __unused int importance_cleared = 0;
1872 switch (error) {
1873 case MACH_SEND_TIMED_OUT:
1874 case MACH_SEND_NO_BUFFER:
1875 case MACH_SEND_INTERRUPTED:
fe8ab488 1876 case MACH_SEND_INVALID_DEST:
39236c6e
A		 1877 /*
1878 * We still have the kmsg and its
1879 * reference on the port. But we
1880 * have to back out the importance
1881 * boost.
1882 *
1883 * The port could have changed hands,
1884 * be inflight to another destination,
1885 * etc... But in those cases our
1886 * back-out will find the new owner
1887 * (and all the operations that
1888 * transferred the right should have
1889 * applied their own boost adjustments
1890 * to the old owner(s)).
1891 */
1892 importance_cleared = 1;
fe8ab488 1893 ipc_importance_clean(kmsg);
39236c6e
A		 1894 break;
1895
39236c6e
A		 1896 case MACH_MSG_SUCCESS:
1897 default:
1898 break;
1899 }
1900#if IMPORTANCE_DEBUG
1901 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_END,
3e170ce0 1902 task_pid(current_task()), sender_pid, imp_msgh_id, importance_cleared, 0);
39236c6e
A		 1903#endif /* IMPORTANCE_DEBUG */
1904 }
1905#endif /* IMPORTANCE_INHERITANCE */
1906
b0d623f7
A		 1907 /*
1908 * If the port has been destroyed while we wait, treat the message
1909 * as a successful delivery (like we do for an inactive port).
1910 */
1911 if (error == MACH_SEND_INVALID_DEST) {
39037602
A		 1912#if MACH_FLIPC
1913 if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport))
1914 flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
1915#endif
fe8ab488 1916 ip_release(port); /* JMM - Future: release right, not just ref */
b0d623f7
A		 1917 kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
1918 ipc_kmsg_destroy(kmsg);
39037602 1919 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_INVALID_DEST);
b0d623f7
A		 1920 return MACH_MSG_SUCCESS;
1921 }
4bd07ac2
A		 1922
1923 if (error != MACH_MSG_SUCCESS && kernel_reply) {
1924 /*
1925 * Kernel reply messages that fail can't be allowed to
1926 * pseudo-receive on error conditions. We need to just treat
1927 * the message as a successful delivery.
1928 */
39037602
A		 1929#if MACH_FLIPC
1930 if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport))
1931 flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);
1932#endif
4bd07ac2
A		 1933 ip_release(port); /* JMM - Future: release right, not just ref */
1934 kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL;
1935 ipc_kmsg_destroy(kmsg);
39037602 1936 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, error);
4bd07ac2
A		 1937 return MACH_MSG_SUCCESS;
1938 }
b0d623f7 1939 return error;
1c79356b
A		 1940}
1941
1942/*
1943 * Routine: ipc_kmsg_put
1944 * Purpose:
1945 * Copies a message buffer to a user message.
1946 * Copies only the specified number of bytes.
1947 * Frees the message buffer.
1948 * Conditions:
1949 * Nothing locked. The message buffer must have clean
1950 * header fields.
1951 * Returns:
1952 * MACH_MSG_SUCCESS Copied data out of message buffer.
1953 * MACH_RCV_INVALID_DATA Couldn't copy to user message.
1954 */
1955
1956mach_msg_return_t
1957ipc_kmsg_put(
1c79356b 1958 ipc_kmsg_t kmsg,
39037602
A		 1959 mach_msg_option_t option,
1960 mach_vm_address_t rcv_addr,
1961 mach_msg_size_t rcv_size,
1962 mach_msg_size_t trailer_size,
1963 mach_msg_size_t *sizep)
1c79356b 1964{
39037602 1965 mach_msg_size_t size = kmsg->ikm_header->msgh_size + trailer_size;
1c79356b
A		 1966 mach_msg_return_t mr;
1967
b0d623f7
A		 1968 DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_put()");
1969
1970
1971 DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_put header:\n"
1972 " size: 0x%.8x\n"
1973 " bits: 0x%.8x\n"
1974 " remote_port: %p\n"
1975 " local_port: %p\n"
fe8ab488 1976 " voucher_port: 0x%.8x\n"
b0d623f7
A		 1977 " id: %.8d\n",
1978 kmsg->ikm_header->msgh_size,
1979 kmsg->ikm_header->msgh_bits,
1980 kmsg->ikm_header->msgh_remote_port,
1981 kmsg->ikm_header->msgh_local_port,
fe8ab488 1982 kmsg->ikm_header->msgh_voucher_port,
b0d623f7
A		 1983 kmsg->ikm_header->msgh_id);
1984
1985#if defined(__LP64__)
1986 if (current_task() != kernel_task) { /* don't if receiver expects fully-cooked in-kernel msg; ux_exception */
1987 mach_msg_legacy_header_t *legacy_header =
1988 (mach_msg_legacy_header_t *)((vm_offset_t)(kmsg->ikm_header) + LEGACY_HEADER_SIZE_DELTA);
1989
1990 mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits;
1991 mach_msg_size_t msg_size = kmsg->ikm_header->msgh_size;
1992 mach_port_name_t remote_port = CAST_MACH_PORT_TO_NAME(kmsg->ikm_header->msgh_remote_port);
1993 mach_port_name_t local_port = CAST_MACH_PORT_TO_NAME(kmsg->ikm_header->msgh_local_port);
fe8ab488 1994 mach_port_name_t voucher_port = kmsg->ikm_header->msgh_voucher_port;
b0d623f7
A		 1995 mach_msg_id_t id = kmsg->ikm_header->msgh_id;
1996
1997 legacy_header->msgh_id = id;
fe8ab488
A		 1998 legacy_header->msgh_local_port = local_port;
1999 legacy_header->msgh_remote_port = remote_port;
2000 legacy_header->msgh_voucher_port = voucher_port;
b0d623f7
A		 2001 legacy_header->msgh_size = msg_size - LEGACY_HEADER_SIZE_DELTA;
2002 legacy_header->msgh_bits = bits;
2003
2004 size -= LEGACY_HEADER_SIZE_DELTA;
2005 kmsg->ikm_header = (mach_msg_header_t *)legacy_header;
2006 }
2007#endif
2008
fe8ab488
A		 2009 /* unreachable if !DEBUG */
2010 __unreachable_ok_push
b0d623f7
A		 2011 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) {
2012 kprintf("ipc_kmsg_put header+body: %d\n", (size));
2013 uint32_t i;
2014 for(i=0;i*4 < size;i++)
2015 {
2016 kprintf("%.4x\n",((uint32_t *)kmsg->ikm_header)[i]);
2017 }
2018 kprintf("type: %d\n", ((mach_msg_type_descriptor_t *)(((mach_msg_base_t *)kmsg->ikm_header)+1))->type);
2019 }
fe8ab488 2020 __unreachable_ok_pop
39037602
A		 2021
2022 /* Re-Compute target address if using stack-style delivery */
2023 if (option & MACH_RCV_STACK) {
2024 rcv_addr += rcv_size - size;
2025 }
2026
2027 if (copyoutmsg((const char *) kmsg->ikm_header, rcv_addr, size)) {
1c79356b 2028 mr = MACH_RCV_INVALID_DATA;
39037602
A		 2029 size = 0;
2030 } else
1c79356b
A		 2031 mr = MACH_MSG_SUCCESS;
2032
39037602
A		 2033 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_LINK) | DBG_FUNC_NONE,
2034 (rcv_addr >= VM_MIN_KERNEL_AND_KEXT_ADDRESS ||
2035 rcv_addr + size >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) ? (uintptr_t)0 : (uintptr_t)rcv_addr,
2036 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
2037 1 /* this is on the receive/copyout path */,
2038 0,
2039 0);
1c79356b 2040 ipc_kmsg_free(kmsg);
39037602
A		 2041
2042 if (sizep)
2043 *sizep = size;
1c79356b
A		 2044 return mr;
2045}
2046
2047/*
2048 * Routine: ipc_kmsg_put_to_kernel
2049 * Purpose:
2050 * Copies a message buffer to a kernel message.
2051 * Frees the message buffer.
2052 * No errors allowed.
2053 * Conditions:
2054 * Nothing locked.
2055 */
2056
2057void
2058ipc_kmsg_put_to_kernel(
2059 mach_msg_header_t *msg,
2060 ipc_kmsg_t kmsg,
2061 mach_msg_size_t size)
2062{
91447636 2063 (void) memcpy((void *) msg, (const void *) kmsg->ikm_header, size);
1c79356b
A		 2064
2065 ipc_kmsg_free(kmsg);
2066}
2067
39037602
A		 2068unsigned long pthread_priority_canonicalize(unsigned long priority, boolean_t propagation);
2069
2070static void
2071ipc_kmsg_set_qos(
2072 ipc_kmsg_t kmsg,
2073 mach_msg_option_t options,
2074 mach_msg_priority_t override)
2075{
2076 kern_return_t kr;
2077
2078 kr = ipc_get_pthpriority_from_kmsg_voucher(kmsg, &kmsg->ikm_qos);
2079 if (kr != KERN_SUCCESS) {
2080 kmsg->ikm_qos = MACH_MSG_PRIORITY_UNSPECIFIED;
2081 }
2082 kmsg->ikm_qos_override = kmsg->ikm_qos;
2083
2084 if (options & MACH_SEND_OVERRIDE) {
2085 unsigned long canonical;
2086 mach_msg_priority_t canon;
2087
2088 canonical = pthread_priority_canonicalize(override, TRUE);
2089 canon = (mach_msg_priority_t)canonical;
2090 if (canon > kmsg->ikm_qos)
2091 kmsg->ikm_qos_override = canon;
2092 }
2093}
2094
1c79356b
A		 2095/*
2096 * Routine: ipc_kmsg_copyin_header
2097 * Purpose:
2098 * "Copy-in" port rights in the header of a message.
2099 * Operates atomically; if it doesn't succeed the
2100 * message header and the space are left untouched.
2101 * If it does succeed the remote/local port fields
2102 * contain object pointers instead of port names,
2103 * and the bits field is updated. The destination port
2104 * will be a valid port pointer.
2105 *
1c79356b
A		 2106 * Conditions:
2107 * Nothing locked.
2108 * Returns:
2109 * MACH_MSG_SUCCESS Successful copyin.
2110 * MACH_SEND_INVALID_HEADER
2111 * Illegal value in the message header bits.
2112 * MACH_SEND_INVALID_DEST The space is dead.
1c79356b
A		 2113 * MACH_SEND_INVALID_DEST Can't copyin destination port.
2114 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
2115 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
2116 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
2117 */
2118
2119mach_msg_return_t
2120ipc_kmsg_copyin_header(
fe8ab488 2121 ipc_kmsg_t kmsg,
1c79356b 2122 ipc_space_t space,
39037602 2123 mach_msg_priority_t override,
39236c6e 2124 mach_msg_option_t *optionp)
1c79356b 2125{
fe8ab488 2126 mach_msg_header_t *msg = kmsg->ikm_header;
1c79356b 2127 mach_msg_bits_t mbits = msg->msgh_bits & MACH_MSGH_BITS_USER;
b0d623f7
A		 2128 mach_port_name_t dest_name = CAST_MACH_PORT_TO_NAME(msg->msgh_remote_port);
2129 mach_port_name_t reply_name = CAST_MACH_PORT_TO_NAME(msg->msgh_local_port);
fe8ab488 2130 mach_port_name_t voucher_name = MACH_PORT_NULL;
1c79356b
A		 2131 kern_return_t kr;
2132
2133 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
2134 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
fe8ab488
A		 2135 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
2136 ipc_object_t dest_port = IO_NULL;
2137 ipc_object_t reply_port = IO_NULL;
2138 ipc_port_t dest_soright = IP_NULL;
2139 ipc_port_t reply_soright = IP_NULL;
2140 ipc_port_t voucher_soright = IP_NULL;
316670eb 2141 ipc_port_t release_port = IP_NULL;
fe8ab488
A		 2142 ipc_port_t voucher_port = IP_NULL;
2143 ipc_port_t voucher_release_port = IP_NULL;
2144 ipc_entry_t dest_entry = IE_NULL;
2145 ipc_entry_t reply_entry = IE_NULL;
2146 ipc_entry_t voucher_entry = IE_NULL;
316670eb 2147
39236c6e 2148 int assertcnt = 0;
3e170ce0 2149#if IMPORTANCE_INHERITANCE
39236c6e
A		 2150 boolean_t needboost = FALSE;
2151#endif /* IMPORTANCE_INHERITANCE */
2152
1c79356b
A		 2153 if ((mbits != msg->msgh_bits) ||
2154 (!MACH_MSG_TYPE_PORT_ANY_SEND(dest_type)) ||
2155 ((reply_type == 0) ?
2156 (reply_name != MACH_PORT_NULL) :
2157 !MACH_MSG_TYPE_PORT_ANY_SEND(reply_type)))
2158 return MACH_SEND_INVALID_HEADER;
2159
fe8ab488
A		 2160 if (!MACH_PORT_VALID(dest_name))
2161 return MACH_SEND_INVALID_DEST;
1c79356b
A		 2162
2163 is_write_lock(space);
fe8ab488
A		 2164 if (!is_active(space)) {
2165 is_write_unlock(space);
2166 return MACH_SEND_INVALID_DEST;
2167 }
2168 /* space locked and active */
1c79356b 2169
2d21ac55 2170 /*
fe8ab488
A		 2171 * If there is a voucher specified, make sure the disposition is
2172 * valid and the entry actually refers to a voucher port. Don't
2173 * actually copy in until we validate destination and reply.
2d21ac55 2174 */
fe8ab488
A		 2175 if (voucher_type != MACH_MSGH_BITS_ZERO) {
2176
2177 voucher_name = msg->msgh_voucher_port;
2178
2179 if (voucher_name == MACH_PORT_DEAD ||
2180 (voucher_type != MACH_MSG_TYPE_MOVE_SEND &&
2181 voucher_type != MACH_MSG_TYPE_COPY_SEND)) {
2182 is_write_unlock(space);
2183 return MACH_SEND_INVALID_VOUCHER;
2184 }
2185
2186 if (voucher_name != MACH_PORT_NULL) {
2187 voucher_entry = ipc_entry_lookup(space, voucher_name);
2188 if (voucher_entry == IE_NULL ||
2189 (voucher_entry->ie_bits & MACH_PORT_TYPE_SEND) == 0 ||
2190 io_kotype(voucher_entry->ie_object) != IKOT_VOUCHER) {
2191 is_write_unlock(space);
2192 return MACH_SEND_INVALID_VOUCHER;
2193 }
2194 } else {
2195 voucher_type = MACH_MSG_TYPE_MOVE_SEND;
2d21ac55 2196 }
2d21ac55 2197 }
1c79356b 2198
fe8ab488
A		 2199 /*
2200 * Handle combinations of validating destination and reply; along
2201 * with copying in destination, reply, and voucher in an atomic way.
2202 */
2203
2204 if (dest_name == voucher_name) {
1c79356b
A		 2205
2206 /*
fe8ab488
A		 2207 * If the destination name is the same as the voucher name,
2208 * the voucher_entry must already be known. Either that or
2209 * the destination name is MACH_PORT_NULL (i.e. invalid).
1c79356b 2210 */
fe8ab488
A		 2211 dest_entry = voucher_entry;
2212 if (dest_entry == IE_NULL) {
1c79356b 2213 goto invalid_dest;
fe8ab488 2214 }
1c79356b 2215
fe8ab488
A		 2216 /*
2217 * Make sure a future copyin of the reply port will succeed.
2218 * Once we start copying in the dest/voucher pair, we can't
2219 * back out.
2220 */
2221 if (MACH_PORT_VALID(reply_name)) {
2222 assert(reply_type != 0); /* because reply_name not null */
1c79356b 2223
fe8ab488
A		 2224 /* It is just WRONG if dest, voucher, and reply are all the same. */
2225 if (voucher_name == reply_name) {
2226 goto invalid_reply;
2227 }
2228 reply_entry = ipc_entry_lookup(space, reply_name);
2229 if (reply_entry == IE_NULL) {
2230 goto invalid_reply;
2231 }
2232 assert(dest_entry != reply_entry); /* names are not equal */
2233 if (!ipc_right_copyin_check(space, reply_name, reply_entry, reply_type)) {
2234 goto invalid_reply;
2235 }
2236 }
1c79356b 2237
fe8ab488
A		 2238 /*
2239 * Do the joint copyin of the dest disposition and
2240 * voucher disposition from the one entry/port. We
2241 * already validated that the voucher copyin would
2242 * succeed (above). So, any failure in combining
2243 * the copyins can be blamed on the destination.
2244 */
2245 kr = ipc_right_copyin_two(space, dest_name, dest_entry,
2246 dest_type, voucher_type,
2247 &dest_port, &dest_soright,
2248 &release_port);
2249 if (kr != KERN_SUCCESS) {
2250 assert(kr != KERN_INVALID_CAPABILITY);
1c79356b 2251 goto invalid_dest;
fe8ab488
A		 2252 }
2253 voucher_port = (ipc_port_t)dest_port;
1c79356b 2254
fe8ab488
A		 2255 /*
2256 * could not have been one of these dispositions,
2257 * validated the port was a true kernel voucher port above,
2258 * AND was successfully able to copyin both dest and voucher.
2259 */
2260 assert(dest_type != MACH_MSG_TYPE_MAKE_SEND);
2261 assert(dest_type != MACH_MSG_TYPE_MAKE_SEND_ONCE);
2262 assert(dest_type != MACH_MSG_TYPE_MOVE_SEND_ONCE);
2263
2264 /*
2265 * Perform the delayed reply right copyin (guaranteed success).
2266 */
2267 if (reply_entry != IE_NULL) {
fe8ab488 2268 kr = ipc_right_copyin(space, reply_name, reply_entry,
39236c6e
A		 2269 reply_type, TRUE,
2270 &reply_port, &reply_soright,
3e170ce0 2271 &release_port, &assertcnt);
39236c6e 2272 assert(assertcnt == 0);
1c79356b 2273 assert(kr == KERN_SUCCESS);
fe8ab488 2274 }
1c79356b 2275
fe8ab488
A		 2276 } else {
2277 if (dest_name == reply_name) {
1c79356b 2278 /*
fe8ab488
A		 2279 * Destination and reply ports are the same!
2280 * This is very similar to the case where the
2281 * destination and voucher ports were the same
2282 * (except the reply port disposition is not
2283 * previously validated).
1c79356b 2284 */
fe8ab488
A		 2285 dest_entry = ipc_entry_lookup(space, dest_name);
2286 if (dest_entry == IE_NULL) {
2287 goto invalid_dest;
2288 }
2289 reply_entry = dest_entry;
2290 assert(reply_type != 0); /* because name not null */
1c79356b 2291
fe8ab488
A		 2292 /*
2293 * Do the joint copyin of the dest disposition and
2294 * reply disposition from the one entry/port.
1c79356b 2295 */
fe8ab488
A		 2296 kr = ipc_right_copyin_two(space, dest_name, dest_entry,
2297 dest_type, reply_type,
316670eb
A		 2298 &dest_port, &dest_soright,
2299 &release_port);
fe8ab488
A		 2300 if (kr == KERN_INVALID_CAPABILITY) {
2301 goto invalid_reply;
2302 } else if (kr != KERN_SUCCESS) {
1c79356b 2303 goto invalid_dest;
6d2010ae 2304 }
1c79356b 2305 reply_port = dest_port;
fe8ab488
A		 2306
2307
1c79356b 2308 } else {
fe8ab488
A		 2309 /*
2310 * Handle destination and reply independently, as
2311 * they are independent entries (even if the entries
2312 * refer to the same port).
2313 *
2314 * This can be the tough case to make atomic.
2315 *
2316 * The difficult problem is serializing with port death.
2317 * The bad case is when dest_port dies after its copyin,
2318 * reply_port dies before its copyin, and dest_port dies before
2319 * reply_port. Then the copyins operated as if dest_port was
2320 * alive and reply_port was dead, which shouldn't have happened
2321 * because they died in the other order.
2322 *
2323 * Note that it is easy for a user task to tell if
2324 * a copyin happened before or after a port died.
2325 * If a port dies before copyin, a dead-name notification
2326 * is generated and the dead name's urefs are incremented,
2327 * and if the copyin happens first, a port-deleted
2328 * notification is generated.
2329 *
2330 * Even so, avoiding that potentially detectable race is too
2331 * expensive - and no known code cares about it. So, we just
2332 * do the expedient thing and copy them in one after the other.
2333 */
1c79356b 2334
fe8ab488
A		 2335 dest_entry = ipc_entry_lookup(space, dest_name);
2336 if (dest_entry == IE_NULL) {
2337 goto invalid_dest;
2338 }
2339 assert(dest_entry != voucher_entry);
1c79356b
A		 2340
2341 /*
fe8ab488 2342 * Make sure reply port entry is valid before dest copyin.
1c79356b 2343 */
fe8ab488
A		 2344 if (MACH_PORT_VALID(reply_name)) {
2345 if (reply_name == voucher_name) {
2346 goto invalid_reply;
2347 }
2348 reply_entry = ipc_entry_lookup(space, reply_name);
2349 if (reply_entry == IE_NULL) {
2350 goto invalid_reply;
2351 }
2352 assert(dest_entry != reply_entry); /* names are not equal */
2353 assert(reply_type != 0); /* because reply_name not null */
2354
2355 if (!ipc_right_copyin_check(space, reply_name, reply_entry, reply_type)) {
2356 goto invalid_reply;
2357 }
2358 }
1c79356b 2359
fe8ab488
A		 2360 /*
2361 * copyin the destination.
2362 */
fe8ab488
A		 2363 kr = ipc_right_copyin(space, dest_name, dest_entry,
2364 dest_type, FALSE,
2365 &dest_port, &dest_soright,
3e170ce0 2366 &release_port, &assertcnt);
39236c6e 2367 assert(assertcnt == 0);
fe8ab488 2368 if (kr != KERN_SUCCESS) {
1c79356b 2369 goto invalid_dest;
6d2010ae 2370 }
fe8ab488
A		 2371 assert(IO_VALID(dest_port));
2372 assert(!IP_VALID(release_port));
1c79356b
A		 2373
2374 /*
fe8ab488
A		 2375 * Copyin the pre-validated reply right.
2376 * It's OK if the reply right has gone dead in the meantime.
1c79356b 2377 */
fe8ab488 2378 if (MACH_PORT_VALID(reply_name)) {
fe8ab488
A		 2379 kr = ipc_right_copyin(space, reply_name, reply_entry,
2380 reply_type, TRUE,
2381 &reply_port, &reply_soright,
3e170ce0 2382 &release_port, &assertcnt);
fe8ab488 2383 assert(assertcnt == 0);
fe8ab488
A		 2384 assert(kr == KERN_SUCCESS);
2385 } else {
2386 /* convert invalid name to equivalent ipc_object type */
2387 reply_port = (ipc_object_t)CAST_MACH_NAME_TO_PORT(reply_name);
2388 }
6d2010ae 2389 }
1c79356b 2390
1c79356b 2391 /*
fe8ab488
A		 2392 * Finally can copyin the voucher right now that dest and reply
2393 * are fully copied in (guaranteed success).
1c79356b 2394 */
fe8ab488 2395 if (IE_NULL != voucher_entry) {
fe8ab488
A		 2396 kr = ipc_right_copyin(space, voucher_name, voucher_entry,
2397 voucher_type, FALSE,
2398 (ipc_object_t *)&voucher_port,
2399 &voucher_soright,
2400 &voucher_release_port,
3e170ce0 2401 &assertcnt);
fe8ab488 2402 assert(assertcnt == 0);
fe8ab488
A		 2403 assert(KERN_SUCCESS == kr);
2404 assert(IP_VALID(voucher_port));
2405 assert(ip_active(voucher_port));
6d2010ae 2406 }
fe8ab488 2407 }
1c79356b 2408
fe8ab488
A		 2409 /* the entry(s) might need to be deallocated */
2410 assert(IE_NULL != dest_entry);
2411 if (IE_BITS_TYPE(dest_entry->ie_bits) == MACH_PORT_TYPE_NONE) {
2412 ipc_entry_dealloc(space, dest_name, dest_entry);
2413 dest_entry = IE_NULL;
2414 }
2415 if (dest_entry != reply_entry && IE_NULL != reply_entry &&
2416 IE_BITS_TYPE(reply_entry->ie_bits) == MACH_PORT_TYPE_NONE) {
2417 ipc_entry_dealloc(space, reply_name, reply_entry);
2418 reply_entry = IE_NULL;
2419 }
2420 if (dest_entry != voucher_entry && IE_NULL != voucher_entry &&
2421 IE_BITS_TYPE(voucher_entry->ie_bits) == MACH_PORT_TYPE_NONE) {
2422 ipc_entry_dealloc(space, voucher_name, voucher_entry);
2423 voucher_entry = IE_NULL;
2424 }
2425
6d2010ae
A		 2426 dest_type = ipc_object_copyin_type(dest_type);
2427 reply_type = ipc_object_copyin_type(reply_type);
2428
1c79356b 2429 /*
6d2010ae
A		 2430 * JMM - Without rdar://problem/6275821, this is the last place we can
2431 * re-arm the send-possible notifications. It may trigger unexpectedly
39236c6e
A		 2432 * early (send may NOT have failed), but better than missing. We assure
2433 * we won't miss by forcing MACH_SEND_ALWAYS if we got past arming.
1c79356b 2434 */
39236c6e
A		 2435 if (((*optionp & MACH_SEND_NOTIFY) != 0) &&
2436 dest_type != MACH_MSG_TYPE_PORT_SEND_ONCE &&
6d2010ae
A		 2437 dest_entry != IE_NULL && dest_entry->ie_request != IE_REQ_NONE) {
2438 ipc_port_t dport = (ipc_port_t)dest_port;
2439
2440 assert(dport != IP_NULL);
2441 ip_lock(dport);
39236c6e
A		 2442 if (ip_active(dport) && dport->ip_receiver != ipc_space_kernel) {
2443 if (ip_full(dport)) {
2444#if IMPORTANCE_INHERITANCE
2445 needboost = ipc_port_request_sparm(dport, dest_name,
39037602
A		 2446 dest_entry->ie_request,
2447 *optionp,
2448 override);
39236c6e
A		 2449 if (needboost == FALSE)
2450 ip_unlock(dport);
2451#else
39037602
A		 2452 ipc_port_request_sparm(dport, dest_name,
2453 dest_entry->ie_request,
2454 *optionp,
2455 override);
39236c6e
A		 2456 ip_unlock(dport);
2457#endif /* IMPORTANCE_INHERITANCE */
2458 } else {
2459 *optionp |= MACH_SEND_ALWAYS;
2460 ip_unlock(dport);
2461 }
2462 } else {
2463 ip_unlock(dport);
6d2010ae 2464 }
1c79356b
A		 2465 }
2466
2467 is_write_unlock(space);
2468
39236c6e
A		 2469#if IMPORTANCE_INHERITANCE
2470 /*
2471 * If our request is the first boosting send-possible
2472 * notification this cycle, push the boost down the
2473 * destination port.
2474 */
2475 if (needboost == TRUE) {
2476 ipc_port_t dport = (ipc_port_t)dest_port;
2477
2478 /* dport still locked from above */
4bd07ac2 2479 if (ipc_port_importance_delta(dport, IPID_OPTION_SENDPOSSIBLE, 1) == FALSE) {
39236c6e 2480 ip_unlock(dport);
fe8ab488 2481 }
39236c6e
A		 2482 }
2483#endif /* IMPORTANCE_INHERITANCE */
2484
fe8ab488 2485 if (dest_soright != IP_NULL) {
1c79356b 2486 ipc_notify_port_deleted(dest_soright, dest_name);
fe8ab488
A		 2487 }
2488 if (reply_soright != IP_NULL) {
1c79356b 2489 ipc_notify_port_deleted(reply_soright, reply_name);
fe8ab488
A		 2490 }
2491 if (voucher_soright != IP_NULL) {
2492 ipc_notify_port_deleted(voucher_soright, voucher_name);
2493 }
39037602
A		 2494
2495 /*
2496 * No room to store voucher port in in-kernel msg header,
2497 * so we store it back in the kmsg itself. Extract the
2498 * qos, and apply any override before we enqueue the kmsg.
2499 */
2500 if (IP_VALID(voucher_port)) {
2501
2502 kmsg->ikm_voucher = voucher_port;
2503 voucher_type = MACH_MSG_TYPE_MOVE_SEND;
2504 }
2505
2506 /* capture the qos value(s) for the kmsg */
2507 ipc_kmsg_set_qos(kmsg, *optionp, override);
2508
fe8ab488 2509 msg->msgh_bits = MACH_MSGH_BITS_SET(dest_type, reply_type, voucher_type, mbits);
1c79356b
A		 2510 msg->msgh_remote_port = (ipc_port_t)dest_port;
2511 msg->msgh_local_port = (ipc_port_t)reply_port;
2512
316670eb
A		 2513 if (release_port != IP_NULL)
2514 ip_release(release_port);
2515
fe8ab488
A		 2516 if (voucher_release_port != IP_NULL)
2517 ip_release(voucher_release_port);
39236c6e 2518
1c79356b
A		 2519 return MACH_MSG_SUCCESS;
2520
2521invalid_reply:
2522 is_write_unlock(space);
316670eb 2523
316670eb
A		 2524 if (release_port != IP_NULL)
2525 ip_release(release_port);
2526
fe8ab488
A		 2527 assert(voucher_port == IP_NULL);
2528 assert(voucher_soright == IP_NULL);
2529
1c79356b
A		 2530 return MACH_SEND_INVALID_REPLY;
2531
2532invalid_dest:
2533 is_write_unlock(space);
316670eb 2534
316670eb
A		 2535 if (release_port != IP_NULL)
2536 ip_release(release_port);
2537
1c79356b
A		 2538 if (reply_soright != IP_NULL)
2539 ipc_notify_port_deleted(reply_soright, reply_name);
316670eb 2540
fe8ab488
A		 2541 assert(voucher_port == IP_NULL);
2542 assert(voucher_soright == IP_NULL);
2543
1c79356b
A		 2544 return MACH_SEND_INVALID_DEST;
2545}
2546
b0d623f7
A		 2547mach_msg_descriptor_t *ipc_kmsg_copyin_port_descriptor(
2548 volatile mach_msg_port_descriptor_t *dsc,
2549 mach_msg_legacy_port_descriptor_t *user_dsc,
2550 ipc_space_t space,
2551 ipc_object_t dest,
2552 ipc_kmsg_t kmsg,
2553 mach_msg_return_t *mr);
2554
2555void ipc_print_type_name(
2556 int type_name);
2557mach_msg_descriptor_t *
2558ipc_kmsg_copyin_port_descriptor(
2559 volatile mach_msg_port_descriptor_t *dsc,
2560 mach_msg_legacy_port_descriptor_t *user_dsc_in,
2561 ipc_space_t space,
2562 ipc_object_t dest,
2563 ipc_kmsg_t kmsg,
2564 mach_msg_return_t *mr)
2565{
2566 volatile mach_msg_legacy_port_descriptor_t *user_dsc = user_dsc_in;
2567 mach_msg_type_name_t user_disp;
2568 mach_msg_type_name_t result_disp;
2569 mach_port_name_t name;
2570 ipc_object_t object;
2571
2572 user_disp = user_dsc->disposition;
2573 result_disp = ipc_object_copyin_type(user_disp);
2574
2575 name = (mach_port_name_t)user_dsc->name;
2576 if (MACH_PORT_VALID(name)) {
2577
2578 kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object);
2579 if (kr != KERN_SUCCESS) {
2580 *mr = MACH_SEND_INVALID_RIGHT;
2581 return NULL;
2582 }
2583
2584 if ((result_disp == MACH_MSG_TYPE_PORT_RECEIVE) &&
2585 ipc_port_check_circularity((ipc_port_t) object,
2586 (ipc_port_t) dest)) {
2587 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
2588 }
2589 dsc->name = (ipc_port_t) object;
2590 } else {
2591 dsc->name = CAST_MACH_NAME_TO_PORT(name);
2592 }
2593 dsc->disposition = result_disp;
2594 dsc->type = MACH_MSG_PORT_DESCRIPTOR;
2595
2596 dsc->pad_end = 0; // debug, unnecessary
2597
2598 return (mach_msg_descriptor_t *)(user_dsc_in+1);
2599}
2600
2601mach_msg_descriptor_t * ipc_kmsg_copyin_ool_descriptor(
2602 mach_msg_ool_descriptor_t *dsc,
2603 mach_msg_descriptor_t *user_dsc,
2604 int is_64bit,
2605 vm_offset_t *paddr,
2606 vm_map_copy_t *copy,
2607 vm_size_t *space_needed,
2608 vm_map_t map,
2609 mach_msg_return_t *mr);
2610mach_msg_descriptor_t *
2611ipc_kmsg_copyin_ool_descriptor(
2612 mach_msg_ool_descriptor_t *dsc,
2613 mach_msg_descriptor_t *user_dsc,
2614 int is_64bit,
2615 vm_offset_t *paddr,
2616 vm_map_copy_t *copy,
2617 vm_size_t *space_needed,
2618 vm_map_t map,
2619 mach_msg_return_t *mr)
2620{
2621 vm_size_t length;
2622 boolean_t dealloc;
2623 mach_msg_copy_options_t copy_options;
2624 mach_vm_offset_t addr;
2625 mach_msg_descriptor_type_t dsc_type;
2626
2627 if (is_64bit) {
2628 mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
2629
2630 addr = (mach_vm_offset_t) user_ool_dsc->address;
2631 length = user_ool_dsc->size;
2632 dealloc = user_ool_dsc->deallocate;
2633 copy_options = user_ool_dsc->copy;
2634 dsc_type = user_ool_dsc->type;
2635
2636 user_dsc = (typeof(user_dsc))(user_ool_dsc+1);
2637 } else {
2638 mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
2639
2640 addr = CAST_USER_ADDR_T(user_ool_dsc->address);
2641 dealloc = user_ool_dsc->deallocate;
2642 copy_options = user_ool_dsc->copy;
2643 dsc_type = user_ool_dsc->type;
2644 length = user_ool_dsc->size;
2645
2646 user_dsc = (typeof(user_dsc))(user_ool_dsc+1);
2647 }
2648
2649 dsc->size = (mach_msg_size_t)length;
2650 dsc->deallocate = dealloc;
2651 dsc->copy = copy_options;
2652 dsc->type = dsc_type;
2653
2654 if (length == 0) {
2655 dsc->address = NULL;
2656 } else if ((length >= MSG_OOL_SIZE_SMALL) &&
2657 (copy_options == MACH_MSG_PHYSICAL_COPY) && !dealloc) {
2658
2659 /*
2660 * If the request is a physical copy and the source
2661 * is not being deallocated, then allocate space
2662 * in the kernel's pageable ipc copy map and copy
2663 * the data in. The semantics guarantee that the
2664 * data will have been physically copied before
2665 * the send operation terminates. Thus if the data
2666 * is not being deallocated, we must be prepared
2667 * to page if the region is sufficiently large.
2668 */
2669 if (copyin(addr, (char *)*paddr, length)) {
2670 *mr = MACH_SEND_INVALID_MEMORY;
2671 return NULL;
2672 }
2673
2674 /*
2675 * The kernel ipc copy map is marked no_zero_fill.
2676 * If the transfer is not a page multiple, we need
2677 * to zero fill the balance.
2678 */
2679 if (!page_aligned(length)) {
2680 (void) memset((void *) (*paddr + length), 0,
2681 round_page(length) - length);
2682 }
2683 if (vm_map_copyin(ipc_kernel_copy_map, (vm_map_address_t)*paddr,
2684 (vm_map_size_t)length, TRUE, copy) != KERN_SUCCESS) {
2685 *mr = MACH_MSG_VM_KERNEL;
2686 return NULL;
2687 }
2688 dsc->address = (void *)*copy;
2689 *paddr += round_page(length);
2690 *space_needed -= round_page(length);
2691 } else {
2692
2693 /*
2694 * Make a vm_map_copy_t of the of the data. If the
2695 * data is small, this will do an optimized physical
2696 * copy. Otherwise, it will do a virtual copy.
2697 *
2698 * NOTE: A virtual copy is OK if the original is being
2699 * deallocted, even if a physical copy was requested.
2700 */
2701 kern_return_t kr = vm_map_copyin(map, addr,
2702 (vm_map_size_t)length, dealloc, copy);
2703 if (kr != KERN_SUCCESS) {
2704 *mr = (kr == KERN_RESOURCE_SHORTAGE) ?
2705 MACH_MSG_VM_KERNEL :
2706 MACH_SEND_INVALID_MEMORY;
2707 return NULL;
2708 }
2709 dsc->address = (void *)*copy;
2710 }
2711 return user_dsc;
2712}
2713
2714mach_msg_descriptor_t * ipc_kmsg_copyin_ool_ports_descriptor(
2715 mach_msg_ool_ports_descriptor_t *dsc,
2716 mach_msg_descriptor_t *user_dsc,
2717 int is_64bit,
2718 vm_map_t map,
2719 ipc_space_t space,
2720 ipc_object_t dest,
2721 ipc_kmsg_t kmsg,
2722 mach_msg_return_t *mr);
2723mach_msg_descriptor_t *
2724ipc_kmsg_copyin_ool_ports_descriptor(
2725 mach_msg_ool_ports_descriptor_t *dsc,
2726 mach_msg_descriptor_t *user_dsc,
2727 int is_64bit,
2728 vm_map_t map,
2729 ipc_space_t space,
2730 ipc_object_t dest,
2731 ipc_kmsg_t kmsg,
2732 mach_msg_return_t *mr)
2733{
2734 void *data;
2735 ipc_object_t *objects;
2736 unsigned int i;
2737 mach_vm_offset_t addr;
2738 mach_msg_type_name_t user_disp;
2739 mach_msg_type_name_t result_disp;
2740 mach_msg_type_number_t count;
2741 mach_msg_copy_options_t copy_option;
2742 boolean_t deallocate;
2743 mach_msg_descriptor_type_t type;
2744 vm_size_t ports_length, names_length;
2745
2746 if (is_64bit) {
2747 mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
2748
2749 addr = (mach_vm_offset_t)user_ool_dsc->address;
2750 count = user_ool_dsc->count;
2751 deallocate = user_ool_dsc->deallocate;
2752 copy_option = user_ool_dsc->copy;
2753 user_disp = user_ool_dsc->disposition;
2754 type = user_ool_dsc->type;
2755
2756 user_dsc = (typeof(user_dsc))(user_ool_dsc+1);
2757 } else {
2758 mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
2759
2760 addr = CAST_USER_ADDR_T(user_ool_dsc->address);
2761 count = user_ool_dsc->count;
2762 deallocate = user_ool_dsc->deallocate;
2763 copy_option = user_ool_dsc->copy;
2764 user_disp = user_ool_dsc->disposition;
2765 type = user_ool_dsc->type;
2766
2767 user_dsc = (typeof(user_dsc))(user_ool_dsc+1);
2768 }
2769
2770 dsc->deallocate = deallocate;
2771 dsc->copy = copy_option;
2772 dsc->type = type;
2773 dsc->count = count;
2774 dsc->address = NULL; /* for now */
2775
2776 result_disp = ipc_object_copyin_type(user_disp);
2777 dsc->disposition = result_disp;
2778
2779 if (count > (INT_MAX / sizeof(mach_port_t))) {
2780 *mr = MACH_SEND_TOO_LARGE;
2781 return NULL;
2782 }
2783
2784 /* calculate length of data in bytes, rounding up */
2785 ports_length = count * sizeof(mach_port_t);
2786 names_length = count * sizeof(mach_port_name_t);
2787
2788 if (ports_length == 0) {
2789 return user_dsc;
2790 }
2791
2792 data = kalloc(ports_length);
2793
2794 if (data == NULL) {
2795 *mr = MACH_SEND_NO_BUFFER;
2796 return NULL;
2797 }
2798
2799#ifdef __LP64__
2800 mach_port_name_t *names = &((mach_port_name_t *)data)[count];
2801#else
2802 mach_port_name_t *names = ((mach_port_name_t *)data);
2803#endif
2804
2805 if (copyinmap(map, addr, names, names_length) != KERN_SUCCESS) {
2806 kfree(data, ports_length);
2807 *mr = MACH_SEND_INVALID_MEMORY;
2808 return NULL;
2809 }
2810
2811 if (deallocate) {
2812 (void) mach_vm_deallocate(map, addr, (mach_vm_size_t)ports_length);
2813 }
2814
2815 objects = (ipc_object_t *) data;
2816 dsc->address = data;
2817
2818 for ( i = 0; i < count; i++) {
2819 mach_port_name_t name = names[i];
2820 ipc_object_t object;
2821
2822 if (!MACH_PORT_VALID(name)) {
2823 objects[i] = (ipc_object_t)CAST_MACH_NAME_TO_PORT(name);
2824 continue;
2825 }
2826
2827 kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object);
2828
2829 if (kr != KERN_SUCCESS) {
2830 unsigned int j;
2831
2832 for(j = 0; j < i; j++) {
2833 object = objects[j];
2834 if (IPC_OBJECT_VALID(object))
2835 ipc_object_destroy(object, result_disp);
2836 }
2837 kfree(data, ports_length);
2838 dsc->address = NULL;
2839 *mr = MACH_SEND_INVALID_RIGHT;
2840 return NULL;
2841 }
2842
2843 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
2844 ipc_port_check_circularity(
2845 (ipc_port_t) object,
2846 (ipc_port_t) dest))
2847 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
2848
2849 objects[i] = object;
2850 }
2851
2852 return user_dsc;
2853}
2854
1c79356b
A		 2855/*
2856 * Routine: ipc_kmsg_copyin_body
2857 * Purpose:
2858 * "Copy-in" port rights and out-of-line memory
2859 * in the message body.
2860 *
2861 * In all failure cases, the message is left holding
2862 * no rights or memory. However, the message buffer
2863 * is not deallocated. If successful, the message
2864 * contains a valid destination port.
2865 * Conditions:
2866 * Nothing locked.
2867 * Returns:
2868 * MACH_MSG_SUCCESS Successful copyin.
2869 * MACH_SEND_INVALID_MEMORY Can't grab out-of-line memory.
2870 * MACH_SEND_INVALID_RIGHT Can't copyin port right in body.
2871 * MACH_SEND_INVALID_TYPE Bad type specification.
2872 * MACH_SEND_MSG_TOO_SMALL Body is too small for types/data.
2873 * MACH_SEND_INVALID_RT_OOL_SIZE OOL Buffer too large for RT
2874 * MACH_MSG_INVALID_RT_DESCRIPTOR Dealloc and RT are incompatible
2875 */
2876
2877mach_msg_return_t
2878ipc_kmsg_copyin_body(
2879 ipc_kmsg_t kmsg,
2880 ipc_space_t space,
2881 vm_map_t map)
2882{
2883 ipc_object_t dest;
2884 mach_msg_body_t *body;
91447636 2885 mach_msg_descriptor_t *daddr, *naddr;
b0d623f7 2886 mach_msg_descriptor_t *user_addr, *kern_addr;
91447636 2887 mach_msg_type_number_t dsc_count;
b0d623f7 2888 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
91447636 2889 boolean_t complex = FALSE;
1c79356b
A		 2890 vm_size_t space_needed = 0;
2891 vm_offset_t paddr = 0;
1c79356b 2892 vm_map_copy_t copy = VM_MAP_COPY_NULL;
91447636
A		 2893 mach_msg_type_number_t i;
2894 mach_msg_return_t mr = MACH_MSG_SUCCESS;
b0d623f7
A		 2895
2896 vm_size_t descriptor_size = 0;
2897
1c79356b
A		 2898 /*
2899 * Determine if the target is a kernel port.
2900 */
91447636
A		 2901 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
2902 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
2903 naddr = (mach_msg_descriptor_t *) (body + 1);
1c79356b 2904
91447636
A		 2905 dsc_count = body->msgh_descriptor_count;
2906 if (dsc_count == 0)
2907 return MACH_MSG_SUCCESS;
2908
1c79356b
A		 2909 /*
2910 * Make an initial pass to determine kernal VM space requirements for
91447636
A		 2911 * physical copies and possible contraction of the descriptors from
2912 * processes with pointers larger than the kernel's.
1c79356b 2913 */
2d21ac55 2914 daddr = NULL;
91447636 2915 for (i = 0; i < dsc_count; i++) {
fe8ab488
A		 2916 mach_msg_size_t size;
2917
91447636
A		 2918 daddr = naddr;
2919
2920 /* make sure the descriptor fits in the message */
b0d623f7 2921 if (is_task_64bit) {
91447636
A		 2922 switch (daddr->type.type) {
2923 case MACH_MSG_OOL_DESCRIPTOR:
2924 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
2925 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
b0d623f7
A		 2926 descriptor_size += 16;
2927 naddr = (typeof(naddr))((vm_offset_t)daddr + 16);
2928 break;
91447636 2929 default:
b0d623f7
A		 2930 descriptor_size += 12;
2931 naddr = (typeof(naddr))((vm_offset_t)daddr + 12);
2932 break;
91447636 2933 }
91447636 2934 } else {
b0d623f7
A		 2935 descriptor_size += 12;
2936 naddr = (typeof(naddr))((vm_offset_t)daddr + 12);
91447636
A		 2937 }
2938
2939 if (naddr > (mach_msg_descriptor_t *)
2940 ((vm_offset_t)kmsg->ikm_header + kmsg->ikm_header->msgh_size)) {
2d21ac55 2941 ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0);
91447636
A		 2942 mr = MACH_SEND_MSG_TOO_SMALL;
2943 goto out;
2944 }
1c79356b 2945
91447636 2946 switch (daddr->type.type) {
91447636
A		 2947 case MACH_MSG_OOL_DESCRIPTOR:
2948 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
b0d623f7
A		 2949 size = (is_task_64bit) ?
2950 ((mach_msg_ool_descriptor64_t *)daddr)->size :
91447636
A		 2951 daddr->out_of_line.size;
2952
2953 if (daddr->out_of_line.copy != MACH_MSG_PHYSICAL_COPY &&
2954 daddr->out_of_line.copy != MACH_MSG_VIRTUAL_COPY) {
2955 /*
2956 * Invalid copy option
2957 */
2d21ac55 2958 ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0);
91447636
A		 2959 mr = MACH_SEND_INVALID_TYPE;
2960 goto out;
2961 }
2962
2963 if ((size >= MSG_OOL_SIZE_SMALL) &&
2964 (daddr->out_of_line.copy == MACH_MSG_PHYSICAL_COPY) &&
2965 !(daddr->out_of_line.deallocate)) {
2966
2967 /*
2968 * Out-of-line memory descriptor, accumulate kernel
2969 * memory requirements
2970 */
39236c6e
A		 2971 if (space_needed + round_page(size) <= space_needed) {
2972 /* Overflow dectected */
2973 ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0);
2974 mr = MACH_MSG_VM_KERNEL;
2975 goto out;
2976 }
2977
91447636
A		 2978 space_needed += round_page(size);
2979 if (space_needed > ipc_kmsg_max_vm_space) {
2980
1c79356b 2981 /*
91447636 2982 * Per message kernel memory limit exceeded
1c79356b 2983 */
2d21ac55 2984 ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0);
91447636
A		 2985 mr = MACH_MSG_VM_KERNEL;
2986 goto out;
1c79356b 2987 }
91447636 2988 }
1c79356b
A		 2989 }
2990 }
2991
2992 /*
2993 * Allocate space in the pageable kernel ipc copy map for all the
2994 * ool data that is to be physically copied. Map is marked wait for
2995 * space.
2996 */
2997 if (space_needed) {
b0d623f7 2998 if (vm_allocate(ipc_kernel_copy_map, &paddr, space_needed,
3e170ce0 2999 VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IPC)) != KERN_SUCCESS) {
b0d623f7
A		 3000 ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0);
3001 mr = MACH_MSG_VM_KERNEL;
3002 goto out;
3003 }
1c79356b
A		 3004 }
3005
b0d623f7
A		 3006 /* user_addr = just after base as it was copied in */
3007 user_addr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t));
fe8ab488
A		 3008
3009 /* Shift the mach_msg_base_t down to make room for dsc_count*16bytes of descriptors */
b0d623f7
A		 3010 if(descriptor_size != 16*dsc_count) {
3011 vm_offset_t dsc_adjust = 16*dsc_count - descriptor_size;
fe8ab488 3012
b0d623f7
A		 3013 memmove((char *)(((vm_offset_t)kmsg->ikm_header) - dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t));
3014 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header - dsc_adjust);
fe8ab488 3015
b0d623f7
A		 3016 /* Update the message size for the larger in-kernel representation */
3017 kmsg->ikm_header->msgh_size += (mach_msg_size_t)dsc_adjust;
3018 }
1c79356b 3019
1c79356b 3020
b0d623f7
A		 3021 /* kern_addr = just after base after it has been (conditionally) moved */
3022 kern_addr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t));
3023
3024 /* handle the OOL regions and port descriptors. */
3025 for(i=0;i<dsc_count;i++) {
3026 switch (user_addr->type.type) {
3027 case MACH_MSG_PORT_DESCRIPTOR:
3028 user_addr = ipc_kmsg_copyin_port_descriptor((mach_msg_port_descriptor_t *)kern_addr,
3029 (mach_msg_legacy_port_descriptor_t *)user_addr, space, dest, kmsg, &mr);
3030 kern_addr++;
3031 complex = TRUE;
3032 break;
3033 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3034 case MACH_MSG_OOL_DESCRIPTOR:
3035 user_addr = ipc_kmsg_copyin_ool_descriptor((mach_msg_ool_descriptor_t *)kern_addr,
3036 user_addr, is_task_64bit, &paddr, &copy, &space_needed, map, &mr);
3037 kern_addr++;
3038 complex = TRUE;
3039 break;
3040 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
3041 user_addr = ipc_kmsg_copyin_ool_ports_descriptor((mach_msg_ool_ports_descriptor_t *)kern_addr,
3042 user_addr, is_task_64bit, map, space, dest, kmsg, &mr);
3043 kern_addr++;
3044 complex = TRUE;
3045 break;
3046 default:
3047 /* Invalid descriptor */
3048 mr = MACH_SEND_INVALID_TYPE;
3049 break;
3050 }
91447636 3051
b0d623f7
A		 3052 if (MACH_MSG_SUCCESS != mr) {
3053 /* clean from start of message descriptors to i */
3054 ipc_kmsg_clean_partial(kmsg, i,
3055 (mach_msg_descriptor_t *)((mach_msg_base_t *)kmsg->ikm_header + 1),
3056 paddr, space_needed);
3057 goto out;
3058 }
3059 } /* End of loop */
1c79356b 3060
91447636
A		 3061 if (!complex) {
3062 kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_COMPLEX;
3063 }
91447636 3064 out:
91447636 3065 return mr;
1c79356b
A		 3066}
3067
3068
3069/*
3070 * Routine: ipc_kmsg_copyin
3071 * Purpose:
3072 * "Copy-in" port rights and out-of-line memory
3073 * in the message.
3074 *
3075 * In all failure cases, the message is left holding
3076 * no rights or memory. However, the message buffer
3077 * is not deallocated. If successful, the message
3078 * contains a valid destination port.
3079 * Conditions:
3080 * Nothing locked.
3081 * Returns:
3082 * MACH_MSG_SUCCESS Successful copyin.
3083 * MACH_SEND_INVALID_HEADER
3084 * Illegal value in the message header bits.
1c79356b
A		 3085 * MACH_SEND_INVALID_DEST Can't copyin destination port.
3086 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
3087 * MACH_SEND_INVALID_MEMORY Can't grab out-of-line memory.
3088 * MACH_SEND_INVALID_RIGHT Can't copyin port right in body.
3089 * MACH_SEND_INVALID_TYPE Bad type specification.
3090 * MACH_SEND_MSG_TOO_SMALL Body is too small for types/data.
3091 */
3092
3093mach_msg_return_t
3094ipc_kmsg_copyin(
3095 ipc_kmsg_t kmsg,
3096 ipc_space_t space,
3097 vm_map_t map,
39037602 3098 mach_msg_priority_t override,
39236c6e 3099 mach_msg_option_t *optionp)
1c79356b
A		 3100{
3101 mach_msg_return_t mr;
39236c6e
A		 3102
3103 kmsg->ikm_header->msgh_bits &= MACH_MSGH_BITS_USER;
3104
39037602 3105 mr = ipc_kmsg_copyin_header(kmsg, space, override, optionp);
39236c6e 3106
1c79356b
A		 3107 if (mr != MACH_MSG_SUCCESS)
3108 return mr;
fe8ab488
A		 3109
3110 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_MSG_SEND) | DBG_FUNC_NONE,
3111 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3112 (uintptr_t)kmsg->ikm_header->msgh_bits,
3113 (uintptr_t)kmsg->ikm_header->msgh_id,
3114 VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(kmsg->ikm_voucher)),
3115 0);
3116
3117 DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_copyin header:\n%.8x\n%.8x\n%p\n%p\n%p\n%.8x\n",
3118 kmsg->ikm_header->msgh_size,
3119 kmsg->ikm_header->msgh_bits,
3120 kmsg->ikm_header->msgh_remote_port,
3121 kmsg->ikm_header->msgh_local_port,
3122 kmsg->ikm_voucher,
3123 kmsg->ikm_header->msgh_id);
b0d623f7 3124
91447636 3125 if ((kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) == 0)
1c79356b
A		 3126 return MACH_MSG_SUCCESS;
3127
b0d623f7
A		 3128 mr = ipc_kmsg_copyin_body( kmsg, space, map);
3129
fe8ab488
A		 3130 /* unreachable if !DEBUG */
3131 __unreachable_ok_push
b0d623f7
A		 3132 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK))
3133 {
3134 kprintf("body:\n");
3135 uint32_t i;
3136 for(i=0;i*4 < (kmsg->ikm_header->msgh_size - sizeof(mach_msg_header_t));i++)
3137 {
3138 kprintf("%.4x\n",((uint32_t *)(kmsg->ikm_header + 1))[i]);
3139 }
3140 }
fe8ab488 3141 __unreachable_ok_pop
39236c6e 3142
b0d623f7 3143 return mr;
1c79356b
A		 3144}
3145
3146/*
3147 * Routine: ipc_kmsg_copyin_from_kernel
3148 * Purpose:
3149 * "Copy-in" port rights and out-of-line memory
3150 * in a message sent from the kernel.
3151 *
3152 * Because the message comes from the kernel,
3153 * the implementation assumes there are no errors
3154 * or peculiarities in the message.
1c79356b
A		 3155 * Conditions:
3156 * Nothing locked.
3157 */
3158
6d2010ae 3159mach_msg_return_t
1c79356b
A		 3160ipc_kmsg_copyin_from_kernel(
3161 ipc_kmsg_t kmsg)
3162{
91447636 3163 mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits;
1c79356b
A		 3164 mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits);
3165 mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits);
91447636
A		 3166 ipc_object_t remote = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
3167 ipc_object_t local = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
1c79356b
A		 3168
3169 /* translate the destination and reply ports */
6d2010ae
A		 3170 if (!IO_VALID(remote))
3171 return MACH_SEND_INVALID_DEST;
1c79356b
A		 3172
3173 ipc_object_copyin_from_kernel(remote, rname);
3174 if (IO_VALID(local))
3175 ipc_object_copyin_from_kernel(local, lname);
3176
3177 /*
3178 * The common case is a complex message with no reply port,
3179 * because that is what the memory_object interface uses.
3180 */
3181
3182 if (bits == (MACH_MSGH_BITS_COMPLEX |
3183 MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) {
3184 bits = (MACH_MSGH_BITS_COMPLEX |
3185 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0));
3186
91447636 3187 kmsg->ikm_header->msgh_bits = bits;
1c79356b
A		 3188 } else {
3189 bits = (MACH_MSGH_BITS_OTHER(bits) |
3190 MACH_MSGH_BITS(ipc_object_copyin_type(rname),
3191 ipc_object_copyin_type(lname)));
3192
91447636 3193 kmsg->ikm_header->msgh_bits = bits;
1c79356b 3194 if ((bits & MACH_MSGH_BITS_COMPLEX) == 0)
6d2010ae 3195 return MACH_MSG_SUCCESS;
1c79356b
A		 3196 }
3197 {
b0d623f7 3198 mach_msg_descriptor_t *saddr;
1c79356b 3199 mach_msg_body_t *body;
b0d623f7 3200 mach_msg_type_number_t i, count;
1c79356b 3201
91447636 3202 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
1c79356b 3203 saddr = (mach_msg_descriptor_t *) (body + 1);
b0d623f7 3204 count = body->msgh_descriptor_count;
1c79356b 3205
b0d623f7 3206 for (i = 0; i < count; i++, saddr++) {
1c79356b
A		 3207
3208 switch (saddr->type.type) {
3209
3210 case MACH_MSG_PORT_DESCRIPTOR: {
3211 mach_msg_type_name_t name;
3212 ipc_object_t object;
3213 mach_msg_port_descriptor_t *dsc;
3214
3215 dsc = &saddr->port;
3216
3217 /* this is really the type SEND, SEND_ONCE, etc. */
3218 name = dsc->disposition;
3219 object = (ipc_object_t) dsc->name;
3220 dsc->disposition = ipc_object_copyin_type(name);
3221
3222 if (!IO_VALID(object)) {
3223 break;
3224 }
3225
3226 ipc_object_copyin_from_kernel(object, name);
3227
3228 /* CDY avoid circularity when the destination is also */
3229 /* the kernel. This check should be changed into an */
3230 /* assert when the new kobject model is in place since*/
3231 /* ports will not be used in kernel to kernel chats */
3232
3233 if (((ipc_port_t)remote)->ip_receiver != ipc_space_kernel) {
3234 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3235 ipc_port_check_circularity((ipc_port_t) object,
3236 (ipc_port_t) remote)) {
91447636 3237 kmsg->ikm_header->msgh_bits |=
1c79356b
A		 3238 MACH_MSGH_BITS_CIRCULAR;
3239 }
3240 }
3241 break;
3242 }
3243 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3244 case MACH_MSG_OOL_DESCRIPTOR: {
3245 /*
3246 * The sender should supply ready-made memory, i.e.
3247 * a vm_map_copy_t, so we don't need to do anything.
3248 */
3249 break;
3250 }
3251 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
3252 ipc_object_t *objects;
91447636 3253 unsigned int j;
1c79356b
A		 3254 mach_msg_type_name_t name;
3255 mach_msg_ool_ports_descriptor_t *dsc;
3256
b0d623f7 3257 dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports;
1c79356b
A		 3258
3259 /* this is really the type SEND, SEND_ONCE, etc. */
3260 name = dsc->disposition;
3261 dsc->disposition = ipc_object_copyin_type(name);
3262
3263 objects = (ipc_object_t *) dsc->address;
3264
3265 for ( j = 0; j < dsc->count; j++) {
3266 ipc_object_t object = objects[j];
3267
3268 if (!IO_VALID(object))
3269 continue;
3270
3271 ipc_object_copyin_from_kernel(object, name);
3272
3273 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3274 ipc_port_check_circularity(
3275 (ipc_port_t) object,
3276 (ipc_port_t) remote))
91447636 3277 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
1c79356b
A		 3278 }
3279 break;
3280 }
3281 default: {
3282#if MACH_ASSERT
3283 panic("ipc_kmsg_copyin_from_kernel: bad descriptor");
3284#endif /* MACH_ASSERT */
3285 }
3286 }
3287 }
3288 }
6d2010ae 3289 return MACH_MSG_SUCCESS;
1c79356b
A		 3290}
3291
b0d623f7 3292#if IKM_SUPPORT_LEGACY
6d2010ae 3293mach_msg_return_t
b0d623f7
A		 3294ipc_kmsg_copyin_from_kernel_legacy(
3295 ipc_kmsg_t kmsg)
3296{
3297 mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits;
3298 mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits);
3299 mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits);
3300 ipc_object_t remote = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
3301 ipc_object_t local = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
3302
3303 /* translate the destination and reply ports */
6d2010ae
A		 3304 if (!IO_VALID(remote))
3305 return MACH_SEND_INVALID_DEST;
b0d623f7
A		 3306
3307 ipc_object_copyin_from_kernel(remote, rname);
3308 if (IO_VALID(local))
3309 ipc_object_copyin_from_kernel(local, lname);
3310
3311 /*
3312 * The common case is a complex message with no reply port,
3313 * because that is what the memory_object interface uses.
3314 */
3315
3316 if (bits == (MACH_MSGH_BITS_COMPLEX |
3317 MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) {
3318 bits = (MACH_MSGH_BITS_COMPLEX |
3319 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0));
3320
3321 kmsg->ikm_header->msgh_bits = bits;
3322 } else {
3323 bits = (MACH_MSGH_BITS_OTHER(bits) |
3324 MACH_MSGH_BITS(ipc_object_copyin_type(rname),
3325 ipc_object_copyin_type(lname)));
3326
3327 kmsg->ikm_header->msgh_bits = bits;
3328 if ((bits & MACH_MSGH_BITS_COMPLEX) == 0)
6d2010ae 3329 return MACH_MSG_SUCCESS;
b0d623f7
A		 3330 }
3331 {
3332 mach_msg_legacy_descriptor_t *saddr;
3333 mach_msg_descriptor_t *daddr;
3334 mach_msg_body_t *body;
3335 mach_msg_type_number_t i, count;
3336
3337 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
3338 saddr = (typeof(saddr)) (body + 1);
3339 count = body->msgh_descriptor_count;
3340
3341 if(count) {
3342 vm_offset_t dsc_adjust = 4*count;
3343 memmove((char *)(((vm_offset_t)kmsg->ikm_header) - dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t));
3344 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header - dsc_adjust);
3345 /* Update the message size for the larger in-kernel representation */
3346 kmsg->ikm_header->msgh_size += dsc_adjust;
3347 }
3348 daddr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t));
3349
3350 for (i = 0; i < count; i++, saddr++, daddr++) {
3351 switch (saddr->type.type) {
3352
3353 case MACH_MSG_PORT_DESCRIPTOR: {
3354 mach_msg_type_name_t name;
3355 ipc_object_t object;
3356 mach_msg_legacy_port_descriptor_t *dsc;
3357 mach_msg_port_descriptor_t *dest_dsc;
3358
3359 dsc = (typeof(dsc))&saddr->port;
3360 dest_dsc = &daddr->port;
3361
3362 /* this is really the type SEND, SEND_ONCE, etc. */
3363 name = dsc->disposition;
3364 object = (ipc_object_t) CAST_MACH_NAME_TO_PORT(dsc->name);
3365 dest_dsc->disposition = ipc_object_copyin_type(name);
3366 dest_dsc->name = (mach_port_t)object;
3367 dest_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
3368
3369 if (!IO_VALID(object)) {
3370 break;
3371 }
3372
3373 ipc_object_copyin_from_kernel(object, name);
3374
3375 /* CDY avoid circularity when the destination is also */
3376 /* the kernel. This check should be changed into an */
3377 /* assert when the new kobject model is in place since*/
3378 /* ports will not be used in kernel to kernel chats */
3379
3380 if (((ipc_port_t)remote)->ip_receiver != ipc_space_kernel) {
3381 if ((dest_dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3382 ipc_port_check_circularity((ipc_port_t) object,
3383 (ipc_port_t) remote)) {
3384 kmsg->ikm_header->msgh_bits |=
3385 MACH_MSGH_BITS_CIRCULAR;
3386 }
3387 }
3388 break;
3389 }
3390 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
3391 case MACH_MSG_OOL_DESCRIPTOR: {
3392 /* The sender should supply ready-made memory, i.e. a vm_map_copy_t
3393 * so we don't need to do anything special. */
3394
3395 mach_msg_ool_descriptor32_t *source_dsc = &saddr->out_of_line32;
3396 mach_msg_ool_descriptor_t *dest_dsc = (typeof(dest_dsc))&daddr->out_of_line;
3397
3398 vm_offset_t address = source_dsc->address;
3399 vm_size_t size = source_dsc->size;
3400 boolean_t deallocate = source_dsc->deallocate;
3401 mach_msg_copy_options_t copy = source_dsc->copy;
3402 mach_msg_descriptor_type_t type = source_dsc->type;
3403
3404 dest_dsc->address = (void *)address;
3405 dest_dsc->size = size;
3406 dest_dsc->deallocate = deallocate;
3407 dest_dsc->copy = copy;
3408 dest_dsc->type = type;
3409 break;
3410 }
3411 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
3412 ipc_object_t *objects;
3413 unsigned int j;
3414 mach_msg_type_name_t name;
3415 mach_msg_ool_ports_descriptor_t *dest_dsc;
3416
3417 mach_msg_ool_ports_descriptor32_t *source_dsc = &saddr->ool_ports32;
3418 dest_dsc = (typeof(dest_dsc))&daddr->ool_ports;
3419
3420 boolean_t deallocate = source_dsc->deallocate;
3421 mach_msg_copy_options_t copy = source_dsc->copy;
3422 mach_msg_size_t port_count = source_dsc->count;
3423 mach_msg_type_name_t disposition = source_dsc->disposition;
3424
3425 /* this is really the type SEND, SEND_ONCE, etc. */
3426 name = disposition;
3427 disposition = ipc_object_copyin_type(name);
3428
3429 objects = (ipc_object_t *) (uintptr_t)source_dsc->address;
3430
3431 for ( j = 0; j < port_count; j++) {
3432 ipc_object_t object = objects[j];
3433
3434 if (!IO_VALID(object))
3435 continue;
3436
3437 ipc_object_copyin_from_kernel(object, name);
3438
3439 if ((disposition == MACH_MSG_TYPE_PORT_RECEIVE) &&
3440 ipc_port_check_circularity(
3441 (ipc_port_t) object,
3442 (ipc_port_t) remote))
3443 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;
3444 }
3445
3446 dest_dsc->address = objects;
3447 dest_dsc->deallocate = deallocate;
3448 dest_dsc->copy = copy;
3449 dest_dsc->disposition = disposition;
3450 dest_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
3451 dest_dsc->count = port_count;
3452 break;
3453 }
3454 default: {
3455#if MACH_ASSERT
3456 panic("ipc_kmsg_copyin_from_kernel: bad descriptor");
3457#endif /* MACH_ASSERT */
3458 }
3459 }
3460 }
3461 }
6d2010ae 3462 return MACH_MSG_SUCCESS;
b0d623f7
A		 3463}
3464#endif /* IKM_SUPPORT_LEGACY */
3465
1c79356b
A		 3466/*
3467 * Routine: ipc_kmsg_copyout_header
3468 * Purpose:
3469 * "Copy-out" port rights in the header of a message.
3470 * Operates atomically; if it doesn't succeed the
3471 * message header and the space are left untouched.
3472 * If it does succeed the remote/local port fields
3473 * contain port names instead of object pointers,
3474 * and the bits field is updated.
1c79356b
A		 3475 * Conditions:
3476 * Nothing locked.
3477 * Returns:
3478 * MACH_MSG_SUCCESS Copied out port rights.
3479 * MACH_RCV_INVALID_NOTIFY
3480 * Notify is non-null and doesn't name a receive right.
3481 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
3482 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE
3483 * The space is dead.
3484 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE
3485 * No room in space for another name.
3486 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_KERNEL
3487 * Couldn't allocate memory for the reply port.
3488 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_KERNEL
3489 * Couldn't allocate memory for the dead-name request.
3490 */
3491
3492mach_msg_return_t
3493ipc_kmsg_copyout_header(
fe8ab488
A		 3494 ipc_kmsg_t kmsg,
3495 ipc_space_t space,
3496 mach_msg_option_t option)
1c79356b 3497{
fe8ab488 3498 mach_msg_header_t *msg = kmsg->ikm_header;
1c79356b
A		 3499 mach_msg_bits_t mbits = msg->msgh_bits;
3500 ipc_port_t dest = (ipc_port_t) msg->msgh_remote_port;
3501
3502 assert(IP_VALID(dest));
3503
6d2010ae
A		 3504 /*
3505 * While we still hold a reference on the received-from port,
3506 * process all send-possible notfications we received along with
3507 * the message.
3508 */
3509 ipc_port_spnotify(dest);
3510
1c79356b
A		 3511 {
3512 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
3513 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
fe8ab488
A		 3514 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
3515 ipc_port_t reply = msg->msgh_local_port;
3516 ipc_port_t release_reply_port = IP_NULL;
1c79356b
A		 3517 mach_port_name_t dest_name, reply_name;
3518
fe8ab488
A		 3519 ipc_port_t voucher = kmsg->ikm_voucher;
3520 ipc_port_t release_voucher_port = IP_NULL;
3521 mach_port_name_t voucher_name;
3522
3523 uint32_t entries_held = 0;
3524 boolean_t need_write_lock = FALSE;
3525 kern_return_t kr;
3526
3527 /*
3528 * Reserve any potentially needed entries in the target space.
3529 * We'll free any unused before unlocking the space.
3530 */
1c79356b 3531 if (IP_VALID(reply)) {
fe8ab488
A		 3532 entries_held++;
3533 need_write_lock = TRUE;
3534 }
3535 if (IP_VALID(voucher)) {
3536 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
1c79356b 3537
fe8ab488
A		 3538 if ((option & MACH_RCV_VOUCHER) != 0)
3539 entries_held++;
3540 need_write_lock = TRUE;
3541 }
3542
3543 if (need_write_lock) {
1c79356b
A		 3544
3545 is_write_lock(space);
3546
fe8ab488 3547 while(entries_held) {
316670eb 3548 if (!is_active(space)) {
1c79356b
A		 3549 is_write_unlock(space);
3550 return (MACH_RCV_HEADER_ERROR|
3551 MACH_MSG_IPC_SPACE);
3552 }
fe8ab488
A		 3553
3554 kr = ipc_entries_hold(space, entries_held);
3555 if (KERN_SUCCESS == kr)
1c79356b 3556 break;
1c79356b 3557
fe8ab488
A		 3558 kr = ipc_entry_grow_table(space, ITS_SIZE_NONE);
3559 if (KERN_SUCCESS != kr)
3560 return(MACH_RCV_HEADER_ERROR|
3561 MACH_MSG_IPC_SPACE);
3562 /* space was unlocked and relocked - retry */
3563 }
1c79356b 3564
fe8ab488
A		 3565 /* Handle reply port. */
3566 if (IP_VALID(reply)) {
3567 ipc_entry_t entry;
1c79356b 3568
fe8ab488
A		 3569 /* Is there already an entry we can use? */
3570 if ((reply_type != MACH_MSG_TYPE_PORT_SEND_ONCE) &&
3571 ipc_right_reverse(space, (ipc_object_t) reply, &reply_name, &entry)) {
3572 /* reply port is locked and active */
3573 assert(entry->ie_bits & MACH_PORT_TYPE_SEND_RECEIVE);
3574 } else {
3575 ip_lock(reply);
3576 if (!ip_active(reply)) {
3577 ip_unlock(reply);
3578
3579 release_reply_port = reply;
3580 reply = IP_DEAD;
3581 reply_name = MACH_PORT_DEAD;
3582 goto done_with_reply;
1c79356b 3583 }
fe8ab488
A		 3584
3585 /* claim a held entry for the reply port */
3586 assert(entries_held > 0);
3587 entries_held--;
3588 ipc_entry_claim(space, &reply_name, &entry);
3589 assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE);
3590 assert(entry->ie_object == IO_NULL);
3591 entry->ie_object = (ipc_object_t) reply;
1c79356b 3592 }
1c79356b 3593
fe8ab488
A		 3594 /* space and reply port are locked and active */
3595 ip_reference(reply); /* hold onto the reply port */
3596
3597 kr = ipc_right_copyout(space, reply_name, entry,
3598 reply_type, TRUE, (ipc_object_t) reply);
3599 assert(kr == KERN_SUCCESS);
3600 /* reply port is unlocked */
3601 } else
3602 reply_name = CAST_MACH_PORT_TO_NAME(reply);
1c79356b 3603
fe8ab488 3604 done_with_reply:
1c79356b 3605
fe8ab488
A		 3606 /* Handle voucher port. */
3607 if (voucher_type != MACH_MSGH_BITS_ZERO) {
3608 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
1c79356b 3609
fe8ab488
A		 3610 if (!IP_VALID(voucher)) {
3611 if ((option & MACH_RCV_VOUCHER) == 0) {
3612 voucher_type = MACH_MSGH_BITS_ZERO;
3613 }
3614 voucher_name = MACH_PORT_NULL;
3615 goto done_with_voucher;
3616 }
3617
3618 /* clear voucher from its hiding place back in the kmsg */
3619 kmsg->ikm_voucher = IP_NULL;
3620
3621 if ((option & MACH_RCV_VOUCHER) != 0) {
3622 ipc_entry_t entry;
3623
3624 if (ipc_right_reverse(space, (ipc_object_t) voucher,
3625 &voucher_name, &entry)) {
3626 /* voucher port locked */
3627 assert(entry->ie_bits & MACH_PORT_TYPE_SEND);
3628 } else {
3629 assert(entries_held > 0);
3630 entries_held--;
3631 ipc_entry_claim(space, &voucher_name, &entry);
3632 assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE);
3633 assert(entry->ie_object == IO_NULL);
3634 entry->ie_object = (ipc_object_t) voucher;
3635 ip_lock(voucher);
3636 }
3637 /* space is locked and active */
3638
3639 assert(ip_active(voucher));
3640 assert(ip_kotype(voucher) == IKOT_VOUCHER);
3641 kr = ipc_right_copyout(space, voucher_name, entry,
3642 MACH_MSG_TYPE_MOVE_SEND, TRUE,
3643 (ipc_object_t) voucher);
3644 /* voucher port is unlocked */
3645 } else {
3646 voucher_type = MACH_MSGH_BITS_ZERO;
3647 release_voucher_port = voucher;
3648 voucher_name = MACH_PORT_NULL;
3649 }
3650 } else {
3651 voucher_name = msg->msgh_voucher_port;
3652 }
39236c6e 3653
fe8ab488 3654 done_with_voucher:
1c79356b 3655
1c79356b
A		 3656 ip_lock(dest);
3657 is_write_unlock(space);
39236c6e 3658
1c79356b
A		 3659 } else {
3660 /*
fe8ab488 3661 * No reply or voucher port! This is an easy case.
1c79356b 3662 * We only need to have the space locked
6d2010ae 3663 * when locking the destination.
1c79356b
A		 3664 */
3665
3666 is_read_lock(space);
316670eb 3667 if (!is_active(space)) {
1c79356b
A		 3668 is_read_unlock(space);
3669 return MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE;
3670 }
3671
1c79356b
A		 3672 ip_lock(dest);
3673 is_read_unlock(space);
3674
b0d623f7 3675 reply_name = CAST_MACH_PORT_TO_NAME(reply);
fe8ab488
A		 3676
3677 if (voucher_type != MACH_MSGH_BITS_ZERO) {
3678 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
3679 if ((option & MACH_RCV_VOUCHER) == 0) {
3680 voucher_type = MACH_MSGH_BITS_ZERO;
3681 }
3682 voucher_name = MACH_PORT_NULL;
3683 } else {
3684 voucher_name = msg->msgh_voucher_port;
3685 }
1c79356b
A		 3686 }
3687
3688 /*
3689 * At this point, the space is unlocked and the destination
3690 * port is locked. (Lock taken while space was locked.)
3691 * reply_name is taken care of; we still need dest_name.
3692 * We still hold a ref for reply (if it is valid).
3693 *
3694 * If the space holds receive rights for the destination,
3695 * we return its name for the right. Otherwise the task
3696 * managed to destroy or give away the receive right between
3697 * receiving the message and this copyout. If the destination
3698 * is dead, return MACH_PORT_DEAD, and if the receive right
3699 * exists somewhere else (another space, in transit)
3700 * return MACH_PORT_NULL.
3701 *
3702 * Making this copyout operation atomic with the previous
3703 * copyout of the reply port is a bit tricky. If there was
3704 * no real reply port (it wasn't IP_VALID) then this isn't
3705 * an issue. If the reply port was dead at copyout time,
3706 * then we are OK, because if dest is dead we serialize
3707 * after the death of both ports and if dest is alive
3708 * we serialize after reply died but before dest's (later) death.
3709 * So assume reply was alive when we copied it out. If dest
3710 * is alive, then we are OK because we serialize before
3711 * the ports' deaths. So assume dest is dead when we look at it.
3712 * If reply dies/died after dest, then we are OK because
3713 * we serialize after dest died but before reply dies.
3714 * So the hard case is when reply is alive at copyout,
3715 * dest is dead at copyout, and reply died before dest died.
3716 * In this case pretend that dest is still alive, so
3717 * we serialize while both ports are alive.
3718 *
3719 * Because the space lock is held across the copyout of reply
3720 * and locking dest, the receive right for dest can't move
3721 * in or out of the space while the copyouts happen, so
3722 * that isn't an atomicity problem. In the last hard case
3723 * above, this implies that when dest is dead that the
3724 * space couldn't have had receive rights for dest at
3725 * the time reply was copied-out, so when we pretend
3726 * that dest is still alive, we can return MACH_PORT_NULL.
3727 *
3728 * If dest == reply, then we have to make it look like
3729 * either both copyouts happened before the port died,
3730 * or both happened after the port died. This special
3731 * case works naturally if the timestamp comparison
3732 * is done correctly.
3733 */
3734
1c79356b
A		 3735 if (ip_active(dest)) {
3736 ipc_object_copyout_dest(space, (ipc_object_t) dest,
3737 dest_type, &dest_name);
3738 /* dest is unlocked */
39236c6e 3739
1c79356b
A		 3740 } else {
3741 ipc_port_timestamp_t timestamp;
3742
3743 timestamp = dest->ip_timestamp;
316670eb 3744 ip_unlock(dest);
1c79356b 3745 ip_release(dest);
1c79356b
A		 3746
3747 if (IP_VALID(reply)) {
3748 ip_lock(reply);
3749 if (ip_active(reply) ||
3750 IP_TIMESTAMP_ORDER(timestamp,
3751 reply->ip_timestamp))
3752 dest_name = MACH_PORT_DEAD;
3753 else
3754 dest_name = MACH_PORT_NULL;
3755 ip_unlock(reply);
3756 } else
3757 dest_name = MACH_PORT_DEAD;
3758 }
3759
3760 if (IP_VALID(reply))
316670eb
A		 3761 ip_release(reply);
3762
fe8ab488
A		 3763 if (IP_VALID(release_reply_port)) {
3764 if (reply_type == MACH_MSG_TYPE_PORT_SEND_ONCE)
3765 ipc_port_release_sonce(release_reply_port);
3766 else
3767 ipc_port_release_send(release_reply_port);
3768 }
3769
3770 if (IP_VALID(release_voucher_port))
3771 ipc_port_release_send(release_voucher_port);
3772
1c79356b 3773
fe8ab488
A		 3774 if ((option & MACH_RCV_VOUCHER) != 0) {
3775 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV) | DBG_FUNC_NONE,
3776 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3777 (uintptr_t)kmsg->ikm_header->msgh_bits,
3778 (uintptr_t)kmsg->ikm_header->msgh_id,
3779 VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(voucher)),
3780 0);
3781 } else {
3782 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV_VOUCHER_REFUSED) | DBG_FUNC_NONE,
3783 VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
3784 (uintptr_t)kmsg->ikm_header->msgh_bits,
3785 (uintptr_t)kmsg->ikm_header->msgh_id,
3786 VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(voucher)),
3787 0);
3788 }
3789
3790 msg->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type,
3791 voucher_type, mbits);
b0d623f7
A		 3792 msg->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
3793 msg->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
fe8ab488 3794 msg->msgh_voucher_port = voucher_name;
1c79356b
A		 3795 }
3796
fe8ab488 3797 return MACH_MSG_SUCCESS;
1c79356b
A		 3798}
3799
3800/*
3801 * Routine: ipc_kmsg_copyout_object
3802 * Purpose:
3803 * Copy-out a port right. Always returns a name,
3804 * even for unsuccessful return codes. Always
3805 * consumes the supplied object.
3806 * Conditions:
3807 * Nothing locked.
3808 * Returns:
3809 * MACH_MSG_SUCCESS The space acquired the right
3810 * (name is valid) or the object is dead (MACH_PORT_DEAD).
3811 * MACH_MSG_IPC_SPACE No room in space for the right,
3812 * or the space is dead. (Name is MACH_PORT_NULL.)
3813 * MACH_MSG_IPC_KERNEL Kernel resource shortage.
3814 * (Name is MACH_PORT_NULL.)
3815 */
3816
3817mach_msg_return_t
3818ipc_kmsg_copyout_object(
3819 ipc_space_t space,
3820 ipc_object_t object,
3821 mach_msg_type_name_t msgt_name,
3822 mach_port_name_t *namep)
3823{
3824 kern_return_t kr;
3825
3826 if (!IO_VALID(object)) {
b0d623f7 3827 *namep = CAST_MACH_PORT_TO_NAME(object);
1c79356b
A		 3828 return MACH_MSG_SUCCESS;
3829 }
3830
3831 kr = ipc_object_copyout(space, object, msgt_name, TRUE, namep);
3832 if (kr != KERN_SUCCESS) {
3833 ipc_object_destroy(object, msgt_name);
3834
3835 if (kr == KERN_INVALID_CAPABILITY)
3836 *namep = MACH_PORT_DEAD;
3837 else {
3838 *namep = MACH_PORT_NULL;
3839
3840 if (kr == KERN_RESOURCE_SHORTAGE)
3841 return MACH_MSG_IPC_KERNEL;
3842 else
3843 return MACH_MSG_IPC_SPACE;
3844 }
3845 }
3846
3847 return MACH_MSG_SUCCESS;
3848}
3849
b0d623f7
A		 3850mach_msg_descriptor_t *
3851ipc_kmsg_copyout_port_descriptor(mach_msg_descriptor_t *dsc,
3852 mach_msg_descriptor_t *user_dsc,
3853 ipc_space_t space,
3854 kern_return_t *mr);
3855mach_msg_descriptor_t *
3856ipc_kmsg_copyout_port_descriptor(mach_msg_descriptor_t *dsc,
3857 mach_msg_descriptor_t *dest_dsc,
3858 ipc_space_t space,
3859 kern_return_t *mr)
3860{
3861 mach_port_t port;
3862 mach_port_name_t name;
3863 mach_msg_type_name_t disp;
3864
3865
3866 /* Copyout port right carried in the message */
3867 port = dsc->port.name;
3868 disp = dsc->port.disposition;
3869 *mr |= ipc_kmsg_copyout_object(space,
3870 (ipc_object_t)port,
3871 disp,
3872 &name);
3873
3874 if(current_task() == kernel_task)
3875 {
3876 mach_msg_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc;
3877 user_dsc--; // point to the start of this port descriptor
3878 user_dsc->name = CAST_MACH_NAME_TO_PORT(name);
3879 user_dsc->disposition = disp;
3880 user_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
3881 dest_dsc = (typeof(dest_dsc))user_dsc;
3882 } else {
3883 mach_msg_legacy_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc;
3884 user_dsc--; // point to the start of this port descriptor
3885 user_dsc->name = CAST_MACH_PORT_TO_NAME(name);
3886 user_dsc->disposition = disp;
3887 user_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
3888 dest_dsc = (typeof(dest_dsc))user_dsc;
3889 }
3890
3891 return (mach_msg_descriptor_t *)dest_dsc;
3892}
3893
3894mach_msg_descriptor_t *
3895ipc_kmsg_copyout_ool_descriptor(mach_msg_ool_descriptor_t *dsc, mach_msg_descriptor_t *user_dsc, int is_64bit, vm_map_t map, mach_msg_return_t *mr);
3896mach_msg_descriptor_t *
3897ipc_kmsg_copyout_ool_descriptor(mach_msg_ool_descriptor_t *dsc, mach_msg_descriptor_t *user_dsc, int is_64bit, vm_map_t map, mach_msg_return_t *mr)
3898{
3899 vm_map_copy_t copy;
316670eb 3900 vm_map_address_t rcv_addr;
b0d623f7 3901 mach_msg_copy_options_t copy_options;
39037602 3902 vm_map_size_t size;
b0d623f7
A		 3903 mach_msg_descriptor_type_t dsc_type;
3904
3905 //SKIP_PORT_DESCRIPTORS(saddr, sdsc_count);
3906
39037602
A		 3907 copy = (vm_map_copy_t)dsc->address;
3908 size = (vm_map_size_t)dsc->size;
b0d623f7
A		 3909 copy_options = dsc->copy;
3910 assert(copy_options != MACH_MSG_KALLOC_COPY_T);
3911 dsc_type = dsc->type;
b0d623f7
A		 3912
3913 if (copy != VM_MAP_COPY_NULL) {
3e170ce0 3914 kern_return_t kr;
b0d623f7 3915
3e170ce0 3916 rcv_addr = 0;
39037602 3917 if (vm_map_copy_validate_size(map, copy, &size) == FALSE)
2dced7af 3918 panic("Inconsistent OOL/copyout size on %p: expected %d, got %lld @%p",
39037602
A		 3919 dsc, dsc->size, (unsigned long long)copy->size, copy);
3920 kr = vm_map_copyout_size(map, &rcv_addr, copy, size);
b0d623f7
A		 3921 if (kr != KERN_SUCCESS) {
3922 if (kr == KERN_RESOURCE_SHORTAGE)
3923 *mr |= MACH_MSG_VM_KERNEL;
3924 else
3925 *mr |= MACH_MSG_VM_SPACE;
3926 vm_map_copy_discard(copy);
3927 rcv_addr = 0;
3928 size = 0;
3929 }
3930 } else {
3931 rcv_addr = 0;
3932 size = 0;
3933 }
3934
3935 /*
3936 * Now update the descriptor as the user would see it.
3937 * This may require expanding the descriptor to the user
3938 * visible size. There is already space allocated for
3939 * this in what naddr points to.
3940 */
3941 if(current_task() == kernel_task)
3942 {
3943 mach_msg_ool_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
3944 user_ool_dsc--;
3945
3946 user_ool_dsc->address = (void *)(uintptr_t)rcv_addr;
3947 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
3948 TRUE : FALSE;
3949 user_ool_dsc->copy = copy_options;
3950 user_ool_dsc->type = dsc_type;
39037602 3951 user_ool_dsc->size = (mach_msg_size_t)size;
b0d623f7
A		 3952
3953 user_dsc = (typeof(user_dsc))user_ool_dsc;
3954 } else if (is_64bit) {
3955 mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
3956 user_ool_dsc--;
3957
3958 user_ool_dsc->address = rcv_addr;
3959 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
3960 TRUE : FALSE;
3961 user_ool_dsc->copy = copy_options;
3962 user_ool_dsc->type = dsc_type;
39037602 3963 user_ool_dsc->size = (mach_msg_size_t)size;
b0d623f7
A		 3964
3965 user_dsc = (typeof(user_dsc))user_ool_dsc;
3966 } else {
3967 mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
3968 user_ool_dsc--;
3969
3970 user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr);
39037602 3971 user_ool_dsc->size = (mach_msg_size_t)size;
b0d623f7
A		 3972 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
3973 TRUE : FALSE;
3974 user_ool_dsc->copy = copy_options;
3975 user_ool_dsc->type = dsc_type;
3976
3977 user_dsc = (typeof(user_dsc))user_ool_dsc;
3978 }
3979 return user_dsc;
3980}
3981
3982mach_msg_descriptor_t *
3983ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc,
3984 mach_msg_descriptor_t *user_dsc,
3985 int is_64bit,
3986 vm_map_t map,
3987 ipc_space_t space,
3988 ipc_kmsg_t kmsg,
3989 mach_msg_return_t *mr);
3990mach_msg_descriptor_t *
3991ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc,
3992 mach_msg_descriptor_t *user_dsc,
3993 int is_64bit,
3994 vm_map_t map,
3995 ipc_space_t space,
3996 ipc_kmsg_t kmsg,
3997 mach_msg_return_t *mr)
3998{
39236c6e 3999 mach_vm_offset_t rcv_addr = 0;
b0d623f7
A		 4000 mach_msg_type_name_t disp;
4001 mach_msg_type_number_t count, i;
4002 vm_size_t ports_length, names_length;
4003
4004 mach_msg_copy_options_t copy_options = MACH_MSG_VIRTUAL_COPY;
4005
4006 //SKIP_PORT_DESCRIPTORS(saddr, sdsc_count);
4007
4008 count = dsc->count;
4009 disp = dsc->disposition;
4010 ports_length = count * sizeof(mach_port_t);
4011 names_length = count * sizeof(mach_port_name_t);
4012
4013 if (ports_length != 0 && dsc->address != 0) {
4014
4015 /*
4016 * Check to see if there is an overwrite descriptor
4017 * specified in the scatter list for this ool data.
4018 * The descriptor has already been verified.
4019 */
4020#if 0
4021 if (saddr != MACH_MSG_DESCRIPTOR_NULL) {
4022 if (differs) {
4023 OTHER_OOL_DESCRIPTOR *scatter_dsc;
4024
4025 scatter_dsc = (OTHER_OOL_DESCRIPTOR *)saddr;
4026 rcv_addr = (mach_vm_offset_t) scatter_dsc->address;
4027 copy_options = scatter_dsc->copy;
4028 } else {
4029 mach_msg_ool_descriptor_t *scatter_dsc;
4030
4031 scatter_dsc = &saddr->out_of_line;
4032 rcv_addr = CAST_USER_ADDR_T(scatter_dsc->address);
4033 copy_options = scatter_dsc->copy;
4034 }
4035 INCREMENT_SCATTER(saddr, sdsc_count, differs);
4036 }
4037#endif
4038
4039 if (copy_options == MACH_MSG_VIRTUAL_COPY) {
4040 /*
4041 * Dynamically allocate the region
4042 */
3e170ce0
A		 4043 int anywhere = VM_FLAGS_ANYWHERE;
4044 if (vm_kernel_map_is_kernel(map)) anywhere |= VM_MAKE_TAG(VM_KERN_MEMORY_IPC);
4045 else anywhere |= VM_MAKE_TAG(VM_MEMORY_MACH_MSG);
b0d623f7
A		 4046
4047 kern_return_t kr;
4048 if ((kr = mach_vm_allocate(map, &rcv_addr,
4049 (mach_vm_size_t)names_length,
4050 anywhere)) != KERN_SUCCESS) {
4051 ipc_kmsg_clean_body(kmsg, 1, (mach_msg_descriptor_t *)dsc);
4052 rcv_addr = 0;
4053
4054 if (kr == KERN_RESOURCE_SHORTAGE){
4055 *mr |= MACH_MSG_VM_KERNEL;
4056 } else {
4057 *mr |= MACH_MSG_VM_SPACE;
4058 }
4059 }
4060 }
4061
4062 /*
4063 * Handle the port rights and copy out the names
4064 * for those rights out to user-space.
4065 */
4066 if (rcv_addr != 0) {
4067 mach_port_t *objects = (mach_port_t *) dsc->address;
4068 mach_port_name_t *names = (mach_port_name_t *) dsc->address;
4069
4070 /* copyout port rights carried in the message */
4071
4072 for ( i = 0; i < count ; i++) {
4073 ipc_object_t object = (ipc_object_t)objects[i];
4074
4075 *mr |= ipc_kmsg_copyout_object(space, object,
4076 disp, &names[i]);
4077 }
4078
4079 /* copyout to memory allocated above */
4080 void *data = dsc->address;
4081 if (copyoutmap(map, data, rcv_addr, names_length) != KERN_SUCCESS)
4082 *mr |= MACH_MSG_VM_SPACE;
4083 kfree(data, ports_length);
4084 }
4085 } else {
4086 rcv_addr = 0;
4087 }
4088
4089 /*
4090 * Now update the descriptor based on the information
4091 * calculated above.
4092 */
4093 if(current_task() == kernel_task) {
4094 mach_msg_ool_ports_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4095 user_ool_dsc--;
4096
4097 user_ool_dsc->address = (void *)(uintptr_t)rcv_addr;
4098 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4099 TRUE : FALSE;
4100 user_ool_dsc->copy = copy_options;
4101 user_ool_dsc->disposition = disp;
4102 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
4103 user_ool_dsc->count = count;
4104
4105 user_dsc = (typeof(user_dsc))user_ool_dsc;
4106 } if (is_64bit) {
4107 mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4108 user_ool_dsc--;
4109
4110 user_ool_dsc->address = rcv_addr;
4111 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4112 TRUE : FALSE;
4113 user_ool_dsc->copy = copy_options;
4114 user_ool_dsc->disposition = disp;
4115 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
4116 user_ool_dsc->count = count;
4117
4118 user_dsc = (typeof(user_dsc))user_ool_dsc;
4119 } else {
4120 mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc;
4121 user_ool_dsc--;
4122
4123 user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr);
4124 user_ool_dsc->count = count;
4125 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ?
4126 TRUE : FALSE;
4127 user_ool_dsc->copy = copy_options;
4128 user_ool_dsc->disposition = disp;
4129 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR;
4130
4131 user_dsc = (typeof(user_dsc))user_ool_dsc;
4132 }
4133 return user_dsc;
4134}
4135
1c79356b
A		 4136/*
4137 * Routine: ipc_kmsg_copyout_body
4138 * Purpose:
4139 * "Copy-out" port rights and out-of-line memory
4140 * in the body of a message.
4141 *
4142 * The error codes are a combination of special bits.
4143 * The copyout proceeds despite errors.
4144 * Conditions:
4145 * Nothing locked.
4146 * Returns:
4147 * MACH_MSG_SUCCESS Successful copyout.
4148 * MACH_MSG_IPC_SPACE No room for port right in name space.
4149 * MACH_MSG_VM_SPACE No room for memory in address space.
4150 * MACH_MSG_IPC_KERNEL Resource shortage handling port right.
4151 * MACH_MSG_VM_KERNEL Resource shortage handling memory.
4152 * MACH_MSG_INVALID_RT_DESCRIPTOR Descriptor incompatible with RT
4153 */
4154
4155mach_msg_return_t
4156ipc_kmsg_copyout_body(
4157 ipc_kmsg_t kmsg,
4158 ipc_space_t space,
4159 vm_map_t map,
4160 mach_msg_body_t *slist)
4161{
4162 mach_msg_body_t *body;
b0d623f7 4163 mach_msg_descriptor_t *kern_dsc, *user_dsc;
91447636 4164 mach_msg_descriptor_t *saddr;
b0d623f7
A		 4165 mach_msg_type_number_t dsc_count, sdsc_count;
4166 int i;
1c79356b 4167 mach_msg_return_t mr = MACH_MSG_SUCCESS;
b0d623f7 4168 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
1c79356b 4169
91447636
A		 4170 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
4171 dsc_count = body->msgh_descriptor_count;
b0d623f7
A		 4172 kern_dsc = (mach_msg_descriptor_t *) (body + 1);
4173 /* Point user_dsc just after the end of all the descriptors */
4174 user_dsc = &kern_dsc[dsc_count];
1c79356b 4175
b0d623f7 4176 /* Do scatter list setup */
1c79356b 4177 if (slist != MACH_MSG_BODY_NULL) {
b0d623f7 4178 panic("Scatter lists disabled");
91447636
A		 4179 saddr = (mach_msg_descriptor_t *) (slist + 1);
4180 sdsc_count = slist->msgh_descriptor_count;
1c79356b
A		 4181 }
4182 else {
91447636
A		 4183 saddr = MACH_MSG_DESCRIPTOR_NULL;
4184 sdsc_count = 0;
1c79356b
A		 4185 }
4186
b0d623f7
A		 4187 /* Now process the descriptors */
4188 for (i = dsc_count-1; i >= 0; i--) {
4189 switch (kern_dsc[i].type.type) {
4190
4191 case MACH_MSG_PORT_DESCRIPTOR:
4192 user_dsc = ipc_kmsg_copyout_port_descriptor(&kern_dsc[i], user_dsc, space, &mr);
4193 break;
4194 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
4195 case MACH_MSG_OOL_DESCRIPTOR :
4196 user_dsc = ipc_kmsg_copyout_ool_descriptor(
4197 (mach_msg_ool_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, map, &mr);
4198 break;
4199 case MACH_MSG_OOL_PORTS_DESCRIPTOR :
4200 user_dsc = ipc_kmsg_copyout_ool_ports_descriptor(
4201 (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, map, space, kmsg, &mr);
4202 break;
4203 default : {
4204 panic("untyped IPC copyout body: invalid message descriptor");
4205 }
4206 }
91447636
A		 4207 }
4208
b0d623f7
A		 4209 if(user_dsc != kern_dsc) {
4210 vm_offset_t dsc_adjust = (vm_offset_t)user_dsc - (vm_offset_t)kern_dsc;
4211 memmove((char *)((vm_offset_t)kmsg->ikm_header + dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t));
4212 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header + dsc_adjust);
4213 /* Update the message size for the smaller user representation */
4214 kmsg->ikm_header->msgh_size -= (mach_msg_size_t)dsc_adjust;
1c79356b 4215 }
b0d623f7 4216
1c79356b
A		 4217 return mr;
4218}
4219
91447636
A		 4220/*
4221 * Routine: ipc_kmsg_copyout_size
4222 * Purpose:
4223 * Compute the size of the message as copied out to the given
4224 * map. If the destination map's pointers are a different size
4225 * than the kernel's, we have to allow for expansion/
4226 * contraction of the descriptors as appropriate.
4227 * Conditions:
4228 * Nothing locked.
4229 * Returns:
4230 * size of the message as it would be received.
4231 */
4232
4233mach_msg_size_t
4234ipc_kmsg_copyout_size(
4235 ipc_kmsg_t kmsg,
4236 vm_map_t map)
4237{
b0d623f7 4238 mach_msg_size_t send_size;
91447636 4239
b0d623f7 4240 send_size = kmsg->ikm_header->msgh_size;
91447636 4241
b0d623f7 4242 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS);
91447636 4243
b0d623f7
A		 4244#if defined(__LP64__)
4245 send_size -= LEGACY_HEADER_SIZE_DELTA;
4246#endif
91447636 4247
b0d623f7 4248 if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
91447636 4249
b0d623f7
A		 4250 mach_msg_body_t *body;
4251 mach_msg_descriptor_t *saddr, *eaddr;
91447636 4252
b0d623f7
A		 4253 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
4254 saddr = (mach_msg_descriptor_t *) (body + 1);
4255 eaddr = saddr + body->msgh_descriptor_count;
4256
4257 for ( ; saddr < eaddr; saddr++ ) {
4258 switch (saddr->type.type) {
4259 case MACH_MSG_OOL_DESCRIPTOR:
4260 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
4261 case MACH_MSG_OOL_PORTS_DESCRIPTOR:
4262 if(!is_task_64bit)
4263 send_size -= DESC_SIZE_ADJUSTMENT;
4264 break;
4265 case MACH_MSG_PORT_DESCRIPTOR:
4266 send_size -= DESC_SIZE_ADJUSTMENT;
4267 break;
4268 default:
4269 break;
4270 }
4271 }
4272 }
4273 return send_size;
91447636
A		 4274}
4275
1c79356b
A		 4276/*
4277 * Routine: ipc_kmsg_copyout
4278 * Purpose:
4279 * "Copy-out" port rights and out-of-line memory
4280 * in the message.
4281 * Conditions:
4282 * Nothing locked.
4283 * Returns:
4284 * MACH_MSG_SUCCESS Copied out all rights and memory.
1c79356b
A		 4285 * MACH_RCV_HEADER_ERROR + special bits
4286 * Rights and memory in the message are intact.
4287 * MACH_RCV_BODY_ERROR + special bits
4288 * The message header was successfully copied out.
4289 * As much of the body was handled as possible.
4290 */
4291
4292mach_msg_return_t
4293ipc_kmsg_copyout(
4294 ipc_kmsg_t kmsg,
4295 ipc_space_t space,
4296 vm_map_t map,
fe8ab488
A		 4297 mach_msg_body_t *slist,
4298 mach_msg_option_t option)
1c79356b
A		 4299{
4300 mach_msg_return_t mr;
4301
fe8ab488 4302 mr = ipc_kmsg_copyout_header(kmsg, space, option);
2d21ac55 4303 if (mr != MACH_MSG_SUCCESS) {
1c79356b 4304 return mr;
2d21ac55 4305 }
1c79356b 4306
91447636 4307 if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
1c79356b
A		 4308 mr = ipc_kmsg_copyout_body(kmsg, space, map, slist);
4309
4310 if (mr != MACH_MSG_SUCCESS)
4311 mr |= MACH_RCV_BODY_ERROR;
4312 }
4313
4314 return mr;
4315}
4316
4317/*
4318 * Routine: ipc_kmsg_copyout_pseudo
4319 * Purpose:
4320 * Does a pseudo-copyout of the message.
4321 * This is like a regular copyout, except
4322 * that the ports in the header are handled
4323 * as if they are in the body. They aren't reversed.
4324 *
4325 * The error codes are a combination of special bits.
4326 * The copyout proceeds despite errors.
4327 * Conditions:
4328 * Nothing locked.
4329 * Returns:
4330 * MACH_MSG_SUCCESS Successful copyout.
4331 * MACH_MSG_IPC_SPACE No room for port right in name space.
4332 * MACH_MSG_VM_SPACE No room for memory in address space.
4333 * MACH_MSG_IPC_KERNEL Resource shortage handling port right.
4334 * MACH_MSG_VM_KERNEL Resource shortage handling memory.
4335 */
4336
4337mach_msg_return_t
4338ipc_kmsg_copyout_pseudo(
4339 ipc_kmsg_t kmsg,
4340 ipc_space_t space,
4341 vm_map_t map,
4342 mach_msg_body_t *slist)
4343{
91447636
A		 4344 mach_msg_bits_t mbits = kmsg->ikm_header->msgh_bits;
4345 ipc_object_t dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4346 ipc_object_t reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
fe8ab488 4347 ipc_object_t voucher = (ipc_object_t) kmsg->ikm_voucher;
1c79356b
A		 4348 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
4349 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
fe8ab488
A		 4350 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
4351 mach_port_name_t voucher_name = kmsg->ikm_header->msgh_voucher_port;
1c79356b
A		 4352 mach_port_name_t dest_name, reply_name;
4353 mach_msg_return_t mr;
4354
4355 assert(IO_VALID(dest));
4356
fe8ab488
A		 4357#if 0
4358 /*
4359 * If we did this here, it looks like we wouldn't need the undo logic
4360 * at the end of ipc_kmsg_send() in the error cases. Not sure which
4361 * would be more elegant to keep.
4362 */
4363 ipc_importance_clean(kmsg);
4364#else
4365 /* just assert it is already clean */
4366 ipc_importance_assert_clean(kmsg);
4367#endif
4368
1c79356b
A		 4369 mr = (ipc_kmsg_copyout_object(space, dest, dest_type, &dest_name) |
4370 ipc_kmsg_copyout_object(space, reply, reply_type, &reply_name));
4371
39236c6e 4372 kmsg->ikm_header->msgh_bits = mbits & MACH_MSGH_BITS_USER;
b0d623f7
A		 4373 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(dest_name);
4374 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(reply_name);
1c79356b 4375
fe8ab488
A		 4376 if (IO_VALID(voucher)) {
4377 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
4378
4379 kmsg->ikm_voucher = IP_NULL;
4380 mr |= ipc_kmsg_copyout_object(space, voucher, voucher_type, &voucher_name);
4381 kmsg->ikm_header->msgh_voucher_port = voucher_name;
4382 }
4383
1c79356b
A		 4384 if (mbits & MACH_MSGH_BITS_COMPLEX) {
4385 mr |= ipc_kmsg_copyout_body(kmsg, space, map, slist);
4386 }
4387
4388 return mr;
4389}
4390
4391/*
4392 * Routine: ipc_kmsg_copyout_dest
4393 * Purpose:
4394 * Copies out the destination port in the message.
4395 * Destroys all other rights and memory in the message.
4396 * Conditions:
4397 * Nothing locked.
4398 */
4399
4400void
4401ipc_kmsg_copyout_dest(
4402 ipc_kmsg_t kmsg,
4403 ipc_space_t space)
4404{
4405 mach_msg_bits_t mbits;
4406 ipc_object_t dest;
4407 ipc_object_t reply;
fe8ab488 4408 ipc_object_t voucher;
1c79356b
A		 4409 mach_msg_type_name_t dest_type;
4410 mach_msg_type_name_t reply_type;
fe8ab488
A		 4411 mach_msg_type_name_t voucher_type;
4412 mach_port_name_t dest_name, reply_name, voucher_name;
1c79356b 4413
91447636
A		 4414 mbits = kmsg->ikm_header->msgh_bits;
4415 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4416 reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
fe8ab488
A		 4417 voucher = (ipc_object_t) kmsg->ikm_voucher;
4418 voucher_name = kmsg->ikm_header->msgh_voucher_port;
1c79356b
A		 4419 dest_type = MACH_MSGH_BITS_REMOTE(mbits);
4420 reply_type = MACH_MSGH_BITS_LOCAL(mbits);
fe8ab488 4421 voucher_type = MACH_MSGH_BITS_VOUCHER(mbits);
1c79356b
A		 4422
4423 assert(IO_VALID(dest));
4424
fe8ab488
A		 4425 ipc_importance_assert_clean(kmsg);
4426
1c79356b
A		 4427 io_lock(dest);
4428 if (io_active(dest)) {
4429 ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
4430 /* dest is unlocked */
4431 } else {
316670eb 4432 io_unlock(dest);
1c79356b 4433 io_release(dest);
1c79356b
A		 4434 dest_name = MACH_PORT_DEAD;
4435 }
4436
4437 if (IO_VALID(reply)) {
4438 ipc_object_destroy(reply, reply_type);
4439 reply_name = MACH_PORT_NULL;
4440 } else
b0d623f7 4441 reply_name = CAST_MACH_PORT_TO_NAME(reply);
1c79356b 4442
fe8ab488
A		 4443 if (IO_VALID(voucher)) {
4444 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND);
4445
4446 kmsg->ikm_voucher = IP_NULL;
4447 ipc_object_destroy((ipc_object_t)voucher, voucher_type);
4448 voucher_name = MACH_PORT_NULL;
4449 }
4450
4451 kmsg->ikm_header->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type,
4452 voucher_type, mbits);
b0d623f7
A		 4453 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
4454 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
fe8ab488 4455 kmsg->ikm_header->msgh_voucher_port = voucher_name;
1c79356b
A		 4456
4457 if (mbits & MACH_MSGH_BITS_COMPLEX) {
4458 mach_msg_body_t *body;
4459
91447636
A		 4460 body = (mach_msg_body_t *) (kmsg->ikm_header + 1);
4461 ipc_kmsg_clean_body(kmsg, body->msgh_descriptor_count,
4462 (mach_msg_descriptor_t *)(body + 1));
1c79356b
A		 4463 }
4464}
91447636 4465
1c79356b
A		 4466/*
4467 * Routine: ipc_kmsg_copyout_to_kernel
4468 * Purpose:
4469 * Copies out the destination and reply ports in the message.
4470 * Leaves all other rights and memory in the message alone.
4471 * Conditions:
4472 * Nothing locked.
4473 *
4474 * Derived from ipc_kmsg_copyout_dest.
4475 * Use by mach_msg_rpc_from_kernel (which used to use copyout_dest).
4476 * We really do want to save rights and memory.
4477 */
4478
4479void
4480ipc_kmsg_copyout_to_kernel(
4481 ipc_kmsg_t kmsg,
4482 ipc_space_t space)
4483{
4484 ipc_object_t dest;
4485 ipc_object_t reply;
4486 mach_msg_type_name_t dest_type;
4487 mach_msg_type_name_t reply_type;
4488 mach_port_name_t dest_name, reply_name;
4489
91447636
A		 4490 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4491 reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
4492 dest_type = MACH_MSGH_BITS_REMOTE(kmsg->ikm_header->msgh_bits);
4493 reply_type = MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits);
1c79356b
A		 4494
4495 assert(IO_VALID(dest));
4496
4497 io_lock(dest);
4498 if (io_active(dest)) {
4499 ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
4500 /* dest is unlocked */
4501 } else {
316670eb 4502 io_unlock(dest);
1c79356b 4503 io_release(dest);
1c79356b
A		 4504 dest_name = MACH_PORT_DEAD;
4505 }
4506
b0d623f7 4507 reply_name = CAST_MACH_PORT_TO_NAME(reply);
1c79356b 4508
91447636
A		 4509 kmsg->ikm_header->msgh_bits =
4510 (MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) |
1c79356b 4511 MACH_MSGH_BITS(reply_type, dest_type));
b0d623f7
A		 4512 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
4513 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
1c79356b
A		 4514}
4515
b0d623f7
A		 4516#if IKM_SUPPORT_LEGACY
4517void
4518ipc_kmsg_copyout_to_kernel_legacy(
4519 ipc_kmsg_t kmsg,
4520 ipc_space_t space)
4521{
4522 ipc_object_t dest;
4523 ipc_object_t reply;
4524 mach_msg_type_name_t dest_type;
4525 mach_msg_type_name_t reply_type;
4526 mach_port_name_t dest_name, reply_name;
4527
4528 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port;
4529 reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port;
4530 dest_type = MACH_MSGH_BITS_REMOTE(kmsg->ikm_header->msgh_bits);
4531 reply_type = MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits);
4532
4533 assert(IO_VALID(dest));
4534
4535 io_lock(dest);
4536 if (io_active(dest)) {
4537 ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
4538 /* dest is unlocked */
4539 } else {
316670eb 4540 io_unlock(dest);
b0d623f7 4541 io_release(dest);
b0d623f7
A		 4542 dest_name = MACH_PORT_DEAD;
4543 }
4544
4545 reply_name = CAST_MACH_PORT_TO_NAME(reply);
4546
4547 kmsg->ikm_header->msgh_bits =
4548 (MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) |
4549 MACH_MSGH_BITS(reply_type, dest_type));
4550 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name);
4551 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name);
4552
4553 mach_msg_descriptor_t *saddr;
4554 mach_msg_legacy_descriptor_t *daddr;
4555 mach_msg_type_number_t i, count = ((mach_msg_base_t *)kmsg->ikm_header)->body.msgh_descriptor_count;
4556 saddr = (mach_msg_descriptor_t *) (((mach_msg_base_t *)kmsg->ikm_header) + 1);
4557 saddr = &saddr[count-1];
4558 daddr = (mach_msg_legacy_descriptor_t *)&saddr[count];
4559 daddr--;
4560
4561 vm_offset_t dsc_adjust = 0;
4562
4563 for (i = 0; i < count; i++, saddr--, daddr--) {
4564 switch (saddr->type.type) {
4565 case MACH_MSG_PORT_DESCRIPTOR: {
4566 mach_msg_port_descriptor_t *dsc = &saddr->port;
4567 mach_msg_legacy_port_descriptor_t *dest_dsc = &daddr->port;
4568
4569 mach_port_t name = dsc->name;
4570 mach_msg_type_name_t disposition = dsc->disposition;
4571
4572 dest_dsc->name = CAST_MACH_PORT_TO_NAME(name);
4573 dest_dsc->disposition = disposition;
4574 dest_dsc->type = MACH_MSG_PORT_DESCRIPTOR;
4575 break;
4576 }
4577 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR:
4578 case MACH_MSG_OOL_DESCRIPTOR: {
4579 /* The sender should supply ready-made memory, i.e. a vm_map_copy_t
4580 * so we don't need to do anything special. */
4581
4582 mach_msg_ool_descriptor_t *source_dsc = (typeof(source_dsc))&saddr->out_of_line;
4583
4584 mach_msg_ool_descriptor32_t *dest_dsc = &daddr->out_of_line32;
4585
4586 vm_offset_t address = (vm_offset_t)source_dsc->address;
4587 vm_size_t size = source_dsc->size;
4588 boolean_t deallocate = source_dsc->deallocate;
4589 mach_msg_copy_options_t copy = source_dsc->copy;
4590 mach_msg_descriptor_type_t type = source_dsc->type;
4591
4592 dest_dsc->address = address;
4593 dest_dsc->size = size;
4594 dest_dsc->deallocate = deallocate;
4595 dest_dsc->copy = copy;
4596 dest_dsc->type = type;
4597 break;
4598 }
4599 case MACH_MSG_OOL_PORTS_DESCRIPTOR: {
4600 mach_msg_ool_ports_descriptor_t *source_dsc = (typeof(source_dsc))&saddr->ool_ports;
4601
4602 mach_msg_ool_ports_descriptor32_t *dest_dsc = &daddr->ool_ports32;
4603
4604 vm_offset_t address = (vm_offset_t)source_dsc->address;
4605 vm_size_t port_count = source_dsc->count;
4606 boolean_t deallocate = source_dsc->deallocate;
4607 mach_msg_copy_options_t copy = source_dsc->copy;
4608 mach_msg_descriptor_type_t type = source_dsc->type;
4609
4610 dest_dsc->address = address;
4611 dest_dsc->count = port_count;
4612 dest_dsc->deallocate = deallocate;
4613 dest_dsc->copy = copy;
4614 dest_dsc->type = type;
4615 break;
4616 }
4617 default: {
4618#if MACH_ASSERT
4619 panic("ipc_kmsg_copyin_from_kernel: bad descriptor");
4620#endif /* MACH_ASSERT */
4621 }
4622 }
4623 }
4624
4625 if(count) {
4626 dsc_adjust = 4*count;
4627 memmove((char *)((vm_offset_t)kmsg->ikm_header + dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t));
4628 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header + dsc_adjust);
4629 /* Update the message size for the smaller user representation */
4630 kmsg->ikm_header->msgh_size -= dsc_adjust;
4631 }
4632}
4633#endif /* IKM_SUPPORT_LEGACY */
4634
fe8ab488 4635
316670eb 4636mach_msg_trailer_size_t
fe8ab488 4637ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused,
316670eb
A		 4638 mach_msg_option_t option, thread_t thread,
4639 mach_port_seqno_t seqno, boolean_t minimal_trailer,
4640 mach_vm_offset_t context)
1c79356b 4641{
316670eb 4642 mach_msg_max_trailer_t *trailer;
1c79356b 4643
316670eb
A		 4644 (void)thread;
4645 trailer = (mach_msg_max_trailer_t *)
4646 ((vm_offset_t)kmsg->ikm_header +
4647 round_msg(kmsg->ikm_header->msgh_size));
1c79356b 4648
316670eb
A		 4649 if (!(option & MACH_RCV_TRAILER_MASK)) {
4650 return trailer->msgh_trailer_size;
1c79356b 4651 }
1c79356b 4652
316670eb
A		 4653 trailer->msgh_seqno = seqno;
4654 trailer->msgh_context = context;
4655 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(thread_is_64bit(thread), option);
1c79356b 4656
316670eb
A		 4657 if (minimal_trailer) {
4658 goto done;
1c79356b
A		 4659 }
4660
316670eb
A		 4661 if (MACH_RCV_TRAILER_ELEMENTS(option) >=
4662 MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_AV)){
fe8ab488 4663 trailer->msgh_ad = 0;
1c79356b 4664 }
1c79356b 4665
316670eb
A		 4666 /*
4667 * The ipc_kmsg_t holds a reference to the label of a label
4668 * handle, not the port. We must get a reference to the port
4669 * and a send right to copyout to the receiver.
4670 */
1c79356b 4671
316670eb 4672 if (option & MACH_RCV_TRAILER_ELEMENTS (MACH_RCV_TRAILER_LABELS)) {
316670eb 4673 trailer->msgh_labels.sender = 0;
1c79356b 4674 }
1c79356b 4675
316670eb 4676done:
39236c6e 4677
316670eb 4678 return trailer->msgh_trailer_size;
1c79356b 4679}
mirror of XNU