]> git.saurik.com Git - apple/xnu.git/blob - osfmk/ipc/mach_msg.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-1228.3.13.tar.gz
[apple/xnu.git] / osfmk / ipc / mach_msg.c
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 * NOTICE: This file was modified by McAfee Research in 2004 to introduce
58 * support for mandatory and extensible security protections. This notice
59 * is included in support of clause 2.2 (b) of the Apple Public License,
60 * Version 2.0.
61 * Copyright (c) 2005 SPARTA, Inc.
62 */
63 /*
64 */
65 /*
66 * File: ipc/mach_msg.c
67 * Author: Rich Draves
68 * Date: 1989
69 *
70 * Exported message traps. See mach/message.h.
71 */
72
73 #include <mach/mach_types.h>
74 #include <mach/kern_return.h>
75 #include <mach/port.h>
76 #include <mach/message.h>
77 #include <mach/mig_errors.h>
78 #include <mach/mach_traps.h>
79
80 #include <kern/kern_types.h>
81 #include <kern/assert.h>
82 #include <kern/counters.h>
83 #include <kern/cpu_number.h>
84 #include <kern/ipc_kobject.h>
85 #include <kern/ipc_mig.h>
86 #include <kern/task.h>
87 #include <kern/thread.h>
88 #include <kern/lock.h>
89 #include <kern/sched_prim.h>
90 #include <kern/exception.h>
91 #include <kern/misc_protos.h>
92 #include <kern/kalloc.h>
93 #include <kern/processor.h>
94 #include <kern/syscall_subr.h>
95
96 #include <vm/vm_map.h>
97
98 #include <ipc/ipc_types.h>
99 #include <ipc/ipc_kmsg.h>
100 #include <ipc/ipc_mqueue.h>
101 #include <ipc/ipc_object.h>
102 #include <ipc/ipc_notify.h>
103 #include <ipc/ipc_port.h>
104 #include <ipc/ipc_pset.h>
105 #include <ipc/ipc_space.h>
106 #include <ipc/ipc_entry.h>
107
108 #include <machine/machine_routines.h>
109 #include <security/mac_mach_internal.h>
110
111 #include <sys/kdebug.h>
112
113
114 #ifndef offsetof
115 #define offsetof(type, member) ((size_t)(&((type *)0)->member))
116 #endif /* offsetof */
117
118 /*
119 * Forward declarations - kernel internal routines
120 */
121
122 mach_msg_return_t mach_msg_send(
123 mach_msg_header_t *msg,
124 mach_msg_option_t option,
125 mach_msg_size_t send_size,
126 mach_msg_timeout_t send_timeout,
127 mach_port_name_t notify);
128
129 mach_msg_return_t mach_msg_receive(
130 mach_msg_header_t *msg,
131 mach_msg_option_t option,
132 mach_msg_size_t rcv_size,
133 mach_port_name_t rcv_name,
134 mach_msg_timeout_t rcv_timeout,
135 void (*continuation)(mach_msg_return_t),
136 mach_msg_size_t slist_size);
137
138
139 mach_msg_return_t mach_msg_receive_results(void);
140
141 mach_msg_return_t msg_receive_error(
142 ipc_kmsg_t kmsg,
143 mach_vm_address_t msg_addr,
144 mach_msg_option_t option,
145 mach_port_seqno_t seqno,
146 ipc_space_t space);
147
148 security_token_t KERNEL_SECURITY_TOKEN = KERNEL_SECURITY_TOKEN_VALUE;
149 audit_token_t KERNEL_AUDIT_TOKEN = KERNEL_AUDIT_TOKEN_VALUE;
150
151 mach_msg_format_0_trailer_t trailer_template = {
152 /* mach_msg_trailer_type_t */ MACH_MSG_TRAILER_FORMAT_0,
153 /* mach_msg_trailer_size_t */ MACH_MSG_TRAILER_MINIMUM_SIZE,
154 /* mach_port_seqno_t */ 0,
155 /* security_token_t */ KERNEL_SECURITY_TOKEN_VALUE
156 };
157
158 /*
159 * Routine: mach_msg_send
160 * Purpose:
161 * Send a message.
162 * Conditions:
163 * Nothing locked.
164 * Returns:
165 * MACH_MSG_SUCCESS Sent the message.
166 * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
167 * MACH_SEND_NO_BUFFER Couldn't allocate buffer.
168 * MACH_SEND_INVALID_DATA Couldn't copy message data.
169 * MACH_SEND_INVALID_HEADER
170 * Illegal value in the message header bits.
171 * MACH_SEND_INVALID_DEST The space is dead.
172 * MACH_SEND_INVALID_NOTIFY Bad notify port.
173 * MACH_SEND_INVALID_DEST Can't copyin destination port.
174 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
175 * MACH_SEND_TIMED_OUT Timeout expired without delivery.
176 * MACH_SEND_INTERRUPTED Delivery interrupted.
177 * MACH_SEND_NO_NOTIFY Can't allocate a msg-accepted request.
178 * MACH_SEND_WILL_NOTIFY Msg-accepted notif. requested.
179 * MACH_SEND_NOTIFY_IN_PROGRESS
180 * This space has already forced a message to this port.
181 */
182
183 mach_msg_return_t
184 mach_msg_send(
185 mach_msg_header_t *msg,
186 mach_msg_option_t option,
187 mach_msg_size_t send_size,
188 mach_msg_timeout_t send_timeout,
189 mach_port_name_t notify)
190 {
191 ipc_space_t space = current_space();
192 vm_map_t map = current_map();
193 ipc_kmsg_t kmsg;
194 mach_msg_return_t mr;
195 mach_msg_size_t msg_and_trailer_size;
196 mach_msg_max_trailer_t *trailer;
197
198 if ((send_size < sizeof(mach_msg_header_t)) || (send_size & 3))
199 return MACH_SEND_MSG_TOO_SMALL;
200
201 if (send_size > MACH_MSG_SIZE_MAX - MAX_TRAILER_SIZE)
202 return MACH_SEND_TOO_LARGE;
203
204 msg_and_trailer_size = send_size + MAX_TRAILER_SIZE;
205
206 kmsg = ipc_kmsg_alloc(msg_and_trailer_size);
207
208 if (kmsg == IKM_NULL)
209 return MACH_SEND_NO_BUFFER;
210
211 (void) memcpy((void *) kmsg->ikm_header, (const void *) msg, send_size);
212
213 kmsg->ikm_header->msgh_size = send_size;
214
215 /*
216 * reserve for the trailer the largest space (MAX_TRAILER_SIZE)
217 * However, the internal size field of the trailer (msgh_trailer_size)
218 * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to optimize
219 * the cases where no implicit data is requested.
220 */
221 trailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + send_size);
222 trailer->msgh_sender = current_thread()->task->sec_token;
223 trailer->msgh_audit = current_thread()->task->audit_token;
224 trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
225 trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
226
227 if (option & MACH_SEND_CANCEL) {
228 if (notify == MACH_PORT_NULL)
229 mr = MACH_SEND_INVALID_NOTIFY;
230 else
231 mr = ipc_kmsg_copyin(kmsg, space, map, notify);
232 } else
233 mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
234 if (mr != MACH_MSG_SUCCESS) {
235 ipc_kmsg_free(kmsg);
236 return mr;
237 }
238
239 mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, send_timeout);
240
241 if (mr != MACH_MSG_SUCCESS) {
242 mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
243 (void) memcpy((void *) msg, (const void *) kmsg->ikm_header,
244 kmsg->ikm_header->msgh_size);
245 ipc_kmsg_free(kmsg);
246 }
247
248 return mr;
249 }
250
251 /*
252 * Routine: mach_msg_receive
253 * Purpose:
254 * Receive a message.
255 * Conditions:
256 * Nothing locked.
257 * Returns:
258 * MACH_MSG_SUCCESS Received a message.
259 * MACH_RCV_INVALID_NAME The name doesn't denote a right,
260 * or the denoted right is not receive or port set.
261 * MACH_RCV_IN_SET Receive right is a member of a set.
262 * MACH_RCV_TOO_LARGE Message wouldn't fit into buffer.
263 * MACH_RCV_TIMED_OUT Timeout expired without a message.
264 * MACH_RCV_INTERRUPTED Reception interrupted.
265 * MACH_RCV_PORT_DIED Port/set died while receiving.
266 * MACH_RCV_PORT_CHANGED Port moved into set while receiving.
267 * MACH_RCV_INVALID_DATA Couldn't copy to user buffer.
268 * MACH_RCV_INVALID_NOTIFY Bad notify port.
269 * MACH_RCV_HEADER_ERROR
270 */
271
272 mach_msg_return_t
273 mach_msg_receive_results(void)
274 {
275 thread_t self = current_thread();
276 ipc_space_t space = current_space();
277 vm_map_t map = current_map();
278
279 ipc_object_t object = self->ith_object;
280 mach_msg_return_t mr = self->ith_state;
281 mach_vm_address_t msg_addr = self->ith_msg_addr;
282 mach_msg_option_t option = self->ith_option;
283 ipc_kmsg_t kmsg = self->ith_kmsg;
284 mach_port_seqno_t seqno = self->ith_seqno;
285
286 mach_msg_max_trailer_t *trailer;
287
288 ipc_object_release(object);
289
290 if (mr != MACH_MSG_SUCCESS) {
291
292 if (mr == MACH_RCV_TOO_LARGE ) {
293 if (option & MACH_RCV_LARGE) {
294 /*
295 * We need to inform the user-level code that it needs more
296 * space. The value for how much space was returned in the
297 * msize save area instead of the message (which was left on
298 * the queue).
299 */
300 if (copyout((char *) &self->ith_msize,
301 msg_addr + offsetof(mach_msg_header_t, msgh_size),
302 sizeof(mach_msg_size_t)))
303 mr = MACH_RCV_INVALID_DATA;
304 goto out;
305 }
306
307 if (msg_receive_error(kmsg, msg_addr, option, seqno, space)
308 == MACH_RCV_INVALID_DATA)
309 mr = MACH_RCV_INVALID_DATA;
310 }
311 goto out;
312 }
313
314 trailer = (mach_msg_max_trailer_t *)
315 ((vm_offset_t)kmsg->ikm_header +
316 round_msg(kmsg->ikm_header->msgh_size));
317 if (option & MACH_RCV_TRAILER_MASK) {
318 trailer->msgh_seqno = seqno;
319 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
320
321
322 if (option & MACH_RCV_TRAILER_ELEMENTS (MACH_RCV_TRAILER_AV)) {
323 #if CONFIG_MACF_MACH
324 if (kmsg->ikm_sender != NULL &&
325 IP_VALID(kmsg->ikm_header->msgh_remote_port) &&
326 mac_port_check_method(kmsg->ikm_sender,
327 &kmsg->ikm_sender->maclabel,
328 &((ipc_port_t)kmsg->ikm_header->msgh_remote_port)->ip_label,
329 kmsg->ikm_header->msgh_id) == 0)
330 trailer->msgh_ad = 1;
331 else
332 #endif
333 trailer->msgh_ad = 0;
334 }
335
336 /*
337 * The ipc_kmsg_t holds a reference to the label of a label
338 * handle, not the port. We must get a reference to the port
339 * and a send right to copyout to the receiver.
340 */
341
342 if (option & MACH_RCV_TRAILER_ELEMENTS (MACH_RCV_TRAILER_LABELS)) {
343 #if CONFIG_MACF_MACH
344 if (kmsg->ikm_sender != NULL) {
345 ipc_labelh_t lh = kmsg->ikm_sender->label;
346 kern_return_t kr;
347
348 ip_lock(lh->lh_port);
349 lh->lh_port->ip_mscount++;
350 lh->lh_port->ip_srights++;
351 ip_reference(lh->lh_port);
352 ip_unlock(lh->lh_port);
353
354 kr = ipc_object_copyout(space, (ipc_object_t)lh->lh_port,
355 MACH_MSG_TYPE_PORT_SEND, 0,
356 &trailer->msgh_labels.sender);
357 if (kr != KERN_SUCCESS) {
358 ip_lock(lh->lh_port);
359 ip_release(lh->lh_port);
360 ip_check_unlock(lh->lh_port);
361
362 trailer->msgh_labels.sender = 0;
363 }
364 } else {
365 trailer->msgh_labels.sender = 0;
366 }
367 #else
368 trailer->msgh_labels.sender = 0;
369 #endif
370 }
371 }
372
373 /*
374 * If MACH_RCV_OVERWRITE was specified, try to get the scatter
375 * list and verify it against the contents of the message. If
376 * there is any problem with it, we will continue without it as
377 * normal.
378 */
379 if (option & MACH_RCV_OVERWRITE) {
380 mach_msg_size_t slist_size = self->ith_scatter_list_size;
381 mach_msg_body_t *slist;
382
383 slist = ipc_kmsg_get_scatter(msg_addr, slist_size, kmsg);
384 mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL, slist);
385 ipc_kmsg_free_scatter(slist, slist_size);
386 } else {
387 mr = ipc_kmsg_copyout(kmsg, space, map,
388 MACH_PORT_NULL, MACH_MSG_BODY_NULL);
389 }
390
391 if (mr != MACH_MSG_SUCCESS) {
392 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
393 if (ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size +
394 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
395 mr = MACH_RCV_INVALID_DATA;
396 }
397 else {
398 if (msg_receive_error(kmsg, msg_addr, option, seqno, space)
399 == MACH_RCV_INVALID_DATA)
400 mr = MACH_RCV_INVALID_DATA;
401 }
402 goto out;
403 }
404 mr = ipc_kmsg_put(msg_addr,
405 kmsg,
406 kmsg->ikm_header->msgh_size +
407 trailer->msgh_trailer_size);
408 out:
409 return mr;
410 }
411
412 mach_msg_return_t
413 mach_msg_receive(
414 mach_msg_header_t *msg,
415 mach_msg_option_t option,
416 mach_msg_size_t rcv_size,
417 mach_port_name_t rcv_name,
418 mach_msg_timeout_t rcv_timeout,
419 void (*continuation)(mach_msg_return_t),
420 mach_msg_size_t slist_size)
421 {
422 thread_t self = current_thread();
423 ipc_space_t space = current_space();
424 ipc_object_t object;
425 ipc_mqueue_t mqueue;
426 mach_msg_return_t mr;
427
428 mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
429 if (mr != MACH_MSG_SUCCESS) {
430 return mr;
431 }
432 /* hold ref for object */
433
434 self->ith_msg_addr = CAST_DOWN(mach_vm_address_t, msg);
435 self->ith_object = object;
436 self->ith_msize = rcv_size;
437 self->ith_option = option;
438 self->ith_scatter_list_size = slist_size;
439 self->ith_continuation = continuation;
440
441 ipc_mqueue_receive(mqueue, option, rcv_size, rcv_timeout, THREAD_ABORTSAFE);
442 if ((option & MACH_RCV_TIMEOUT) && rcv_timeout == 0)
443 thread_poll_yield(self);
444 return mach_msg_receive_results();
445 }
446
447 void
448 mach_msg_receive_continue(void)
449 {
450 thread_t self = current_thread();
451
452 (*self->ith_continuation)(mach_msg_receive_results());
453 }
454
455 /*
456 * Toggle this to compile the hotpath in/out
457 * If compiled in, the run-time toggle "enable_hotpath" below
458 * eases testing & debugging
459 */
460 #define ENABLE_HOTPATH 1 /* Hacked on for now */
461
462 #if ENABLE_HOTPATH
463 /*
464 * These counters allow tracing of hotpath behavior under test loads.
465 * A couple key counters are unconditional (see below).
466 */
467 #define HOTPATH_DEBUG 0 /* Toggle to include lots of counters */
468 #if HOTPATH_DEBUG
469 #define HOT(expr) expr
470
471 unsigned int c_mmot_FIRST = 0; /* Unused First Counter */
472 unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
473 unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
474 unsigned int c_mmot_kernel_send = 0; /* kernel server */
475 unsigned int c_mmot_cold_000 = 0; /* see below ... */
476 unsigned int c_mmot_smallsendsize = 0;
477 unsigned int c_mmot_oddsendsize = 0;
478 unsigned int c_mmot_bigsendsize = 0;
479 unsigned int c_mmot_copyinmsg_fail = 0;
480 unsigned int c_mmot_g_slow_copyin3 = 0;
481 unsigned int c_mmot_cold_006 = 0;
482 unsigned int c_mmot_cold_007 = 0;
483 unsigned int c_mmot_cold_008 = 0;
484 unsigned int c_mmot_cold_009 = 0;
485 unsigned int c_mmot_cold_010 = 0;
486 unsigned int c_mmot_cold_012 = 0;
487 unsigned int c_mmot_cold_013 = 0;
488 unsigned int c_mmot_cold_014 = 0;
489 unsigned int c_mmot_cold_016 = 0;
490 unsigned int c_mmot_cold_018 = 0;
491 unsigned int c_mmot_cold_019 = 0;
492 unsigned int c_mmot_cold_020 = 0;
493 unsigned int c_mmot_cold_021 = 0;
494 unsigned int c_mmot_cold_022 = 0;
495 unsigned int c_mmot_cold_023 = 0;
496 unsigned int c_mmot_cold_024 = 0;
497 unsigned int c_mmot_cold_025 = 0;
498 unsigned int c_mmot_cold_026 = 0;
499 unsigned int c_mmot_cold_027 = 0;
500 unsigned int c_mmot_hot_fSR_ok = 0;
501 unsigned int c_mmot_cold_029 = 0;
502 unsigned int c_mmot_cold_030 = 0;
503 unsigned int c_mmot_cold_031 = 0;
504 unsigned int c_mmot_cold_032 = 0;
505 unsigned int c_mmot_cold_033 = 0;
506 unsigned int c_mmot_bad_rcvr = 0;
507 unsigned int c_mmot_rcvr_swapped = 0;
508 unsigned int c_mmot_rcvr_locked = 0;
509 unsigned int c_mmot_rcvr_tswapped = 0;
510 unsigned int c_mmot_rcvr_freed = 0;
511 unsigned int c_mmot_g_slow_copyout6 = 0;
512 unsigned int c_mmot_g_slow_copyout5 = 0;
513 unsigned int c_mmot_cold_037 = 0;
514 unsigned int c_mmot_cold_038 = 0;
515 unsigned int c_mmot_cold_039 = 0;
516 unsigned int c_mmot_g_slow_copyout4 = 0;
517 unsigned int c_mmot_g_slow_copyout3 = 0;
518 unsigned int c_mmot_hot_ok1 = 0;
519 unsigned int c_mmot_hot_ok2 = 0;
520 unsigned int c_mmot_hot_ok3 = 0;
521 unsigned int c_mmot_g_slow_copyout1 = 0;
522 unsigned int c_mmot_g_slow_copyout2 = 0;
523 unsigned int c_mmot_getback_fast_copyin = 0;
524 unsigned int c_mmot_cold_048 = 0;
525 unsigned int c_mmot_getback_FastSR = 0;
526 unsigned int c_mmot_cold_050 = 0;
527 unsigned int c_mmot_cold_051 = 0;
528 unsigned int c_mmot_cold_052 = 0;
529 unsigned int c_mmot_cold_053 = 0;
530 unsigned int c_mmot_fastkernelreply = 0;
531 unsigned int c_mmot_cold_055 = 0;
532 unsigned int c_mmot_getback_fast_put = 0;
533 unsigned int c_mmot_LAST = 0; /* End Marker - Unused */
534
535 void db_mmot_zero_counters(void); /* forward; */
536 void db_mmot_show_counters(void); /* forward; */
537
538 void /* Call from the debugger to clear all counters */
539 db_mmot_zero_counters(void)
540 {
541 register unsigned int *ip = &c_mmot_FIRST;
542 while (ip <= &c_mmot_LAST)
543 *ip++ = 0;
544 }
545
546 void /* Call from the debugger to show all counters */
547 db_mmot_show_counters(void)
548 {
549 #define xx(str) printf("%s: %d\n", # str, str);
550
551 xx(c_mmot_combined_S_R);
552 xx(c_mach_msg_trap_switch_fast);
553 xx(c_mmot_kernel_send);
554 xx(c_mmot_cold_000);
555 xx(c_mmot_smallsendsize);
556 xx(c_mmot_oddsendsize);
557 xx(c_mmot_bigsendsize);
558 xx(c_mmot_copyinmsg_fail);
559 xx(c_mmot_g_slow_copyin3);
560 xx(c_mmot_cold_006);
561 xx(c_mmot_cold_007);
562 xx(c_mmot_cold_008);
563 xx(c_mmot_cold_009);
564 xx(c_mmot_cold_010);
565 xx(c_mmot_cold_012);
566 xx(c_mmot_cold_013);
567 xx(c_mmot_cold_014);
568 xx(c_mmot_cold_016);
569 xx(c_mmot_cold_018);
570 xx(c_mmot_cold_019);
571 xx(c_mmot_cold_020);
572 xx(c_mmot_cold_021);
573 xx(c_mmot_cold_022);
574 xx(c_mmot_cold_023);
575 xx(c_mmot_cold_024);
576 xx(c_mmot_cold_025);
577 xx(c_mmot_cold_026);
578 xx(c_mmot_cold_027);
579 xx(c_mmot_hot_fSR_ok);
580 xx(c_mmot_cold_029);
581 xx(c_mmot_cold_030);
582 xx(c_mmot_cold_031);
583 xx(c_mmot_cold_032);
584 xx(c_mmot_cold_033);
585 xx(c_mmot_bad_rcvr);
586 xx(c_mmot_rcvr_swapped);
587 xx(c_mmot_rcvr_locked);
588 xx(c_mmot_rcvr_tswapped);
589 xx(c_mmot_rcvr_freed);
590 xx(c_mmot_g_slow_copyout6);
591 xx(c_mmot_g_slow_copyout5);
592 xx(c_mmot_cold_037);
593 xx(c_mmot_cold_038);
594 xx(c_mmot_cold_039);
595 xx(c_mmot_g_slow_copyout4);
596 xx(c_mmot_g_slow_copyout3);
597 xx(c_mmot_g_slow_copyout1);
598 xx(c_mmot_hot_ok3);
599 xx(c_mmot_hot_ok2);
600 xx(c_mmot_hot_ok1);
601 xx(c_mmot_g_slow_copyout2);
602 xx(c_mmot_getback_fast_copyin);
603 xx(c_mmot_cold_048);
604 xx(c_mmot_getback_FastSR);
605 xx(c_mmot_cold_050);
606 xx(c_mmot_cold_051);
607 xx(c_mmot_cold_052);
608 xx(c_mmot_cold_053);
609 xx(c_mmot_fastkernelreply);
610 xx(c_mmot_cold_055);
611 xx(c_mmot_getback_fast_put);
612
613 #undef xx
614 }
615
616 #else /* !HOTPATH_DEBUG */
617
618 /*
619 * Duplicate just these few so we can always do a quick sanity check
620 */
621 unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
622 unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
623 unsigned int c_mmot_kernel_send = 0; /* kernel server calls */
624 #define HOT(expr) /* no optional counters */
625
626 #endif /* !HOTPATH_DEBUG */
627
628 #if CONFIG_MACF_MACH
629 boolean_t enable_hotpath = FALSE; /* XXX - push MAC into HOTPATH too */
630 #else
631 boolean_t enable_hotpath = TRUE; /* Patchable, just in case ... */
632 #endif
633 #endif /* HOTPATH_ENABLE */
634
635 /*
636 * Routine: mach_msg_overwrite_trap [mach trap]
637 * Purpose:
638 * Possibly send a message; possibly receive a message.
639 * Conditions:
640 * Nothing locked.
641 * Returns:
642 * All of mach_msg_send and mach_msg_receive error codes.
643 */
644
645 mach_msg_return_t
646 mach_msg_overwrite_trap(
647 struct mach_msg_overwrite_trap_args *args)
648 {
649 mach_vm_address_t msg_addr = args->msg;
650 mach_msg_option_t option = args->option;
651 mach_msg_size_t send_size = args->send_size;
652 mach_msg_size_t rcv_size = args->rcv_size;
653 mach_port_name_t rcv_name = args->rcv_name;
654 mach_msg_timeout_t msg_timeout = args->timeout;
655 mach_port_name_t notify = args->notify;
656 mach_vm_address_t rcv_msg_addr = args->rcv_msg;
657 mach_msg_size_t scatter_list_size = 0; /* NOT INITIALIZED - but not used in pactice */
658 __unused mach_port_seqno_t temp_seqno = 0;
659
660 mach_msg_return_t mr = MACH_MSG_SUCCESS;
661 #if ENABLE_HOTPATH
662 /* mask out some of the options before entering the hot path */
663 mach_msg_option_t masked_option =
664 option & ~(MACH_SEND_TRAILER|MACH_RCV_TRAILER_MASK|MACH_RCV_LARGE);
665 register mach_msg_header_t *hdr;
666
667 if ((masked_option == (MACH_SEND_MSG|MACH_RCV_MSG)) && enable_hotpath) {
668 thread_t self = current_thread();
669 mach_msg_format_0_trailer_t *trailer;
670 ipc_space_t space = self->task->itk_space;
671 ipc_kmsg_t kmsg;
672 register ipc_port_t dest_port;
673 ipc_object_t rcv_object;
674 ipc_mqueue_t rcv_mqueue;
675 mach_msg_size_t reply_size;
676
677 c_mmot_combined_S_R++;
678
679 /*
680 * This case is divided into ten sections, each
681 * with a label. There are five optimized
682 * sections and six unoptimized sections, which
683 * do the same thing but handle all possible
684 * cases and are slower.
685 *
686 * The five sections for an RPC are
687 * 1) Get request message into a buffer.
688 * 2) Copyin request message and rcv_name.
689 * (fast_copyin or slow_copyin)
690 * 3) Enqueue request and dequeue reply.
691 * (fast_send_receive or
692 * slow_send and slow_receive)
693 * 4) Copyout reply message.
694 * (fast_copyout or slow_copyout)
695 * 5) Put reply message to user's buffer.
696 *
697 * Keep the locking hierarchy firmly in mind.
698 * (First spaces, then ports, then port sets,
699 * then message queues.) Only a non-blocking
700 * attempt can be made to acquire locks out of
701 * order, or acquire two locks on the same level.
702 * Acquiring two locks on the same level will
703 * fail if the objects are really the same,
704 * unless simple locking is disabled. This is OK,
705 * because then the extra unlock does nothing.
706 *
707 * There are two major reasons these RPCs can't use
708 * ipc_thread_switch, and use slow_send/slow_receive:
709 * 1) Kernel RPCs.
710 * 2) Servers fall behind clients, so
711 * client doesn't find a blocked server thread and
712 * server finds waiting messages and can't block.
713 */
714
715 mr = ipc_kmsg_get(msg_addr, send_size, &kmsg);
716 if (mr != KERN_SUCCESS) {
717 return mr;
718 }
719 hdr = kmsg->ikm_header;
720 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
721 send_size);
722
723 /*
724 * fast_copyin:
725 *
726 * optimized ipc_kmsg_copyin/ipc_mqueue_copyin
727 *
728 * We have the request message data in kmsg.
729 * Must still do copyin, send, receive, etc.
730 *
731 * If the message isn't simple, we can't combine
732 * ipc_kmsg_copyin_header and ipc_mqueue_copyin,
733 * because copyin of the message body might
734 * affect rcv_name.
735 */
736
737 switch (hdr->msgh_bits) {
738 case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
739 MACH_MSG_TYPE_MAKE_SEND_ONCE): {
740 register ipc_entry_t table;
741 register ipc_entry_num_t size;
742 register ipc_port_t reply_port;
743
744 /* sending a request message */
745
746 {
747 register mach_port_index_t index;
748 register mach_port_gen_t gen;
749
750 {
751 register mach_port_name_t reply_name =
752 (mach_port_name_t)hdr->msgh_local_port;
753
754 if (reply_name != rcv_name) {
755 HOT(c_mmot_g_slow_copyin3++);
756 goto slow_copyin;
757 }
758
759 /* optimized ipc_entry_lookup of reply_name */
760
761 index = MACH_PORT_INDEX(reply_name);
762 gen = MACH_PORT_GEN(reply_name);
763
764 is_read_lock(space);
765 assert(space->is_active);
766
767 size = space->is_table_size;
768 table = space->is_table;
769
770 {
771 register ipc_entry_t entry;
772 register ipc_entry_bits_t bits;
773
774 if (index < size) {
775 entry = &table[index];
776 bits = entry->ie_bits;
777 if (IE_BITS_GEN(bits) != gen ||
778 (bits & IE_BITS_COLLISION)) {
779 entry = IE_NULL;
780 }
781 } else {
782 entry = IE_NULL;
783 bits = 0;
784 }
785 if (entry == IE_NULL) {
786 entry = ipc_entry_lookup(space, reply_name);
787 if (entry == IE_NULL) {
788 HOT(c_mmot_cold_006++);
789 goto abort_request_copyin;
790 }
791 bits = entry->ie_bits;
792 }
793
794 /* check type bit */
795
796 if (! (bits & MACH_PORT_TYPE_RECEIVE)) {
797 HOT(c_mmot_cold_007++);
798 goto abort_request_copyin;
799 }
800
801 reply_port = (ipc_port_t) entry->ie_object;
802 assert(reply_port != IP_NULL);
803 }
804 }
805 }
806
807 /* optimized ipc_entry_lookup of dest_name */
808
809 {
810 register mach_port_index_t index;
811 register mach_port_gen_t gen;
812
813 {
814 register mach_port_name_t dest_name =
815 (mach_port_name_t)hdr->msgh_remote_port;
816
817 index = MACH_PORT_INDEX(dest_name);
818 gen = MACH_PORT_GEN(dest_name);
819
820 {
821 register ipc_entry_t entry;
822 register ipc_entry_bits_t bits;
823
824 if (index < size) {
825 entry = &table[index];
826 bits = entry->ie_bits;
827 if (IE_BITS_GEN(bits) != gen ||
828 (bits & IE_BITS_COLLISION)) {
829 entry = IE_NULL;
830 }
831 } else {
832 entry = IE_NULL;
833 bits = 0;
834 }
835 if (entry == IE_NULL) {
836 entry = ipc_entry_lookup(space, dest_name);
837 if (entry == IE_NULL) {
838 HOT(c_mmot_cold_008++);
839 goto abort_request_copyin;
840 }
841 bits = entry->ie_bits;
842 }
843
844 /* check type bit */
845
846 if (! (bits & MACH_PORT_TYPE_SEND)) {
847 HOT(c_mmot_cold_009++);
848 goto abort_request_copyin;
849 }
850
851 assert(IE_BITS_UREFS(bits) > 0);
852
853 dest_port = (ipc_port_t) entry->ie_object;
854 assert(dest_port != IP_NULL);
855 }
856 }
857 }
858
859 /*
860 * To do an atomic copyin, need simultaneous
861 * locks on both ports and the space. If
862 * dest_port == reply_port, and simple locking is
863 * enabled, then we will abort. Otherwise it's
864 * OK to unlock twice.
865 */
866
867 ip_lock(dest_port);
868 if (!ip_active(dest_port) ||
869 !ip_lock_try(reply_port)) {
870 ip_unlock(dest_port);
871 HOT(c_mmot_cold_010++);
872 goto abort_request_copyin;
873 }
874 is_read_unlock(space);
875
876 assert(dest_port->ip_srights > 0);
877 dest_port->ip_srights++;
878 ip_reference(dest_port);
879
880 assert(ip_active(reply_port));
881 assert(reply_port->ip_receiver_name ==
882 (mach_port_name_t)hdr->msgh_local_port);
883 assert(reply_port->ip_receiver == space);
884
885 reply_port->ip_sorights++;
886 ip_reference(reply_port);
887
888 hdr->msgh_bits =
889 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
890 MACH_MSG_TYPE_PORT_SEND_ONCE);
891 hdr->msgh_remote_port = dest_port;
892 hdr->msgh_local_port = reply_port;
893
894 /* make sure we can queue to the destination */
895
896 if (dest_port->ip_receiver == ipc_space_kernel) {
897 /*
898 * The kernel server has a reference to
899 * the reply port, which it hands back
900 * to us in the reply message. We do
901 * not need to keep another reference to
902 * it.
903 */
904 ip_unlock(reply_port);
905
906 assert(ip_active(dest_port));
907 dest_port->ip_messages.imq_seqno++;
908 ip_unlock(dest_port);
909 goto kernel_send;
910 }
911
912 if (imq_full(&dest_port->ip_messages)) {
913 HOT(c_mmot_cold_013++);
914 goto abort_request_send_receive;
915 }
916
917 /* optimized ipc_mqueue_copyin */
918
919 rcv_object = (ipc_object_t) reply_port;
920 io_reference(rcv_object);
921 rcv_mqueue = &reply_port->ip_messages;
922 io_unlock(rcv_object);
923 HOT(c_mmot_hot_fSR_ok++);
924 goto fast_send_receive;
925
926 abort_request_copyin:
927 is_read_unlock(space);
928 goto slow_copyin;
929
930 abort_request_send_receive:
931 ip_unlock(dest_port);
932 ip_unlock(reply_port);
933 goto slow_send;
934 }
935
936 case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
937 register ipc_entry_num_t size;
938 register ipc_entry_t table;
939
940 /* sending a reply message */
941
942 {
943 register mach_port_name_t reply_name =
944 (mach_port_name_t)hdr->msgh_local_port;
945
946 if (reply_name != MACH_PORT_NULL) {
947 HOT(c_mmot_cold_018++);
948 goto slow_copyin;
949 }
950 }
951
952 is_write_lock(space);
953 assert(space->is_active);
954
955 /* optimized ipc_entry_lookup */
956
957 size = space->is_table_size;
958 table = space->is_table;
959
960 {
961 register ipc_entry_t entry;
962 register mach_port_gen_t gen;
963 register mach_port_index_t index;
964
965 {
966 register mach_port_name_t dest_name =
967 (mach_port_name_t)hdr->msgh_remote_port;
968
969 index = MACH_PORT_INDEX(dest_name);
970 gen = MACH_PORT_GEN(dest_name);
971 }
972
973 if (index >= size) {
974 HOT(c_mmot_cold_019++);
975 goto abort_reply_dest_copyin;
976 }
977
978 entry = &table[index];
979
980 /* check generation, collision bit, and type bit */
981
982 if ((entry->ie_bits & (IE_BITS_GEN_MASK|
983 IE_BITS_COLLISION|
984 MACH_PORT_TYPE_SEND_ONCE)) !=
985 (gen | MACH_PORT_TYPE_SEND_ONCE)) {
986 HOT(c_mmot_cold_020++);
987 goto abort_reply_dest_copyin;
988 }
989
990 /* optimized ipc_right_copyin */
991
992 assert(IE_BITS_TYPE(entry->ie_bits) ==
993 MACH_PORT_TYPE_SEND_ONCE);
994 assert(IE_BITS_UREFS(entry->ie_bits) == 1);
995
996 if (entry->ie_request != 0) {
997 HOT(c_mmot_cold_021++);
998 goto abort_reply_dest_copyin;
999 }
1000
1001 dest_port = (ipc_port_t) entry->ie_object;
1002 assert(dest_port != IP_NULL);
1003
1004 ip_lock(dest_port);
1005 if (!ip_active(dest_port)) {
1006 ip_unlock(dest_port);
1007 HOT(c_mmot_cold_022++);
1008 goto abort_reply_dest_copyin;
1009 }
1010
1011 assert(dest_port->ip_sorights > 0);
1012
1013 /* optimized ipc_entry_dealloc */
1014
1015
1016 entry->ie_bits = gen;
1017 entry->ie_next = table->ie_next;
1018 table->ie_next = index;
1019 entry->ie_object = IO_NULL;
1020 }
1021
1022 hdr->msgh_bits =
1023 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
1024 0);
1025 hdr->msgh_remote_port = dest_port;
1026
1027 /* make sure we can queue to the destination */
1028
1029 assert(dest_port->ip_receiver != ipc_space_kernel);
1030
1031 /* optimized ipc_entry_lookup/ipc_mqueue_copyin */
1032
1033 {
1034 register ipc_entry_t entry;
1035 register ipc_entry_bits_t bits;
1036
1037 {
1038 register mach_port_index_t index;
1039 register mach_port_gen_t gen;
1040
1041 index = MACH_PORT_INDEX(rcv_name);
1042 gen = MACH_PORT_GEN(rcv_name);
1043
1044 if (index < size) {
1045 entry = &table[index];
1046 bits = entry->ie_bits;
1047 if (IE_BITS_GEN(bits) != gen ||
1048 (bits & IE_BITS_COLLISION)) {
1049 entry = IE_NULL;
1050 }
1051 } else {
1052 entry = IE_NULL;
1053 bits = 0;
1054 }
1055 if (entry == IE_NULL) {
1056 entry = ipc_entry_lookup(space, rcv_name);
1057 if (entry == IE_NULL) {
1058 HOT(c_mmot_cold_024++);
1059 goto abort_reply_rcv_copyin;
1060 }
1061 bits = entry->ie_bits;
1062 }
1063
1064 }
1065
1066 /* check type bits; looking for receive or set */
1067 #if 0
1068 /*
1069 * JMM - The check below for messages in the receive
1070 * mqueue is insufficient to work with port sets, since
1071 * the messages stay in the port queues. For now, don't
1072 * allow portsets (but receiving on portsets when sending
1073 * a message to a send-once right is actually a very
1074 * common case (so we should re-enable).
1075 */
1076 if (bits & MACH_PORT_TYPE_PORT_SET) {
1077 register ipc_pset_t rcv_pset;
1078
1079 rcv_pset = (ipc_pset_t) entry->ie_object;
1080 assert(rcv_pset != IPS_NULL);
1081
1082 ips_lock(rcv_pset);
1083 assert(ips_active(rcv_pset));
1084
1085 rcv_object = (ipc_object_t) rcv_pset;
1086 rcv_mqueue = &rcv_pset->ips_messages;
1087 } else
1088 #endif /* 0 */
1089 if (bits & MACH_PORT_TYPE_RECEIVE) {
1090 register ipc_port_t rcv_port;
1091
1092 rcv_port = (ipc_port_t) entry->ie_object;
1093 assert(rcv_port != IP_NULL);
1094
1095 if (!ip_lock_try(rcv_port)) {
1096 HOT(c_mmot_cold_025++);
1097 goto abort_reply_rcv_copyin;
1098 }
1099 assert(ip_active(rcv_port));
1100
1101 if (rcv_port->ip_pset_count != 0) {
1102 ip_unlock(rcv_port);
1103 HOT(c_mmot_cold_026++);
1104 goto abort_reply_rcv_copyin;
1105 }
1106
1107 rcv_object = (ipc_object_t) rcv_port;
1108 rcv_mqueue = &rcv_port->ip_messages;
1109 } else {
1110 HOT(c_mmot_cold_027++);
1111 goto abort_reply_rcv_copyin;
1112 }
1113 }
1114
1115 is_write_unlock(space);
1116 io_reference(rcv_object);
1117 io_unlock(rcv_object);
1118 HOT(c_mmot_hot_fSR_ok++);
1119 goto fast_send_receive;
1120
1121 abort_reply_dest_copyin:
1122 is_write_unlock(space);
1123 HOT(c_mmot_cold_029++);
1124 goto slow_copyin;
1125
1126 abort_reply_rcv_copyin:
1127 ip_unlock(dest_port);
1128 is_write_unlock(space);
1129 HOT(c_mmot_cold_030++);
1130 goto slow_send;
1131 }
1132
1133 default:
1134 HOT(c_mmot_cold_031++);
1135 goto slow_copyin;
1136 }
1137 /*NOTREACHED*/
1138
1139 fast_send_receive:
1140 /*
1141 * optimized ipc_mqueue_send/ipc_mqueue_receive
1142 *
1143 * Finished get/copyin of kmsg and copyin of rcv_name.
1144 * space is unlocked, dest_port is locked,
1145 * we can queue kmsg to dest_port,
1146 * rcv_mqueue is set, and rcv_object holds a ref
1147 * so the mqueue cannot go away.
1148 *
1149 * JMM - For now, rcv_object is just a port. Portsets
1150 * are disabled for the time being.
1151 */
1152
1153 assert(ip_active(dest_port));
1154 assert(dest_port->ip_receiver != ipc_space_kernel);
1155 // assert(!imq_full(&dest_port->ip_messages) ||
1156 // (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1157 // MACH_MSG_TYPE_PORT_SEND_ONCE));
1158 assert((hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR) == 0);
1159
1160 {
1161 register ipc_mqueue_t dest_mqueue;
1162 wait_queue_t waitq;
1163 thread_t receiver;
1164 spl_t s;
1165
1166 s = splsched();
1167 dest_mqueue = &dest_port->ip_messages;
1168 waitq = &dest_mqueue->imq_wait_queue;
1169 imq_lock(dest_mqueue);
1170
1171 get_next_receiver:
1172 receiver = wait_queue_wakeup64_identity_locked(waitq,
1173 IPC_MQUEUE_RECEIVE,
1174 THREAD_AWAKENED,
1175 FALSE);
1176 /* queue still locked, receiver thread locked (if any) */
1177
1178 if ( receiver == THREAD_NULL ) {
1179 imq_unlock(dest_mqueue);
1180 splx(s);
1181
1182 ip_unlock(dest_port);
1183 ipc_object_release(rcv_object);
1184 HOT(c_mmot_cold_032++);
1185 goto slow_send;
1186 }
1187
1188 /*
1189 * Check that the receiver can handle the size of the message.
1190 * If not, and the receiver just wants to be informed of that
1191 * fact, set it running and try to find another thread.
1192 *
1193 * If he didn't want the "too large" message left on the queue,
1194 * give it to him anyway, he'll consume it as part of his receive
1195 * processing.
1196 */
1197 if (receiver->ith_msize <
1198 ipc_kmsg_copyout_size(kmsg, receiver->map) +
1199 REQUESTED_TRAILER_SIZE(receiver->ith_option))
1200 {
1201 receiver->ith_msize = kmsg->ikm_header->msgh_size;
1202 receiver->ith_state = MACH_RCV_TOO_LARGE;
1203
1204 if ((receiver->ith_option & MACH_RCV_LARGE) != 0) {
1205 receiver->ith_kmsg = IKM_NULL;
1206 receiver->ith_seqno = 0;
1207 thread_unlock(receiver);
1208 HOT(c_mmot_bad_rcvr++);
1209 goto get_next_receiver;
1210 }
1211 } else {
1212 receiver->ith_state = MACH_MSG_SUCCESS;
1213 }
1214
1215 /* At this point we are committed to do the message handoff. */
1216 c_mach_msg_trap_switch_fast++;
1217
1218 /*
1219 * Store the kmsg and seqno where the receiver can pick it up.
1220 * and set it running.
1221 */
1222 receiver->ith_kmsg = kmsg;
1223 receiver->ith_seqno = dest_mqueue->imq_seqno++;
1224 thread_unlock(receiver);
1225
1226 imq_unlock(dest_mqueue);
1227 ip_unlock(dest_port);
1228 current_task()->messages_sent++;
1229
1230 /*
1231 * Now prepare to wait on our receive queue. But we have to make
1232 * sure the queue doesn't already have messages. If it does, we'll
1233 * have to do a slow receive.
1234 *
1235 * JMM - Need to make this check appropriate for portsets as
1236 * well before re-enabling them.
1237 */
1238 imq_lock(rcv_mqueue);
1239
1240 if (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL) {
1241 imq_unlock(rcv_mqueue);
1242 splx(s);
1243 HOT(c_mmot_cold_033++);
1244 goto slow_receive;
1245 }
1246
1247 /*
1248 * Put self on receive port's queue.
1249 * Also save state that the sender of
1250 * our reply message needs to determine if it
1251 * can hand off directly back to us.
1252 */
1253 thread_lock(self);
1254 self->ith_msg_addr = (rcv_msg_addr) ? rcv_msg_addr : msg_addr;
1255 self->ith_object = rcv_object; /* still holds reference */
1256 self->ith_msize = rcv_size;
1257 self->ith_option = option;
1258 self->ith_scatter_list_size = scatter_list_size;
1259 self->ith_continuation = thread_syscall_return;
1260
1261 waitq = &rcv_mqueue->imq_wait_queue;
1262 (void)wait_queue_assert_wait64_locked(waitq,
1263 IPC_MQUEUE_RECEIVE,
1264 THREAD_ABORTSAFE, 0,
1265 self);
1266 thread_unlock(self);
1267 imq_unlock(rcv_mqueue);
1268 splx(s);
1269 thread_block(ipc_mqueue_receive_continue);
1270 /* NOTREACHED */
1271 }
1272
1273 fast_copyout:
1274 /*
1275 * Nothing locked and no references held, except
1276 * we have kmsg with msgh_seqno filled in. Must
1277 * still check against rcv_size and do
1278 * ipc_kmsg_copyout/ipc_kmsg_put.
1279 */
1280
1281 reply_size = send_size + trailer->msgh_trailer_size;
1282 if (rcv_size < reply_size) {
1283 HOT(c_mmot_g_slow_copyout6++);
1284 goto slow_copyout;
1285 }
1286
1287 /* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */
1288
1289 switch (hdr->msgh_bits) {
1290 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
1291 MACH_MSG_TYPE_PORT_SEND_ONCE): {
1292 ipc_port_t reply_port =
1293 (ipc_port_t) hdr->msgh_local_port;
1294 mach_port_name_t dest_name, reply_name;
1295
1296 /* receiving a request message */
1297
1298 if (!IP_VALID(reply_port)) {
1299 HOT(c_mmot_g_slow_copyout5++);
1300 goto slow_copyout;
1301 }
1302
1303 is_write_lock(space);
1304 assert(space->is_active);
1305
1306 /*
1307 * To do an atomic copyout, need simultaneous
1308 * locks on both ports and the space. If
1309 * dest_port == reply_port, and simple locking is
1310 * enabled, then we will abort. Otherwise it's
1311 * OK to unlock twice.
1312 */
1313
1314 ip_lock(dest_port);
1315 if (!ip_active(dest_port) ||
1316 !ip_lock_try(reply_port)) {
1317 HOT(c_mmot_cold_037++);
1318 goto abort_request_copyout;
1319 }
1320
1321 if (!ip_active(reply_port)) {
1322 ip_unlock(reply_port);
1323 HOT(c_mmot_cold_038++);
1324 goto abort_request_copyout;
1325 }
1326
1327 assert(reply_port->ip_sorights > 0);
1328 ip_unlock(reply_port);
1329
1330 {
1331 register ipc_entry_t table;
1332 register ipc_entry_t entry;
1333 register mach_port_index_t index;
1334
1335 /* optimized ipc_entry_get */
1336
1337 table = space->is_table;
1338 index = table->ie_next;
1339
1340 if (index == 0) {
1341 HOT(c_mmot_cold_039++);
1342 goto abort_request_copyout;
1343 }
1344
1345 entry = &table[index];
1346 table->ie_next = entry->ie_next;
1347 entry->ie_request = 0;
1348
1349 {
1350 register mach_port_gen_t gen;
1351
1352 assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
1353 gen = IE_BITS_NEW_GEN(entry->ie_bits);
1354
1355 reply_name = MACH_PORT_MAKE(index, gen);
1356
1357 /* optimized ipc_right_copyout */
1358
1359 entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
1360 }
1361
1362 assert(MACH_PORT_VALID(reply_name));
1363 entry->ie_object = (ipc_object_t) reply_port;
1364 is_write_unlock(space);
1365 }
1366
1367 /* optimized ipc_object_copyout_dest */
1368
1369 assert(dest_port->ip_srights > 0);
1370 ip_release(dest_port);
1371
1372 if (dest_port->ip_receiver == space)
1373 dest_name = dest_port->ip_receiver_name;
1374 else
1375 dest_name = MACH_PORT_NULL;
1376
1377 if ((--dest_port->ip_srights == 0) &&
1378 (dest_port->ip_nsrequest != IP_NULL)) {
1379 ipc_port_t nsrequest;
1380 mach_port_mscount_t mscount;
1381
1382 /* a rather rare case */
1383
1384 nsrequest = dest_port->ip_nsrequest;
1385 mscount = dest_port->ip_mscount;
1386 dest_port->ip_nsrequest = IP_NULL;
1387 ip_unlock(dest_port);
1388 ipc_notify_no_senders(nsrequest, mscount);
1389 } else
1390 ip_unlock(dest_port);
1391
1392 hdr->msgh_bits =
1393 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
1394 MACH_MSG_TYPE_PORT_SEND);
1395 hdr->msgh_remote_port = (mach_port_t)reply_name;
1396 hdr->msgh_local_port = (mach_port_t)dest_name;
1397 HOT(c_mmot_hot_ok1++);
1398 goto fast_put;
1399
1400 abort_request_copyout:
1401 ip_unlock(dest_port);
1402 is_write_unlock(space);
1403 HOT(c_mmot_g_slow_copyout4++);
1404 goto slow_copyout;
1405 }
1406
1407 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1408 register mach_port_name_t dest_name;
1409
1410 /* receiving a reply message */
1411
1412 ip_lock(dest_port);
1413 if (!ip_active(dest_port)) {
1414 ip_unlock(dest_port);
1415 HOT(c_mmot_g_slow_copyout3++);
1416 goto slow_copyout;
1417 }
1418
1419 /* optimized ipc_object_copyout_dest */
1420
1421 assert(dest_port->ip_sorights > 0);
1422
1423 if (dest_port->ip_receiver == space) {
1424 ip_release(dest_port);
1425 dest_port->ip_sorights--;
1426 dest_name = dest_port->ip_receiver_name;
1427 ip_unlock(dest_port);
1428 } else {
1429 ip_unlock(dest_port);
1430
1431 ipc_notify_send_once(dest_port);
1432 dest_name = MACH_PORT_NULL;
1433 }
1434
1435 hdr->msgh_bits = MACH_MSGH_BITS(0,
1436 MACH_MSG_TYPE_PORT_SEND_ONCE);
1437 hdr->msgh_remote_port = MACH_PORT_NULL;
1438 hdr->msgh_local_port = (ipc_port_t)dest_name;
1439 HOT(c_mmot_hot_ok2++);
1440 goto fast_put;
1441 }
1442
1443 case MACH_MSGH_BITS_COMPLEX|
1444 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1445 register mach_port_name_t dest_name;
1446
1447 /* receiving a complex reply message */
1448
1449 ip_lock(dest_port);
1450 if (!ip_active(dest_port)) {
1451 ip_unlock(dest_port);
1452 HOT(c_mmot_g_slow_copyout1++);
1453 goto slow_copyout;
1454 }
1455
1456 /* optimized ipc_object_copyout_dest */
1457
1458 assert(dest_port->ip_sorights > 0);
1459
1460 if (dest_port->ip_receiver == space) {
1461 ip_release(dest_port);
1462 dest_port->ip_sorights--;
1463 dest_name = dest_port->ip_receiver_name;
1464 ip_unlock(dest_port);
1465 } else {
1466 ip_unlock(dest_port);
1467
1468 ipc_notify_send_once(dest_port);
1469 dest_name = MACH_PORT_NULL;
1470 }
1471
1472 hdr->msgh_bits =
1473 MACH_MSGH_BITS_COMPLEX |
1474 MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND_ONCE);
1475 hdr->msgh_remote_port = MACH_PORT_NULL;
1476 hdr->msgh_local_port = (mach_port_t)dest_name;
1477
1478 mr = ipc_kmsg_copyout_body(kmsg, space,
1479 current_map(),
1480 MACH_MSG_BODY_NULL);
1481 /* hdr and send_size may be invalid now - done use */
1482 if (mr != MACH_MSG_SUCCESS) {
1483 if (ipc_kmsg_put(msg_addr, kmsg,
1484 kmsg->ikm_header->msgh_size +
1485 trailer->msgh_trailer_size) ==
1486 MACH_RCV_INVALID_DATA)
1487 return MACH_RCV_INVALID_DATA;
1488 else
1489 return mr | MACH_RCV_BODY_ERROR;
1490 }
1491 HOT(c_mmot_hot_ok3++);
1492 goto fast_put;
1493 }
1494
1495 default:
1496 HOT(c_mmot_g_slow_copyout2++);
1497 goto slow_copyout;
1498 }
1499 /*NOTREACHED*/
1500
1501 fast_put:
1502 mr = ipc_kmsg_put(rcv_msg_addr ? rcv_msg_addr : msg_addr,
1503 kmsg,
1504 kmsg->ikm_header->msgh_size +
1505 trailer->msgh_trailer_size);
1506 if (mr != MACH_MSG_SUCCESS) {
1507 return MACH_RCV_INVALID_DATA;
1508 }
1509 current_task()->messages_received++;
1510 return mr;
1511
1512
1513 /* BEGINNING OF WARM PATH */
1514
1515 /*
1516 * The slow path has a few non-register temporary
1517 * variables used only for call-by-reference.
1518 */
1519
1520 slow_copyin:
1521 {
1522 register mach_port_name_t reply_name =
1523 (mach_port_name_t)hdr->msgh_local_port;
1524
1525
1526 /*
1527 * We have the message data in kmsg, but
1528 * we still need to copyin, send it,
1529 * receive a reply, and do copyout.
1530 */
1531
1532 mr = ipc_kmsg_copyin(kmsg, space, current_map(),
1533 MACH_PORT_NULL);
1534 if (mr != MACH_MSG_SUCCESS) {
1535 ipc_kmsg_free(kmsg);
1536 return(mr);
1537 }
1538
1539 /*
1540 * LP64support - We have to recompute the header pointer
1541 * and send_size - as they could have changed during the
1542 * complex copyin.
1543 */
1544 hdr = kmsg->ikm_header;
1545 send_size = hdr->msgh_size;
1546
1547 /* try to get back on optimized path */
1548 if ((reply_name != rcv_name) ||
1549 (hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR)) {
1550 HOT(c_mmot_cold_048++);
1551 goto slow_send;
1552 }
1553
1554 dest_port = (ipc_port_t) hdr->msgh_remote_port;
1555 assert(IP_VALID(dest_port));
1556
1557 ip_lock(dest_port);
1558 if (!ip_active(dest_port)) {
1559 ip_unlock(dest_port);
1560 goto slow_send;
1561 }
1562
1563 if (dest_port->ip_receiver == ipc_space_kernel) {
1564 dest_port->ip_messages.imq_seqno++;
1565 ip_unlock(dest_port);
1566 goto kernel_send;
1567 }
1568
1569 if (!imq_full(&dest_port->ip_messages) ||
1570 (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1571 MACH_MSG_TYPE_PORT_SEND_ONCE))
1572 {
1573 /*
1574 * Try an optimized ipc_mqueue_copyin.
1575 * It will work if this is a request message.
1576 */
1577
1578 register ipc_port_t reply_port;
1579
1580 reply_port = (ipc_port_t) hdr->msgh_local_port;
1581 if (IP_VALID(reply_port)) {
1582 if (ip_lock_try(reply_port)) {
1583 if (ip_active(reply_port) &&
1584 reply_port->ip_receiver == space &&
1585 reply_port->ip_receiver_name == rcv_name &&
1586 reply_port->ip_pset_count == 0)
1587 {
1588 /* Grab a reference to the reply port. */
1589 rcv_object = (ipc_object_t) reply_port;
1590 io_reference(rcv_object);
1591 rcv_mqueue = &reply_port->ip_messages;
1592 io_unlock(rcv_object);
1593 HOT(c_mmot_getback_FastSR++);
1594 goto fast_send_receive;
1595 }
1596 ip_unlock(reply_port);
1597 }
1598 }
1599 }
1600
1601 ip_unlock(dest_port);
1602 HOT(c_mmot_cold_050++);
1603 goto slow_send;
1604
1605 kernel_send:
1606 /*
1607 * Special case: send message to kernel services.
1608 * The request message has been copied into the
1609 * kmsg. Nothing is locked.
1610 */
1611
1612 {
1613 register ipc_port_t reply_port;
1614 spl_t s;
1615
1616 /*
1617 * Perform the kernel function.
1618 */
1619 c_mmot_kernel_send++;
1620
1621 current_task()->messages_sent++;
1622
1623 kmsg = ipc_kobject_server(kmsg);
1624 if (kmsg == IKM_NULL) {
1625 /*
1626 * No reply. Take the
1627 * slow receive path.
1628 */
1629 HOT(c_mmot_cold_051++);
1630 goto slow_get_rcv_port;
1631 }
1632
1633 /*
1634 * Check that:
1635 * the reply port is alive
1636 * we hold the receive right
1637 * the name has not changed.
1638 * the port is not in a set
1639 * If any of these are not true,
1640 * we cannot directly receive the reply
1641 * message.
1642 */
1643 hdr = kmsg->ikm_header;
1644 send_size = hdr->msgh_size;
1645 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1646 round_msg(send_size));
1647 reply_port = (ipc_port_t) hdr->msgh_remote_port;
1648 ip_lock(reply_port);
1649
1650 if ((!ip_active(reply_port)) ||
1651 (reply_port->ip_receiver != space) ||
1652 (reply_port->ip_receiver_name != rcv_name) ||
1653 (reply_port->ip_pset_count != 0))
1654 {
1655 ip_unlock(reply_port);
1656 ipc_kmsg_send_always(kmsg);
1657 HOT(c_mmot_cold_052++);
1658 goto slow_get_rcv_port;
1659 }
1660
1661 s = splsched();
1662 rcv_mqueue = &reply_port->ip_messages;
1663 imq_lock(rcv_mqueue);
1664
1665 /* keep port locked, and don`t change ref count yet */
1666
1667 /*
1668 * If there are messages on the port
1669 * or other threads waiting for a message,
1670 * we cannot directly receive the reply.
1671 */
1672 if (!wait_queue_empty(&rcv_mqueue->imq_wait_queue) ||
1673 (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL))
1674 {
1675 imq_unlock(rcv_mqueue);
1676 splx(s);
1677 ip_unlock(reply_port);
1678 ipc_kmsg_send_always(kmsg);
1679 HOT(c_mmot_cold_053++);
1680 goto slow_get_rcv_port;
1681 }
1682
1683 /*
1684 * We can directly receive this reply.
1685 * Since there were no messages queued
1686 * on the reply port, there should be
1687 * no threads blocked waiting to send.
1688 */
1689 dest_port = reply_port;
1690 temp_seqno = rcv_mqueue->imq_seqno++;
1691 imq_unlock(rcv_mqueue);
1692 splx(s);
1693
1694 /*
1695 * inline ipc_object_release.
1696 * Port is still locked.
1697 * Reference count was not incremented.
1698 */
1699 ip_check_unlock(reply_port);
1700
1701 if (option & MACH_RCV_TRAILER_MASK) {
1702 trailer->msgh_seqno = temp_seqno;
1703 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1704 }
1705 /* copy out the kernel reply */
1706 HOT(c_mmot_fastkernelreply++);
1707 goto fast_copyout;
1708 }
1709
1710 slow_send:
1711 /*
1712 * Nothing is locked. We have acquired kmsg, but
1713 * we still need to send it and receive a reply.
1714 */
1715
1716 mr = ipc_kmsg_send(kmsg, MACH_MSG_OPTION_NONE,
1717 MACH_MSG_TIMEOUT_NONE);
1718 if (mr != MACH_MSG_SUCCESS) {
1719 mr |= ipc_kmsg_copyout_pseudo(kmsg, space,
1720 current_map(),
1721 MACH_MSG_BODY_NULL);
1722
1723 (void) ipc_kmsg_put(msg_addr, kmsg,
1724 kmsg->ikm_header->msgh_size);
1725 return(mr);
1726 }
1727
1728 slow_get_rcv_port:
1729 /*
1730 * We have sent the message. Copy in the receive port.
1731 */
1732 mr = ipc_mqueue_copyin(space, rcv_name,
1733 &rcv_mqueue, &rcv_object);
1734 if (mr != MACH_MSG_SUCCESS) {
1735 return(mr);
1736 }
1737 /* hold ref for rcv_object */
1738
1739 /*
1740 *
1741 * Now we have sent the request and copied in rcv_name,
1742 * and hold ref for rcv_object (to keep mqueue alive).
1743 * Just receive a reply and try to get back to fast path.
1744 */
1745
1746 slow_receive:
1747 self->ith_continuation = (void (*)(mach_msg_return_t))0;
1748 ipc_mqueue_receive(rcv_mqueue,
1749 MACH_MSG_OPTION_NONE,
1750 MACH_MSG_SIZE_MAX,
1751 MACH_MSG_TIMEOUT_NONE,
1752 THREAD_ABORTSAFE);
1753
1754 mr = self->ith_state;
1755 temp_seqno = self->ith_seqno;
1756
1757 ipc_object_release(rcv_object);
1758
1759 if (mr != MACH_MSG_SUCCESS) {
1760 return(mr);
1761 }
1762
1763 kmsg = self->ith_kmsg;
1764 hdr = kmsg->ikm_header;
1765 send_size = hdr->msgh_size;
1766 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1767 round_msg(send_size));
1768 if (option & MACH_RCV_TRAILER_MASK) {
1769 trailer->msgh_seqno = temp_seqno;
1770 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1771 }
1772 dest_port = (ipc_port_t) hdr->msgh_remote_port;
1773 HOT(c_mmot_cold_055++);
1774 goto fast_copyout;
1775
1776 slow_copyout:
1777 /*
1778 * Nothing locked and no references held, except
1779 * we have kmsg with msgh_seqno filled in. Must
1780 * still check against rcv_size and do
1781 * ipc_kmsg_copyout/ipc_kmsg_put.
1782 */
1783
1784 /* LP64support - have to compute real size as it would be received */
1785 reply_size = ipc_kmsg_copyout_size(kmsg, current_map()) +
1786 REQUESTED_TRAILER_SIZE(option);
1787 temp_seqno = trailer->msgh_seqno;
1788 if (rcv_size < reply_size) {
1789 if (msg_receive_error(kmsg, msg_addr, option, temp_seqno,
1790 space) == MACH_RCV_INVALID_DATA) {
1791 mr = MACH_RCV_INVALID_DATA;
1792 return(mr);
1793 }
1794 else {
1795 mr = MACH_RCV_TOO_LARGE;
1796 return(mr);
1797 }
1798 }
1799
1800 mr = ipc_kmsg_copyout(kmsg, space, current_map(),
1801 MACH_PORT_NULL, MACH_MSG_BODY_NULL);
1802 if (mr != MACH_MSG_SUCCESS) {
1803 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1804 if (ipc_kmsg_put(msg_addr, kmsg, reply_size) ==
1805 MACH_RCV_INVALID_DATA)
1806 mr = MACH_RCV_INVALID_DATA;
1807 }
1808 else {
1809 if (msg_receive_error(kmsg, msg_addr, option,
1810 temp_seqno, space) == MACH_RCV_INVALID_DATA)
1811 mr = MACH_RCV_INVALID_DATA;
1812 }
1813
1814 return(mr);
1815 }
1816
1817 /* try to get back on optimized path */
1818 HOT(c_mmot_getback_fast_put++);
1819 goto fast_put;
1820
1821 /*NOTREACHED*/
1822 }
1823 } /* END OF HOT PATH */
1824 #endif /* ENABLE_HOTPATH */
1825
1826 if (option & MACH_SEND_MSG) {
1827 ipc_space_t space = current_space();
1828 vm_map_t map = current_map();
1829 ipc_kmsg_t kmsg;
1830
1831 mr = ipc_kmsg_get(msg_addr, send_size, &kmsg);
1832
1833 if (mr != MACH_MSG_SUCCESS)
1834 return mr;
1835
1836 if (option & MACH_SEND_CANCEL) {
1837 if (notify == MACH_PORT_NULL)
1838 mr = MACH_SEND_INVALID_NOTIFY;
1839 else
1840 mr = ipc_kmsg_copyin(kmsg, space, map, notify);
1841 } else
1842 mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
1843 if (mr != MACH_MSG_SUCCESS) {
1844 ipc_kmsg_free(kmsg);
1845 return mr;
1846 }
1847
1848 mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, msg_timeout);
1849
1850 if (mr != MACH_MSG_SUCCESS) {
1851 mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
1852 (void) ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size);
1853 return mr;
1854 }
1855
1856 }
1857
1858 if (option & MACH_RCV_MSG) {
1859 thread_t self = current_thread();
1860 ipc_space_t space = current_space();
1861 ipc_object_t object;
1862 ipc_mqueue_t mqueue;
1863
1864 mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
1865 if (mr != MACH_MSG_SUCCESS) {
1866 return mr;
1867 }
1868 /* hold ref for object */
1869
1870 /*
1871 * 1. MACH_RCV_OVERWRITE is on, and rcv_msg is our scatter list
1872 * and receive buffer
1873 * 2. MACH_RCV_OVERWRITE is off, and rcv_msg might be the
1874 * alternate receive buffer (separate send and receive buffers).
1875 */
1876 if (option & MACH_RCV_OVERWRITE)
1877 self->ith_msg_addr = rcv_msg_addr;
1878 else if (rcv_msg_addr != (mach_vm_address_t)0)
1879 self->ith_msg_addr = rcv_msg_addr;
1880 else
1881 self->ith_msg_addr = msg_addr;
1882 self->ith_object = object;
1883 self->ith_msize = rcv_size;
1884 self->ith_option = option;
1885 self->ith_scatter_list_size = scatter_list_size;
1886 self->ith_continuation = thread_syscall_return;
1887
1888 ipc_mqueue_receive(mqueue, option, rcv_size, msg_timeout, THREAD_ABORTSAFE);
1889 if ((option & MACH_RCV_TIMEOUT) && msg_timeout == 0)
1890 thread_poll_yield(self);
1891 return mach_msg_receive_results();
1892 }
1893
1894 return MACH_MSG_SUCCESS;
1895 }
1896
1897 /*
1898 * Routine: mach_msg_trap [mach trap]
1899 * Purpose:
1900 * Possibly send a message; possibly receive a message.
1901 * Conditions:
1902 * Nothing locked.
1903 * Returns:
1904 * All of mach_msg_send and mach_msg_receive error codes.
1905 */
1906
1907 mach_msg_return_t
1908 mach_msg_trap(
1909 struct mach_msg_overwrite_trap_args *args)
1910 {
1911 kern_return_t kr;
1912 args->rcv_msg = (mach_vm_address_t)0;
1913
1914 kr = mach_msg_overwrite_trap(args);
1915 return kr;
1916 }
1917
1918
1919 /*
1920 * Routine: msg_receive_error [internal]
1921 * Purpose:
1922 * Builds a minimal header/trailer and copies it to
1923 * the user message buffer. Invoked when in the case of a
1924 * MACH_RCV_TOO_LARGE or MACH_RCV_BODY_ERROR error.
1925 * Conditions:
1926 * Nothing locked.
1927 * Returns:
1928 * MACH_MSG_SUCCESS minimal header/trailer copied
1929 * MACH_RCV_INVALID_DATA copyout to user buffer failed
1930 */
1931
1932 mach_msg_return_t
1933 msg_receive_error(
1934 ipc_kmsg_t kmsg,
1935 mach_vm_address_t msg_addr,
1936 mach_msg_option_t option,
1937 mach_port_seqno_t seqno,
1938 ipc_space_t space)
1939 {
1940 mach_msg_format_0_trailer_t *trailer;
1941
1942 /*
1943 * Copy out the destination port in the message.
1944 * Destroy all other rights and memory in the message.
1945 */
1946 ipc_kmsg_copyout_dest(kmsg, space);
1947
1948 /*
1949 * Build a minimal message with the requested trailer.
1950 */
1951 trailer = (mach_msg_format_0_trailer_t *)
1952 ((vm_offset_t)kmsg->ikm_header +
1953 round_msg(sizeof(mach_msg_header_t)));
1954 kmsg->ikm_header->msgh_size = sizeof(mach_msg_header_t);
1955 bcopy( (char *)&trailer_template,
1956 (char *)trailer,
1957 sizeof(trailer_template));
1958 if (option & MACH_RCV_TRAILER_MASK) {
1959 trailer->msgh_seqno = seqno;
1960 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1961 }
1962
1963 /*
1964 * Copy the message to user space
1965 */
1966 if (ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size +
1967 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
1968 return(MACH_RCV_INVALID_DATA);
1969 else
1970 return(MACH_MSG_SUCCESS);
1971 }
mirror of XNU