]> git.saurik.com Git - apple/xnu.git/blob - osfmk/ipc/mach_msg.c
projects / apple / xnu.git / blob
? search:


332034f3128589b63970e1ee9356d3d2ef8a0362
[apple/xnu.git] / osfmk / ipc / mach_msg.c
1 /*
2 * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the
10 * License may not be used to create, or enable the creation or
11 * redistribution of, unlawful or unlicensed copies of an Apple operating
12 * system, or to circumvent, violate, or enable the circumvention or
13 * violation of, any terms of an Apple operating system software license
14 * agreement.
15 *
16 * Please obtain a copy of the License at
17 * http://www.opensource.apple.com/apsl/ and read it before using this
18 * file.
19 *
20 * The Original Code and all software distributed under the License are
21 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
22 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
23 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
24 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
25 * Please see the License for the specific language governing rights and
26 * limitations under the License.
27 *
28 * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
29 */
30 /*
31 * @OSF_COPYRIGHT@
32 */
33 /*
34 * Mach Operating System
35 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
36 * All Rights Reserved.
37 *
38 * Permission to use, copy, modify and distribute this software and its
39 * documentation is hereby granted, provided that both the copyright
40 * notice and this permission notice appear in all copies of the
41 * software, derivative works or modified versions, and any portions
42 * thereof, and that both notices appear in supporting documentation.
43 *
44 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
45 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
46 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
47 *
48 * Carnegie Mellon requests users of this software to return to
49 *
50 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
51 * School of Computer Science
52 * Carnegie Mellon University
53 * Pittsburgh PA 15213-3890
54 *
55 * any improvements or extensions that they make and grant Carnegie Mellon
56 * the rights to redistribute these changes.
57 */
58 /*
59 */
60 /*
61 * File: ipc/mach_msg.c
62 * Author: Rich Draves
63 * Date: 1989
64 *
65 * Exported message traps. See mach/message.h.
66 */
67
68 #include <mach/mach_types.h>
69 #include <mach/kern_return.h>
70 #include <mach/port.h>
71 #include <mach/message.h>
72 #include <mach/mig_errors.h>
73 #include <mach/mach_traps.h>
74
75 #include <kern/kern_types.h>
76 #include <kern/assert.h>
77 #include <kern/counters.h>
78 #include <kern/cpu_number.h>
79 #include <kern/ipc_kobject.h>
80 #include <kern/ipc_mig.h>
81 #include <kern/task.h>
82 #include <kern/thread.h>
83 #include <kern/lock.h>
84 #include <kern/sched_prim.h>
85 #include <kern/exception.h>
86 #include <kern/misc_protos.h>
87 #include <kern/kalloc.h>
88 #include <kern/processor.h>
89 #include <kern/syscall_subr.h>
90
91 #include <vm/vm_map.h>
92
93 #include <ipc/ipc_types.h>
94 #include <ipc/ipc_kmsg.h>
95 #include <ipc/ipc_mqueue.h>
96 #include <ipc/ipc_object.h>
97 #include <ipc/ipc_notify.h>
98 #include <ipc/ipc_port.h>
99 #include <ipc/ipc_pset.h>
100 #include <ipc/ipc_space.h>
101 #include <ipc/ipc_entry.h>
102
103 #include <machine/machine_routines.h>
104
105 #include <sys/kdebug.h>
106
107 #ifndef offsetof
108 #define offsetof(type, member) ((size_t)(&((type *)0)->member))
109 #endif /* offsetof */
110
111 /*
112 * Forward declarations - kernel internal routines
113 */
114
115 mach_msg_return_t mach_msg_send(
116 mach_msg_header_t *msg,
117 mach_msg_option_t option,
118 mach_msg_size_t send_size,
119 mach_msg_timeout_t send_timeout,
120 mach_port_name_t notify);
121
122 mach_msg_return_t mach_msg_receive(
123 mach_msg_header_t *msg,
124 mach_msg_option_t option,
125 mach_msg_size_t rcv_size,
126 mach_port_name_t rcv_name,
127 mach_msg_timeout_t rcv_timeout,
128 void (*continuation)(mach_msg_return_t),
129 mach_msg_size_t slist_size);
130
131
132 mach_msg_return_t mach_msg_receive_results(void);
133
134 mach_msg_return_t msg_receive_error(
135 ipc_kmsg_t kmsg,
136 mach_vm_address_t msg_addr,
137 mach_msg_option_t option,
138 mach_port_seqno_t seqno,
139 ipc_space_t space);
140
141 security_token_t KERNEL_SECURITY_TOKEN = KERNEL_SECURITY_TOKEN_VALUE;
142 audit_token_t KERNEL_AUDIT_TOKEN = KERNEL_AUDIT_TOKEN_VALUE;
143
144 mach_msg_format_0_trailer_t trailer_template = {
145 /* mach_msg_trailer_type_t */ MACH_MSG_TRAILER_FORMAT_0,
146 /* mach_msg_trailer_size_t */ MACH_MSG_TRAILER_MINIMUM_SIZE,
147 /* mach_port_seqno_t */ 0,
148 /* security_token_t */ KERNEL_SECURITY_TOKEN_VALUE
149 };
150
151 /*
152 * Routine: mach_msg_send
153 * Purpose:
154 * Send a message.
155 * Conditions:
156 * Nothing locked.
157 * Returns:
158 * MACH_MSG_SUCCESS Sent the message.
159 * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
160 * MACH_SEND_NO_BUFFER Couldn't allocate buffer.
161 * MACH_SEND_INVALID_DATA Couldn't copy message data.
162 * MACH_SEND_INVALID_HEADER
163 * Illegal value in the message header bits.
164 * MACH_SEND_INVALID_DEST The space is dead.
165 * MACH_SEND_INVALID_NOTIFY Bad notify port.
166 * MACH_SEND_INVALID_DEST Can't copyin destination port.
167 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
168 * MACH_SEND_TIMED_OUT Timeout expired without delivery.
169 * MACH_SEND_INTERRUPTED Delivery interrupted.
170 * MACH_SEND_NO_NOTIFY Can't allocate a msg-accepted request.
171 * MACH_SEND_WILL_NOTIFY Msg-accepted notif. requested.
172 * MACH_SEND_NOTIFY_IN_PROGRESS
173 * This space has already forced a message to this port.
174 */
175
176 mach_msg_return_t
177 mach_msg_send(
178 mach_msg_header_t *msg,
179 mach_msg_option_t option,
180 mach_msg_size_t send_size,
181 mach_msg_timeout_t send_timeout,
182 mach_port_name_t notify)
183 {
184 ipc_space_t space = current_space();
185 vm_map_t map = current_map();
186 ipc_kmsg_t kmsg;
187 mach_msg_return_t mr;
188 mach_msg_size_t msg_and_trailer_size;
189 mach_msg_max_trailer_t *trailer;
190
191 if ((send_size < sizeof(mach_msg_header_t)) || (send_size & 3))
192 return MACH_SEND_MSG_TOO_SMALL;
193
194 if (send_size > MACH_MSG_SIZE_MAX - MAX_TRAILER_SIZE)
195 return MACH_SEND_TOO_LARGE;
196
197 msg_and_trailer_size = send_size + MAX_TRAILER_SIZE;
198
199 kmsg = ipc_kmsg_alloc(msg_and_trailer_size);
200
201 if (kmsg == IKM_NULL)
202 return MACH_SEND_NO_BUFFER;
203
204 (void) memcpy((void *) kmsg->ikm_header, (const void *) msg, send_size);
205
206 kmsg->ikm_header->msgh_size = send_size;
207
208 /*
209 * reserve for the trailer the largest space (MAX_TRAILER_SIZE)
210 * However, the internal size field of the trailer (msgh_trailer_size)
211 * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to optimize
212 * the cases where no implicit data is requested.
213 */
214 trailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + send_size);
215 trailer->msgh_sender = current_thread()->task->sec_token;
216 trailer->msgh_audit = current_thread()->task->audit_token;
217 trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
218 trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
219
220 if (option & MACH_SEND_CANCEL) {
221 if (notify == MACH_PORT_NULL)
222 mr = MACH_SEND_INVALID_NOTIFY;
223 else
224 mr = ipc_kmsg_copyin(kmsg, space, map, notify);
225 } else
226 mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
227 if (mr != MACH_MSG_SUCCESS) {
228 ipc_kmsg_free(kmsg);
229 return mr;
230 }
231
232 mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, send_timeout);
233
234 if (mr != MACH_MSG_SUCCESS) {
235 mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
236 (void) memcpy((void *) msg, (const void *) kmsg->ikm_header,
237 kmsg->ikm_header->msgh_size);
238 ipc_kmsg_free(kmsg);
239 }
240
241 return mr;
242 }
243
244 /*
245 * Routine: mach_msg_receive
246 * Purpose:
247 * Receive a message.
248 * Conditions:
249 * Nothing locked.
250 * Returns:
251 * MACH_MSG_SUCCESS Received a message.
252 * MACH_RCV_INVALID_NAME The name doesn't denote a right,
253 * or the denoted right is not receive or port set.
254 * MACH_RCV_IN_SET Receive right is a member of a set.
255 * MACH_RCV_TOO_LARGE Message wouldn't fit into buffer.
256 * MACH_RCV_TIMED_OUT Timeout expired without a message.
257 * MACH_RCV_INTERRUPTED Reception interrupted.
258 * MACH_RCV_PORT_DIED Port/set died while receiving.
259 * MACH_RCV_PORT_CHANGED Port moved into set while receiving.
260 * MACH_RCV_INVALID_DATA Couldn't copy to user buffer.
261 * MACH_RCV_INVALID_NOTIFY Bad notify port.
262 * MACH_RCV_HEADER_ERROR
263 */
264
265 mach_msg_return_t
266 mach_msg_receive_results(void)
267 {
268 thread_t self = current_thread();
269 ipc_space_t space = current_space();
270 vm_map_t map = current_map();
271
272 ipc_object_t object = self->ith_object;
273 mach_msg_return_t mr = self->ith_state;
274 mach_vm_address_t msg_addr = self->ith_msg_addr;
275 mach_msg_option_t option = self->ith_option;
276 ipc_kmsg_t kmsg = self->ith_kmsg;
277 mach_port_seqno_t seqno = self->ith_seqno;
278
279 mach_msg_format_0_trailer_t *trailer;
280
281 ipc_object_release(object);
282
283 if (mr != MACH_MSG_SUCCESS) {
284
285 if (mr == MACH_RCV_TOO_LARGE ) {
286 if (option & MACH_RCV_LARGE) {
287 /*
288 * We need to inform the user-level code that it needs more
289 * space. The value for how much space was returned in the
290 * msize save area instead of the message (which was left on
291 * the queue).
292 */
293 if (copyout((char *) &self->ith_msize,
294 msg_addr + offsetof(mach_msg_header_t, msgh_size),
295 sizeof(mach_msg_size_t)))
296 mr = MACH_RCV_INVALID_DATA;
297 goto out;
298 }
299
300 if (msg_receive_error(kmsg, msg_addr, option, seqno, space)
301 == MACH_RCV_INVALID_DATA)
302 mr = MACH_RCV_INVALID_DATA;
303 }
304 goto out;
305 }
306
307 trailer = (mach_msg_format_0_trailer_t *)
308 ((vm_offset_t)kmsg->ikm_header +
309 round_msg(kmsg->ikm_header->msgh_size));
310 if (option & MACH_RCV_TRAILER_MASK) {
311 trailer->msgh_seqno = seqno;
312 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
313 }
314
315 /*
316 * If MACH_RCV_OVERWRITE was specified, try to get the scatter
317 * list and verify it against the contents of the message. If
318 * there is any problem with it, we will continue without it as
319 * normal.
320 */
321 if (option & MACH_RCV_OVERWRITE) {
322 mach_msg_size_t slist_size = self->ith_scatter_list_size;
323 mach_msg_body_t *slist;
324
325 slist = ipc_kmsg_get_scatter(msg_addr, slist_size, kmsg);
326 mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL, slist);
327 ipc_kmsg_free_scatter(slist, slist_size);
328 } else {
329 mr = ipc_kmsg_copyout(kmsg, space, map,
330 MACH_PORT_NULL, MACH_MSG_BODY_NULL);
331 }
332
333 if (mr != MACH_MSG_SUCCESS) {
334 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
335 if (ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size +
336 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
337 mr = MACH_RCV_INVALID_DATA;
338 }
339 else {
340 if (msg_receive_error(kmsg, msg_addr, option, seqno, space)
341 == MACH_RCV_INVALID_DATA)
342 mr = MACH_RCV_INVALID_DATA;
343 }
344 goto out;
345 }
346 mr = ipc_kmsg_put(msg_addr,
347 kmsg,
348 kmsg->ikm_header->msgh_size +
349 trailer->msgh_trailer_size);
350 out:
351 return mr;
352 }
353
354 mach_msg_return_t
355 mach_msg_receive(
356 mach_msg_header_t *msg,
357 mach_msg_option_t option,
358 mach_msg_size_t rcv_size,
359 mach_port_name_t rcv_name,
360 mach_msg_timeout_t rcv_timeout,
361 void (*continuation)(mach_msg_return_t),
362 mach_msg_size_t slist_size)
363 {
364 thread_t self = current_thread();
365 ipc_space_t space = current_space();
366 ipc_object_t object;
367 ipc_mqueue_t mqueue;
368 mach_msg_return_t mr;
369
370 mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
371 if (mr != MACH_MSG_SUCCESS) {
372 return mr;
373 }
374 /* hold ref for object */
375
376 self->ith_msg_addr = CAST_DOWN(mach_vm_address_t, msg);
377 self->ith_object = object;
378 self->ith_msize = rcv_size;
379 self->ith_option = option;
380 self->ith_scatter_list_size = slist_size;
381 self->ith_continuation = continuation;
382
383 ipc_mqueue_receive(mqueue, option, rcv_size, rcv_timeout, THREAD_ABORTSAFE);
384 if ((option & MACH_RCV_TIMEOUT) && rcv_timeout == 0)
385 thread_poll_yield(self);
386 return mach_msg_receive_results();
387 }
388
389 void
390 mach_msg_receive_continue(void)
391 {
392 thread_t self = current_thread();
393
394 (*self->ith_continuation)(mach_msg_receive_results());
395 }
396
397 /*
398 * Toggle this to compile the hotpath in/out
399 * If compiled in, the run-time toggle "enable_hotpath" below
400 * eases testing & debugging
401 */
402 #define ENABLE_HOTPATH 1 /* Hacked on for now */
403
404 #if ENABLE_HOTPATH
405 /*
406 * These counters allow tracing of hotpath behavior under test loads.
407 * A couple key counters are unconditional (see below).
408 */
409 #define HOTPATH_DEBUG 0 /* Toggle to include lots of counters */
410 #if HOTPATH_DEBUG
411 #define HOT(expr) expr
412
413 unsigned int c_mmot_FIRST = 0; /* Unused First Counter */
414 unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
415 unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
416 unsigned int c_mmot_kernel_send = 0; /* kernel server */
417 unsigned int c_mmot_cold_000 = 0; /* see below ... */
418 unsigned int c_mmot_smallsendsize = 0;
419 unsigned int c_mmot_oddsendsize = 0;
420 unsigned int c_mmot_bigsendsize = 0;
421 unsigned int c_mmot_copyinmsg_fail = 0;
422 unsigned int c_mmot_g_slow_copyin3 = 0;
423 unsigned int c_mmot_cold_006 = 0;
424 unsigned int c_mmot_cold_007 = 0;
425 unsigned int c_mmot_cold_008 = 0;
426 unsigned int c_mmot_cold_009 = 0;
427 unsigned int c_mmot_cold_010 = 0;
428 unsigned int c_mmot_cold_012 = 0;
429 unsigned int c_mmot_cold_013 = 0;
430 unsigned int c_mmot_cold_014 = 0;
431 unsigned int c_mmot_cold_016 = 0;
432 unsigned int c_mmot_cold_018 = 0;
433 unsigned int c_mmot_cold_019 = 0;
434 unsigned int c_mmot_cold_020 = 0;
435 unsigned int c_mmot_cold_021 = 0;
436 unsigned int c_mmot_cold_022 = 0;
437 unsigned int c_mmot_cold_023 = 0;
438 unsigned int c_mmot_cold_024 = 0;
439 unsigned int c_mmot_cold_025 = 0;
440 unsigned int c_mmot_cold_026 = 0;
441 unsigned int c_mmot_cold_027 = 0;
442 unsigned int c_mmot_hot_fSR_ok = 0;
443 unsigned int c_mmot_cold_029 = 0;
444 unsigned int c_mmot_cold_030 = 0;
445 unsigned int c_mmot_cold_031 = 0;
446 unsigned int c_mmot_cold_032 = 0;
447 unsigned int c_mmot_cold_033 = 0;
448 unsigned int c_mmot_bad_rcvr = 0;
449 unsigned int c_mmot_rcvr_swapped = 0;
450 unsigned int c_mmot_rcvr_locked = 0;
451 unsigned int c_mmot_rcvr_tswapped = 0;
452 unsigned int c_mmot_rcvr_freed = 0;
453 unsigned int c_mmot_g_slow_copyout6 = 0;
454 unsigned int c_mmot_g_slow_copyout5 = 0;
455 unsigned int c_mmot_cold_037 = 0;
456 unsigned int c_mmot_cold_038 = 0;
457 unsigned int c_mmot_cold_039 = 0;
458 unsigned int c_mmot_g_slow_copyout4 = 0;
459 unsigned int c_mmot_g_slow_copyout3 = 0;
460 unsigned int c_mmot_hot_ok1 = 0;
461 unsigned int c_mmot_hot_ok2 = 0;
462 unsigned int c_mmot_hot_ok3 = 0;
463 unsigned int c_mmot_g_slow_copyout1 = 0;
464 unsigned int c_mmot_g_slow_copyout2 = 0;
465 unsigned int c_mmot_getback_fast_copyin = 0;
466 unsigned int c_mmot_cold_048 = 0;
467 unsigned int c_mmot_getback_FastSR = 0;
468 unsigned int c_mmot_cold_050 = 0;
469 unsigned int c_mmot_cold_051 = 0;
470 unsigned int c_mmot_cold_052 = 0;
471 unsigned int c_mmot_cold_053 = 0;
472 unsigned int c_mmot_fastkernelreply = 0;
473 unsigned int c_mmot_cold_055 = 0;
474 unsigned int c_mmot_getback_fast_put = 0;
475 unsigned int c_mmot_LAST = 0; /* End Marker - Unused */
476
477 void db_mmot_zero_counters(void); /* forward; */
478 void db_mmot_show_counters(void); /* forward; */
479
480 void /* Call from the debugger to clear all counters */
481 db_mmot_zero_counters(void)
482 {
483 register unsigned int *ip = &c_mmot_FIRST;
484 while (ip <= &c_mmot_LAST)
485 *ip++ = 0;
486 }
487
488 void /* Call from the debugger to show all counters */
489 db_mmot_show_counters(void)
490 {
491 #define xx(str) printf("%s: %d\n", # str, str);
492
493 xx(c_mmot_combined_S_R);
494 xx(c_mach_msg_trap_switch_fast);
495 xx(c_mmot_kernel_send);
496 xx(c_mmot_cold_000);
497 xx(c_mmot_smallsendsize);
498 xx(c_mmot_oddsendsize);
499 xx(c_mmot_bigsendsize);
500 xx(c_mmot_copyinmsg_fail);
501 xx(c_mmot_g_slow_copyin3);
502 xx(c_mmot_cold_006);
503 xx(c_mmot_cold_007);
504 xx(c_mmot_cold_008);
505 xx(c_mmot_cold_009);
506 xx(c_mmot_cold_010);
507 xx(c_mmot_cold_012);
508 xx(c_mmot_cold_013);
509 xx(c_mmot_cold_014);
510 xx(c_mmot_cold_016);
511 xx(c_mmot_cold_018);
512 xx(c_mmot_cold_019);
513 xx(c_mmot_cold_020);
514 xx(c_mmot_cold_021);
515 xx(c_mmot_cold_022);
516 xx(c_mmot_cold_023);
517 xx(c_mmot_cold_024);
518 xx(c_mmot_cold_025);
519 xx(c_mmot_cold_026);
520 xx(c_mmot_cold_027);
521 xx(c_mmot_hot_fSR_ok);
522 xx(c_mmot_cold_029);
523 xx(c_mmot_cold_030);
524 xx(c_mmot_cold_031);
525 xx(c_mmot_cold_032);
526 xx(c_mmot_cold_033);
527 xx(c_mmot_bad_rcvr);
528 xx(c_mmot_rcvr_swapped);
529 xx(c_mmot_rcvr_locked);
530 xx(c_mmot_rcvr_tswapped);
531 xx(c_mmot_rcvr_freed);
532 xx(c_mmot_g_slow_copyout6);
533 xx(c_mmot_g_slow_copyout5);
534 xx(c_mmot_cold_037);
535 xx(c_mmot_cold_038);
536 xx(c_mmot_cold_039);
537 xx(c_mmot_g_slow_copyout4);
538 xx(c_mmot_g_slow_copyout3);
539 xx(c_mmot_g_slow_copyout1);
540 xx(c_mmot_hot_ok3);
541 xx(c_mmot_hot_ok2);
542 xx(c_mmot_hot_ok1);
543 xx(c_mmot_g_slow_copyout2);
544 xx(c_mmot_getback_fast_copyin);
545 xx(c_mmot_cold_048);
546 xx(c_mmot_getback_FastSR);
547 xx(c_mmot_cold_050);
548 xx(c_mmot_cold_051);
549 xx(c_mmot_cold_052);
550 xx(c_mmot_cold_053);
551 xx(c_mmot_fastkernelreply);
552 xx(c_mmot_cold_055);
553 xx(c_mmot_getback_fast_put);
554
555 #undef xx
556 }
557
558 #else /* !HOTPATH_DEBUG */
559
560 /*
561 * Duplicate just these few so we can always do a quick sanity check
562 */
563 unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
564 unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
565 unsigned int c_mmot_kernel_send = 0; /* kernel server calls */
566 #define HOT(expr) /* no optional counters */
567
568 #endif /* !HOTPATH_DEBUG */
569
570 boolean_t enable_hotpath = TRUE; /* Patchable, just in case ... */
571 #endif /* HOTPATH_ENABLE */
572
573 /*
574 * Routine: mach_msg_overwrite_trap [mach trap]
575 * Purpose:
576 * Possibly send a message; possibly receive a message.
577 * Conditions:
578 * Nothing locked.
579 * Returns:
580 * All of mach_msg_send and mach_msg_receive error codes.
581 */
582
583 mach_msg_return_t
584 mach_msg_overwrite_trap(
585 struct mach_msg_overwrite_trap_args *args)
586 {
587 mach_vm_address_t msg_addr = args->msg;
588 mach_msg_option_t option = args->option;
589 mach_msg_size_t send_size = args->send_size;
590 mach_msg_size_t rcv_size = args->rcv_size;
591 mach_port_name_t rcv_name = args->rcv_name;
592 mach_msg_timeout_t msg_timeout = args->timeout;
593 mach_port_name_t notify = args->notify;
594 mach_vm_address_t rcv_msg_addr = args->rcv_msg;
595 mach_msg_size_t scatter_list_size = 0; /* NOT INITIALIZED - but not used in pactice */
596
597 register mach_msg_header_t *hdr;
598 mach_msg_return_t mr = MACH_MSG_SUCCESS;
599 /* mask out some of the options before entering the hot path */
600 mach_msg_option_t masked_option =
601 option & ~(MACH_SEND_TRAILER|MACH_RCV_TRAILER_MASK|MACH_RCV_LARGE);
602
603 #if ENABLE_HOTPATH
604 /* BEGINNING OF HOT PATH */
605 if ((masked_option == (MACH_SEND_MSG|MACH_RCV_MSG)) && enable_hotpath) {
606 thread_t self = current_thread();
607 mach_msg_format_0_trailer_t *trailer;
608 ipc_space_t space = self->task->itk_space;
609 ipc_kmsg_t kmsg;
610 register ipc_port_t dest_port;
611 ipc_object_t rcv_object;
612 ipc_mqueue_t rcv_mqueue;
613 mach_msg_size_t reply_size;
614
615 c_mmot_combined_S_R++;
616
617 /*
618 * This case is divided into ten sections, each
619 * with a label. There are five optimized
620 * sections and six unoptimized sections, which
621 * do the same thing but handle all possible
622 * cases and are slower.
623 *
624 * The five sections for an RPC are
625 * 1) Get request message into a buffer.
626 * 2) Copyin request message and rcv_name.
627 * (fast_copyin or slow_copyin)
628 * 3) Enqueue request and dequeue reply.
629 * (fast_send_receive or
630 * slow_send and slow_receive)
631 * 4) Copyout reply message.
632 * (fast_copyout or slow_copyout)
633 * 5) Put reply message to user's buffer.
634 *
635 * Keep the locking hierarchy firmly in mind.
636 * (First spaces, then ports, then port sets,
637 * then message queues.) Only a non-blocking
638 * attempt can be made to acquire locks out of
639 * order, or acquire two locks on the same level.
640 * Acquiring two locks on the same level will
641 * fail if the objects are really the same,
642 * unless simple locking is disabled. This is OK,
643 * because then the extra unlock does nothing.
644 *
645 * There are two major reasons these RPCs can't use
646 * ipc_thread_switch, and use slow_send/slow_receive:
647 * 1) Kernel RPCs.
648 * 2) Servers fall behind clients, so
649 * client doesn't find a blocked server thread and
650 * server finds waiting messages and can't block.
651 */
652
653 mr = ipc_kmsg_get(msg_addr, send_size, &kmsg);
654 if (mr != KERN_SUCCESS) {
655 return mr;
656 }
657 hdr = kmsg->ikm_header;
658 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
659 send_size);
660
661 /*
662 * fast_copyin:
663 *
664 * optimized ipc_kmsg_copyin/ipc_mqueue_copyin
665 *
666 * We have the request message data in kmsg.
667 * Must still do copyin, send, receive, etc.
668 *
669 * If the message isn't simple, we can't combine
670 * ipc_kmsg_copyin_header and ipc_mqueue_copyin,
671 * because copyin of the message body might
672 * affect rcv_name.
673 */
674
675 switch (hdr->msgh_bits) {
676 case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
677 MACH_MSG_TYPE_MAKE_SEND_ONCE): {
678 register ipc_entry_t table;
679 register ipc_entry_num_t size;
680 register ipc_port_t reply_port;
681
682 /* sending a request message */
683
684 {
685 register mach_port_index_t index;
686 register mach_port_gen_t gen;
687
688 {
689 register mach_port_name_t reply_name =
690 (mach_port_name_t)hdr->msgh_local_port;
691
692 if (reply_name != rcv_name) {
693 HOT(c_mmot_g_slow_copyin3++);
694 goto slow_copyin;
695 }
696
697 /* optimized ipc_entry_lookup of reply_name */
698
699 index = MACH_PORT_INDEX(reply_name);
700 gen = MACH_PORT_GEN(reply_name);
701
702 is_read_lock(space);
703 assert(space->is_active);
704
705 size = space->is_table_size;
706 table = space->is_table;
707
708 {
709 register ipc_entry_t entry;
710 register ipc_entry_bits_t bits;
711
712 if (index < size) {
713 entry = &table[index];
714 bits = entry->ie_bits;
715 if (IE_BITS_GEN(bits) != gen ||
716 (bits & IE_BITS_COLLISION)) {
717 entry = IE_NULL;
718 }
719 } else {
720 entry = IE_NULL;
721 bits = 0;
722 }
723 if (entry == IE_NULL) {
724 entry = ipc_entry_lookup(space, reply_name);
725 if (entry == IE_NULL) {
726 HOT(c_mmot_cold_006++);
727 goto abort_request_copyin;
728 }
729 bits = entry->ie_bits;
730 }
731
732 /* check type bit */
733
734 if (! (bits & MACH_PORT_TYPE_RECEIVE)) {
735 HOT(c_mmot_cold_007++);
736 goto abort_request_copyin;
737 }
738
739 reply_port = (ipc_port_t) entry->ie_object;
740 assert(reply_port != IP_NULL);
741 }
742 }
743 }
744
745 /* optimized ipc_entry_lookup of dest_name */
746
747 {
748 register mach_port_index_t index;
749 register mach_port_gen_t gen;
750
751 {
752 register mach_port_name_t dest_name =
753 (mach_port_name_t)hdr->msgh_remote_port;
754
755 index = MACH_PORT_INDEX(dest_name);
756 gen = MACH_PORT_GEN(dest_name);
757
758 {
759 register ipc_entry_t entry;
760 register ipc_entry_bits_t bits;
761
762 if (index < size) {
763 entry = &table[index];
764 bits = entry->ie_bits;
765 if (IE_BITS_GEN(bits) != gen ||
766 (bits & IE_BITS_COLLISION)) {
767 entry = IE_NULL;
768 }
769 } else {
770 entry = IE_NULL;
771 bits = 0;
772 }
773 if (entry == IE_NULL) {
774 entry = ipc_entry_lookup(space, dest_name);
775 if (entry == IE_NULL) {
776 HOT(c_mmot_cold_008++);
777 goto abort_request_copyin;
778 }
779 bits = entry->ie_bits;
780 }
781
782 /* check type bit */
783
784 if (! (bits & MACH_PORT_TYPE_SEND)) {
785 HOT(c_mmot_cold_009++);
786 goto abort_request_copyin;
787 }
788
789 assert(IE_BITS_UREFS(bits) > 0);
790
791 dest_port = (ipc_port_t) entry->ie_object;
792 assert(dest_port != IP_NULL);
793 }
794 }
795 }
796
797 /*
798 * To do an atomic copyin, need simultaneous
799 * locks on both ports and the space. If
800 * dest_port == reply_port, and simple locking is
801 * enabled, then we will abort. Otherwise it's
802 * OK to unlock twice.
803 */
804
805 ip_lock(dest_port);
806 if (!ip_active(dest_port) ||
807 !ip_lock_try(reply_port)) {
808 ip_unlock(dest_port);
809 HOT(c_mmot_cold_010++);
810 goto abort_request_copyin;
811 }
812 is_read_unlock(space);
813
814 assert(dest_port->ip_srights > 0);
815 dest_port->ip_srights++;
816 ip_reference(dest_port);
817
818 assert(ip_active(reply_port));
819 assert(reply_port->ip_receiver_name ==
820 (mach_port_name_t)hdr->msgh_local_port);
821 assert(reply_port->ip_receiver == space);
822
823 reply_port->ip_sorights++;
824 ip_reference(reply_port);
825
826 hdr->msgh_bits =
827 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
828 MACH_MSG_TYPE_PORT_SEND_ONCE);
829 hdr->msgh_remote_port = dest_port;
830 hdr->msgh_local_port = reply_port;
831
832 /* make sure we can queue to the destination */
833
834 if (dest_port->ip_receiver == ipc_space_kernel) {
835 /*
836 * The kernel server has a reference to
837 * the reply port, which it hands back
838 * to us in the reply message. We do
839 * not need to keep another reference to
840 * it.
841 */
842 ip_unlock(reply_port);
843
844 assert(ip_active(dest_port));
845 dest_port->ip_messages.imq_seqno++;
846 ip_unlock(dest_port);
847 goto kernel_send;
848 }
849
850 if (imq_full(&dest_port->ip_messages)) {
851 HOT(c_mmot_cold_013++);
852 goto abort_request_send_receive;
853 }
854
855 /* optimized ipc_mqueue_copyin */
856
857 rcv_object = (ipc_object_t) reply_port;
858 io_reference(rcv_object);
859 rcv_mqueue = &reply_port->ip_messages;
860 io_unlock(rcv_object);
861 HOT(c_mmot_hot_fSR_ok++);
862 goto fast_send_receive;
863
864 abort_request_copyin:
865 is_read_unlock(space);
866 goto slow_copyin;
867
868 abort_request_send_receive:
869 ip_unlock(dest_port);
870 ip_unlock(reply_port);
871 goto slow_send;
872 }
873
874 case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
875 register ipc_entry_num_t size;
876 register ipc_entry_t table;
877
878 /* sending a reply message */
879
880 {
881 register mach_port_name_t reply_name =
882 (mach_port_name_t)hdr->msgh_local_port;
883
884 if (reply_name != MACH_PORT_NULL) {
885 HOT(c_mmot_cold_018++);
886 goto slow_copyin;
887 }
888 }
889
890 is_write_lock(space);
891 assert(space->is_active);
892
893 /* optimized ipc_entry_lookup */
894
895 size = space->is_table_size;
896 table = space->is_table;
897
898 {
899 register ipc_entry_t entry;
900 register mach_port_gen_t gen;
901 register mach_port_index_t index;
902
903 {
904 register mach_port_name_t dest_name =
905 (mach_port_name_t)hdr->msgh_remote_port;
906
907 index = MACH_PORT_INDEX(dest_name);
908 gen = MACH_PORT_GEN(dest_name);
909 }
910
911 if (index >= size) {
912 HOT(c_mmot_cold_019++);
913 goto abort_reply_dest_copyin;
914 }
915
916 entry = &table[index];
917
918 /* check generation, collision bit, and type bit */
919
920 if ((entry->ie_bits & (IE_BITS_GEN_MASK|
921 IE_BITS_COLLISION|
922 MACH_PORT_TYPE_SEND_ONCE)) !=
923 (gen | MACH_PORT_TYPE_SEND_ONCE)) {
924 HOT(c_mmot_cold_020++);
925 goto abort_reply_dest_copyin;
926 }
927
928 /* optimized ipc_right_copyin */
929
930 assert(IE_BITS_TYPE(entry->ie_bits) ==
931 MACH_PORT_TYPE_SEND_ONCE);
932 assert(IE_BITS_UREFS(entry->ie_bits) == 1);
933
934 if (entry->ie_request != 0) {
935 HOT(c_mmot_cold_021++);
936 goto abort_reply_dest_copyin;
937 }
938
939 dest_port = (ipc_port_t) entry->ie_object;
940 assert(dest_port != IP_NULL);
941
942 ip_lock(dest_port);
943 if (!ip_active(dest_port)) {
944 ip_unlock(dest_port);
945 HOT(c_mmot_cold_022++);
946 goto abort_reply_dest_copyin;
947 }
948
949 assert(dest_port->ip_sorights > 0);
950
951 /* optimized ipc_entry_dealloc */
952
953
954 entry->ie_bits = gen;
955 entry->ie_next = table->ie_next;
956 table->ie_next = index;
957 entry->ie_object = IO_NULL;
958 }
959
960 hdr->msgh_bits =
961 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
962 0);
963 hdr->msgh_remote_port = dest_port;
964
965 /* make sure we can queue to the destination */
966
967 assert(dest_port->ip_receiver != ipc_space_kernel);
968
969 /* optimized ipc_entry_lookup/ipc_mqueue_copyin */
970
971 {
972 register ipc_entry_t entry;
973 register ipc_entry_bits_t bits;
974
975 {
976 register mach_port_index_t index;
977 register mach_port_gen_t gen;
978
979 index = MACH_PORT_INDEX(rcv_name);
980 gen = MACH_PORT_GEN(rcv_name);
981
982 if (index < size) {
983 entry = &table[index];
984 bits = entry->ie_bits;
985 if (IE_BITS_GEN(bits) != gen ||
986 (bits & IE_BITS_COLLISION)) {
987 entry = IE_NULL;
988 }
989 } else {
990 entry = IE_NULL;
991 bits = 0;
992 }
993 if (entry == IE_NULL) {
994 entry = ipc_entry_lookup(space, rcv_name);
995 if (entry == IE_NULL) {
996 HOT(c_mmot_cold_024++);
997 goto abort_reply_rcv_copyin;
998 }
999 bits = entry->ie_bits;
1000 }
1001
1002 }
1003
1004 /* check type bits; looking for receive or set */
1005 #if 0
1006 /*
1007 * JMM - The check below for messages in the receive
1008 * mqueue is insufficient to work with port sets, since
1009 * the messages stay in the port queues. For now, don't
1010 * allow portsets (but receiving on portsets when sending
1011 * a message to a send-once right is actually a very
1012 * common case (so we should re-enable).
1013 */
1014 if (bits & MACH_PORT_TYPE_PORT_SET) {
1015 register ipc_pset_t rcv_pset;
1016
1017 rcv_pset = (ipc_pset_t) entry->ie_object;
1018 assert(rcv_pset != IPS_NULL);
1019
1020 ips_lock(rcv_pset);
1021 assert(ips_active(rcv_pset));
1022
1023 rcv_object = (ipc_object_t) rcv_pset;
1024 rcv_mqueue = &rcv_pset->ips_messages;
1025 } else
1026 #endif /* 0 */
1027 if (bits & MACH_PORT_TYPE_RECEIVE) {
1028 register ipc_port_t rcv_port;
1029
1030 rcv_port = (ipc_port_t) entry->ie_object;
1031 assert(rcv_port != IP_NULL);
1032
1033 if (!ip_lock_try(rcv_port)) {
1034 HOT(c_mmot_cold_025++);
1035 goto abort_reply_rcv_copyin;
1036 }
1037 assert(ip_active(rcv_port));
1038
1039 if (rcv_port->ip_pset_count != 0) {
1040 ip_unlock(rcv_port);
1041 HOT(c_mmot_cold_026++);
1042 goto abort_reply_rcv_copyin;
1043 }
1044
1045 rcv_object = (ipc_object_t) rcv_port;
1046 rcv_mqueue = &rcv_port->ip_messages;
1047 } else {
1048 HOT(c_mmot_cold_027++);
1049 goto abort_reply_rcv_copyin;
1050 }
1051 }
1052
1053 is_write_unlock(space);
1054 io_reference(rcv_object);
1055 io_unlock(rcv_object);
1056 HOT(c_mmot_hot_fSR_ok++);
1057 goto fast_send_receive;
1058
1059 abort_reply_dest_copyin:
1060 is_write_unlock(space);
1061 HOT(c_mmot_cold_029++);
1062 goto slow_copyin;
1063
1064 abort_reply_rcv_copyin:
1065 ip_unlock(dest_port);
1066 is_write_unlock(space);
1067 HOT(c_mmot_cold_030++);
1068 goto slow_send;
1069 }
1070
1071 default:
1072 HOT(c_mmot_cold_031++);
1073 goto slow_copyin;
1074 }
1075 /*NOTREACHED*/
1076
1077 fast_send_receive:
1078 /*
1079 * optimized ipc_mqueue_send/ipc_mqueue_receive
1080 *
1081 * Finished get/copyin of kmsg and copyin of rcv_name.
1082 * space is unlocked, dest_port is locked,
1083 * we can queue kmsg to dest_port,
1084 * rcv_mqueue is set, and rcv_object holds a ref
1085 * so the mqueue cannot go away.
1086 *
1087 * JMM - For now, rcv_object is just a port. Portsets
1088 * are disabled for the time being.
1089 */
1090
1091 assert(ip_active(dest_port));
1092 assert(dest_port->ip_receiver != ipc_space_kernel);
1093 // assert(!imq_full(&dest_port->ip_messages) ||
1094 // (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1095 // MACH_MSG_TYPE_PORT_SEND_ONCE));
1096 assert((hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR) == 0);
1097
1098 {
1099 register ipc_mqueue_t dest_mqueue;
1100 wait_queue_t waitq;
1101 thread_t receiver;
1102 processor_t processor;
1103 boolean_t still_running;
1104 spl_t s;
1105
1106 s = splsched();
1107 processor = current_processor();
1108 if (processor->current_pri >= BASEPRI_RTQUEUES)
1109 goto abort_send_receive1;
1110
1111 dest_mqueue = &dest_port->ip_messages;
1112 waitq = &dest_mqueue->imq_wait_queue;
1113 imq_lock(dest_mqueue);
1114
1115 wait_queue_peek64_locked(waitq, IPC_MQUEUE_RECEIVE, &receiver, &waitq);
1116 /* queue still locked, thread locked - but still on q */
1117
1118 if ( receiver == THREAD_NULL ) {
1119 abort_send_receive:
1120 imq_unlock(dest_mqueue);
1121 abort_send_receive1:
1122 splx(s);
1123 ip_unlock(dest_port);
1124 ipc_object_release(rcv_object);
1125 HOT(c_mmot_cold_032++);
1126 goto slow_send;
1127 }
1128
1129 assert(receiver->state & TH_WAIT);
1130 assert(receiver->wait_queue == waitq);
1131 assert(receiver->wait_event == IPC_MQUEUE_RECEIVE);
1132
1133 /*
1134 * Make sure that the scheduling restrictions of the receiver
1135 * are consistent with a handoff here (if it comes down to that).
1136 */
1137 if ( receiver->sched_pri >= BASEPRI_RTQUEUES ||
1138 receiver->processor_set != processor->processor_set ||
1139 (receiver->bound_processor != PROCESSOR_NULL &&
1140 receiver->bound_processor != processor)) {
1141 HOT(c_mmot_cold_033++);
1142 fall_off:
1143 thread_unlock(receiver);
1144 if (waitq != &dest_mqueue->imq_wait_queue)
1145 wait_queue_unlock(waitq);
1146 goto abort_send_receive;
1147 }
1148
1149 /*
1150 * Check that the receiver can stay on the hot path.
1151 */
1152 if (ipc_kmsg_copyout_size(kmsg, receiver->map) +
1153 REQUESTED_TRAILER_SIZE(receiver->ith_option) > receiver->ith_msize) {
1154 /*
1155 * The receiver can't accept the message.
1156 */
1157 HOT(c_mmot_bad_rcvr++);
1158 goto fall_off;
1159 }
1160
1161 /*
1162 * Before committing to the handoff, make sure that we are
1163 * really going to block (i.e. there are no messages already
1164 * queued for us. This violates lock ordering, so make sure
1165 * we don't deadlock. After the trylock succeeds below, we
1166 * may have up to 3 message queues locked:
1167 * - the dest port mqueue
1168 * - a portset mqueue (where waiting receiver was found)
1169 * - finally our own rcv_mqueue
1170 *
1171 * JMM - Need to make this check appropriate for portsets as
1172 * well before re-enabling them.
1173 */
1174 if (!imq_lock_try(rcv_mqueue)) {
1175 goto fall_off;
1176 }
1177 if (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL) {
1178 imq_unlock(rcv_mqueue);
1179 HOT(c_mmot_cold_033++);
1180 goto fall_off;
1181 }
1182
1183 /* At this point we are committed to do the "handoff". */
1184 c_mach_msg_trap_switch_fast++;
1185
1186 /*
1187 * Go ahead and pull the receiver from the waitq. If the
1188 * waitq wasn't the one for the mqueue, unlock it.
1189 */
1190 wait_queue_pull_thread_locked(waitq,
1191 receiver,
1192 (waitq != &dest_mqueue->imq_wait_queue));
1193
1194 /*
1195 * Store the kmsg and seqno where the receiver can pick it up.
1196 */
1197 receiver->ith_state = MACH_MSG_SUCCESS;
1198 receiver->ith_kmsg = kmsg;
1199 receiver->ith_seqno = dest_mqueue->imq_seqno++;
1200
1201 /*
1202 * Unblock the receiver. If it was still running on another
1203 * CPU, we'll give it a chance to run with the message where
1204 * it is (and just select someother thread to run here).
1205 * Otherwise, we'll invoke it here as part of the handoff.
1206 */
1207 still_running = thread_unblock(receiver, THREAD_AWAKENED);
1208
1209 thread_unlock(receiver);
1210
1211 imq_unlock(dest_mqueue);
1212 ip_unlock(dest_port);
1213 current_task()->messages_sent++;
1214
1215
1216 /*
1217 * Put self on receive port's queue.
1218 * Also save state that the sender of
1219 * our reply message needs to determine if it
1220 * can hand off directly back to us.
1221 */
1222 thread_lock(self);
1223 self->ith_msg_addr = (rcv_msg_addr) ? rcv_msg_addr : msg_addr;
1224 self->ith_object = rcv_object; /* still holds reference */
1225 self->ith_msize = rcv_size;
1226 self->ith_option = option;
1227 self->ith_scatter_list_size = scatter_list_size;
1228 self->ith_continuation = thread_syscall_return;
1229
1230 waitq = &rcv_mqueue->imq_wait_queue;
1231 (void)wait_queue_assert_wait64_locked(waitq,
1232 IPC_MQUEUE_RECEIVE,
1233 THREAD_ABORTSAFE, 0,
1234 self);
1235 thread_unlock(self);
1236 imq_unlock(rcv_mqueue);
1237
1238 /*
1239 * If the receiving thread wasn't still running, we switch directly
1240 * to it here. Otherwise we let the scheduler pick something for
1241 * here. In either case, block this thread as though it had called
1242 * ipc_mqueue_receive.
1243 */
1244 if (still_running) {
1245 splx(s);
1246 thread_block(ipc_mqueue_receive_continue);
1247 } else {
1248 thread_run(self, ipc_mqueue_receive_continue, NULL, receiver);
1249 }
1250 /* NOTREACHED */
1251 }
1252
1253 fast_copyout:
1254 /*
1255 * Nothing locked and no references held, except
1256 * we have kmsg with msgh_seqno filled in. Must
1257 * still check against rcv_size and do
1258 * ipc_kmsg_copyout/ipc_kmsg_put.
1259 */
1260
1261 reply_size = send_size + trailer->msgh_trailer_size;
1262 if (rcv_size < reply_size) {
1263 HOT(c_mmot_g_slow_copyout6++);
1264 goto slow_copyout;
1265 }
1266
1267 /* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */
1268
1269 switch (hdr->msgh_bits) {
1270 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
1271 MACH_MSG_TYPE_PORT_SEND_ONCE): {
1272 ipc_port_t reply_port =
1273 (ipc_port_t) hdr->msgh_local_port;
1274 mach_port_name_t dest_name, reply_name;
1275
1276 /* receiving a request message */
1277
1278 if (!IP_VALID(reply_port)) {
1279 HOT(c_mmot_g_slow_copyout5++);
1280 goto slow_copyout;
1281 }
1282
1283 is_write_lock(space);
1284 assert(space->is_active);
1285
1286 /*
1287 * To do an atomic copyout, need simultaneous
1288 * locks on both ports and the space. If
1289 * dest_port == reply_port, and simple locking is
1290 * enabled, then we will abort. Otherwise it's
1291 * OK to unlock twice.
1292 */
1293
1294 ip_lock(dest_port);
1295 if (!ip_active(dest_port) ||
1296 !ip_lock_try(reply_port)) {
1297 HOT(c_mmot_cold_037++);
1298 goto abort_request_copyout;
1299 }
1300
1301 if (!ip_active(reply_port)) {
1302 ip_unlock(reply_port);
1303 HOT(c_mmot_cold_038++);
1304 goto abort_request_copyout;
1305 }
1306
1307 assert(reply_port->ip_sorights > 0);
1308 ip_unlock(reply_port);
1309
1310 {
1311 register ipc_entry_t table;
1312 register ipc_entry_t entry;
1313 register mach_port_index_t index;
1314
1315 /* optimized ipc_entry_get */
1316
1317 table = space->is_table;
1318 index = table->ie_next;
1319
1320 if (index == 0) {
1321 HOT(c_mmot_cold_039++);
1322 goto abort_request_copyout;
1323 }
1324
1325 entry = &table[index];
1326 table->ie_next = entry->ie_next;
1327 entry->ie_request = 0;
1328
1329 {
1330 register mach_port_gen_t gen;
1331
1332 assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
1333 gen = IE_BITS_NEW_GEN(entry->ie_bits);
1334
1335 reply_name = MACH_PORT_MAKE(index, gen);
1336
1337 /* optimized ipc_right_copyout */
1338
1339 entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
1340 }
1341
1342 assert(MACH_PORT_VALID(reply_name));
1343 entry->ie_object = (ipc_object_t) reply_port;
1344 is_write_unlock(space);
1345 }
1346
1347 /* optimized ipc_object_copyout_dest */
1348
1349 assert(dest_port->ip_srights > 0);
1350 ip_release(dest_port);
1351
1352 if (dest_port->ip_receiver == space)
1353 dest_name = dest_port->ip_receiver_name;
1354 else
1355 dest_name = MACH_PORT_NULL;
1356
1357 if ((--dest_port->ip_srights == 0) &&
1358 (dest_port->ip_nsrequest != IP_NULL)) {
1359 ipc_port_t nsrequest;
1360 mach_port_mscount_t mscount;
1361
1362 /* a rather rare case */
1363
1364 nsrequest = dest_port->ip_nsrequest;
1365 mscount = dest_port->ip_mscount;
1366 dest_port->ip_nsrequest = IP_NULL;
1367 ip_unlock(dest_port);
1368 ipc_notify_no_senders(nsrequest, mscount);
1369 } else
1370 ip_unlock(dest_port);
1371
1372 hdr->msgh_bits =
1373 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
1374 MACH_MSG_TYPE_PORT_SEND);
1375 hdr->msgh_remote_port = (mach_port_t)reply_name;
1376 hdr->msgh_local_port = (mach_port_t)dest_name;
1377 HOT(c_mmot_hot_ok1++);
1378 goto fast_put;
1379
1380 abort_request_copyout:
1381 ip_unlock(dest_port);
1382 is_write_unlock(space);
1383 HOT(c_mmot_g_slow_copyout4++);
1384 goto slow_copyout;
1385 }
1386
1387 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1388 register mach_port_name_t dest_name;
1389
1390 /* receiving a reply message */
1391
1392 ip_lock(dest_port);
1393 if (!ip_active(dest_port)) {
1394 ip_unlock(dest_port);
1395 HOT(c_mmot_g_slow_copyout3++);
1396 goto slow_copyout;
1397 }
1398
1399 /* optimized ipc_object_copyout_dest */
1400
1401 assert(dest_port->ip_sorights > 0);
1402
1403 if (dest_port->ip_receiver == space) {
1404 ip_release(dest_port);
1405 dest_port->ip_sorights--;
1406 dest_name = dest_port->ip_receiver_name;
1407 ip_unlock(dest_port);
1408 } else {
1409 ip_unlock(dest_port);
1410
1411 ipc_notify_send_once(dest_port);
1412 dest_name = MACH_PORT_NULL;
1413 }
1414
1415 hdr->msgh_bits = MACH_MSGH_BITS(0,
1416 MACH_MSG_TYPE_PORT_SEND_ONCE);
1417 hdr->msgh_remote_port = MACH_PORT_NULL;
1418 hdr->msgh_local_port = (ipc_port_t)dest_name;
1419 HOT(c_mmot_hot_ok2++);
1420 goto fast_put;
1421 }
1422
1423 case MACH_MSGH_BITS_COMPLEX|
1424 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1425 register mach_port_name_t dest_name;
1426
1427 /* receiving a complex reply message */
1428
1429 ip_lock(dest_port);
1430 if (!ip_active(dest_port)) {
1431 ip_unlock(dest_port);
1432 HOT(c_mmot_g_slow_copyout1++);
1433 goto slow_copyout;
1434 }
1435
1436 /* optimized ipc_object_copyout_dest */
1437
1438 assert(dest_port->ip_sorights > 0);
1439
1440 if (dest_port->ip_receiver == space) {
1441 ip_release(dest_port);
1442 dest_port->ip_sorights--;
1443 dest_name = dest_port->ip_receiver_name;
1444 ip_unlock(dest_port);
1445 } else {
1446 ip_unlock(dest_port);
1447
1448 ipc_notify_send_once(dest_port);
1449 dest_name = MACH_PORT_NULL;
1450 }
1451
1452 hdr->msgh_bits =
1453 MACH_MSGH_BITS_COMPLEX |
1454 MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND_ONCE);
1455 hdr->msgh_remote_port = MACH_PORT_NULL;
1456 hdr->msgh_local_port = (mach_port_t)dest_name;
1457
1458 mr = ipc_kmsg_copyout_body(kmsg, space,
1459 current_map(),
1460 MACH_MSG_BODY_NULL);
1461 /* hdr and send_size may be invalid now - done use */
1462 if (mr != MACH_MSG_SUCCESS) {
1463 if (ipc_kmsg_put(msg_addr, kmsg,
1464 kmsg->ikm_header->msgh_size +
1465 trailer->msgh_trailer_size) ==
1466 MACH_RCV_INVALID_DATA)
1467 return MACH_RCV_INVALID_DATA;
1468 else
1469 return mr | MACH_RCV_BODY_ERROR;
1470 }
1471 HOT(c_mmot_hot_ok3++);
1472 goto fast_put;
1473 }
1474
1475 default:
1476 HOT(c_mmot_g_slow_copyout2++);
1477 goto slow_copyout;
1478 }
1479 /*NOTREACHED*/
1480
1481 fast_put:
1482 mr = ipc_kmsg_put(rcv_msg_addr ? rcv_msg_addr : msg_addr,
1483 kmsg,
1484 kmsg->ikm_header->msgh_size +
1485 trailer->msgh_trailer_size);
1486 if (mr != MACH_MSG_SUCCESS) {
1487 return MACH_RCV_INVALID_DATA;
1488 }
1489 current_task()->messages_received++;
1490 return mr;
1491
1492
1493 /* BEGINNING OF WARM PATH */
1494
1495 /*
1496 * The slow path has a few non-register temporary
1497 * variables used only for call-by-reference.
1498 */
1499
1500 slow_copyin:
1501 {
1502 mach_port_seqno_t temp_seqno = 0;
1503 register mach_port_name_t reply_name =
1504 (mach_port_name_t)hdr->msgh_local_port;
1505
1506
1507 /*
1508 * We have the message data in kmsg, but
1509 * we still need to copyin, send it,
1510 * receive a reply, and do copyout.
1511 */
1512
1513 mr = ipc_kmsg_copyin(kmsg, space, current_map(),
1514 MACH_PORT_NULL);
1515 if (mr != MACH_MSG_SUCCESS) {
1516 ipc_kmsg_free(kmsg);
1517 return(mr);
1518 }
1519
1520 /*
1521 * LP64support - We have to recompute the header pointer
1522 * and send_size - as they could have changed during the
1523 * complex copyin.
1524 */
1525 hdr = kmsg->ikm_header;
1526 send_size = hdr->msgh_size;
1527
1528 /* try to get back on optimized path */
1529 if ((reply_name != rcv_name) ||
1530 (hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR)) {
1531 HOT(c_mmot_cold_048++);
1532 goto slow_send;
1533 }
1534
1535 dest_port = (ipc_port_t) hdr->msgh_remote_port;
1536 assert(IP_VALID(dest_port));
1537
1538 ip_lock(dest_port);
1539 if (!ip_active(dest_port)) {
1540 ip_unlock(dest_port);
1541 goto slow_send;
1542 }
1543
1544 if (dest_port->ip_receiver == ipc_space_kernel) {
1545 dest_port->ip_messages.imq_seqno++;
1546 ip_unlock(dest_port);
1547 goto kernel_send;
1548 }
1549
1550 if (!imq_full(&dest_port->ip_messages) ||
1551 (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1552 MACH_MSG_TYPE_PORT_SEND_ONCE))
1553 {
1554 /*
1555 * Try an optimized ipc_mqueue_copyin.
1556 * It will work if this is a request message.
1557 */
1558
1559 register ipc_port_t reply_port;
1560
1561 reply_port = (ipc_port_t) hdr->msgh_local_port;
1562 if (IP_VALID(reply_port)) {
1563 if (ip_lock_try(reply_port)) {
1564 if (ip_active(reply_port) &&
1565 reply_port->ip_receiver == space &&
1566 reply_port->ip_receiver_name == rcv_name &&
1567 reply_port->ip_pset_count == 0)
1568 {
1569 /* Grab a reference to the reply port. */
1570 rcv_object = (ipc_object_t) reply_port;
1571 io_reference(rcv_object);
1572 rcv_mqueue = &reply_port->ip_messages;
1573 io_unlock(rcv_object);
1574 HOT(c_mmot_getback_FastSR++);
1575 goto fast_send_receive;
1576 }
1577 ip_unlock(reply_port);
1578 }
1579 }
1580 }
1581
1582 ip_unlock(dest_port);
1583 HOT(c_mmot_cold_050++);
1584 goto slow_send;
1585
1586 kernel_send:
1587 /*
1588 * Special case: send message to kernel services.
1589 * The request message has been copied into the
1590 * kmsg. Nothing is locked.
1591 */
1592
1593 {
1594 register ipc_port_t reply_port;
1595 mach_port_seqno_t local_seqno;
1596 spl_t s;
1597
1598 /*
1599 * Perform the kernel function.
1600 */
1601 c_mmot_kernel_send++;
1602
1603 current_task()->messages_sent++;
1604
1605 kmsg = ipc_kobject_server(kmsg);
1606 if (kmsg == IKM_NULL) {
1607 /*
1608 * No reply. Take the
1609 * slow receive path.
1610 */
1611 HOT(c_mmot_cold_051++);
1612 goto slow_get_rcv_port;
1613 }
1614
1615 /*
1616 * Check that:
1617 * the reply port is alive
1618 * we hold the receive right
1619 * the name has not changed.
1620 * the port is not in a set
1621 * If any of these are not true,
1622 * we cannot directly receive the reply
1623 * message.
1624 */
1625 hdr = kmsg->ikm_header;
1626 send_size = hdr->msgh_size;
1627 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1628 round_msg(send_size));
1629 reply_port = (ipc_port_t) hdr->msgh_remote_port;
1630 ip_lock(reply_port);
1631
1632 if ((!ip_active(reply_port)) ||
1633 (reply_port->ip_receiver != space) ||
1634 (reply_port->ip_receiver_name != rcv_name) ||
1635 (reply_port->ip_pset_count != 0))
1636 {
1637 ip_unlock(reply_port);
1638 ipc_kmsg_send_always(kmsg);
1639 HOT(c_mmot_cold_052++);
1640 goto slow_get_rcv_port;
1641 }
1642
1643 s = splsched();
1644 rcv_mqueue = &reply_port->ip_messages;
1645 imq_lock(rcv_mqueue);
1646
1647 /* keep port locked, and don`t change ref count yet */
1648
1649 /*
1650 * If there are messages on the port
1651 * or other threads waiting for a message,
1652 * we cannot directly receive the reply.
1653 */
1654 if (!wait_queue_empty(&rcv_mqueue->imq_wait_queue) ||
1655 (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL))
1656 {
1657 imq_unlock(rcv_mqueue);
1658 splx(s);
1659 ip_unlock(reply_port);
1660 ipc_kmsg_send_always(kmsg);
1661 HOT(c_mmot_cold_053++);
1662 goto slow_get_rcv_port;
1663 }
1664
1665 /*
1666 * We can directly receive this reply.
1667 * Since there were no messages queued
1668 * on the reply port, there should be
1669 * no threads blocked waiting to send.
1670 */
1671 dest_port = reply_port;
1672 local_seqno = rcv_mqueue->imq_seqno++;
1673 imq_unlock(rcv_mqueue);
1674 splx(s);
1675
1676 /*
1677 * inline ipc_object_release.
1678 * Port is still locked.
1679 * Reference count was not incremented.
1680 */
1681 ip_check_unlock(reply_port);
1682
1683 if (option & MACH_RCV_TRAILER_MASK) {
1684 trailer->msgh_seqno = local_seqno;
1685 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1686 }
1687 /* copy out the kernel reply */
1688 HOT(c_mmot_fastkernelreply++);
1689 goto fast_copyout;
1690 }
1691
1692 slow_send:
1693 /*
1694 * Nothing is locked. We have acquired kmsg, but
1695 * we still need to send it and receive a reply.
1696 */
1697
1698 mr = ipc_kmsg_send(kmsg, MACH_MSG_OPTION_NONE,
1699 MACH_MSG_TIMEOUT_NONE);
1700 if (mr != MACH_MSG_SUCCESS) {
1701 mr |= ipc_kmsg_copyout_pseudo(kmsg, space,
1702 current_map(),
1703 MACH_MSG_BODY_NULL);
1704
1705 (void) ipc_kmsg_put(msg_addr, kmsg,
1706 kmsg->ikm_header->msgh_size);
1707 return(mr);
1708 }
1709
1710 slow_get_rcv_port:
1711 /*
1712 * We have sent the message. Copy in the receive port.
1713 */
1714 mr = ipc_mqueue_copyin(space, rcv_name,
1715 &rcv_mqueue, &rcv_object);
1716 if (mr != MACH_MSG_SUCCESS) {
1717 return(mr);
1718 }
1719 /* hold ref for rcv_object */
1720
1721 /*
1722 * slow_receive:
1723 *
1724 * Now we have sent the request and copied in rcv_name,
1725 * and hold ref for rcv_object (to keep mqueue alive).
1726 * Just receive a reply and try to get back to fast path.
1727 */
1728
1729 self->ith_continuation = (void (*)(mach_msg_return_t))0;
1730 ipc_mqueue_receive(rcv_mqueue,
1731 MACH_MSG_OPTION_NONE,
1732 MACH_MSG_SIZE_MAX,
1733 MACH_MSG_TIMEOUT_NONE,
1734 THREAD_ABORTSAFE);
1735
1736 mr = self->ith_state;
1737 temp_seqno = self->ith_seqno;
1738
1739 ipc_object_release(rcv_object);
1740
1741 if (mr != MACH_MSG_SUCCESS) {
1742 return(mr);
1743 }
1744
1745 kmsg = self->ith_kmsg;
1746 hdr = kmsg->ikm_header;
1747 send_size = hdr->msgh_size;
1748 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1749 round_msg(send_size));
1750 if (option & MACH_RCV_TRAILER_MASK) {
1751 trailer->msgh_seqno = temp_seqno;
1752 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1753 }
1754 dest_port = (ipc_port_t) hdr->msgh_remote_port;
1755 HOT(c_mmot_cold_055++);
1756 goto fast_copyout;
1757
1758 slow_copyout:
1759 /*
1760 * Nothing locked and no references held, except
1761 * we have kmsg with msgh_seqno filled in. Must
1762 * still check against rcv_size and do
1763 * ipc_kmsg_copyout/ipc_kmsg_put.
1764 */
1765
1766 /* LP64support - have to compute real size as it would be received */
1767 reply_size = ipc_kmsg_copyout_size(kmsg, current_map()) +
1768 REQUESTED_TRAILER_SIZE(option);
1769 if (rcv_size < reply_size) {
1770 if (msg_receive_error(kmsg, msg_addr, option, temp_seqno,
1771 space) == MACH_RCV_INVALID_DATA) {
1772 mr = MACH_RCV_INVALID_DATA;
1773 return(mr);
1774 }
1775 else {
1776 mr = MACH_RCV_TOO_LARGE;
1777 return(mr);
1778 }
1779 }
1780
1781 mr = ipc_kmsg_copyout(kmsg, space, current_map(),
1782 MACH_PORT_NULL, MACH_MSG_BODY_NULL);
1783 if (mr != MACH_MSG_SUCCESS) {
1784 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1785 if (ipc_kmsg_put(msg_addr, kmsg, reply_size) ==
1786 MACH_RCV_INVALID_DATA)
1787 mr = MACH_RCV_INVALID_DATA;
1788 }
1789 else {
1790 if (msg_receive_error(kmsg, msg_addr, option,
1791 temp_seqno, space) == MACH_RCV_INVALID_DATA)
1792 mr = MACH_RCV_INVALID_DATA;
1793 }
1794
1795 return(mr);
1796 }
1797
1798 /* try to get back on optimized path */
1799 HOT(c_mmot_getback_fast_put++);
1800 goto fast_put;
1801
1802 /*NOTREACHED*/
1803 }
1804 } /* END OF HOT PATH */
1805 #endif /* ENABLE_HOTPATH */
1806
1807 if (option & MACH_SEND_MSG) {
1808 ipc_space_t space = current_space();
1809 vm_map_t map = current_map();
1810 ipc_kmsg_t kmsg;
1811
1812 mr = ipc_kmsg_get(msg_addr, send_size, &kmsg);
1813
1814 if (mr != MACH_MSG_SUCCESS)
1815 return mr;
1816
1817 if (option & MACH_SEND_CANCEL) {
1818 if (notify == MACH_PORT_NULL)
1819 mr = MACH_SEND_INVALID_NOTIFY;
1820 else
1821 mr = ipc_kmsg_copyin(kmsg, space, map, notify);
1822 } else
1823 mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
1824 if (mr != MACH_MSG_SUCCESS) {
1825 ipc_kmsg_free(kmsg);
1826 return mr;
1827 }
1828
1829 mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, msg_timeout);
1830
1831 if (mr != MACH_MSG_SUCCESS) {
1832 mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
1833 (void) ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size);
1834 return mr;
1835 }
1836
1837 }
1838
1839 if (option & MACH_RCV_MSG) {
1840 thread_t self = current_thread();
1841 ipc_space_t space = current_space();
1842 ipc_object_t object;
1843 ipc_mqueue_t mqueue;
1844
1845 mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
1846 if (mr != MACH_MSG_SUCCESS) {
1847 return mr;
1848 }
1849 /* hold ref for object */
1850
1851 /*
1852 * 1. MACH_RCV_OVERWRITE is on, and rcv_msg is our scatter list
1853 * and receive buffer
1854 * 2. MACH_RCV_OVERWRITE is off, and rcv_msg might be the
1855 * alternate receive buffer (separate send and receive buffers).
1856 */
1857 if (option & MACH_RCV_OVERWRITE)
1858 self->ith_msg_addr = rcv_msg_addr;
1859 else if (rcv_msg_addr != (mach_vm_address_t)0)
1860 self->ith_msg_addr = rcv_msg_addr;
1861 else
1862 self->ith_msg_addr = msg_addr;
1863 self->ith_object = object;
1864 self->ith_msize = rcv_size;
1865 self->ith_option = option;
1866 self->ith_scatter_list_size = scatter_list_size;
1867 self->ith_continuation = thread_syscall_return;
1868
1869 ipc_mqueue_receive(mqueue, option, rcv_size, msg_timeout, THREAD_ABORTSAFE);
1870 if ((option & MACH_RCV_TIMEOUT) && msg_timeout == 0)
1871 thread_poll_yield(self);
1872 return mach_msg_receive_results();
1873 }
1874
1875 return MACH_MSG_SUCCESS;
1876 }
1877
1878 /*
1879 * Routine: mach_msg_trap [mach trap]
1880 * Purpose:
1881 * Possibly send a message; possibly receive a message.
1882 * Conditions:
1883 * Nothing locked.
1884 * Returns:
1885 * All of mach_msg_send and mach_msg_receive error codes.
1886 */
1887
1888 mach_msg_return_t
1889 mach_msg_trap(
1890 struct mach_msg_overwrite_trap_args *args)
1891 {
1892 kern_return_t kr;
1893 args->rcv_msg = (mach_vm_address_t)0;
1894
1895 kr = mach_msg_overwrite_trap(args);
1896 return kr;
1897 }
1898
1899
1900 /*
1901 * Routine: msg_receive_error [internal]
1902 * Purpose:
1903 * Builds a minimal header/trailer and copies it to
1904 * the user message buffer. Invoked when in the case of a
1905 * MACH_RCV_TOO_LARGE or MACH_RCV_BODY_ERROR error.
1906 * Conditions:
1907 * Nothing locked.
1908 * Returns:
1909 * MACH_MSG_SUCCESS minimal header/trailer copied
1910 * MACH_RCV_INVALID_DATA copyout to user buffer failed
1911 */
1912
1913 mach_msg_return_t
1914 msg_receive_error(
1915 ipc_kmsg_t kmsg,
1916 mach_vm_address_t msg_addr,
1917 mach_msg_option_t option,
1918 mach_port_seqno_t seqno,
1919 ipc_space_t space)
1920 {
1921 mach_msg_format_0_trailer_t *trailer;
1922
1923 /*
1924 * Copy out the destination port in the message.
1925 * Destroy all other rights and memory in the message.
1926 */
1927 ipc_kmsg_copyout_dest(kmsg, space);
1928
1929 /*
1930 * Build a minimal message with the requested trailer.
1931 */
1932 trailer = (mach_msg_format_0_trailer_t *)
1933 ((vm_offset_t)kmsg->ikm_header +
1934 round_msg(sizeof(mach_msg_header_t)));
1935 kmsg->ikm_header->msgh_size = sizeof(mach_msg_header_t);
1936 bcopy( (char *)&trailer_template,
1937 (char *)trailer,
1938 sizeof(trailer_template));
1939 if (option & MACH_RCV_TRAILER_MASK) {
1940 trailer->msgh_seqno = seqno;
1941 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1942 }
1943
1944 /*
1945 * Copy the message to user space
1946 */
1947 if (ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size +
1948 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
1949 return(MACH_RCV_INVALID_DATA);
1950 else
1951 return(MACH_MSG_SUCCESS);
1952 }
mirror of XNU