]> git.saurik.com Git - apple/xnu.git/blob - osfmk/ipc/mach_msg.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-792.24.17.tar.gz
[apple/xnu.git] / osfmk / ipc / mach_msg.c
1 /*
2 * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22 /*
23 * @OSF_COPYRIGHT@
24 */
25 /*
26 * Mach Operating System
27 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
28 * All Rights Reserved.
29 *
30 * Permission to use, copy, modify and distribute this software and its
31 * documentation is hereby granted, provided that both the copyright
32 * notice and this permission notice appear in all copies of the
33 * software, derivative works or modified versions, and any portions
34 * thereof, and that both notices appear in supporting documentation.
35 *
36 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
39 *
40 * Carnegie Mellon requests users of this software to return to
41 *
42 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
43 * School of Computer Science
44 * Carnegie Mellon University
45 * Pittsburgh PA 15213-3890
46 *
47 * any improvements or extensions that they make and grant Carnegie Mellon
48 * the rights to redistribute these changes.
49 */
50 /*
51 */
52 /*
53 * File: ipc/mach_msg.c
54 * Author: Rich Draves
55 * Date: 1989
56 *
57 * Exported message traps. See mach/message.h.
58 */
59
60 #include <mach/mach_types.h>
61 #include <mach/kern_return.h>
62 #include <mach/port.h>
63 #include <mach/message.h>
64 #include <mach/mig_errors.h>
65 #include <mach/mach_traps.h>
66
67 #include <kern/kern_types.h>
68 #include <kern/assert.h>
69 #include <kern/counters.h>
70 #include <kern/cpu_number.h>
71 #include <kern/ipc_kobject.h>
72 #include <kern/ipc_mig.h>
73 #include <kern/task.h>
74 #include <kern/thread.h>
75 #include <kern/lock.h>
76 #include <kern/sched_prim.h>
77 #include <kern/exception.h>
78 #include <kern/misc_protos.h>
79 #include <kern/kalloc.h>
80 #include <kern/processor.h>
81 #include <kern/syscall_subr.h>
82
83 #include <vm/vm_map.h>
84
85 #include <ipc/ipc_types.h>
86 #include <ipc/ipc_kmsg.h>
87 #include <ipc/ipc_mqueue.h>
88 #include <ipc/ipc_object.h>
89 #include <ipc/ipc_notify.h>
90 #include <ipc/ipc_port.h>
91 #include <ipc/ipc_pset.h>
92 #include <ipc/ipc_space.h>
93 #include <ipc/ipc_entry.h>
94
95 #include <machine/machine_routines.h>
96
97 #include <sys/kdebug.h>
98
99 #ifndef offsetof
100 #define offsetof(type, member) ((size_t)(&((type *)0)->member))
101 #endif /* offsetof */
102
103 /*
104 * Forward declarations - kernel internal routines
105 */
106
107 mach_msg_return_t mach_msg_send(
108 mach_msg_header_t *msg,
109 mach_msg_option_t option,
110 mach_msg_size_t send_size,
111 mach_msg_timeout_t send_timeout,
112 mach_port_name_t notify);
113
114 mach_msg_return_t mach_msg_receive(
115 mach_msg_header_t *msg,
116 mach_msg_option_t option,
117 mach_msg_size_t rcv_size,
118 mach_port_name_t rcv_name,
119 mach_msg_timeout_t rcv_timeout,
120 void (*continuation)(mach_msg_return_t),
121 mach_msg_size_t slist_size);
122
123
124 mach_msg_return_t mach_msg_receive_results(void);
125
126 mach_msg_return_t msg_receive_error(
127 ipc_kmsg_t kmsg,
128 mach_vm_address_t msg_addr,
129 mach_msg_option_t option,
130 mach_port_seqno_t seqno,
131 ipc_space_t space);
132
133 security_token_t KERNEL_SECURITY_TOKEN = KERNEL_SECURITY_TOKEN_VALUE;
134 audit_token_t KERNEL_AUDIT_TOKEN = KERNEL_AUDIT_TOKEN_VALUE;
135
136 mach_msg_format_0_trailer_t trailer_template = {
137 /* mach_msg_trailer_type_t */ MACH_MSG_TRAILER_FORMAT_0,
138 /* mach_msg_trailer_size_t */ MACH_MSG_TRAILER_MINIMUM_SIZE,
139 /* mach_port_seqno_t */ 0,
140 /* security_token_t */ KERNEL_SECURITY_TOKEN_VALUE
141 };
142
143 /*
144 * Routine: mach_msg_send
145 * Purpose:
146 * Send a message.
147 * Conditions:
148 * Nothing locked.
149 * Returns:
150 * MACH_MSG_SUCCESS Sent the message.
151 * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
152 * MACH_SEND_NO_BUFFER Couldn't allocate buffer.
153 * MACH_SEND_INVALID_DATA Couldn't copy message data.
154 * MACH_SEND_INVALID_HEADER
155 * Illegal value in the message header bits.
156 * MACH_SEND_INVALID_DEST The space is dead.
157 * MACH_SEND_INVALID_NOTIFY Bad notify port.
158 * MACH_SEND_INVALID_DEST Can't copyin destination port.
159 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
160 * MACH_SEND_TIMED_OUT Timeout expired without delivery.
161 * MACH_SEND_INTERRUPTED Delivery interrupted.
162 * MACH_SEND_NO_NOTIFY Can't allocate a msg-accepted request.
163 * MACH_SEND_WILL_NOTIFY Msg-accepted notif. requested.
164 * MACH_SEND_NOTIFY_IN_PROGRESS
165 * This space has already forced a message to this port.
166 */
167
168 mach_msg_return_t
169 mach_msg_send(
170 mach_msg_header_t *msg,
171 mach_msg_option_t option,
172 mach_msg_size_t send_size,
173 mach_msg_timeout_t send_timeout,
174 mach_port_name_t notify)
175 {
176 ipc_space_t space = current_space();
177 vm_map_t map = current_map();
178 ipc_kmsg_t kmsg;
179 mach_msg_return_t mr;
180 mach_msg_size_t msg_and_trailer_size;
181 mach_msg_max_trailer_t *trailer;
182
183 if ((send_size < sizeof(mach_msg_header_t)) || (send_size & 3))
184 return MACH_SEND_MSG_TOO_SMALL;
185
186 if (send_size > MACH_MSG_SIZE_MAX - MAX_TRAILER_SIZE)
187 return MACH_SEND_TOO_LARGE;
188
189 msg_and_trailer_size = send_size + MAX_TRAILER_SIZE;
190
191 kmsg = ipc_kmsg_alloc(msg_and_trailer_size);
192
193 if (kmsg == IKM_NULL)
194 return MACH_SEND_NO_BUFFER;
195
196 (void) memcpy((void *) kmsg->ikm_header, (const void *) msg, send_size);
197
198 kmsg->ikm_header->msgh_size = send_size;
199
200 /*
201 * reserve for the trailer the largest space (MAX_TRAILER_SIZE)
202 * However, the internal size field of the trailer (msgh_trailer_size)
203 * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to optimize
204 * the cases where no implicit data is requested.
205 */
206 trailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + send_size);
207 trailer->msgh_sender = current_thread()->task->sec_token;
208 trailer->msgh_audit = current_thread()->task->audit_token;
209 trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
210 trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
211
212 if (option & MACH_SEND_CANCEL) {
213 if (notify == MACH_PORT_NULL)
214 mr = MACH_SEND_INVALID_NOTIFY;
215 else
216 mr = ipc_kmsg_copyin(kmsg, space, map, notify);
217 } else
218 mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
219 if (mr != MACH_MSG_SUCCESS) {
220 ipc_kmsg_free(kmsg);
221 return mr;
222 }
223
224 mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, send_timeout);
225
226 if (mr != MACH_MSG_SUCCESS) {
227 mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
228 (void) memcpy((void *) msg, (const void *) kmsg->ikm_header,
229 kmsg->ikm_header->msgh_size);
230 ipc_kmsg_free(kmsg);
231 }
232
233 return mr;
234 }
235
236 /*
237 * Routine: mach_msg_receive
238 * Purpose:
239 * Receive a message.
240 * Conditions:
241 * Nothing locked.
242 * Returns:
243 * MACH_MSG_SUCCESS Received a message.
244 * MACH_RCV_INVALID_NAME The name doesn't denote a right,
245 * or the denoted right is not receive or port set.
246 * MACH_RCV_IN_SET Receive right is a member of a set.
247 * MACH_RCV_TOO_LARGE Message wouldn't fit into buffer.
248 * MACH_RCV_TIMED_OUT Timeout expired without a message.
249 * MACH_RCV_INTERRUPTED Reception interrupted.
250 * MACH_RCV_PORT_DIED Port/set died while receiving.
251 * MACH_RCV_PORT_CHANGED Port moved into set while receiving.
252 * MACH_RCV_INVALID_DATA Couldn't copy to user buffer.
253 * MACH_RCV_INVALID_NOTIFY Bad notify port.
254 * MACH_RCV_HEADER_ERROR
255 */
256
257 mach_msg_return_t
258 mach_msg_receive_results(void)
259 {
260 thread_t self = current_thread();
261 ipc_space_t space = current_space();
262 vm_map_t map = current_map();
263
264 ipc_object_t object = self->ith_object;
265 mach_msg_return_t mr = self->ith_state;
266 mach_vm_address_t msg_addr = self->ith_msg_addr;
267 mach_msg_option_t option = self->ith_option;
268 ipc_kmsg_t kmsg = self->ith_kmsg;
269 mach_port_seqno_t seqno = self->ith_seqno;
270
271 mach_msg_format_0_trailer_t *trailer;
272
273 ipc_object_release(object);
274
275 if (mr != MACH_MSG_SUCCESS) {
276
277 if (mr == MACH_RCV_TOO_LARGE ) {
278 if (option & MACH_RCV_LARGE) {
279 /*
280 * We need to inform the user-level code that it needs more
281 * space. The value for how much space was returned in the
282 * msize save area instead of the message (which was left on
283 * the queue).
284 */
285 if (copyout((char *) &self->ith_msize,
286 msg_addr + offsetof(mach_msg_header_t, msgh_size),
287 sizeof(mach_msg_size_t)))
288 mr = MACH_RCV_INVALID_DATA;
289 goto out;
290 }
291
292 if (msg_receive_error(kmsg, msg_addr, option, seqno, space)
293 == MACH_RCV_INVALID_DATA)
294 mr = MACH_RCV_INVALID_DATA;
295 }
296 goto out;
297 }
298
299 trailer = (mach_msg_format_0_trailer_t *)
300 ((vm_offset_t)kmsg->ikm_header +
301 round_msg(kmsg->ikm_header->msgh_size));
302 if (option & MACH_RCV_TRAILER_MASK) {
303 trailer->msgh_seqno = seqno;
304 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
305 }
306
307 /*
308 * If MACH_RCV_OVERWRITE was specified, try to get the scatter
309 * list and verify it against the contents of the message. If
310 * there is any problem with it, we will continue without it as
311 * normal.
312 */
313 if (option & MACH_RCV_OVERWRITE) {
314 mach_msg_size_t slist_size = self->ith_scatter_list_size;
315 mach_msg_body_t *slist;
316
317 slist = ipc_kmsg_get_scatter(msg_addr, slist_size, kmsg);
318 mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL, slist);
319 ipc_kmsg_free_scatter(slist, slist_size);
320 } else {
321 mr = ipc_kmsg_copyout(kmsg, space, map,
322 MACH_PORT_NULL, MACH_MSG_BODY_NULL);
323 }
324
325 if (mr != MACH_MSG_SUCCESS) {
326 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
327 if (ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size +
328 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
329 mr = MACH_RCV_INVALID_DATA;
330 }
331 else {
332 if (msg_receive_error(kmsg, msg_addr, option, seqno, space)
333 == MACH_RCV_INVALID_DATA)
334 mr = MACH_RCV_INVALID_DATA;
335 }
336 goto out;
337 }
338 mr = ipc_kmsg_put(msg_addr,
339 kmsg,
340 kmsg->ikm_header->msgh_size +
341 trailer->msgh_trailer_size);
342 out:
343 return mr;
344 }
345
346 mach_msg_return_t
347 mach_msg_receive(
348 mach_msg_header_t *msg,
349 mach_msg_option_t option,
350 mach_msg_size_t rcv_size,
351 mach_port_name_t rcv_name,
352 mach_msg_timeout_t rcv_timeout,
353 void (*continuation)(mach_msg_return_t),
354 mach_msg_size_t slist_size)
355 {
356 thread_t self = current_thread();
357 ipc_space_t space = current_space();
358 ipc_object_t object;
359 ipc_mqueue_t mqueue;
360 mach_msg_return_t mr;
361
362 mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
363 if (mr != MACH_MSG_SUCCESS) {
364 return mr;
365 }
366 /* hold ref for object */
367
368 self->ith_msg_addr = CAST_DOWN(mach_vm_address_t, msg);
369 self->ith_object = object;
370 self->ith_msize = rcv_size;
371 self->ith_option = option;
372 self->ith_scatter_list_size = slist_size;
373 self->ith_continuation = continuation;
374
375 ipc_mqueue_receive(mqueue, option, rcv_size, rcv_timeout, THREAD_ABORTSAFE);
376 if ((option & MACH_RCV_TIMEOUT) && rcv_timeout == 0)
377 thread_poll_yield(self);
378 return mach_msg_receive_results();
379 }
380
381 void
382 mach_msg_receive_continue(void)
383 {
384 thread_t self = current_thread();
385
386 (*self->ith_continuation)(mach_msg_receive_results());
387 }
388
389 /*
390 * Toggle this to compile the hotpath in/out
391 * If compiled in, the run-time toggle "enable_hotpath" below
392 * eases testing & debugging
393 */
394 #define ENABLE_HOTPATH 1 /* Hacked on for now */
395
396 #if ENABLE_HOTPATH
397 /*
398 * These counters allow tracing of hotpath behavior under test loads.
399 * A couple key counters are unconditional (see below).
400 */
401 #define HOTPATH_DEBUG 0 /* Toggle to include lots of counters */
402 #if HOTPATH_DEBUG
403 #define HOT(expr) expr
404
405 unsigned int c_mmot_FIRST = 0; /* Unused First Counter */
406 unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
407 unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
408 unsigned int c_mmot_kernel_send = 0; /* kernel server */
409 unsigned int c_mmot_cold_000 = 0; /* see below ... */
410 unsigned int c_mmot_smallsendsize = 0;
411 unsigned int c_mmot_oddsendsize = 0;
412 unsigned int c_mmot_bigsendsize = 0;
413 unsigned int c_mmot_copyinmsg_fail = 0;
414 unsigned int c_mmot_g_slow_copyin3 = 0;
415 unsigned int c_mmot_cold_006 = 0;
416 unsigned int c_mmot_cold_007 = 0;
417 unsigned int c_mmot_cold_008 = 0;
418 unsigned int c_mmot_cold_009 = 0;
419 unsigned int c_mmot_cold_010 = 0;
420 unsigned int c_mmot_cold_012 = 0;
421 unsigned int c_mmot_cold_013 = 0;
422 unsigned int c_mmot_cold_014 = 0;
423 unsigned int c_mmot_cold_016 = 0;
424 unsigned int c_mmot_cold_018 = 0;
425 unsigned int c_mmot_cold_019 = 0;
426 unsigned int c_mmot_cold_020 = 0;
427 unsigned int c_mmot_cold_021 = 0;
428 unsigned int c_mmot_cold_022 = 0;
429 unsigned int c_mmot_cold_023 = 0;
430 unsigned int c_mmot_cold_024 = 0;
431 unsigned int c_mmot_cold_025 = 0;
432 unsigned int c_mmot_cold_026 = 0;
433 unsigned int c_mmot_cold_027 = 0;
434 unsigned int c_mmot_hot_fSR_ok = 0;
435 unsigned int c_mmot_cold_029 = 0;
436 unsigned int c_mmot_cold_030 = 0;
437 unsigned int c_mmot_cold_031 = 0;
438 unsigned int c_mmot_cold_032 = 0;
439 unsigned int c_mmot_cold_033 = 0;
440 unsigned int c_mmot_bad_rcvr = 0;
441 unsigned int c_mmot_rcvr_swapped = 0;
442 unsigned int c_mmot_rcvr_locked = 0;
443 unsigned int c_mmot_rcvr_tswapped = 0;
444 unsigned int c_mmot_rcvr_freed = 0;
445 unsigned int c_mmot_g_slow_copyout6 = 0;
446 unsigned int c_mmot_g_slow_copyout5 = 0;
447 unsigned int c_mmot_cold_037 = 0;
448 unsigned int c_mmot_cold_038 = 0;
449 unsigned int c_mmot_cold_039 = 0;
450 unsigned int c_mmot_g_slow_copyout4 = 0;
451 unsigned int c_mmot_g_slow_copyout3 = 0;
452 unsigned int c_mmot_hot_ok1 = 0;
453 unsigned int c_mmot_hot_ok2 = 0;
454 unsigned int c_mmot_hot_ok3 = 0;
455 unsigned int c_mmot_g_slow_copyout1 = 0;
456 unsigned int c_mmot_g_slow_copyout2 = 0;
457 unsigned int c_mmot_getback_fast_copyin = 0;
458 unsigned int c_mmot_cold_048 = 0;
459 unsigned int c_mmot_getback_FastSR = 0;
460 unsigned int c_mmot_cold_050 = 0;
461 unsigned int c_mmot_cold_051 = 0;
462 unsigned int c_mmot_cold_052 = 0;
463 unsigned int c_mmot_cold_053 = 0;
464 unsigned int c_mmot_fastkernelreply = 0;
465 unsigned int c_mmot_cold_055 = 0;
466 unsigned int c_mmot_getback_fast_put = 0;
467 unsigned int c_mmot_LAST = 0; /* End Marker - Unused */
468
469 void db_mmot_zero_counters(void); /* forward; */
470 void db_mmot_show_counters(void); /* forward; */
471
472 void /* Call from the debugger to clear all counters */
473 db_mmot_zero_counters(void)
474 {
475 register unsigned int *ip = &c_mmot_FIRST;
476 while (ip <= &c_mmot_LAST)
477 *ip++ = 0;
478 }
479
480 void /* Call from the debugger to show all counters */
481 db_mmot_show_counters(void)
482 {
483 #define xx(str) printf("%s: %d\n", # str, str);
484
485 xx(c_mmot_combined_S_R);
486 xx(c_mach_msg_trap_switch_fast);
487 xx(c_mmot_kernel_send);
488 xx(c_mmot_cold_000);
489 xx(c_mmot_smallsendsize);
490 xx(c_mmot_oddsendsize);
491 xx(c_mmot_bigsendsize);
492 xx(c_mmot_copyinmsg_fail);
493 xx(c_mmot_g_slow_copyin3);
494 xx(c_mmot_cold_006);
495 xx(c_mmot_cold_007);
496 xx(c_mmot_cold_008);
497 xx(c_mmot_cold_009);
498 xx(c_mmot_cold_010);
499 xx(c_mmot_cold_012);
500 xx(c_mmot_cold_013);
501 xx(c_mmot_cold_014);
502 xx(c_mmot_cold_016);
503 xx(c_mmot_cold_018);
504 xx(c_mmot_cold_019);
505 xx(c_mmot_cold_020);
506 xx(c_mmot_cold_021);
507 xx(c_mmot_cold_022);
508 xx(c_mmot_cold_023);
509 xx(c_mmot_cold_024);
510 xx(c_mmot_cold_025);
511 xx(c_mmot_cold_026);
512 xx(c_mmot_cold_027);
513 xx(c_mmot_hot_fSR_ok);
514 xx(c_mmot_cold_029);
515 xx(c_mmot_cold_030);
516 xx(c_mmot_cold_031);
517 xx(c_mmot_cold_032);
518 xx(c_mmot_cold_033);
519 xx(c_mmot_bad_rcvr);
520 xx(c_mmot_rcvr_swapped);
521 xx(c_mmot_rcvr_locked);
522 xx(c_mmot_rcvr_tswapped);
523 xx(c_mmot_rcvr_freed);
524 xx(c_mmot_g_slow_copyout6);
525 xx(c_mmot_g_slow_copyout5);
526 xx(c_mmot_cold_037);
527 xx(c_mmot_cold_038);
528 xx(c_mmot_cold_039);
529 xx(c_mmot_g_slow_copyout4);
530 xx(c_mmot_g_slow_copyout3);
531 xx(c_mmot_g_slow_copyout1);
532 xx(c_mmot_hot_ok3);
533 xx(c_mmot_hot_ok2);
534 xx(c_mmot_hot_ok1);
535 xx(c_mmot_g_slow_copyout2);
536 xx(c_mmot_getback_fast_copyin);
537 xx(c_mmot_cold_048);
538 xx(c_mmot_getback_FastSR);
539 xx(c_mmot_cold_050);
540 xx(c_mmot_cold_051);
541 xx(c_mmot_cold_052);
542 xx(c_mmot_cold_053);
543 xx(c_mmot_fastkernelreply);
544 xx(c_mmot_cold_055);
545 xx(c_mmot_getback_fast_put);
546
547 #undef xx
548 }
549
550 #else /* !HOTPATH_DEBUG */
551
552 /*
553 * Duplicate just these few so we can always do a quick sanity check
554 */
555 unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
556 unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
557 unsigned int c_mmot_kernel_send = 0; /* kernel server calls */
558 #define HOT(expr) /* no optional counters */
559
560 #endif /* !HOTPATH_DEBUG */
561
562 boolean_t enable_hotpath = TRUE; /* Patchable, just in case ... */
563 #endif /* HOTPATH_ENABLE */
564
565 /*
566 * Routine: mach_msg_overwrite_trap [mach trap]
567 * Purpose:
568 * Possibly send a message; possibly receive a message.
569 * Conditions:
570 * Nothing locked.
571 * Returns:
572 * All of mach_msg_send and mach_msg_receive error codes.
573 */
574
575 mach_msg_return_t
576 mach_msg_overwrite_trap(
577 struct mach_msg_overwrite_trap_args *args)
578 {
579 mach_vm_address_t msg_addr = args->msg;
580 mach_msg_option_t option = args->option;
581 mach_msg_size_t send_size = args->send_size;
582 mach_msg_size_t rcv_size = args->rcv_size;
583 mach_port_name_t rcv_name = args->rcv_name;
584 mach_msg_timeout_t msg_timeout = args->timeout;
585 mach_port_name_t notify = args->notify;
586 mach_vm_address_t rcv_msg_addr = args->rcv_msg;
587 mach_msg_size_t scatter_list_size = 0; /* NOT INITIALIZED - but not used in pactice */
588
589 register mach_msg_header_t *hdr;
590 mach_msg_return_t mr = MACH_MSG_SUCCESS;
591 /* mask out some of the options before entering the hot path */
592 mach_msg_option_t masked_option =
593 option & ~(MACH_SEND_TRAILER|MACH_RCV_TRAILER_MASK|MACH_RCV_LARGE);
594
595 #if ENABLE_HOTPATH
596 /* BEGINNING OF HOT PATH */
597 if ((masked_option == (MACH_SEND_MSG|MACH_RCV_MSG)) && enable_hotpath) {
598 thread_t self = current_thread();
599 mach_msg_format_0_trailer_t *trailer;
600 ipc_space_t space = self->task->itk_space;
601 ipc_kmsg_t kmsg;
602 register ipc_port_t dest_port;
603 ipc_object_t rcv_object;
604 ipc_mqueue_t rcv_mqueue;
605 mach_msg_size_t reply_size;
606
607 c_mmot_combined_S_R++;
608
609 /*
610 * This case is divided into ten sections, each
611 * with a label. There are five optimized
612 * sections and six unoptimized sections, which
613 * do the same thing but handle all possible
614 * cases and are slower.
615 *
616 * The five sections for an RPC are
617 * 1) Get request message into a buffer.
618 * 2) Copyin request message and rcv_name.
619 * (fast_copyin or slow_copyin)
620 * 3) Enqueue request and dequeue reply.
621 * (fast_send_receive or
622 * slow_send and slow_receive)
623 * 4) Copyout reply message.
624 * (fast_copyout or slow_copyout)
625 * 5) Put reply message to user's buffer.
626 *
627 * Keep the locking hierarchy firmly in mind.
628 * (First spaces, then ports, then port sets,
629 * then message queues.) Only a non-blocking
630 * attempt can be made to acquire locks out of
631 * order, or acquire two locks on the same level.
632 * Acquiring two locks on the same level will
633 * fail if the objects are really the same,
634 * unless simple locking is disabled. This is OK,
635 * because then the extra unlock does nothing.
636 *
637 * There are two major reasons these RPCs can't use
638 * ipc_thread_switch, and use slow_send/slow_receive:
639 * 1) Kernel RPCs.
640 * 2) Servers fall behind clients, so
641 * client doesn't find a blocked server thread and
642 * server finds waiting messages and can't block.
643 */
644
645 mr = ipc_kmsg_get(msg_addr, send_size, &kmsg);
646 if (mr != KERN_SUCCESS) {
647 return mr;
648 }
649 hdr = kmsg->ikm_header;
650 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
651 send_size);
652
653 /*
654 * fast_copyin:
655 *
656 * optimized ipc_kmsg_copyin/ipc_mqueue_copyin
657 *
658 * We have the request message data in kmsg.
659 * Must still do copyin, send, receive, etc.
660 *
661 * If the message isn't simple, we can't combine
662 * ipc_kmsg_copyin_header and ipc_mqueue_copyin,
663 * because copyin of the message body might
664 * affect rcv_name.
665 */
666
667 switch (hdr->msgh_bits) {
668 case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
669 MACH_MSG_TYPE_MAKE_SEND_ONCE): {
670 register ipc_entry_t table;
671 register ipc_entry_num_t size;
672 register ipc_port_t reply_port;
673
674 /* sending a request message */
675
676 {
677 register mach_port_index_t index;
678 register mach_port_gen_t gen;
679
680 {
681 register mach_port_name_t reply_name =
682 (mach_port_name_t)hdr->msgh_local_port;
683
684 if (reply_name != rcv_name) {
685 HOT(c_mmot_g_slow_copyin3++);
686 goto slow_copyin;
687 }
688
689 /* optimized ipc_entry_lookup of reply_name */
690
691 index = MACH_PORT_INDEX(reply_name);
692 gen = MACH_PORT_GEN(reply_name);
693
694 is_read_lock(space);
695 assert(space->is_active);
696
697 size = space->is_table_size;
698 table = space->is_table;
699
700 {
701 register ipc_entry_t entry;
702 register ipc_entry_bits_t bits;
703
704 if (index < size) {
705 entry = &table[index];
706 bits = entry->ie_bits;
707 if (IE_BITS_GEN(bits) != gen ||
708 (bits & IE_BITS_COLLISION)) {
709 entry = IE_NULL;
710 }
711 } else {
712 entry = IE_NULL;
713 bits = 0;
714 }
715 if (entry == IE_NULL) {
716 entry = ipc_entry_lookup(space, reply_name);
717 if (entry == IE_NULL) {
718 HOT(c_mmot_cold_006++);
719 goto abort_request_copyin;
720 }
721 bits = entry->ie_bits;
722 }
723
724 /* check type bit */
725
726 if (! (bits & MACH_PORT_TYPE_RECEIVE)) {
727 HOT(c_mmot_cold_007++);
728 goto abort_request_copyin;
729 }
730
731 reply_port = (ipc_port_t) entry->ie_object;
732 assert(reply_port != IP_NULL);
733 }
734 }
735 }
736
737 /* optimized ipc_entry_lookup of dest_name */
738
739 {
740 register mach_port_index_t index;
741 register mach_port_gen_t gen;
742
743 {
744 register mach_port_name_t dest_name =
745 (mach_port_name_t)hdr->msgh_remote_port;
746
747 index = MACH_PORT_INDEX(dest_name);
748 gen = MACH_PORT_GEN(dest_name);
749
750 {
751 register ipc_entry_t entry;
752 register ipc_entry_bits_t bits;
753
754 if (index < size) {
755 entry = &table[index];
756 bits = entry->ie_bits;
757 if (IE_BITS_GEN(bits) != gen ||
758 (bits & IE_BITS_COLLISION)) {
759 entry = IE_NULL;
760 }
761 } else {
762 entry = IE_NULL;
763 bits = 0;
764 }
765 if (entry == IE_NULL) {
766 entry = ipc_entry_lookup(space, dest_name);
767 if (entry == IE_NULL) {
768 HOT(c_mmot_cold_008++);
769 goto abort_request_copyin;
770 }
771 bits = entry->ie_bits;
772 }
773
774 /* check type bit */
775
776 if (! (bits & MACH_PORT_TYPE_SEND)) {
777 HOT(c_mmot_cold_009++);
778 goto abort_request_copyin;
779 }
780
781 assert(IE_BITS_UREFS(bits) > 0);
782
783 dest_port = (ipc_port_t) entry->ie_object;
784 assert(dest_port != IP_NULL);
785 }
786 }
787 }
788
789 /*
790 * To do an atomic copyin, need simultaneous
791 * locks on both ports and the space. If
792 * dest_port == reply_port, and simple locking is
793 * enabled, then we will abort. Otherwise it's
794 * OK to unlock twice.
795 */
796
797 ip_lock(dest_port);
798 if (!ip_active(dest_port) ||
799 !ip_lock_try(reply_port)) {
800 ip_unlock(dest_port);
801 HOT(c_mmot_cold_010++);
802 goto abort_request_copyin;
803 }
804 is_read_unlock(space);
805
806 assert(dest_port->ip_srights > 0);
807 dest_port->ip_srights++;
808 ip_reference(dest_port);
809
810 assert(ip_active(reply_port));
811 assert(reply_port->ip_receiver_name ==
812 (mach_port_name_t)hdr->msgh_local_port);
813 assert(reply_port->ip_receiver == space);
814
815 reply_port->ip_sorights++;
816 ip_reference(reply_port);
817
818 hdr->msgh_bits =
819 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
820 MACH_MSG_TYPE_PORT_SEND_ONCE);
821 hdr->msgh_remote_port = dest_port;
822 hdr->msgh_local_port = reply_port;
823
824 /* make sure we can queue to the destination */
825
826 if (dest_port->ip_receiver == ipc_space_kernel) {
827 /*
828 * The kernel server has a reference to
829 * the reply port, which it hands back
830 * to us in the reply message. We do
831 * not need to keep another reference to
832 * it.
833 */
834 ip_unlock(reply_port);
835
836 assert(ip_active(dest_port));
837 dest_port->ip_messages.imq_seqno++;
838 ip_unlock(dest_port);
839 goto kernel_send;
840 }
841
842 if (imq_full(&dest_port->ip_messages)) {
843 HOT(c_mmot_cold_013++);
844 goto abort_request_send_receive;
845 }
846
847 /* optimized ipc_mqueue_copyin */
848
849 rcv_object = (ipc_object_t) reply_port;
850 io_reference(rcv_object);
851 rcv_mqueue = &reply_port->ip_messages;
852 io_unlock(rcv_object);
853 HOT(c_mmot_hot_fSR_ok++);
854 goto fast_send_receive;
855
856 abort_request_copyin:
857 is_read_unlock(space);
858 goto slow_copyin;
859
860 abort_request_send_receive:
861 ip_unlock(dest_port);
862 ip_unlock(reply_port);
863 goto slow_send;
864 }
865
866 case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
867 register ipc_entry_num_t size;
868 register ipc_entry_t table;
869
870 /* sending a reply message */
871
872 {
873 register mach_port_name_t reply_name =
874 (mach_port_name_t)hdr->msgh_local_port;
875
876 if (reply_name != MACH_PORT_NULL) {
877 HOT(c_mmot_cold_018++);
878 goto slow_copyin;
879 }
880 }
881
882 is_write_lock(space);
883 assert(space->is_active);
884
885 /* optimized ipc_entry_lookup */
886
887 size = space->is_table_size;
888 table = space->is_table;
889
890 {
891 register ipc_entry_t entry;
892 register mach_port_gen_t gen;
893 register mach_port_index_t index;
894
895 {
896 register mach_port_name_t dest_name =
897 (mach_port_name_t)hdr->msgh_remote_port;
898
899 index = MACH_PORT_INDEX(dest_name);
900 gen = MACH_PORT_GEN(dest_name);
901 }
902
903 if (index >= size) {
904 HOT(c_mmot_cold_019++);
905 goto abort_reply_dest_copyin;
906 }
907
908 entry = &table[index];
909
910 /* check generation, collision bit, and type bit */
911
912 if ((entry->ie_bits & (IE_BITS_GEN_MASK|
913 IE_BITS_COLLISION|
914 MACH_PORT_TYPE_SEND_ONCE)) !=
915 (gen | MACH_PORT_TYPE_SEND_ONCE)) {
916 HOT(c_mmot_cold_020++);
917 goto abort_reply_dest_copyin;
918 }
919
920 /* optimized ipc_right_copyin */
921
922 assert(IE_BITS_TYPE(entry->ie_bits) ==
923 MACH_PORT_TYPE_SEND_ONCE);
924 assert(IE_BITS_UREFS(entry->ie_bits) == 1);
925
926 if (entry->ie_request != 0) {
927 HOT(c_mmot_cold_021++);
928 goto abort_reply_dest_copyin;
929 }
930
931 dest_port = (ipc_port_t) entry->ie_object;
932 assert(dest_port != IP_NULL);
933
934 ip_lock(dest_port);
935 if (!ip_active(dest_port)) {
936 ip_unlock(dest_port);
937 HOT(c_mmot_cold_022++);
938 goto abort_reply_dest_copyin;
939 }
940
941 assert(dest_port->ip_sorights > 0);
942
943 /* optimized ipc_entry_dealloc */
944
945
946 entry->ie_bits = gen;
947 entry->ie_next = table->ie_next;
948 table->ie_next = index;
949 entry->ie_object = IO_NULL;
950 }
951
952 hdr->msgh_bits =
953 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
954 0);
955 hdr->msgh_remote_port = dest_port;
956
957 /* make sure we can queue to the destination */
958
959 assert(dest_port->ip_receiver != ipc_space_kernel);
960
961 /* optimized ipc_entry_lookup/ipc_mqueue_copyin */
962
963 {
964 register ipc_entry_t entry;
965 register ipc_entry_bits_t bits;
966
967 {
968 register mach_port_index_t index;
969 register mach_port_gen_t gen;
970
971 index = MACH_PORT_INDEX(rcv_name);
972 gen = MACH_PORT_GEN(rcv_name);
973
974 if (index < size) {
975 entry = &table[index];
976 bits = entry->ie_bits;
977 if (IE_BITS_GEN(bits) != gen ||
978 (bits & IE_BITS_COLLISION)) {
979 entry = IE_NULL;
980 }
981 } else {
982 entry = IE_NULL;
983 bits = 0;
984 }
985 if (entry == IE_NULL) {
986 entry = ipc_entry_lookup(space, rcv_name);
987 if (entry == IE_NULL) {
988 HOT(c_mmot_cold_024++);
989 goto abort_reply_rcv_copyin;
990 }
991 bits = entry->ie_bits;
992 }
993
994 }
995
996 /* check type bits; looking for receive or set */
997 #if 0
998 /*
999 * JMM - The check below for messages in the receive
1000 * mqueue is insufficient to work with port sets, since
1001 * the messages stay in the port queues. For now, don't
1002 * allow portsets (but receiving on portsets when sending
1003 * a message to a send-once right is actually a very
1004 * common case (so we should re-enable).
1005 */
1006 if (bits & MACH_PORT_TYPE_PORT_SET) {
1007 register ipc_pset_t rcv_pset;
1008
1009 rcv_pset = (ipc_pset_t) entry->ie_object;
1010 assert(rcv_pset != IPS_NULL);
1011
1012 ips_lock(rcv_pset);
1013 assert(ips_active(rcv_pset));
1014
1015 rcv_object = (ipc_object_t) rcv_pset;
1016 rcv_mqueue = &rcv_pset->ips_messages;
1017 } else
1018 #endif /* 0 */
1019 if (bits & MACH_PORT_TYPE_RECEIVE) {
1020 register ipc_port_t rcv_port;
1021
1022 rcv_port = (ipc_port_t) entry->ie_object;
1023 assert(rcv_port != IP_NULL);
1024
1025 if (!ip_lock_try(rcv_port)) {
1026 HOT(c_mmot_cold_025++);
1027 goto abort_reply_rcv_copyin;
1028 }
1029 assert(ip_active(rcv_port));
1030
1031 if (rcv_port->ip_pset_count != 0) {
1032 ip_unlock(rcv_port);
1033 HOT(c_mmot_cold_026++);
1034 goto abort_reply_rcv_copyin;
1035 }
1036
1037 rcv_object = (ipc_object_t) rcv_port;
1038 rcv_mqueue = &rcv_port->ip_messages;
1039 } else {
1040 HOT(c_mmot_cold_027++);
1041 goto abort_reply_rcv_copyin;
1042 }
1043 }
1044
1045 is_write_unlock(space);
1046 io_reference(rcv_object);
1047 io_unlock(rcv_object);
1048 HOT(c_mmot_hot_fSR_ok++);
1049 goto fast_send_receive;
1050
1051 abort_reply_dest_copyin:
1052 is_write_unlock(space);
1053 HOT(c_mmot_cold_029++);
1054 goto slow_copyin;
1055
1056 abort_reply_rcv_copyin:
1057 ip_unlock(dest_port);
1058 is_write_unlock(space);
1059 HOT(c_mmot_cold_030++);
1060 goto slow_send;
1061 }
1062
1063 default:
1064 HOT(c_mmot_cold_031++);
1065 goto slow_copyin;
1066 }
1067 /*NOTREACHED*/
1068
1069 fast_send_receive:
1070 /*
1071 * optimized ipc_mqueue_send/ipc_mqueue_receive
1072 *
1073 * Finished get/copyin of kmsg and copyin of rcv_name.
1074 * space is unlocked, dest_port is locked,
1075 * we can queue kmsg to dest_port,
1076 * rcv_mqueue is set, and rcv_object holds a ref
1077 * so the mqueue cannot go away.
1078 *
1079 * JMM - For now, rcv_object is just a port. Portsets
1080 * are disabled for the time being.
1081 */
1082
1083 assert(ip_active(dest_port));
1084 assert(dest_port->ip_receiver != ipc_space_kernel);
1085 // assert(!imq_full(&dest_port->ip_messages) ||
1086 // (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1087 // MACH_MSG_TYPE_PORT_SEND_ONCE));
1088 assert((hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR) == 0);
1089
1090 {
1091 register ipc_mqueue_t dest_mqueue;
1092 wait_queue_t waitq;
1093 thread_t receiver;
1094 processor_t processor;
1095 boolean_t still_running;
1096 spl_t s;
1097
1098 s = splsched();
1099 processor = current_processor();
1100 if (processor->current_pri >= BASEPRI_RTQUEUES)
1101 goto abort_send_receive1;
1102
1103 dest_mqueue = &dest_port->ip_messages;
1104 waitq = &dest_mqueue->imq_wait_queue;
1105 imq_lock(dest_mqueue);
1106
1107 wait_queue_peek64_locked(waitq, IPC_MQUEUE_RECEIVE, &receiver, &waitq);
1108 /* queue still locked, thread locked - but still on q */
1109
1110 if ( receiver == THREAD_NULL ) {
1111 abort_send_receive:
1112 imq_unlock(dest_mqueue);
1113 abort_send_receive1:
1114 splx(s);
1115 ip_unlock(dest_port);
1116 ipc_object_release(rcv_object);
1117 HOT(c_mmot_cold_032++);
1118 goto slow_send;
1119 }
1120
1121 assert(receiver->state & TH_WAIT);
1122 assert(receiver->wait_queue == waitq);
1123 assert(receiver->wait_event == IPC_MQUEUE_RECEIVE);
1124
1125 /*
1126 * Make sure that the scheduling restrictions of the receiver
1127 * are consistent with a handoff here (if it comes down to that).
1128 */
1129 if ( receiver->sched_pri >= BASEPRI_RTQUEUES ||
1130 receiver->processor_set != processor->processor_set ||
1131 (receiver->bound_processor != PROCESSOR_NULL &&
1132 receiver->bound_processor != processor)) {
1133 HOT(c_mmot_cold_033++);
1134 fall_off:
1135 thread_unlock(receiver);
1136 if (waitq != &dest_mqueue->imq_wait_queue)
1137 wait_queue_unlock(waitq);
1138 goto abort_send_receive;
1139 }
1140
1141 /*
1142 * Check that the receiver can stay on the hot path.
1143 */
1144 if (ipc_kmsg_copyout_size(kmsg, receiver->map) +
1145 REQUESTED_TRAILER_SIZE(receiver->ith_option) > receiver->ith_msize) {
1146 /*
1147 * The receiver can't accept the message.
1148 */
1149 HOT(c_mmot_bad_rcvr++);
1150 goto fall_off;
1151 }
1152
1153 /*
1154 * Before committing to the handoff, make sure that we are
1155 * really going to block (i.e. there are no messages already
1156 * queued for us. This violates lock ordering, so make sure
1157 * we don't deadlock. After the trylock succeeds below, we
1158 * may have up to 3 message queues locked:
1159 * - the dest port mqueue
1160 * - a portset mqueue (where waiting receiver was found)
1161 * - finally our own rcv_mqueue
1162 *
1163 * JMM - Need to make this check appropriate for portsets as
1164 * well before re-enabling them.
1165 */
1166 if (!imq_lock_try(rcv_mqueue)) {
1167 goto fall_off;
1168 }
1169 if (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL) {
1170 imq_unlock(rcv_mqueue);
1171 HOT(c_mmot_cold_033++);
1172 goto fall_off;
1173 }
1174
1175 /* At this point we are committed to do the "handoff". */
1176 c_mach_msg_trap_switch_fast++;
1177
1178 /*
1179 * Go ahead and pull the receiver from the waitq. If the
1180 * waitq wasn't the one for the mqueue, unlock it.
1181 */
1182 wait_queue_pull_thread_locked(waitq,
1183 receiver,
1184 (waitq != &dest_mqueue->imq_wait_queue));
1185
1186 /*
1187 * Store the kmsg and seqno where the receiver can pick it up.
1188 */
1189 receiver->ith_state = MACH_MSG_SUCCESS;
1190 receiver->ith_kmsg = kmsg;
1191 receiver->ith_seqno = dest_mqueue->imq_seqno++;
1192
1193 /*
1194 * Unblock the receiver. If it was still running on another
1195 * CPU, we'll give it a chance to run with the message where
1196 * it is (and just select someother thread to run here).
1197 * Otherwise, we'll invoke it here as part of the handoff.
1198 */
1199 still_running = thread_unblock(receiver, THREAD_AWAKENED);
1200
1201 thread_unlock(receiver);
1202
1203 imq_unlock(dest_mqueue);
1204 ip_unlock(dest_port);
1205 current_task()->messages_sent++;
1206
1207
1208 /*
1209 * Put self on receive port's queue.
1210 * Also save state that the sender of
1211 * our reply message needs to determine if it
1212 * can hand off directly back to us.
1213 */
1214 thread_lock(self);
1215 self->ith_msg_addr = (rcv_msg_addr) ? rcv_msg_addr : msg_addr;
1216 self->ith_object = rcv_object; /* still holds reference */
1217 self->ith_msize = rcv_size;
1218 self->ith_option = option;
1219 self->ith_scatter_list_size = scatter_list_size;
1220 self->ith_continuation = thread_syscall_return;
1221
1222 waitq = &rcv_mqueue->imq_wait_queue;
1223 (void)wait_queue_assert_wait64_locked(waitq,
1224 IPC_MQUEUE_RECEIVE,
1225 THREAD_ABORTSAFE, 0,
1226 self);
1227 thread_unlock(self);
1228 imq_unlock(rcv_mqueue);
1229
1230 /*
1231 * If the receiving thread wasn't still running, we switch directly
1232 * to it here. Otherwise we let the scheduler pick something for
1233 * here. In either case, block this thread as though it had called
1234 * ipc_mqueue_receive.
1235 */
1236 if (still_running) {
1237 splx(s);
1238 thread_block(ipc_mqueue_receive_continue);
1239 } else {
1240 thread_run(self, ipc_mqueue_receive_continue, NULL, receiver);
1241 }
1242 /* NOTREACHED */
1243 }
1244
1245 fast_copyout:
1246 /*
1247 * Nothing locked and no references held, except
1248 * we have kmsg with msgh_seqno filled in. Must
1249 * still check against rcv_size and do
1250 * ipc_kmsg_copyout/ipc_kmsg_put.
1251 */
1252
1253 reply_size = send_size + trailer->msgh_trailer_size;
1254 if (rcv_size < reply_size) {
1255 HOT(c_mmot_g_slow_copyout6++);
1256 goto slow_copyout;
1257 }
1258
1259 /* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */
1260
1261 switch (hdr->msgh_bits) {
1262 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
1263 MACH_MSG_TYPE_PORT_SEND_ONCE): {
1264 ipc_port_t reply_port =
1265 (ipc_port_t) hdr->msgh_local_port;
1266 mach_port_name_t dest_name, reply_name;
1267
1268 /* receiving a request message */
1269
1270 if (!IP_VALID(reply_port)) {
1271 HOT(c_mmot_g_slow_copyout5++);
1272 goto slow_copyout;
1273 }
1274
1275 is_write_lock(space);
1276 assert(space->is_active);
1277
1278 /*
1279 * To do an atomic copyout, need simultaneous
1280 * locks on both ports and the space. If
1281 * dest_port == reply_port, and simple locking is
1282 * enabled, then we will abort. Otherwise it's
1283 * OK to unlock twice.
1284 */
1285
1286 ip_lock(dest_port);
1287 if (!ip_active(dest_port) ||
1288 !ip_lock_try(reply_port)) {
1289 HOT(c_mmot_cold_037++);
1290 goto abort_request_copyout;
1291 }
1292
1293 if (!ip_active(reply_port)) {
1294 ip_unlock(reply_port);
1295 HOT(c_mmot_cold_038++);
1296 goto abort_request_copyout;
1297 }
1298
1299 assert(reply_port->ip_sorights > 0);
1300 ip_unlock(reply_port);
1301
1302 {
1303 register ipc_entry_t table;
1304 register ipc_entry_t entry;
1305 register mach_port_index_t index;
1306
1307 /* optimized ipc_entry_get */
1308
1309 table = space->is_table;
1310 index = table->ie_next;
1311
1312 if (index == 0) {
1313 HOT(c_mmot_cold_039++);
1314 goto abort_request_copyout;
1315 }
1316
1317 entry = &table[index];
1318 table->ie_next = entry->ie_next;
1319 entry->ie_request = 0;
1320
1321 {
1322 register mach_port_gen_t gen;
1323
1324 assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
1325 gen = IE_BITS_NEW_GEN(entry->ie_bits);
1326
1327 reply_name = MACH_PORT_MAKE(index, gen);
1328
1329 /* optimized ipc_right_copyout */
1330
1331 entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
1332 }
1333
1334 assert(MACH_PORT_VALID(reply_name));
1335 entry->ie_object = (ipc_object_t) reply_port;
1336 is_write_unlock(space);
1337 }
1338
1339 /* optimized ipc_object_copyout_dest */
1340
1341 assert(dest_port->ip_srights > 0);
1342 ip_release(dest_port);
1343
1344 if (dest_port->ip_receiver == space)
1345 dest_name = dest_port->ip_receiver_name;
1346 else
1347 dest_name = MACH_PORT_NULL;
1348
1349 if ((--dest_port->ip_srights == 0) &&
1350 (dest_port->ip_nsrequest != IP_NULL)) {
1351 ipc_port_t nsrequest;
1352 mach_port_mscount_t mscount;
1353
1354 /* a rather rare case */
1355
1356 nsrequest = dest_port->ip_nsrequest;
1357 mscount = dest_port->ip_mscount;
1358 dest_port->ip_nsrequest = IP_NULL;
1359 ip_unlock(dest_port);
1360 ipc_notify_no_senders(nsrequest, mscount);
1361 } else
1362 ip_unlock(dest_port);
1363
1364 hdr->msgh_bits =
1365 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
1366 MACH_MSG_TYPE_PORT_SEND);
1367 hdr->msgh_remote_port = (mach_port_t)reply_name;
1368 hdr->msgh_local_port = (mach_port_t)dest_name;
1369 HOT(c_mmot_hot_ok1++);
1370 goto fast_put;
1371
1372 abort_request_copyout:
1373 ip_unlock(dest_port);
1374 is_write_unlock(space);
1375 HOT(c_mmot_g_slow_copyout4++);
1376 goto slow_copyout;
1377 }
1378
1379 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1380 register mach_port_name_t dest_name;
1381
1382 /* receiving a reply message */
1383
1384 ip_lock(dest_port);
1385 if (!ip_active(dest_port)) {
1386 ip_unlock(dest_port);
1387 HOT(c_mmot_g_slow_copyout3++);
1388 goto slow_copyout;
1389 }
1390
1391 /* optimized ipc_object_copyout_dest */
1392
1393 assert(dest_port->ip_sorights > 0);
1394
1395 if (dest_port->ip_receiver == space) {
1396 ip_release(dest_port);
1397 dest_port->ip_sorights--;
1398 dest_name = dest_port->ip_receiver_name;
1399 ip_unlock(dest_port);
1400 } else {
1401 ip_unlock(dest_port);
1402
1403 ipc_notify_send_once(dest_port);
1404 dest_name = MACH_PORT_NULL;
1405 }
1406
1407 hdr->msgh_bits = MACH_MSGH_BITS(0,
1408 MACH_MSG_TYPE_PORT_SEND_ONCE);
1409 hdr->msgh_remote_port = MACH_PORT_NULL;
1410 hdr->msgh_local_port = (ipc_port_t)dest_name;
1411 HOT(c_mmot_hot_ok2++);
1412 goto fast_put;
1413 }
1414
1415 case MACH_MSGH_BITS_COMPLEX|
1416 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1417 register mach_port_name_t dest_name;
1418
1419 /* receiving a complex reply message */
1420
1421 ip_lock(dest_port);
1422 if (!ip_active(dest_port)) {
1423 ip_unlock(dest_port);
1424 HOT(c_mmot_g_slow_copyout1++);
1425 goto slow_copyout;
1426 }
1427
1428 /* optimized ipc_object_copyout_dest */
1429
1430 assert(dest_port->ip_sorights > 0);
1431
1432 if (dest_port->ip_receiver == space) {
1433 ip_release(dest_port);
1434 dest_port->ip_sorights--;
1435 dest_name = dest_port->ip_receiver_name;
1436 ip_unlock(dest_port);
1437 } else {
1438 ip_unlock(dest_port);
1439
1440 ipc_notify_send_once(dest_port);
1441 dest_name = MACH_PORT_NULL;
1442 }
1443
1444 hdr->msgh_bits =
1445 MACH_MSGH_BITS_COMPLEX |
1446 MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND_ONCE);
1447 hdr->msgh_remote_port = MACH_PORT_NULL;
1448 hdr->msgh_local_port = (mach_port_t)dest_name;
1449
1450 mr = ipc_kmsg_copyout_body(kmsg, space,
1451 current_map(),
1452 MACH_MSG_BODY_NULL);
1453 /* hdr and send_size may be invalid now - done use */
1454 if (mr != MACH_MSG_SUCCESS) {
1455 if (ipc_kmsg_put(msg_addr, kmsg,
1456 kmsg->ikm_header->msgh_size +
1457 trailer->msgh_trailer_size) ==
1458 MACH_RCV_INVALID_DATA)
1459 return MACH_RCV_INVALID_DATA;
1460 else
1461 return mr | MACH_RCV_BODY_ERROR;
1462 }
1463 HOT(c_mmot_hot_ok3++);
1464 goto fast_put;
1465 }
1466
1467 default:
1468 HOT(c_mmot_g_slow_copyout2++);
1469 goto slow_copyout;
1470 }
1471 /*NOTREACHED*/
1472
1473 fast_put:
1474 mr = ipc_kmsg_put(rcv_msg_addr ? rcv_msg_addr : msg_addr,
1475 kmsg,
1476 kmsg->ikm_header->msgh_size +
1477 trailer->msgh_trailer_size);
1478 if (mr != MACH_MSG_SUCCESS) {
1479 return MACH_RCV_INVALID_DATA;
1480 }
1481 current_task()->messages_received++;
1482 return mr;
1483
1484
1485 /* BEGINNING OF WARM PATH */
1486
1487 /*
1488 * The slow path has a few non-register temporary
1489 * variables used only for call-by-reference.
1490 */
1491
1492 slow_copyin:
1493 {
1494 mach_port_seqno_t temp_seqno = 0;
1495 register mach_port_name_t reply_name =
1496 (mach_port_name_t)hdr->msgh_local_port;
1497
1498
1499 /*
1500 * We have the message data in kmsg, but
1501 * we still need to copyin, send it,
1502 * receive a reply, and do copyout.
1503 */
1504
1505 mr = ipc_kmsg_copyin(kmsg, space, current_map(),
1506 MACH_PORT_NULL);
1507 if (mr != MACH_MSG_SUCCESS) {
1508 ipc_kmsg_free(kmsg);
1509 return(mr);
1510 }
1511
1512 /*
1513 * LP64support - We have to recompute the header pointer
1514 * and send_size - as they could have changed during the
1515 * complex copyin.
1516 */
1517 hdr = kmsg->ikm_header;
1518 send_size = hdr->msgh_size;
1519
1520 /* try to get back on optimized path */
1521 if ((reply_name != rcv_name) ||
1522 (hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR)) {
1523 HOT(c_mmot_cold_048++);
1524 goto slow_send;
1525 }
1526
1527 dest_port = (ipc_port_t) hdr->msgh_remote_port;
1528 assert(IP_VALID(dest_port));
1529
1530 ip_lock(dest_port);
1531 if (!ip_active(dest_port)) {
1532 ip_unlock(dest_port);
1533 goto slow_send;
1534 }
1535
1536 if (dest_port->ip_receiver == ipc_space_kernel) {
1537 dest_port->ip_messages.imq_seqno++;
1538 ip_unlock(dest_port);
1539 goto kernel_send;
1540 }
1541
1542 if (!imq_full(&dest_port->ip_messages) ||
1543 (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1544 MACH_MSG_TYPE_PORT_SEND_ONCE))
1545 {
1546 /*
1547 * Try an optimized ipc_mqueue_copyin.
1548 * It will work if this is a request message.
1549 */
1550
1551 register ipc_port_t reply_port;
1552
1553 reply_port = (ipc_port_t) hdr->msgh_local_port;
1554 if (IP_VALID(reply_port)) {
1555 if (ip_lock_try(reply_port)) {
1556 if (ip_active(reply_port) &&
1557 reply_port->ip_receiver == space &&
1558 reply_port->ip_receiver_name == rcv_name &&
1559 reply_port->ip_pset_count == 0)
1560 {
1561 /* Grab a reference to the reply port. */
1562 rcv_object = (ipc_object_t) reply_port;
1563 io_reference(rcv_object);
1564 rcv_mqueue = &reply_port->ip_messages;
1565 io_unlock(rcv_object);
1566 HOT(c_mmot_getback_FastSR++);
1567 goto fast_send_receive;
1568 }
1569 ip_unlock(reply_port);
1570 }
1571 }
1572 }
1573
1574 ip_unlock(dest_port);
1575 HOT(c_mmot_cold_050++);
1576 goto slow_send;
1577
1578 kernel_send:
1579 /*
1580 * Special case: send message to kernel services.
1581 * The request message has been copied into the
1582 * kmsg. Nothing is locked.
1583 */
1584
1585 {
1586 register ipc_port_t reply_port;
1587 mach_port_seqno_t local_seqno;
1588 spl_t s;
1589
1590 /*
1591 * Perform the kernel function.
1592 */
1593 c_mmot_kernel_send++;
1594
1595 current_task()->messages_sent++;
1596
1597 kmsg = ipc_kobject_server(kmsg);
1598 if (kmsg == IKM_NULL) {
1599 /*
1600 * No reply. Take the
1601 * slow receive path.
1602 */
1603 HOT(c_mmot_cold_051++);
1604 goto slow_get_rcv_port;
1605 }
1606
1607 /*
1608 * Check that:
1609 * the reply port is alive
1610 * we hold the receive right
1611 * the name has not changed.
1612 * the port is not in a set
1613 * If any of these are not true,
1614 * we cannot directly receive the reply
1615 * message.
1616 */
1617 hdr = kmsg->ikm_header;
1618 send_size = hdr->msgh_size;
1619 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1620 round_msg(send_size));
1621 reply_port = (ipc_port_t) hdr->msgh_remote_port;
1622 ip_lock(reply_port);
1623
1624 if ((!ip_active(reply_port)) ||
1625 (reply_port->ip_receiver != space) ||
1626 (reply_port->ip_receiver_name != rcv_name) ||
1627 (reply_port->ip_pset_count != 0))
1628 {
1629 ip_unlock(reply_port);
1630 ipc_kmsg_send_always(kmsg);
1631 HOT(c_mmot_cold_052++);
1632 goto slow_get_rcv_port;
1633 }
1634
1635 s = splsched();
1636 rcv_mqueue = &reply_port->ip_messages;
1637 imq_lock(rcv_mqueue);
1638
1639 /* keep port locked, and don`t change ref count yet */
1640
1641 /*
1642 * If there are messages on the port
1643 * or other threads waiting for a message,
1644 * we cannot directly receive the reply.
1645 */
1646 if (!wait_queue_empty(&rcv_mqueue->imq_wait_queue) ||
1647 (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL))
1648 {
1649 imq_unlock(rcv_mqueue);
1650 splx(s);
1651 ip_unlock(reply_port);
1652 ipc_kmsg_send_always(kmsg);
1653 HOT(c_mmot_cold_053++);
1654 goto slow_get_rcv_port;
1655 }
1656
1657 /*
1658 * We can directly receive this reply.
1659 * Since there were no messages queued
1660 * on the reply port, there should be
1661 * no threads blocked waiting to send.
1662 */
1663 dest_port = reply_port;
1664 local_seqno = rcv_mqueue->imq_seqno++;
1665 imq_unlock(rcv_mqueue);
1666 splx(s);
1667
1668 /*
1669 * inline ipc_object_release.
1670 * Port is still locked.
1671 * Reference count was not incremented.
1672 */
1673 ip_check_unlock(reply_port);
1674
1675 if (option & MACH_RCV_TRAILER_MASK) {
1676 trailer->msgh_seqno = local_seqno;
1677 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1678 }
1679 /* copy out the kernel reply */
1680 HOT(c_mmot_fastkernelreply++);
1681 goto fast_copyout;
1682 }
1683
1684 slow_send:
1685 /*
1686 * Nothing is locked. We have acquired kmsg, but
1687 * we still need to send it and receive a reply.
1688 */
1689
1690 mr = ipc_kmsg_send(kmsg, MACH_MSG_OPTION_NONE,
1691 MACH_MSG_TIMEOUT_NONE);
1692 if (mr != MACH_MSG_SUCCESS) {
1693 mr |= ipc_kmsg_copyout_pseudo(kmsg, space,
1694 current_map(),
1695 MACH_MSG_BODY_NULL);
1696
1697 (void) ipc_kmsg_put(msg_addr, kmsg,
1698 kmsg->ikm_header->msgh_size);
1699 return(mr);
1700 }
1701
1702 slow_get_rcv_port:
1703 /*
1704 * We have sent the message. Copy in the receive port.
1705 */
1706 mr = ipc_mqueue_copyin(space, rcv_name,
1707 &rcv_mqueue, &rcv_object);
1708 if (mr != MACH_MSG_SUCCESS) {
1709 return(mr);
1710 }
1711 /* hold ref for rcv_object */
1712
1713 /*
1714 * slow_receive:
1715 *
1716 * Now we have sent the request and copied in rcv_name,
1717 * and hold ref for rcv_object (to keep mqueue alive).
1718 * Just receive a reply and try to get back to fast path.
1719 */
1720
1721 self->ith_continuation = (void (*)(mach_msg_return_t))0;
1722 ipc_mqueue_receive(rcv_mqueue,
1723 MACH_MSG_OPTION_NONE,
1724 MACH_MSG_SIZE_MAX,
1725 MACH_MSG_TIMEOUT_NONE,
1726 THREAD_ABORTSAFE);
1727
1728 mr = self->ith_state;
1729 temp_seqno = self->ith_seqno;
1730
1731 ipc_object_release(rcv_object);
1732
1733 if (mr != MACH_MSG_SUCCESS) {
1734 return(mr);
1735 }
1736
1737 kmsg = self->ith_kmsg;
1738 hdr = kmsg->ikm_header;
1739 send_size = hdr->msgh_size;
1740 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1741 round_msg(send_size));
1742 if (option & MACH_RCV_TRAILER_MASK) {
1743 trailer->msgh_seqno = temp_seqno;
1744 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1745 }
1746 dest_port = (ipc_port_t) hdr->msgh_remote_port;
1747 HOT(c_mmot_cold_055++);
1748 goto fast_copyout;
1749
1750 slow_copyout:
1751 /*
1752 * Nothing locked and no references held, except
1753 * we have kmsg with msgh_seqno filled in. Must
1754 * still check against rcv_size and do
1755 * ipc_kmsg_copyout/ipc_kmsg_put.
1756 */
1757
1758 /* LP64support - have to compute real size as it would be received */
1759 reply_size = ipc_kmsg_copyout_size(kmsg, current_map()) +
1760 REQUESTED_TRAILER_SIZE(option);
1761 if (rcv_size < reply_size) {
1762 if (msg_receive_error(kmsg, msg_addr, option, temp_seqno,
1763 space) == MACH_RCV_INVALID_DATA) {
1764 mr = MACH_RCV_INVALID_DATA;
1765 return(mr);
1766 }
1767 else {
1768 mr = MACH_RCV_TOO_LARGE;
1769 return(mr);
1770 }
1771 }
1772
1773 mr = ipc_kmsg_copyout(kmsg, space, current_map(),
1774 MACH_PORT_NULL, MACH_MSG_BODY_NULL);
1775 if (mr != MACH_MSG_SUCCESS) {
1776 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1777 if (ipc_kmsg_put(msg_addr, kmsg, reply_size) ==
1778 MACH_RCV_INVALID_DATA)
1779 mr = MACH_RCV_INVALID_DATA;
1780 }
1781 else {
1782 if (msg_receive_error(kmsg, msg_addr, option,
1783 temp_seqno, space) == MACH_RCV_INVALID_DATA)
1784 mr = MACH_RCV_INVALID_DATA;
1785 }
1786
1787 return(mr);
1788 }
1789
1790 /* try to get back on optimized path */
1791 HOT(c_mmot_getback_fast_put++);
1792 goto fast_put;
1793
1794 /*NOTREACHED*/
1795 }
1796 } /* END OF HOT PATH */
1797 #endif /* ENABLE_HOTPATH */
1798
1799 if (option & MACH_SEND_MSG) {
1800 ipc_space_t space = current_space();
1801 vm_map_t map = current_map();
1802 ipc_kmsg_t kmsg;
1803
1804 mr = ipc_kmsg_get(msg_addr, send_size, &kmsg);
1805
1806 if (mr != MACH_MSG_SUCCESS)
1807 return mr;
1808
1809 if (option & MACH_SEND_CANCEL) {
1810 if (notify == MACH_PORT_NULL)
1811 mr = MACH_SEND_INVALID_NOTIFY;
1812 else
1813 mr = ipc_kmsg_copyin(kmsg, space, map, notify);
1814 } else
1815 mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
1816 if (mr != MACH_MSG_SUCCESS) {
1817 ipc_kmsg_free(kmsg);
1818 return mr;
1819 }
1820
1821 mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, msg_timeout);
1822
1823 if (mr != MACH_MSG_SUCCESS) {
1824 mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
1825 (void) ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size);
1826 return mr;
1827 }
1828
1829 }
1830
1831 if (option & MACH_RCV_MSG) {
1832 thread_t self = current_thread();
1833 ipc_space_t space = current_space();
1834 ipc_object_t object;
1835 ipc_mqueue_t mqueue;
1836
1837 mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
1838 if (mr != MACH_MSG_SUCCESS) {
1839 return mr;
1840 }
1841 /* hold ref for object */
1842
1843 /*
1844 * 1. MACH_RCV_OVERWRITE is on, and rcv_msg is our scatter list
1845 * and receive buffer
1846 * 2. MACH_RCV_OVERWRITE is off, and rcv_msg might be the
1847 * alternate receive buffer (separate send and receive buffers).
1848 */
1849 if (option & MACH_RCV_OVERWRITE)
1850 self->ith_msg_addr = rcv_msg_addr;
1851 else if (rcv_msg_addr != (mach_vm_address_t)0)
1852 self->ith_msg_addr = rcv_msg_addr;
1853 else
1854 self->ith_msg_addr = msg_addr;
1855 self->ith_object = object;
1856 self->ith_msize = rcv_size;
1857 self->ith_option = option;
1858 self->ith_scatter_list_size = scatter_list_size;
1859 self->ith_continuation = thread_syscall_return;
1860
1861 ipc_mqueue_receive(mqueue, option, rcv_size, msg_timeout, THREAD_ABORTSAFE);
1862 if ((option & MACH_RCV_TIMEOUT) && msg_timeout == 0)
1863 thread_poll_yield(self);
1864 return mach_msg_receive_results();
1865 }
1866
1867 return MACH_MSG_SUCCESS;
1868 }
1869
1870 /*
1871 * Routine: mach_msg_trap [mach trap]
1872 * Purpose:
1873 * Possibly send a message; possibly receive a message.
1874 * Conditions:
1875 * Nothing locked.
1876 * Returns:
1877 * All of mach_msg_send and mach_msg_receive error codes.
1878 */
1879
1880 mach_msg_return_t
1881 mach_msg_trap(
1882 struct mach_msg_overwrite_trap_args *args)
1883 {
1884 kern_return_t kr;
1885 args->rcv_msg = (mach_vm_address_t)0;
1886
1887 kr = mach_msg_overwrite_trap(args);
1888 return kr;
1889 }
1890
1891
1892 /*
1893 * Routine: msg_receive_error [internal]
1894 * Purpose:
1895 * Builds a minimal header/trailer and copies it to
1896 * the user message buffer. Invoked when in the case of a
1897 * MACH_RCV_TOO_LARGE or MACH_RCV_BODY_ERROR error.
1898 * Conditions:
1899 * Nothing locked.
1900 * Returns:
1901 * MACH_MSG_SUCCESS minimal header/trailer copied
1902 * MACH_RCV_INVALID_DATA copyout to user buffer failed
1903 */
1904
1905 mach_msg_return_t
1906 msg_receive_error(
1907 ipc_kmsg_t kmsg,
1908 mach_vm_address_t msg_addr,
1909 mach_msg_option_t option,
1910 mach_port_seqno_t seqno,
1911 ipc_space_t space)
1912 {
1913 mach_msg_format_0_trailer_t *trailer;
1914
1915 /*
1916 * Copy out the destination port in the message.
1917 * Destroy all other rights and memory in the message.
1918 */
1919 ipc_kmsg_copyout_dest(kmsg, space);
1920
1921 /*
1922 * Build a minimal message with the requested trailer.
1923 */
1924 trailer = (mach_msg_format_0_trailer_t *)
1925 ((vm_offset_t)kmsg->ikm_header +
1926 round_msg(sizeof(mach_msg_header_t)));
1927 kmsg->ikm_header->msgh_size = sizeof(mach_msg_header_t);
1928 bcopy( (char *)&trailer_template,
1929 (char *)trailer,
1930 sizeof(trailer_template));
1931 if (option & MACH_RCV_TRAILER_MASK) {
1932 trailer->msgh_seqno = seqno;
1933 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1934 }
1935
1936 /*
1937 * Copy the message to user space
1938 */
1939 if (ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size +
1940 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
1941 return(MACH_RCV_INVALID_DATA);
1942 else
1943 return(MACH_MSG_SUCCESS);
1944 }
mirror of XNU