]> git.saurik.com Git - apple/xnu.git/blob - osfmk/ipc/mach_msg.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-344.21.74.tar.gz
[apple/xnu.git] / osfmk / ipc / mach_msg.c
1 /*
2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
7 *
8 * This file contains Original Code and/or Modifications of Original Code
9 * as defined in and that are subject to the Apple Public Source License
10 * Version 2.0 (the 'License'). You may not use this file except in
11 * compliance with the License. Please obtain a copy of the License at
12 * http://www.opensource.apple.com/apsl/ and read it before using this
13 * file.
14 *
15 * The Original Code and all software distributed under the License are
16 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
17 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
18 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
20 * Please see the License for the specific language governing rights and
21 * limitations under the License.
22 *
23 * @APPLE_LICENSE_HEADER_END@
24 */
25 /*
26 * @OSF_COPYRIGHT@
27 */
28 /*
29 * Mach Operating System
30 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
31 * All Rights Reserved.
32 *
33 * Permission to use, copy, modify and distribute this software and its
34 * documentation is hereby granted, provided that both the copyright
35 * notice and this permission notice appear in all copies of the
36 * software, derivative works or modified versions, and any portions
37 * thereof, and that both notices appear in supporting documentation.
38 *
39 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
40 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
41 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
42 *
43 * Carnegie Mellon requests users of this software to return to
44 *
45 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
46 * School of Computer Science
47 * Carnegie Mellon University
48 * Pittsburgh PA 15213-3890
49 *
50 * any improvements or extensions that they make and grant Carnegie Mellon
51 * the rights to redistribute these changes.
52 */
53 /*
54 */
55 /*
56 * File: ipc/mach_msg.c
57 * Author: Rich Draves
58 * Date: 1989
59 *
60 * Exported message traps. See mach/message.h.
61 */
62
63 #include <cpus.h>
64
65 #include <mach/kern_return.h>
66 #include <mach/port.h>
67 #include <mach/message.h>
68 #include <mach/mig_errors.h>
69 #include <kern/assert.h>
70 #include <kern/counters.h>
71 #include <kern/cpu_number.h>
72 #include <kern/task.h>
73 #include <kern/thread.h>
74 #include <kern/lock.h>
75 #include <kern/sched_prim.h>
76 #include <kern/exception.h>
77 #include <kern/misc_protos.h>
78 #include <vm/vm_map.h>
79 #include <ipc/ipc_kmsg.h>
80 #include <ipc/ipc_mqueue.h>
81 #include <ipc/ipc_object.h>
82 #include <ipc/ipc_notify.h>
83 #include <ipc/ipc_port.h>
84 #include <ipc/ipc_pset.h>
85 #include <ipc/ipc_space.h>
86 #include <ipc/ipc_entry.h>
87 #include <kern/kalloc.h>
88 #include <kern/thread_swap.h>
89 #include <kern/processor.h>
90
91 #include <kern/mk_sp.h>
92
93 #include <machine/machine_routines.h>
94 #include <sys/kdebug.h>
95
96 /*
97 * Forward declarations
98 */
99
100 mach_msg_return_t mach_msg_send(
101 mach_msg_header_t *msg,
102 mach_msg_option_t option,
103 mach_msg_size_t send_size,
104 mach_msg_timeout_t timeout,
105 mach_port_name_t notify);
106
107 mach_msg_return_t mach_msg_receive(
108 mach_msg_header_t *msg,
109 mach_msg_option_t option,
110 mach_msg_size_t rcv_size,
111 mach_port_name_t rcv_name,
112 mach_msg_timeout_t timeout,
113 void (*continuation)(mach_msg_return_t),
114 mach_msg_size_t slist_size);
115
116
117 mach_msg_return_t msg_receive_error(
118 ipc_kmsg_t kmsg,
119 mach_msg_header_t *msg,
120 mach_msg_option_t option,
121 mach_port_seqno_t seqno,
122 ipc_space_t space);
123
124 security_token_t KERNEL_SECURITY_TOKEN = KERNEL_SECURITY_TOKEN_VALUE;
125
126 mach_msg_format_0_trailer_t trailer_template = {
127 /* mach_msg_trailer_type_t */ MACH_MSG_TRAILER_FORMAT_0,
128 /* mach_msg_trailer_size_t */ MACH_MSG_TRAILER_MINIMUM_SIZE,
129 /* mach_port_seqno_t */ 0,
130 /* security_token_t */ KERNEL_SECURITY_TOKEN_VALUE
131 };
132
133 /*
134 * Routine: mach_msg_send
135 * Purpose:
136 * Send a message.
137 * Conditions:
138 * Nothing locked.
139 * Returns:
140 * MACH_MSG_SUCCESS Sent the message.
141 * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
142 * MACH_SEND_NO_BUFFER Couldn't allocate buffer.
143 * MACH_SEND_INVALID_DATA Couldn't copy message data.
144 * MACH_SEND_INVALID_HEADER
145 * Illegal value in the message header bits.
146 * MACH_SEND_INVALID_DEST The space is dead.
147 * MACH_SEND_INVALID_NOTIFY Bad notify port.
148 * MACH_SEND_INVALID_DEST Can't copyin destination port.
149 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
150 * MACH_SEND_TIMED_OUT Timeout expired without delivery.
151 * MACH_SEND_INTERRUPTED Delivery interrupted.
152 * MACH_SEND_NO_NOTIFY Can't allocate a msg-accepted request.
153 * MACH_SEND_WILL_NOTIFY Msg-accepted notif. requested.
154 * MACH_SEND_NOTIFY_IN_PROGRESS
155 * This space has already forced a message to this port.
156 */
157
158 mach_msg_return_t
159 mach_msg_send(
160 mach_msg_header_t *msg,
161 mach_msg_option_t option,
162 mach_msg_size_t send_size,
163 mach_msg_timeout_t timeout,
164 mach_port_name_t notify)
165 {
166 ipc_space_t space = current_space();
167 vm_map_t map = current_map();
168 ipc_kmsg_t kmsg;
169 mach_msg_return_t mr;
170
171 mr = ipc_kmsg_get(msg, send_size, &kmsg);
172
173 if (mr != MACH_MSG_SUCCESS)
174 return mr;
175
176 if (option & MACH_SEND_CANCEL) {
177 if (notify == MACH_PORT_NULL)
178 mr = MACH_SEND_INVALID_NOTIFY;
179 else
180 mr = ipc_kmsg_copyin(kmsg, space, map, notify);
181 } else
182 mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
183 if (mr != MACH_MSG_SUCCESS) {
184 ipc_kmsg_free(kmsg);
185 return mr;
186 }
187
188 mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, timeout);
189
190 if (mr != MACH_MSG_SUCCESS) {
191 mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
192 (void) ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
193 }
194
195 return mr;
196 }
197
198 /*
199 * Routine: mach_msg_receive
200 * Purpose:
201 * Receive a message.
202 * Conditions:
203 * Nothing locked.
204 * Returns:
205 * MACH_MSG_SUCCESS Received a message.
206 * MACH_RCV_INVALID_NAME The name doesn't denote a right,
207 * or the denoted right is not receive or port set.
208 * MACH_RCV_IN_SET Receive right is a member of a set.
209 * MACH_RCV_TOO_LARGE Message wouldn't fit into buffer.
210 * MACH_RCV_TIMED_OUT Timeout expired without a message.
211 * MACH_RCV_INTERRUPTED Reception interrupted.
212 * MACH_RCV_PORT_DIED Port/set died while receiving.
213 * MACH_RCV_PORT_CHANGED Port moved into set while receiving.
214 * MACH_RCV_INVALID_DATA Couldn't copy to user buffer.
215 * MACH_RCV_INVALID_NOTIFY Bad notify port.
216 * MACH_RCV_HEADER_ERROR
217 */
218
219 mach_msg_return_t
220 mach_msg_receive_results(void)
221 {
222 thread_t self = current_thread();
223 ipc_space_t space = current_space();
224 vm_map_t map = current_map();
225
226 ipc_object_t object = self->ith_object;
227 mach_msg_return_t mr = self->ith_state;
228 mach_msg_header_t *msg = self->ith_msg;
229 mach_msg_option_t option = self->ith_option;
230 ipc_kmsg_t kmsg = self->ith_kmsg;
231 mach_port_seqno_t seqno = self->ith_seqno;
232 mach_msg_size_t slist_size = self->ith_scatter_list_size;
233
234 mach_msg_format_0_trailer_t *trailer;
235
236 ipc_object_release(object);
237
238 if (mr != MACH_MSG_SUCCESS) {
239
240 if (mr == MACH_RCV_TOO_LARGE ) {
241 if (option & MACH_RCV_LARGE) {
242 /*
243 * We need to inform the user-level code that it needs more
244 * space. The value for how much space was returned in the
245 * msize save area instead of the message (which was left on
246 * the queue).
247 */
248 if (copyout((char *) &self->ith_msize,
249 (char *) &msg->msgh_size,
250 sizeof(mach_msg_size_t)))
251 mr = MACH_RCV_INVALID_DATA;
252 goto out;
253 }
254
255 if (msg_receive_error(kmsg, msg, option, seqno, space)
256 == MACH_RCV_INVALID_DATA)
257 mr = MACH_RCV_INVALID_DATA;
258 }
259 goto out;
260 }
261
262 trailer = (mach_msg_format_0_trailer_t *)
263 ((vm_offset_t)&kmsg->ikm_header +
264 round_msg(kmsg->ikm_header.msgh_size));
265 if (option & MACH_RCV_TRAILER_MASK) {
266 trailer->msgh_seqno = seqno;
267 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
268 }
269
270 /*
271 * If MACH_RCV_OVERWRITE was specified, try to get the scatter
272 * list and verify it against the contents of the message. If
273 * there is any problem with it, we will continue without it as
274 * normal.
275 */
276 if (option & MACH_RCV_OVERWRITE) {
277 mach_msg_size_t slist_size = self->ith_scatter_list_size;
278 mach_msg_body_t *slist;
279
280 slist = ipc_kmsg_copyin_scatter(msg, slist_size, kmsg);
281 mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL, slist);
282 ipc_kmsg_free_scatter(slist, slist_size);
283 } else {
284 mr = ipc_kmsg_copyout(kmsg, space, map,
285 MACH_PORT_NULL, MACH_MSG_BODY_NULL);
286 }
287
288 if (mr != MACH_MSG_SUCCESS) {
289 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
290 if (ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size +
291 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
292 mr = MACH_RCV_INVALID_DATA;
293 }
294 else {
295 if (msg_receive_error(kmsg, msg, option, seqno, space)
296 == MACH_RCV_INVALID_DATA)
297 mr = MACH_RCV_INVALID_DATA;
298 }
299 goto out;
300 }
301 mr = ipc_kmsg_put(msg,
302 kmsg,
303 kmsg->ikm_header.msgh_size +
304 trailer->msgh_trailer_size);
305 out:
306 return mr;
307 }
308
309 mach_msg_return_t
310 mach_msg_receive(
311 mach_msg_header_t *msg,
312 mach_msg_option_t option,
313 mach_msg_size_t rcv_size,
314 mach_port_name_t rcv_name,
315 mach_msg_timeout_t timeout,
316 void (*continuation)(mach_msg_return_t),
317 mach_msg_size_t slist_size)
318 {
319 thread_t self = current_thread();
320 ipc_space_t space = current_space();
321 vm_map_t map = current_map();
322 ipc_object_t object;
323 ipc_mqueue_t mqueue;
324 ipc_kmsg_t kmsg;
325 mach_port_seqno_t seqno;
326 mach_msg_return_t mr;
327 mach_msg_body_t *slist;
328 mach_msg_format_0_trailer_t *trailer;
329
330 mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
331 if (mr != MACH_MSG_SUCCESS) {
332 return mr;
333 }
334 /* hold ref for object */
335
336 self->ith_msg = msg;
337 self->ith_object = object;
338 self->ith_msize = rcv_size;
339 self->ith_option = option;
340 self->ith_scatter_list_size = slist_size;
341 self->ith_continuation = continuation;
342
343 ipc_mqueue_receive(mqueue, option, rcv_size, timeout, THREAD_ABORTSAFE);
344 if ((option & MACH_RCV_TIMEOUT) && timeout == 0)
345 _mk_sp_thread_perhaps_yield(self);
346 return mach_msg_receive_results();
347 }
348
349 void
350 mach_msg_receive_continue(void)
351 {
352 thread_t self = current_thread();
353
354 (*self->ith_continuation)(mach_msg_receive_results());
355 }
356
357 /*
358 * Toggle this to compile the hotpath in/out
359 * If compiled in, the run-time toggle "enable_hotpath" below
360 * eases testing & debugging
361 */
362 #define ENABLE_HOTPATH 1 /* Hacked on for now */
363
364 #if ENABLE_HOTPATH
365 /*
366 * These counters allow tracing of hotpath behavior under test loads.
367 * A couple key counters are unconditional (see below).
368 */
369 #define HOTPATH_DEBUG 0 /* Toggle to include lots of counters */
370 #if HOTPATH_DEBUG
371 #define HOT(expr) expr
372
373 unsigned int c_mmot_FIRST = 0; /* Unused First Counter */
374 unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
375 unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
376 unsigned int c_mmot_kernel_send = 0; /* kernel server */
377 unsigned int c_mmot_cold_000 = 0; /* see below ... */
378 unsigned int c_mmot_smallsendsize = 0;
379 unsigned int c_mmot_oddsendsize = 0;
380 unsigned int c_mmot_bigsendsize = 0;
381 unsigned int c_mmot_copyinmsg_fail = 0;
382 unsigned int c_mmot_g_slow_copyin3 = 0;
383 unsigned int c_mmot_cold_006 = 0;
384 unsigned int c_mmot_cold_007 = 0;
385 unsigned int c_mmot_cold_008 = 0;
386 unsigned int c_mmot_cold_009 = 0;
387 unsigned int c_mmot_cold_010 = 0;
388 unsigned int c_mmot_cold_012 = 0;
389 unsigned int c_mmot_cold_013 = 0;
390 unsigned int c_mmot_cold_014 = 0;
391 unsigned int c_mmot_cold_016 = 0;
392 unsigned int c_mmot_cold_018 = 0;
393 unsigned int c_mmot_cold_019 = 0;
394 unsigned int c_mmot_cold_020 = 0;
395 unsigned int c_mmot_cold_021 = 0;
396 unsigned int c_mmot_cold_022 = 0;
397 unsigned int c_mmot_cold_023 = 0;
398 unsigned int c_mmot_cold_024 = 0;
399 unsigned int c_mmot_cold_025 = 0;
400 unsigned int c_mmot_cold_026 = 0;
401 unsigned int c_mmot_cold_027 = 0;
402 unsigned int c_mmot_hot_fSR_ok = 0;
403 unsigned int c_mmot_cold_029 = 0;
404 unsigned int c_mmot_cold_030 = 0;
405 unsigned int c_mmot_cold_031 = 0;
406 unsigned int c_mmot_cold_032 = 0;
407 unsigned int c_mmot_cold_033 = 0;
408 unsigned int c_mmot_bad_rcvr = 0;
409 unsigned int c_mmot_rcvr_swapped = 0;
410 unsigned int c_mmot_rcvr_locked = 0;
411 unsigned int c_mmot_rcvr_tswapped = 0;
412 unsigned int c_mmot_rcvr_freed = 0;
413 unsigned int c_mmot_g_slow_copyout6 = 0;
414 unsigned int c_mmot_g_slow_copyout5 = 0;
415 unsigned int c_mmot_cold_037 = 0;
416 unsigned int c_mmot_cold_038 = 0;
417 unsigned int c_mmot_cold_039 = 0;
418 unsigned int c_mmot_g_slow_copyout4 = 0;
419 unsigned int c_mmot_g_slow_copyout3 = 0;
420 unsigned int c_mmot_hot_ok1 = 0;
421 unsigned int c_mmot_hot_ok2 = 0;
422 unsigned int c_mmot_hot_ok3 = 0;
423 unsigned int c_mmot_g_slow_copyout1 = 0;
424 unsigned int c_mmot_g_slow_copyout2 = 0;
425 unsigned int c_mmot_getback_fast_copyin = 0;
426 unsigned int c_mmot_cold_048 = 0;
427 unsigned int c_mmot_getback_FastSR = 0;
428 unsigned int c_mmot_cold_050 = 0;
429 unsigned int c_mmot_cold_051 = 0;
430 unsigned int c_mmot_cold_052 = 0;
431 unsigned int c_mmot_cold_053 = 0;
432 unsigned int c_mmot_fastkernelreply = 0;
433 unsigned int c_mmot_cold_055 = 0;
434 unsigned int c_mmot_getback_fast_put = 0;
435 unsigned int c_mmot_LAST = 0; /* End Marker - Unused */
436
437 void db_mmot_zero_counters(void); /* forward; */
438 void db_mmot_show_counters(void); /* forward; */
439
440 void /* Call from the debugger to clear all counters */
441 db_mmot_zero_counters(void)
442 {
443 register unsigned int *ip = &c_mmot_FIRST;
444 while (ip <= &c_mmot_LAST)
445 *ip++ = 0;
446 }
447
448 void /* Call from the debugger to show all counters */
449 db_mmot_show_counters(void)
450 {
451 #define xx(str) printf("%s: %d\n", # str, str);
452
453 xx(c_mmot_combined_S_R);
454 xx(c_mach_msg_trap_switch_fast);
455 xx(c_mmot_kernel_send);
456 xx(c_mmot_cold_000);
457 xx(c_mmot_smallsendsize);
458 xx(c_mmot_oddsendsize);
459 xx(c_mmot_bigsendsize);
460 xx(c_mmot_copyinmsg_fail);
461 xx(c_mmot_g_slow_copyin3);
462 xx(c_mmot_cold_006);
463 xx(c_mmot_cold_007);
464 xx(c_mmot_cold_008);
465 xx(c_mmot_cold_009);
466 xx(c_mmot_cold_010);
467 xx(c_mmot_cold_012);
468 xx(c_mmot_cold_013);
469 xx(c_mmot_cold_014);
470 xx(c_mmot_cold_016);
471 xx(c_mmot_cold_018);
472 xx(c_mmot_cold_019);
473 xx(c_mmot_cold_020);
474 xx(c_mmot_cold_021);
475 xx(c_mmot_cold_022);
476 xx(c_mmot_cold_023);
477 xx(c_mmot_cold_024);
478 xx(c_mmot_cold_025);
479 xx(c_mmot_cold_026);
480 xx(c_mmot_cold_027);
481 xx(c_mmot_hot_fSR_ok);
482 xx(c_mmot_cold_029);
483 xx(c_mmot_cold_030);
484 xx(c_mmot_cold_031);
485 xx(c_mmot_cold_032);
486 xx(c_mmot_cold_033);
487 xx(c_mmot_bad_rcvr);
488 xx(c_mmot_rcvr_swapped);
489 xx(c_mmot_rcvr_locked);
490 xx(c_mmot_rcvr_tswapped);
491 xx(c_mmot_rcvr_freed);
492 xx(c_mmot_g_slow_copyout6);
493 xx(c_mmot_g_slow_copyout5);
494 xx(c_mmot_cold_037);
495 xx(c_mmot_cold_038);
496 xx(c_mmot_cold_039);
497 xx(c_mmot_g_slow_copyout4);
498 xx(c_mmot_g_slow_copyout3);
499 xx(c_mmot_g_slow_copyout1);
500 xx(c_mmot_hot_ok3);
501 xx(c_mmot_hot_ok2);
502 xx(c_mmot_hot_ok1);
503 xx(c_mmot_g_slow_copyout2);
504 xx(c_mmot_getback_fast_copyin);
505 xx(c_mmot_cold_048);
506 xx(c_mmot_getback_FastSR);
507 xx(c_mmot_cold_050);
508 xx(c_mmot_cold_051);
509 xx(c_mmot_cold_052);
510 xx(c_mmot_cold_053);
511 xx(c_mmot_fastkernelreply);
512 xx(c_mmot_cold_055);
513 xx(c_mmot_getback_fast_put);
514
515 #undef xx
516 }
517
518 #else /* !HOTPATH_DEBUG */
519
520 /*
521 * Duplicate just these few so we can always do a quick sanity check
522 */
523 unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
524 unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
525 unsigned int c_mmot_kernel_send = 0; /* kernel server calls */
526 #define HOT(expr) /* no optional counters */
527
528 #endif /* !HOTPATH_DEBUG */
529
530 boolean_t enable_hotpath = TRUE; /* Patchable, just in case ... */
531 #endif /* HOTPATH_ENABLE */
532
533 /*
534 * Routine: mach_msg_overwrite_trap [mach trap]
535 * Purpose:
536 * Possibly send a message; possibly receive a message.
537 * Conditions:
538 * Nothing locked.
539 * Returns:
540 * All of mach_msg_send and mach_msg_receive error codes.
541 */
542
543 mach_msg_return_t
544 mach_msg_overwrite_trap(
545 mach_msg_header_t *msg,
546 mach_msg_option_t option,
547 mach_msg_size_t send_size,
548 mach_msg_size_t rcv_size,
549 mach_port_name_t rcv_name,
550 mach_msg_timeout_t timeout,
551 mach_port_name_t notify,
552 mach_msg_header_t *rcv_msg,
553 mach_msg_size_t scatter_list_size)
554 {
555 register mach_msg_header_t *hdr;
556 mach_msg_return_t mr = MACH_MSG_SUCCESS;
557 /* mask out some of the options before entering the hot path */
558 mach_msg_option_t masked_option =
559 option & ~(MACH_SEND_TRAILER|MACH_RCV_TRAILER_MASK|MACH_RCV_LARGE);
560 int i;
561
562 #if ENABLE_HOTPATH
563 /* BEGINNING OF HOT PATH */
564 if ((masked_option == (MACH_SEND_MSG|MACH_RCV_MSG)) && enable_hotpath) {
565 register thread_t self = current_thread();
566 register mach_msg_format_0_trailer_t *trailer;
567
568 ipc_space_t space = current_act()->task->itk_space;
569 ipc_kmsg_t kmsg;
570 register ipc_port_t dest_port;
571 ipc_object_t rcv_object;
572 register ipc_mqueue_t rcv_mqueue;
573 mach_msg_size_t reply_size;
574 ipc_kmsg_t rcv_kmsg;
575
576 c_mmot_combined_S_R++;
577
578 /*
579 * This case is divided into ten sections, each
580 * with a label. There are five optimized
581 * sections and six unoptimized sections, which
582 * do the same thing but handle all possible
583 * cases and are slower.
584 *
585 * The five sections for an RPC are
586 * 1) Get request message into a buffer.
587 * 2) Copyin request message and rcv_name.
588 * (fast_copyin or slow_copyin)
589 * 3) Enqueue request and dequeue reply.
590 * (fast_send_receive or
591 * slow_send and slow_receive)
592 * 4) Copyout reply message.
593 * (fast_copyout or slow_copyout)
594 * 5) Put reply message to user's buffer.
595 *
596 * Keep the locking hierarchy firmly in mind.
597 * (First spaces, then ports, then port sets,
598 * then message queues.) Only a non-blocking
599 * attempt can be made to acquire locks out of
600 * order, or acquire two locks on the same level.
601 * Acquiring two locks on the same level will
602 * fail if the objects are really the same,
603 * unless simple locking is disabled. This is OK,
604 * because then the extra unlock does nothing.
605 *
606 * There are two major reasons these RPCs can't use
607 * ipc_thread_switch, and use slow_send/slow_receive:
608 * 1) Kernel RPCs.
609 * 2) Servers fall behind clients, so
610 * client doesn't find a blocked server thread and
611 * server finds waiting messages and can't block.
612 */
613
614 mr = ipc_kmsg_get(msg, send_size, &kmsg);
615 if (mr != KERN_SUCCESS) {
616 return mr;
617 }
618 hdr = &kmsg->ikm_header;
619 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
620 send_size);
621
622 fast_copyin:
623 /*
624 * optimized ipc_kmsg_copyin/ipc_mqueue_copyin
625 *
626 * We have the request message data in kmsg.
627 * Must still do copyin, send, receive, etc.
628 *
629 * If the message isn't simple, we can't combine
630 * ipc_kmsg_copyin_header and ipc_mqueue_copyin,
631 * because copyin of the message body might
632 * affect rcv_name.
633 */
634
635 switch (hdr->msgh_bits) {
636 case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
637 MACH_MSG_TYPE_MAKE_SEND_ONCE): {
638 register ipc_entry_t table;
639 register ipc_entry_num_t size;
640 register ipc_port_t reply_port;
641
642 /* sending a request message */
643
644 {
645 register mach_port_index_t index;
646 register mach_port_gen_t gen;
647
648 {
649 register mach_port_name_t reply_name =
650 (mach_port_name_t)hdr->msgh_local_port;
651
652 if (reply_name != rcv_name) {
653 HOT(c_mmot_g_slow_copyin3++);
654 goto slow_copyin;
655 }
656
657 /* optimized ipc_entry_lookup of reply_name */
658
659 index = MACH_PORT_INDEX(reply_name);
660 gen = MACH_PORT_GEN(reply_name);
661
662 is_read_lock(space);
663 assert(space->is_active);
664
665 size = space->is_table_size;
666 table = space->is_table;
667
668 {
669 register ipc_entry_t entry;
670 register ipc_entry_bits_t bits;
671
672 if (index < size) {
673 entry = &table[index];
674 bits = entry->ie_bits;
675 if (IE_BITS_GEN(bits) != gen ||
676 (bits & IE_BITS_COLLISION)) {
677 entry = IE_NULL;
678 }
679 } else {
680 entry = IE_NULL;
681 }
682 if (entry == IE_NULL) {
683 entry = ipc_entry_lookup(space, reply_name);
684 if (entry == IE_NULL) {
685 HOT(c_mmot_cold_006++);
686 goto abort_request_copyin;
687 }
688 bits = entry->ie_bits;
689 }
690
691 /* check type bit */
692
693 if (! (bits & MACH_PORT_TYPE_RECEIVE)) {
694 HOT(c_mmot_cold_007++);
695 goto abort_request_copyin;
696 }
697
698 reply_port = (ipc_port_t) entry->ie_object;
699 assert(reply_port != IP_NULL);
700 }
701 }
702 }
703
704 /* optimized ipc_entry_lookup of dest_name */
705
706 {
707 register mach_port_index_t index;
708 register mach_port_gen_t gen;
709
710 {
711 register mach_port_name_t dest_name =
712 (mach_port_name_t)hdr->msgh_remote_port;
713
714 index = MACH_PORT_INDEX(dest_name);
715 gen = MACH_PORT_GEN(dest_name);
716
717 {
718 register ipc_entry_t entry;
719 register ipc_entry_bits_t bits;
720
721 if (index < size) {
722 entry = &table[index];
723 bits = entry->ie_bits;
724 if (IE_BITS_GEN(bits) != gen ||
725 (bits & IE_BITS_COLLISION)) {
726 entry = IE_NULL;
727 }
728 } else {
729 entry = IE_NULL;
730 }
731 if (entry == IE_NULL) {
732 entry = ipc_entry_lookup(space, dest_name);
733 if (entry == IE_NULL) {
734 HOT(c_mmot_cold_008++);
735 goto abort_request_copyin;
736 }
737 bits = entry->ie_bits;
738 }
739
740 /* check type bit */
741
742 if (! (bits & MACH_PORT_TYPE_SEND)) {
743 HOT(c_mmot_cold_009++);
744 goto abort_request_copyin;
745 }
746
747 assert(IE_BITS_UREFS(bits) > 0);
748
749 dest_port = (ipc_port_t) entry->ie_object;
750 assert(dest_port != IP_NULL);
751 }
752 }
753 }
754
755 /*
756 * To do an atomic copyin, need simultaneous
757 * locks on both ports and the space. If
758 * dest_port == reply_port, and simple locking is
759 * enabled, then we will abort. Otherwise it's
760 * OK to unlock twice.
761 */
762
763 ip_lock(dest_port);
764 if (!ip_active(dest_port) ||
765 !ip_lock_try(reply_port)) {
766 ip_unlock(dest_port);
767 HOT(c_mmot_cold_010++);
768 goto abort_request_copyin;
769 }
770 is_read_unlock(space);
771
772 assert(dest_port->ip_srights > 0);
773 dest_port->ip_srights++;
774 ip_reference(dest_port);
775
776 assert(ip_active(reply_port));
777 assert(reply_port->ip_receiver_name ==
778 (mach_port_name_t)hdr->msgh_local_port);
779 assert(reply_port->ip_receiver == space);
780
781 reply_port->ip_sorights++;
782 ip_reference(reply_port);
783
784 hdr->msgh_bits =
785 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
786 MACH_MSG_TYPE_PORT_SEND_ONCE);
787 hdr->msgh_remote_port = dest_port;
788 hdr->msgh_local_port = reply_port;
789
790 /* make sure we can queue to the destination */
791
792 if (dest_port->ip_receiver == ipc_space_kernel) {
793 /*
794 * The kernel server has a reference to
795 * the reply port, which it hands back
796 * to us in the reply message. We do
797 * not need to keep another reference to
798 * it.
799 */
800 ip_unlock(reply_port);
801
802 assert(ip_active(dest_port));
803 dest_port->ip_messages.imq_seqno++;
804 ip_unlock(dest_port);
805 goto kernel_send;
806 }
807
808 if (imq_full(&dest_port->ip_messages)) {
809 HOT(c_mmot_cold_013++);
810 goto abort_request_send_receive;
811 }
812
813 /* optimized ipc_mqueue_copyin */
814
815 rcv_object = (ipc_object_t) reply_port;
816 io_reference(rcv_object);
817 rcv_mqueue = &reply_port->ip_messages;
818 io_unlock(rcv_object);
819 HOT(c_mmot_hot_fSR_ok++);
820 goto fast_send_receive;
821
822 abort_request_copyin:
823 is_read_unlock(space);
824 goto slow_copyin;
825
826 abort_request_send_receive:
827 ip_unlock(dest_port);
828 ip_unlock(reply_port);
829 goto slow_send;
830 }
831
832 case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
833 register ipc_entry_num_t size;
834 register ipc_entry_t table;
835
836 /* sending a reply message */
837
838 {
839 register mach_port_name_t reply_name =
840 (mach_port_name_t)hdr->msgh_local_port;
841
842 if (reply_name != MACH_PORT_NULL) {
843 HOT(c_mmot_cold_018++);
844 goto slow_copyin;
845 }
846 }
847
848 is_write_lock(space);
849 assert(space->is_active);
850
851 /* optimized ipc_entry_lookup */
852
853 size = space->is_table_size;
854 table = space->is_table;
855
856 {
857 register ipc_entry_t entry;
858 register mach_port_gen_t gen;
859 register mach_port_index_t index;
860 ipc_table_index_t *requests;
861
862 {
863 register mach_port_name_t dest_name =
864 (mach_port_name_t)hdr->msgh_remote_port;
865
866 index = MACH_PORT_INDEX(dest_name);
867 gen = MACH_PORT_GEN(dest_name);
868 }
869
870 if (index >= size) {
871 HOT(c_mmot_cold_019++);
872 goto abort_reply_dest_copyin;
873 }
874
875 entry = &table[index];
876
877 /* check generation, collision bit, and type bit */
878
879 if ((entry->ie_bits & (IE_BITS_GEN_MASK|
880 IE_BITS_COLLISION|
881 MACH_PORT_TYPE_SEND_ONCE)) !=
882 (gen | MACH_PORT_TYPE_SEND_ONCE)) {
883 HOT(c_mmot_cold_020++);
884 goto abort_reply_dest_copyin;
885 }
886
887 /* optimized ipc_right_copyin */
888
889 assert(IE_BITS_TYPE(entry->ie_bits) ==
890 MACH_PORT_TYPE_SEND_ONCE);
891 assert(IE_BITS_UREFS(entry->ie_bits) == 1);
892
893 if (entry->ie_request != 0) {
894 HOT(c_mmot_cold_021++);
895 goto abort_reply_dest_copyin;
896 }
897
898 dest_port = (ipc_port_t) entry->ie_object;
899 assert(dest_port != IP_NULL);
900
901 ip_lock(dest_port);
902 if (!ip_active(dest_port)) {
903 ip_unlock(dest_port);
904 HOT(c_mmot_cold_022++);
905 goto abort_reply_dest_copyin;
906 }
907
908 assert(dest_port->ip_sorights > 0);
909
910 /* optimized ipc_entry_dealloc */
911
912
913 entry->ie_bits = gen;
914 entry->ie_next = table->ie_next;
915 table->ie_next = index;
916 entry->ie_object = IO_NULL;
917 }
918
919 hdr->msgh_bits =
920 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
921 0);
922 hdr->msgh_remote_port = dest_port;
923
924 /* make sure we can queue to the destination */
925
926 assert(dest_port->ip_receiver != ipc_space_kernel);
927
928 /* optimized ipc_entry_lookup/ipc_mqueue_copyin */
929
930 {
931 register ipc_entry_t entry;
932 register ipc_entry_bits_t bits;
933
934 {
935 register mach_port_index_t index;
936 register mach_port_gen_t gen;
937
938 index = MACH_PORT_INDEX(rcv_name);
939 gen = MACH_PORT_GEN(rcv_name);
940
941 if (index < size) {
942 entry = &table[index];
943 bits = entry->ie_bits;
944 if (IE_BITS_GEN(bits) != gen ||
945 (bits & IE_BITS_COLLISION)) {
946 entry = IE_NULL;
947 }
948 } else {
949 entry = IE_NULL;
950 }
951 if (entry == IE_NULL) {
952 entry = ipc_entry_lookup(space, rcv_name);
953 if (entry == IE_NULL) {
954 HOT(c_mmot_cold_024++);
955 goto abort_reply_rcv_copyin;
956 }
957 bits = entry->ie_bits;
958 }
959
960 }
961
962 /* check type bits; looking for receive or set */
963 #if 0
964 /*
965 * JMM - The check below for messages in the receive
966 * mqueue is insufficient to work with port sets, since
967 * the messages stay in the port queues. For now, don't
968 * allow portsets (but receiving on portsets when sending
969 * a message to a send-once right is actually a very
970 * common case (so we should re-enable).
971 */
972 if (bits & MACH_PORT_TYPE_PORT_SET) {
973 register ipc_pset_t rcv_pset;
974
975 rcv_pset = (ipc_pset_t) entry->ie_object;
976 assert(rcv_pset != IPS_NULL);
977
978 ips_lock(rcv_pset);
979 assert(ips_active(rcv_pset));
980
981 rcv_object = (ipc_object_t) rcv_pset;
982 rcv_mqueue = &rcv_pset->ips_messages;
983 } else
984 #endif /* 0 */
985 if (bits & MACH_PORT_TYPE_RECEIVE) {
986 register ipc_port_t rcv_port;
987
988 rcv_port = (ipc_port_t) entry->ie_object;
989 assert(rcv_port != IP_NULL);
990
991 if (!ip_lock_try(rcv_port)) {
992 HOT(c_mmot_cold_025++);
993 goto abort_reply_rcv_copyin;
994 }
995 assert(ip_active(rcv_port));
996
997 if (rcv_port->ip_pset_count != 0) {
998 ip_unlock(rcv_port);
999 HOT(c_mmot_cold_026++);
1000 goto abort_reply_rcv_copyin;
1001 }
1002
1003 rcv_object = (ipc_object_t) rcv_port;
1004 rcv_mqueue = &rcv_port->ip_messages;
1005 } else {
1006 HOT(c_mmot_cold_027++);
1007 goto abort_reply_rcv_copyin;
1008 }
1009 }
1010
1011 is_write_unlock(space);
1012 io_reference(rcv_object);
1013 io_unlock(rcv_object);
1014 HOT(c_mmot_hot_fSR_ok++);
1015 goto fast_send_receive;
1016
1017 abort_reply_dest_copyin:
1018 is_write_unlock(space);
1019 HOT(c_mmot_cold_029++);
1020 goto slow_copyin;
1021
1022 abort_reply_rcv_copyin:
1023 ip_unlock(dest_port);
1024 is_write_unlock(space);
1025 HOT(c_mmot_cold_030++);
1026 goto slow_send;
1027 }
1028
1029 default:
1030 HOT(c_mmot_cold_031++);
1031 goto slow_copyin;
1032 }
1033 /*NOTREACHED*/
1034
1035 fast_send_receive:
1036 /*
1037 * optimized ipc_mqueue_send/ipc_mqueue_receive
1038 *
1039 * Finished get/copyin of kmsg and copyin of rcv_name.
1040 * space is unlocked, dest_port is locked,
1041 * we can queue kmsg to dest_port,
1042 * rcv_mqueue is set, and rcv_object holds a ref
1043 * so the mqueue cannot go away.
1044 *
1045 * JMM - For now, rcv_object is just a port. Portsets
1046 * are disabled for the time being.
1047 */
1048
1049 assert(ip_active(dest_port));
1050 assert(dest_port->ip_receiver != ipc_space_kernel);
1051 // assert(!imq_full(&dest_port->ip_messages) ||
1052 // (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1053 // MACH_MSG_TYPE_PORT_SEND_ONCE));
1054 assert((hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR) == 0);
1055
1056 {
1057 register ipc_mqueue_t dest_mqueue;
1058 wait_queue_t waitq;
1059 thread_t receiver;
1060 #if THREAD_SWAPPER
1061 thread_act_t rcv_act;
1062 #endif
1063 spl_t s;
1064
1065 s = splsched();
1066 dest_mqueue = &dest_port->ip_messages;
1067 waitq = &dest_mqueue->imq_wait_queue;
1068 imq_lock(dest_mqueue);
1069
1070 wait_queue_peek64_locked(waitq, IPC_MQUEUE_RECEIVE, &receiver, &waitq);
1071 /* queue still locked, thread locked - but still on q */
1072
1073 if (receiver == THREAD_NULL) {
1074 abort_send_receive:
1075 imq_unlock(dest_mqueue);
1076 splx(s);
1077 ip_unlock(dest_port);
1078 ipc_object_release(rcv_object);
1079 HOT(c_mmot_cold_032++);
1080 goto slow_send;
1081 }
1082
1083 assert(receiver->wait_queue == waitq);
1084 assert(receiver->wait_event == IPC_MQUEUE_RECEIVE);
1085
1086 /*
1087 * See if it is still running on another processor (trying to
1088 * block itself). If so, fall off.
1089 *
1090 * JMM - We have an opportunity here. Since the thread is locked
1091 * and we find it runnable, it must still be trying to get into
1092 * thread_block on itself. We could just "hand him the message"
1093 * and let him go (thread_go_locked()) and then fall down into a
1094 * slow receive for ourselves. Only his RECEIVE_TOO_LARGE handling
1095 * runs afoul of that. Clean this up!
1096 */
1097 if ((receiver->state & (TH_RUN|TH_WAIT)) != TH_WAIT) {
1098 assert(NCPUS > 1);
1099 HOT(c_mmot_cold_033++);
1100 fall_off:
1101 thread_unlock(receiver);
1102 if (waitq != &dest_mqueue->imq_wait_queue)
1103 wait_queue_unlock(waitq);
1104 goto abort_send_receive;
1105 }
1106
1107 /*
1108 * Check that the receiver can stay on the hot path.
1109 */
1110 if (send_size + REQUESTED_TRAILER_SIZE(receiver->ith_option) >
1111 receiver->ith_msize) {
1112 /*
1113 * The receiver can't accept the message.
1114 */
1115 HOT(c_mmot_bad_rcvr++);
1116 goto fall_off;
1117 }
1118
1119 #if THREAD_SWAPPER
1120 /*
1121 * Receiver looks okay -- is it swapped in?
1122 */
1123 rcv_act = receiver->top_act;
1124 if (rcv_act->swap_state != TH_SW_IN &&
1125 rcv_act->swap_state != TH_SW_UNSWAPPABLE) {
1126 HOT(c_mmot_rcvr_swapped++);
1127 goto fall_off;
1128 }
1129
1130 /*
1131 * Make sure receiver stays swapped in (if we can).
1132 */
1133 if (!act_lock_try(rcv_act)) { /* out of order! */
1134 HOT(c_mmot_rcvr_locked++);
1135 goto fall_off;
1136 }
1137
1138 /*
1139 * Check for task swapping in progress affecting
1140 * receiver. Since rcv_act is attached to a shuttle,
1141 * its swap_state is covered by shuttle's thread_lock()
1142 * (sigh).
1143 */
1144 if ((rcv_act->swap_state != TH_SW_IN &&
1145 rcv_act->swap_state != TH_SW_UNSWAPPABLE) ||
1146 rcv_act->ast & AST_SWAPOUT) {
1147 act_unlock(rcv_act);
1148 HOT(c_mmot_rcvr_tswapped++);
1149 goto fall_off;
1150 }
1151
1152 /*
1153 * We don't need to make receiver unswappable here -- holding
1154 * act_lock() of rcv_act is sufficient to prevent either thread
1155 * or task swapping from changing its state (see swapout_scan(),
1156 * task_swapout()). Don't release lock till receiver's state
1157 * is consistent. Its task may then be marked for swapout,
1158 * but that's life.
1159 */
1160 /*
1161 * NB: act_lock(rcv_act) still held
1162 */
1163 #endif /* THREAD_SWAPPER */
1164
1165 /*
1166 * Before committing to the handoff, make sure that we are
1167 * really going to block (i.e. there are no messages already
1168 * queued for us. This violates lock ordering, so make sure
1169 * we don't deadlock. After the trylock succeeds below, we
1170 * may have up to 3 message queues locked:
1171 * - the dest port mqueue
1172 * - a portset mqueue (where waiting receiver was found)
1173 * - finally our own rcv_mqueue
1174 *
1175 * JMM - Need to make this check appropriate for portsets as
1176 * well before re-enabling them.
1177 */
1178 if (!imq_lock_try(rcv_mqueue)) {
1179 goto fall_off;
1180 }
1181 if (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL) {
1182 imq_unlock(rcv_mqueue);
1183 HOT(c_mmot_cold_033++);
1184 goto fall_off;
1185 }
1186
1187 /* At this point we are committed to do the "handoff". */
1188 c_mach_msg_trap_switch_fast++;
1189
1190 /*
1191 * JMM - Go ahead and pull the receiver from the runq. If the
1192 * runq wasn't the one for the mqueue, unlock it.
1193 */
1194 wait_queue_pull_thread_locked(waitq,
1195 receiver,
1196 (waitq != &dest_mqueue->imq_wait_queue));
1197
1198 /*
1199 * Store the kmsg and seqno where the receiver can pick it up.
1200 */
1201 receiver->ith_state = MACH_MSG_SUCCESS;
1202 receiver->ith_kmsg = kmsg;
1203 receiver->ith_seqno = dest_mqueue->imq_seqno++;
1204
1205 /*
1206 * Inline thread_go_locked
1207 *
1208 * JMM - Including hacked in version of setrun scheduler op
1209 * that doesn't try to put thread on a runq.
1210 */
1211 {
1212 receiver->state &= ~(TH_WAIT|TH_UNINT);
1213 hw_atomic_add(&receiver->processor_set->run_count, 1);
1214 receiver->state |= TH_RUN;
1215 receiver->wait_result = THREAD_AWAKENED;
1216
1217 receiver->computation_metered = 0;
1218 receiver->reason = AST_NONE;
1219 }
1220
1221 thread_unlock(receiver);
1222 #if THREAD_SWAPPER
1223 act_unlock(rcv_act);
1224 #endif /* THREAD_SWAPPER */
1225
1226 imq_unlock(dest_mqueue);
1227 ip_unlock(dest_port);
1228 current_task()->messages_sent++;
1229
1230
1231 /*
1232 * Put self on receive port's queue.
1233 * Also save state that the sender of
1234 * our reply message needs to determine if it
1235 * can hand off directly back to us.
1236 */
1237 self->ith_msg = (rcv_msg) ? rcv_msg : msg;
1238 self->ith_object = rcv_object; /* still holds reference */
1239 self->ith_msize = rcv_size;
1240 self->ith_option = option;
1241 self->ith_scatter_list_size = scatter_list_size;
1242 self->ith_continuation = thread_syscall_return;
1243
1244 waitq = &rcv_mqueue->imq_wait_queue;
1245 (void)wait_queue_assert_wait64_locked(waitq,
1246 IPC_MQUEUE_RECEIVE,
1247 THREAD_ABORTSAFE,
1248 TRUE); /* unlock? */
1249 /* rcv_mqueue is unlocked */
1250
1251 /*
1252 * Switch directly to receiving thread, and block
1253 * this thread as though it had called ipc_mqueue_receive.
1254 */
1255 thread_run(self, ipc_mqueue_receive_continue, receiver);
1256 /* NOTREACHED */
1257 }
1258
1259 fast_copyout:
1260 /*
1261 * Nothing locked and no references held, except
1262 * we have kmsg with msgh_seqno filled in. Must
1263 * still check against rcv_size and do
1264 * ipc_kmsg_copyout/ipc_kmsg_put.
1265 */
1266
1267 reply_size = send_size + trailer->msgh_trailer_size;
1268 if (rcv_size < reply_size) {
1269 HOT(c_mmot_g_slow_copyout6++);
1270 goto slow_copyout;
1271 }
1272
1273 /* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */
1274
1275 switch (hdr->msgh_bits) {
1276 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
1277 MACH_MSG_TYPE_PORT_SEND_ONCE): {
1278 ipc_port_t reply_port =
1279 (ipc_port_t) hdr->msgh_local_port;
1280 mach_port_name_t dest_name, reply_name;
1281
1282 /* receiving a request message */
1283
1284 if (!IP_VALID(reply_port)) {
1285 HOT(c_mmot_g_slow_copyout5++);
1286 goto slow_copyout;
1287 }
1288
1289 is_write_lock(space);
1290 assert(space->is_active);
1291
1292 /*
1293 * To do an atomic copyout, need simultaneous
1294 * locks on both ports and the space. If
1295 * dest_port == reply_port, and simple locking is
1296 * enabled, then we will abort. Otherwise it's
1297 * OK to unlock twice.
1298 */
1299
1300 ip_lock(dest_port);
1301 if (!ip_active(dest_port) ||
1302 !ip_lock_try(reply_port)) {
1303 HOT(c_mmot_cold_037++);
1304 goto abort_request_copyout;
1305 }
1306
1307 if (!ip_active(reply_port)) {
1308 ip_unlock(reply_port);
1309 HOT(c_mmot_cold_038++);
1310 goto abort_request_copyout;
1311 }
1312
1313 assert(reply_port->ip_sorights > 0);
1314 ip_unlock(reply_port);
1315
1316 {
1317 register ipc_entry_t table;
1318 register ipc_entry_t entry;
1319 register mach_port_index_t index;
1320
1321 /* optimized ipc_entry_get */
1322
1323 table = space->is_table;
1324 index = table->ie_next;
1325
1326 if (index == 0) {
1327 HOT(c_mmot_cold_039++);
1328 goto abort_request_copyout;
1329 }
1330
1331 entry = &table[index];
1332 table->ie_next = entry->ie_next;
1333 entry->ie_request = 0;
1334
1335 {
1336 register mach_port_gen_t gen;
1337
1338 assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
1339 gen = IE_BITS_NEW_GEN(entry->ie_bits);
1340
1341 reply_name = MACH_PORT_MAKE(index, gen);
1342
1343 /* optimized ipc_right_copyout */
1344
1345 entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
1346 }
1347
1348 assert(MACH_PORT_VALID(reply_name));
1349 entry->ie_object = (ipc_object_t) reply_port;
1350 is_write_unlock(space);
1351 }
1352
1353 /* optimized ipc_object_copyout_dest */
1354
1355 assert(dest_port->ip_srights > 0);
1356 ip_release(dest_port);
1357
1358 if (dest_port->ip_receiver == space)
1359 dest_name = dest_port->ip_receiver_name;
1360 else
1361 dest_name = MACH_PORT_NULL;
1362
1363 if ((--dest_port->ip_srights == 0) &&
1364 (dest_port->ip_nsrequest != IP_NULL)) {
1365 ipc_port_t nsrequest;
1366 mach_port_mscount_t mscount;
1367
1368 /* a rather rare case */
1369
1370 nsrequest = dest_port->ip_nsrequest;
1371 mscount = dest_port->ip_mscount;
1372 dest_port->ip_nsrequest = IP_NULL;
1373 ip_unlock(dest_port);
1374 ipc_notify_no_senders(nsrequest, mscount);
1375 } else
1376 ip_unlock(dest_port);
1377
1378 hdr->msgh_bits =
1379 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
1380 MACH_MSG_TYPE_PORT_SEND);
1381 hdr->msgh_remote_port = (mach_port_t)reply_name;
1382 hdr->msgh_local_port = (mach_port_t)dest_name;
1383 HOT(c_mmot_hot_ok1++);
1384 goto fast_put;
1385
1386 abort_request_copyout:
1387 ip_unlock(dest_port);
1388 is_write_unlock(space);
1389 HOT(c_mmot_g_slow_copyout4++);
1390 goto slow_copyout;
1391 }
1392
1393 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1394 register mach_port_name_t dest_name;
1395
1396 /* receiving a reply message */
1397
1398 ip_lock(dest_port);
1399 if (!ip_active(dest_port)) {
1400 ip_unlock(dest_port);
1401 HOT(c_mmot_g_slow_copyout3++);
1402 goto slow_copyout;
1403 }
1404
1405 /* optimized ipc_object_copyout_dest */
1406
1407 assert(dest_port->ip_sorights > 0);
1408
1409 if (dest_port->ip_receiver == space) {
1410 ip_release(dest_port);
1411 dest_port->ip_sorights--;
1412 dest_name = dest_port->ip_receiver_name;
1413 ip_unlock(dest_port);
1414 } else {
1415 ip_unlock(dest_port);
1416
1417 ipc_notify_send_once(dest_port);
1418 dest_name = MACH_PORT_NULL;
1419 }
1420
1421 hdr->msgh_bits = MACH_MSGH_BITS(0,
1422 MACH_MSG_TYPE_PORT_SEND_ONCE);
1423 hdr->msgh_remote_port = MACH_PORT_NULL;
1424 hdr->msgh_local_port = (ipc_port_t)dest_name;
1425 HOT(c_mmot_hot_ok2++);
1426 goto fast_put;
1427 }
1428
1429 case MACH_MSGH_BITS_COMPLEX|
1430 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1431 register mach_port_name_t dest_name;
1432
1433 /* receiving a complex reply message */
1434
1435 ip_lock(dest_port);
1436 if (!ip_active(dest_port)) {
1437 ip_unlock(dest_port);
1438 HOT(c_mmot_g_slow_copyout1++);
1439 goto slow_copyout;
1440 }
1441
1442 /* optimized ipc_object_copyout_dest */
1443
1444 assert(dest_port->ip_sorights > 0);
1445
1446 if (dest_port->ip_receiver == space) {
1447 ip_release(dest_port);
1448 dest_port->ip_sorights--;
1449 dest_name = dest_port->ip_receiver_name;
1450 ip_unlock(dest_port);
1451 } else {
1452 ip_unlock(dest_port);
1453
1454 ipc_notify_send_once(dest_port);
1455 dest_name = MACH_PORT_NULL;
1456 }
1457
1458 hdr->msgh_bits =
1459 MACH_MSGH_BITS_COMPLEX |
1460 MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND_ONCE);
1461 hdr->msgh_remote_port = MACH_PORT_NULL;
1462 hdr->msgh_local_port = (mach_port_t)dest_name;
1463
1464 mr = ipc_kmsg_copyout_body(kmsg, space,
1465 current_map(),
1466 MACH_MSG_BODY_NULL);
1467 if (mr != MACH_MSG_SUCCESS) {
1468 if (ipc_kmsg_put(msg, kmsg, hdr->msgh_size +
1469 trailer->msgh_trailer_size) ==
1470 MACH_RCV_INVALID_DATA)
1471 return MACH_RCV_INVALID_DATA;
1472 else
1473 return mr | MACH_RCV_BODY_ERROR;
1474 }
1475 HOT(c_mmot_hot_ok3++);
1476 goto fast_put;
1477 }
1478
1479 default:
1480 HOT(c_mmot_g_slow_copyout2++);
1481 goto slow_copyout;
1482 }
1483 /*NOTREACHED*/
1484
1485 fast_put:
1486 mr = ipc_kmsg_put(rcv_msg ? rcv_msg : msg,
1487 kmsg,
1488 hdr->msgh_size + trailer->msgh_trailer_size);
1489 if (mr != MACH_MSG_SUCCESS) {
1490 return MACH_RCV_INVALID_DATA;
1491 }
1492 current_task()->messages_received++;
1493 return mr;
1494
1495
1496 /* BEGINNING OF WARM PATH */
1497
1498 /*
1499 * The slow path has a few non-register temporary
1500 * variables used only for call-by-reference.
1501 */
1502
1503 slow_copyin:
1504 {
1505 ipc_kmsg_t temp_kmsg;
1506 mach_port_seqno_t temp_seqno;
1507 ipc_object_t temp_rcv_object;
1508 ipc_mqueue_t temp_rcv_mqueue;
1509 register mach_port_name_t reply_name =
1510 (mach_port_name_t)hdr->msgh_local_port;
1511
1512
1513 /*
1514 * We have the message data in kmsg, but
1515 * we still need to copyin, send it,
1516 * receive a reply, and do copyout.
1517 */
1518
1519 mr = ipc_kmsg_copyin(kmsg, space, current_map(),
1520 MACH_PORT_NULL);
1521 if (mr != MACH_MSG_SUCCESS) {
1522 ipc_kmsg_free(kmsg);
1523 return(mr);
1524 }
1525
1526 /* try to get back on optimized path */
1527
1528 if ((reply_name != rcv_name) ||
1529 (hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR)) {
1530 HOT(c_mmot_cold_048++);
1531 goto slow_send;
1532 }
1533
1534 dest_port = (ipc_port_t) hdr->msgh_remote_port;
1535 assert(IP_VALID(dest_port));
1536
1537 ip_lock(dest_port);
1538 if (!ip_active(dest_port)) {
1539 ip_unlock(dest_port);
1540 goto slow_send;
1541 }
1542
1543 if (dest_port->ip_receiver == ipc_space_kernel) {
1544 dest_port->ip_messages.imq_seqno++;
1545 ip_unlock(dest_port);
1546 goto kernel_send;
1547 }
1548
1549 if (!imq_full(&dest_port->ip_messages) ||
1550 (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1551 MACH_MSG_TYPE_PORT_SEND_ONCE))
1552 {
1553 /*
1554 * Try an optimized ipc_mqueue_copyin.
1555 * It will work if this is a request message.
1556 */
1557
1558 register ipc_port_t reply_port;
1559
1560 reply_port = (ipc_port_t) hdr->msgh_local_port;
1561 if (IP_VALID(reply_port)) {
1562 if (ip_lock_try(reply_port)) {
1563 if (ip_active(reply_port) &&
1564 reply_port->ip_receiver == space &&
1565 reply_port->ip_receiver_name == rcv_name &&
1566 reply_port->ip_pset_count == 0)
1567 {
1568 /* Grab a reference to the reply port. */
1569 rcv_object = (ipc_object_t) reply_port;
1570 io_reference(rcv_object);
1571 rcv_mqueue = &reply_port->ip_messages;
1572 io_unlock(rcv_object);
1573 HOT(c_mmot_getback_FastSR++);
1574 goto fast_send_receive;
1575 }
1576 ip_unlock(reply_port);
1577 }
1578 }
1579 }
1580
1581 ip_unlock(dest_port);
1582 HOT(c_mmot_cold_050++);
1583 goto slow_send;
1584
1585 kernel_send:
1586 /*
1587 * Special case: send message to kernel services.
1588 * The request message has been copied into the
1589 * kmsg. Nothing is locked.
1590 */
1591
1592 {
1593 register ipc_port_t reply_port;
1594 mach_port_seqno_t local_seqno;
1595 spl_t s;
1596
1597 /*
1598 * Perform the kernel function.
1599 */
1600 c_mmot_kernel_send++;
1601
1602 current_task()->messages_sent++;
1603
1604 kmsg = ipc_kobject_server(kmsg);
1605 if (kmsg == IKM_NULL) {
1606 /*
1607 * No reply. Take the
1608 * slow receive path.
1609 */
1610 HOT(c_mmot_cold_051++);
1611 goto slow_get_rcv_port;
1612 }
1613
1614 /*
1615 * Check that:
1616 * the reply port is alive
1617 * we hold the receive right
1618 * the name has not changed.
1619 * the port is not in a set
1620 * If any of these are not true,
1621 * we cannot directly receive the reply
1622 * message.
1623 */
1624 hdr = &kmsg->ikm_header;
1625 send_size = hdr->msgh_size;
1626 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1627 round_msg(send_size));
1628 reply_port = (ipc_port_t) hdr->msgh_remote_port;
1629 ip_lock(reply_port);
1630
1631 if ((!ip_active(reply_port)) ||
1632 (reply_port->ip_receiver != space) ||
1633 (reply_port->ip_receiver_name != rcv_name) ||
1634 (reply_port->ip_pset_count != 0))
1635 {
1636 ip_unlock(reply_port);
1637 ipc_kmsg_send_always(kmsg);
1638 HOT(c_mmot_cold_052++);
1639 goto slow_get_rcv_port;
1640 }
1641
1642 s = splsched();
1643 rcv_mqueue = &reply_port->ip_messages;
1644 imq_lock(rcv_mqueue);
1645
1646 /* keep port locked, and don`t change ref count yet */
1647
1648 /*
1649 * If there are messages on the port
1650 * or other threads waiting for a message,
1651 * we cannot directly receive the reply.
1652 */
1653 if (!wait_queue_empty(&rcv_mqueue->imq_wait_queue) ||
1654 (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL))
1655 {
1656 imq_unlock(rcv_mqueue);
1657 splx(s);
1658 ip_unlock(reply_port);
1659 ipc_kmsg_send_always(kmsg);
1660 HOT(c_mmot_cold_053++);
1661 goto slow_get_rcv_port;
1662 }
1663
1664 /*
1665 * We can directly receive this reply.
1666 * Since there were no messages queued
1667 * on the reply port, there should be
1668 * no threads blocked waiting to send.
1669 */
1670 dest_port = reply_port;
1671 local_seqno = rcv_mqueue->imq_seqno++;
1672 imq_unlock(rcv_mqueue);
1673 splx(s);
1674
1675 /*
1676 * inline ipc_object_release.
1677 * Port is still locked.
1678 * Reference count was not incremented.
1679 */
1680 ip_check_unlock(reply_port);
1681
1682 if (option & MACH_RCV_TRAILER_MASK) {
1683 trailer->msgh_seqno = local_seqno;
1684 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1685 }
1686 /* copy out the kernel reply */
1687 HOT(c_mmot_fastkernelreply++);
1688 goto fast_copyout;
1689 }
1690
1691 slow_send:
1692 /*
1693 * Nothing is locked. We have acquired kmsg, but
1694 * we still need to send it and receive a reply.
1695 */
1696
1697 mr = ipc_kmsg_send(kmsg, MACH_MSG_OPTION_NONE,
1698 MACH_MSG_TIMEOUT_NONE);
1699 if (mr != MACH_MSG_SUCCESS) {
1700 mr |= ipc_kmsg_copyout_pseudo(kmsg, space,
1701 current_map(),
1702 MACH_MSG_BODY_NULL);
1703
1704 (void) ipc_kmsg_put(msg, kmsg, hdr->msgh_size);
1705 return(mr);
1706 }
1707
1708 slow_get_rcv_port:
1709 /*
1710 * We have sent the message. Copy in the receive port.
1711 */
1712 mr = ipc_mqueue_copyin(space, rcv_name,
1713 &temp_rcv_mqueue, &temp_rcv_object);
1714 if (mr != MACH_MSG_SUCCESS) {
1715 return(mr);
1716 }
1717 rcv_mqueue = temp_rcv_mqueue;
1718 rcv_object = temp_rcv_object;
1719 /* hold ref for rcv_object */
1720
1721 slow_receive:
1722 /*
1723 * Now we have sent the request and copied in rcv_name,
1724 * and hold ref for rcv_object (to keep mqueue alive).
1725 * Just receive a reply and try to get back to fast path.
1726 */
1727
1728 self->ith_continuation = (void (*)(mach_msg_return_t))0;
1729 ipc_mqueue_receive(rcv_mqueue,
1730 MACH_MSG_OPTION_NONE,
1731 MACH_MSG_SIZE_MAX,
1732 MACH_MSG_TIMEOUT_NONE,
1733 THREAD_ABORTSAFE);
1734
1735 mr = self->ith_state;
1736 temp_kmsg = self->ith_kmsg;
1737 temp_seqno = self->ith_seqno;
1738
1739 ipc_object_release(rcv_object);
1740
1741 if (mr != MACH_MSG_SUCCESS) {
1742 return(mr);
1743 }
1744
1745 kmsg = temp_kmsg;
1746 hdr = &kmsg->ikm_header;
1747 send_size = hdr->msgh_size;
1748 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1749 round_msg(send_size));
1750 if (option & MACH_RCV_TRAILER_MASK) {
1751 trailer->msgh_seqno = temp_seqno;
1752 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1753 }
1754 dest_port = (ipc_port_t) hdr->msgh_remote_port;
1755 HOT(c_mmot_cold_055++);
1756 goto fast_copyout;
1757
1758 slow_copyout:
1759 /*
1760 * Nothing locked and no references held, except
1761 * we have kmsg with msgh_seqno filled in. Must
1762 * still check against rcv_size and do
1763 * ipc_kmsg_copyout/ipc_kmsg_put.
1764 */
1765
1766 reply_size = send_size + trailer->msgh_trailer_size;
1767 if (rcv_size < reply_size) {
1768 if (msg_receive_error(kmsg, msg, option, temp_seqno,
1769 space) == MACH_RCV_INVALID_DATA) {
1770 mr = MACH_RCV_INVALID_DATA;
1771 return(mr);
1772 }
1773 else {
1774 mr = MACH_RCV_TOO_LARGE;
1775 return(mr);
1776 }
1777 }
1778
1779 mr = ipc_kmsg_copyout(kmsg, space, current_map(),
1780 MACH_PORT_NULL, MACH_MSG_BODY_NULL);
1781 if (mr != MACH_MSG_SUCCESS) {
1782 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1783 if (ipc_kmsg_put(msg, kmsg, reply_size) ==
1784 MACH_RCV_INVALID_DATA)
1785 mr = MACH_RCV_INVALID_DATA;
1786 }
1787 else {
1788 if (msg_receive_error(kmsg, msg, option,
1789 temp_seqno, space) == MACH_RCV_INVALID_DATA)
1790 mr = MACH_RCV_INVALID_DATA;
1791 }
1792
1793 return(mr);
1794 }
1795
1796 /* try to get back on optimized path */
1797 HOT(c_mmot_getback_fast_put++);
1798 goto fast_put;
1799
1800 /*NOTREACHED*/
1801 }
1802 } /* END OF HOT PATH */
1803 #endif /* ENABLE_HOTPATH */
1804
1805 if (option & MACH_SEND_MSG) {
1806 mr = mach_msg_send(msg, option, send_size,
1807 timeout, notify);
1808 if (mr != MACH_MSG_SUCCESS) {
1809 return mr;
1810 }
1811 }
1812
1813 if (option & MACH_RCV_MSG) {
1814 mach_msg_header_t *rcv;
1815
1816 /*
1817 * 1. MACH_RCV_OVERWRITE is on, and rcv_msg is our scatter list
1818 * and receive buffer
1819 * 2. MACH_RCV_OVERWRITE is off, and rcv_msg might be the
1820 * alternate receive buffer (separate send and receive buffers).
1821 */
1822 if (option & MACH_RCV_OVERWRITE)
1823 rcv = rcv_msg;
1824 else if (rcv_msg != MACH_MSG_NULL)
1825 rcv = rcv_msg;
1826 else
1827 rcv = msg;
1828 mr = mach_msg_receive(rcv, option, rcv_size, rcv_name,
1829 timeout, thread_syscall_return, scatter_list_size);
1830 thread_syscall_return(mr);
1831 }
1832
1833 return MACH_MSG_SUCCESS;
1834 }
1835
1836 /*
1837 * Routine: mach_msg_trap [mach trap]
1838 * Purpose:
1839 * Possibly send a message; possibly receive a message.
1840 * Conditions:
1841 * Nothing locked.
1842 * Returns:
1843 * All of mach_msg_send and mach_msg_receive error codes.
1844 */
1845
1846 mach_msg_return_t
1847 mach_msg_trap(
1848 mach_msg_header_t *msg,
1849 mach_msg_option_t option,
1850 mach_msg_size_t send_size,
1851 mach_msg_size_t rcv_size,
1852 mach_port_name_t rcv_name,
1853 mach_msg_timeout_t timeout,
1854 mach_port_name_t notify)
1855 {
1856 return mach_msg_overwrite_trap(msg,
1857 option,
1858 send_size,
1859 rcv_size,
1860 rcv_name,
1861 timeout,
1862 notify,
1863 (mach_msg_header_t *)0,
1864 (mach_msg_size_t)0);
1865 }
1866
1867
1868 /*
1869 * Routine: msg_receive_error [internal]
1870 * Purpose:
1871 * Builds a minimal header/trailer and copies it to
1872 * the user message buffer. Invoked when in the case of a
1873 * MACH_RCV_TOO_LARGE or MACH_RCV_BODY_ERROR error.
1874 * Conditions:
1875 * Nothing locked.
1876 * Returns:
1877 * MACH_MSG_SUCCESS minimal header/trailer copied
1878 * MACH_RCV_INVALID_DATA copyout to user buffer failed
1879 */
1880
1881 mach_msg_return_t
1882 msg_receive_error(
1883 ipc_kmsg_t kmsg,
1884 mach_msg_header_t *msg,
1885 mach_msg_option_t option,
1886 mach_port_seqno_t seqno,
1887 ipc_space_t space)
1888 {
1889 mach_msg_format_0_trailer_t *trailer;
1890
1891 /*
1892 * Copy out the destination port in the message.
1893 * Destroy all other rights and memory in the message.
1894 */
1895 ipc_kmsg_copyout_dest(kmsg, space);
1896
1897 /*
1898 * Build a minimal message with the requested trailer.
1899 */
1900 trailer = (mach_msg_format_0_trailer_t *)
1901 ((vm_offset_t)&kmsg->ikm_header +
1902 round_msg(sizeof(mach_msg_header_t)));
1903 kmsg->ikm_header.msgh_size = sizeof(mach_msg_header_t);
1904 bcopy( (char *)&trailer_template,
1905 (char *)trailer,
1906 sizeof(trailer_template));
1907 if (option & MACH_RCV_TRAILER_MASK) {
1908 trailer->msgh_seqno = seqno;
1909 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1910 }
1911
1912 /*
1913 * Copy the message to user space
1914 */
1915 if (ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size +
1916 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
1917 return(MACH_RCV_INVALID_DATA);
1918 else
1919 return(MACH_MSG_SUCCESS);
1920 }
mirror of XNU