]> git.saurik.com Git - apple/xnu.git/blob - osfmk/ipc/mach_msg.c
projects / apple / xnu.git / blob
? search:


4af874090b1b248f1253dae8f5b95fc83a6ab1db
[apple/xnu.git] / osfmk / ipc / mach_msg.c
1 /*
2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22 /*
23 * @OSF_COPYRIGHT@
24 */
25 /*
26 * Mach Operating System
27 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
28 * All Rights Reserved.
29 *
30 * Permission to use, copy, modify and distribute this software and its
31 * documentation is hereby granted, provided that both the copyright
32 * notice and this permission notice appear in all copies of the
33 * software, derivative works or modified versions, and any portions
34 * thereof, and that both notices appear in supporting documentation.
35 *
36 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
39 *
40 * Carnegie Mellon requests users of this software to return to
41 *
42 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
43 * School of Computer Science
44 * Carnegie Mellon University
45 * Pittsburgh PA 15213-3890
46 *
47 * any improvements or extensions that they make and grant Carnegie Mellon
48 * the rights to redistribute these changes.
49 */
50 /*
51 */
52 /*
53 * File: ipc/mach_msg.c
54 * Author: Rich Draves
55 * Date: 1989
56 *
57 * Exported message traps. See mach/message.h.
58 */
59
60 #include <cpus.h>
61
62 #include <mach/kern_return.h>
63 #include <mach/port.h>
64 #include <mach/message.h>
65 #include <mach/mig_errors.h>
66 #include <kern/assert.h>
67 #include <kern/counters.h>
68 #include <kern/cpu_number.h>
69 #include <kern/task.h>
70 #include <kern/thread.h>
71 #include <kern/lock.h>
72 #include <kern/sched_prim.h>
73 #include <kern/exception.h>
74 #include <kern/misc_protos.h>
75 #include <vm/vm_map.h>
76 #include <ipc/ipc_kmsg.h>
77 #include <ipc/ipc_mqueue.h>
78 #include <ipc/ipc_object.h>
79 #include <ipc/ipc_notify.h>
80 #include <ipc/ipc_port.h>
81 #include <ipc/ipc_pset.h>
82 #include <ipc/ipc_space.h>
83 #include <ipc/ipc_entry.h>
84 #include <kern/kalloc.h>
85 #include <kern/thread_swap.h>
86 #include <kern/processor.h>
87
88 #include <kern/mk_sp.h>
89
90 #include <machine/machine_routines.h>
91 #include <sys/kdebug.h>
92
93 /*
94 * Forward declarations
95 */
96
97 mach_msg_return_t mach_msg_send(
98 mach_msg_header_t *msg,
99 mach_msg_option_t option,
100 mach_msg_size_t send_size,
101 mach_msg_timeout_t timeout,
102 mach_port_name_t notify);
103
104 mach_msg_return_t mach_msg_receive(
105 mach_msg_header_t *msg,
106 mach_msg_option_t option,
107 mach_msg_size_t rcv_size,
108 mach_port_name_t rcv_name,
109 mach_msg_timeout_t timeout,
110 void (*continuation)(mach_msg_return_t),
111 mach_msg_size_t slist_size);
112
113
114 mach_msg_return_t msg_receive_error(
115 ipc_kmsg_t kmsg,
116 mach_msg_header_t *msg,
117 mach_msg_option_t option,
118 mach_port_seqno_t seqno,
119 ipc_space_t space);
120
121 /* the size of each trailer has to be listed here for copyout purposes */
122 mach_msg_trailer_size_t trailer_size[] = {
123 sizeof(mach_msg_trailer_t),
124 sizeof(mach_msg_seqno_trailer_t),
125 sizeof(mach_msg_security_trailer_t) };
126
127 security_token_t KERNEL_SECURITY_TOKEN = KERNEL_SECURITY_TOKEN_VALUE;
128
129 mach_msg_format_0_trailer_t trailer_template = {
130 /* mach_msg_trailer_type_t */ MACH_MSG_TRAILER_FORMAT_0,
131 /* mach_msg_trailer_size_t */ MACH_MSG_TRAILER_MINIMUM_SIZE,
132 /* mach_port_seqno_t */ 0,
133 /* security_token_t */ KERNEL_SECURITY_TOKEN_VALUE
134 };
135
136 /*
137 * Routine: mach_msg_send
138 * Purpose:
139 * Send a message.
140 * Conditions:
141 * Nothing locked.
142 * Returns:
143 * MACH_MSG_SUCCESS Sent the message.
144 * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
145 * MACH_SEND_NO_BUFFER Couldn't allocate buffer.
146 * MACH_SEND_INVALID_DATA Couldn't copy message data.
147 * MACH_SEND_INVALID_HEADER
148 * Illegal value in the message header bits.
149 * MACH_SEND_INVALID_DEST The space is dead.
150 * MACH_SEND_INVALID_NOTIFY Bad notify port.
151 * MACH_SEND_INVALID_DEST Can't copyin destination port.
152 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
153 * MACH_SEND_TIMED_OUT Timeout expired without delivery.
154 * MACH_SEND_INTERRUPTED Delivery interrupted.
155 * MACH_SEND_NO_NOTIFY Can't allocate a msg-accepted request.
156 * MACH_SEND_WILL_NOTIFY Msg-accepted notif. requested.
157 * MACH_SEND_NOTIFY_IN_PROGRESS
158 * This space has already forced a message to this port.
159 */
160
161 mach_msg_return_t
162 mach_msg_send(
163 mach_msg_header_t *msg,
164 mach_msg_option_t option,
165 mach_msg_size_t send_size,
166 mach_msg_timeout_t timeout,
167 mach_port_name_t notify)
168 {
169 ipc_space_t space = current_space();
170 vm_map_t map = current_map();
171 ipc_kmsg_t kmsg;
172 mach_msg_return_t mr;
173
174 mr = ipc_kmsg_get(msg, send_size, &kmsg);
175
176 if (mr != MACH_MSG_SUCCESS)
177 return mr;
178
179 if (option & MACH_SEND_CANCEL) {
180 if (notify == MACH_PORT_NULL)
181 mr = MACH_SEND_INVALID_NOTIFY;
182 else
183 mr = ipc_kmsg_copyin(kmsg, space, map, notify);
184 } else
185 mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
186 if (mr != MACH_MSG_SUCCESS) {
187 ipc_kmsg_free(kmsg);
188 return mr;
189 }
190
191 mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, timeout);
192
193 if (mr != MACH_MSG_SUCCESS) {
194 mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
195 (void) ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
196 }
197
198 return mr;
199 }
200
201 /*
202 * Routine: mach_msg_receive
203 * Purpose:
204 * Receive a message.
205 * Conditions:
206 * Nothing locked.
207 * Returns:
208 * MACH_MSG_SUCCESS Received a message.
209 * MACH_RCV_INVALID_NAME The name doesn't denote a right,
210 * or the denoted right is not receive or port set.
211 * MACH_RCV_IN_SET Receive right is a member of a set.
212 * MACH_RCV_TOO_LARGE Message wouldn't fit into buffer.
213 * MACH_RCV_TIMED_OUT Timeout expired without a message.
214 * MACH_RCV_INTERRUPTED Reception interrupted.
215 * MACH_RCV_PORT_DIED Port/set died while receiving.
216 * MACH_RCV_PORT_CHANGED Port moved into set while receiving.
217 * MACH_RCV_INVALID_DATA Couldn't copy to user buffer.
218 * MACH_RCV_INVALID_NOTIFY Bad notify port.
219 * MACH_RCV_HEADER_ERROR
220 */
221
222 mach_msg_return_t
223 mach_msg_receive_results(void)
224 {
225 thread_t self = current_thread();
226 ipc_space_t space = current_space();
227 vm_map_t map = current_map();
228
229 ipc_object_t object = self->ith_object;
230 mach_msg_return_t mr = self->ith_state;
231 mach_msg_header_t *msg = self->ith_msg;
232 mach_msg_option_t option = self->ith_option;
233 ipc_kmsg_t kmsg = self->ith_kmsg;
234 mach_port_seqno_t seqno = self->ith_seqno;
235 mach_msg_size_t slist_size = self->ith_scatter_list_size;
236
237 mach_msg_format_0_trailer_t *trailer;
238
239 ipc_object_release(object);
240
241 if (mr != MACH_MSG_SUCCESS) {
242
243 if (mr == MACH_RCV_TOO_LARGE ) {
244 if (option & MACH_RCV_LARGE) {
245 /*
246 * We need to inform the user-level code that it needs more
247 * space. The value for how much space was returned in the
248 * msize save area instead of the message (which was left on
249 * the queue).
250 */
251 if (copyout((char *) &self->ith_msize,
252 (char *) &msg->msgh_size,
253 sizeof(mach_msg_size_t)))
254 mr = MACH_RCV_INVALID_DATA;
255 goto out;
256 }
257
258 if (msg_receive_error(kmsg, msg, option, seqno, space)
259 == MACH_RCV_INVALID_DATA)
260 mr = MACH_RCV_INVALID_DATA;
261 }
262 goto out;
263 }
264
265 trailer = (mach_msg_format_0_trailer_t *)
266 ((vm_offset_t)&kmsg->ikm_header +
267 round_msg(kmsg->ikm_header.msgh_size));
268 if (option & MACH_RCV_TRAILER_MASK) {
269 trailer->msgh_seqno = seqno;
270 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
271 }
272
273 /*
274 * If MACH_RCV_OVERWRITE was specified, try to get the scatter
275 * list and verify it against the contents of the message. If
276 * there is any problem with it, we will continue without it as
277 * normal.
278 */
279 if (option & MACH_RCV_OVERWRITE) {
280 mach_msg_size_t slist_size = self->ith_scatter_list_size;
281 mach_msg_body_t *slist;
282
283 slist = ipc_kmsg_copyin_scatter(msg, slist_size, kmsg);
284 mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL, slist);
285 ipc_kmsg_free_scatter(slist, slist_size);
286 } else {
287 mr = ipc_kmsg_copyout(kmsg, space, map,
288 MACH_PORT_NULL, MACH_MSG_BODY_NULL);
289 }
290
291 if (mr != MACH_MSG_SUCCESS) {
292 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
293 if (ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size +
294 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
295 mr = MACH_RCV_INVALID_DATA;
296 }
297 else {
298 if (msg_receive_error(kmsg, msg, option, seqno, space)
299 == MACH_RCV_INVALID_DATA)
300 mr = MACH_RCV_INVALID_DATA;
301 }
302 goto out;
303 }
304 mr = ipc_kmsg_put(msg,
305 kmsg,
306 kmsg->ikm_header.msgh_size +
307 trailer->msgh_trailer_size);
308 out:
309 return mr;
310 }
311
312 mach_msg_return_t
313 mach_msg_receive(
314 mach_msg_header_t *msg,
315 mach_msg_option_t option,
316 mach_msg_size_t rcv_size,
317 mach_port_name_t rcv_name,
318 mach_msg_timeout_t timeout,
319 void (*continuation)(mach_msg_return_t),
320 mach_msg_size_t slist_size)
321 {
322 thread_t self = current_thread();
323 ipc_space_t space = current_space();
324 vm_map_t map = current_map();
325 ipc_object_t object;
326 ipc_mqueue_t mqueue;
327 ipc_kmsg_t kmsg;
328 mach_port_seqno_t seqno;
329 mach_msg_return_t mr;
330 mach_msg_body_t *slist;
331 mach_msg_format_0_trailer_t *trailer;
332
333 mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
334 if (mr != MACH_MSG_SUCCESS) {
335 return mr;
336 }
337 /* hold ref for object */
338
339 self->ith_msg = msg;
340 self->ith_object = object;
341 self->ith_msize = rcv_size;
342 self->ith_option = option;
343 self->ith_scatter_list_size = slist_size;
344 self->ith_continuation = continuation;
345
346 ipc_mqueue_receive(mqueue, option, rcv_size, timeout, THREAD_ABORTSAFE);
347 if ((option & MACH_RCV_TIMEOUT) && timeout == 0)
348 _mk_sp_thread_perhaps_yield(self);
349 return mach_msg_receive_results();
350 }
351
352 void
353 mach_msg_receive_continue(void)
354 {
355 thread_t self = current_thread();
356
357 (*self->ith_continuation)(mach_msg_receive_results());
358 }
359
360 /*
361 * Toggle this to compile the hotpath in/out
362 * If compiled in, the run-time toggle "enable_hotpath" below
363 * eases testing & debugging
364 */
365 #define ENABLE_HOTPATH 1 /* Hacked on for now */
366
367 #if ENABLE_HOTPATH
368 /*
369 * These counters allow tracing of hotpath behavior under test loads.
370 * A couple key counters are unconditional (see below).
371 */
372 #define HOTPATH_DEBUG 0 /* Toggle to include lots of counters */
373 #if HOTPATH_DEBUG
374 #define HOT(expr) expr
375
376 unsigned int c_mmot_FIRST = 0; /* Unused First Counter */
377 unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
378 unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
379 unsigned int c_mmot_kernel_send = 0; /* kernel server */
380 unsigned int c_mmot_cold_000 = 0; /* see below ... */
381 unsigned int c_mmot_smallsendsize = 0;
382 unsigned int c_mmot_oddsendsize = 0;
383 unsigned int c_mmot_bigsendsize = 0;
384 unsigned int c_mmot_copyinmsg_fail = 0;
385 unsigned int c_mmot_g_slow_copyin3 = 0;
386 unsigned int c_mmot_cold_006 = 0;
387 unsigned int c_mmot_cold_007 = 0;
388 unsigned int c_mmot_cold_008 = 0;
389 unsigned int c_mmot_cold_009 = 0;
390 unsigned int c_mmot_cold_010 = 0;
391 unsigned int c_mmot_cold_012 = 0;
392 unsigned int c_mmot_cold_013 = 0;
393 unsigned int c_mmot_cold_014 = 0;
394 unsigned int c_mmot_cold_016 = 0;
395 unsigned int c_mmot_cold_018 = 0;
396 unsigned int c_mmot_cold_019 = 0;
397 unsigned int c_mmot_cold_020 = 0;
398 unsigned int c_mmot_cold_021 = 0;
399 unsigned int c_mmot_cold_022 = 0;
400 unsigned int c_mmot_cold_023 = 0;
401 unsigned int c_mmot_cold_024 = 0;
402 unsigned int c_mmot_cold_025 = 0;
403 unsigned int c_mmot_cold_026 = 0;
404 unsigned int c_mmot_cold_027 = 0;
405 unsigned int c_mmot_hot_fSR_ok = 0;
406 unsigned int c_mmot_cold_029 = 0;
407 unsigned int c_mmot_cold_030 = 0;
408 unsigned int c_mmot_cold_031 = 0;
409 unsigned int c_mmot_cold_032 = 0;
410 unsigned int c_mmot_cold_033 = 0;
411 unsigned int c_mmot_bad_rcvr = 0;
412 unsigned int c_mmot_rcvr_swapped = 0;
413 unsigned int c_mmot_rcvr_locked = 0;
414 unsigned int c_mmot_rcvr_tswapped = 0;
415 unsigned int c_mmot_rcvr_freed = 0;
416 unsigned int c_mmot_g_slow_copyout6 = 0;
417 unsigned int c_mmot_g_slow_copyout5 = 0;
418 unsigned int c_mmot_cold_037 = 0;
419 unsigned int c_mmot_cold_038 = 0;
420 unsigned int c_mmot_cold_039 = 0;
421 unsigned int c_mmot_g_slow_copyout4 = 0;
422 unsigned int c_mmot_g_slow_copyout3 = 0;
423 unsigned int c_mmot_hot_ok1 = 0;
424 unsigned int c_mmot_hot_ok2 = 0;
425 unsigned int c_mmot_hot_ok3 = 0;
426 unsigned int c_mmot_g_slow_copyout1 = 0;
427 unsigned int c_mmot_g_slow_copyout2 = 0;
428 unsigned int c_mmot_getback_fast_copyin = 0;
429 unsigned int c_mmot_cold_048 = 0;
430 unsigned int c_mmot_getback_FastSR = 0;
431 unsigned int c_mmot_cold_050 = 0;
432 unsigned int c_mmot_cold_051 = 0;
433 unsigned int c_mmot_cold_052 = 0;
434 unsigned int c_mmot_cold_053 = 0;
435 unsigned int c_mmot_fastkernelreply = 0;
436 unsigned int c_mmot_cold_055 = 0;
437 unsigned int c_mmot_getback_fast_put = 0;
438 unsigned int c_mmot_LAST = 0; /* End Marker - Unused */
439
440 void db_mmot_zero_counters(void); /* forward; */
441 void db_mmot_show_counters(void); /* forward; */
442
443 void /* Call from the debugger to clear all counters */
444 db_mmot_zero_counters(void)
445 {
446 register unsigned int *ip = &c_mmot_FIRST;
447 while (ip <= &c_mmot_LAST)
448 *ip++ = 0;
449 }
450
451 void /* Call from the debugger to show all counters */
452 db_mmot_show_counters(void)
453 {
454 #define xx(str) printf("%s: %d\n", # str, str);
455
456 xx(c_mmot_combined_S_R);
457 xx(c_mach_msg_trap_switch_fast);
458 xx(c_mmot_kernel_send);
459 xx(c_mmot_cold_000);
460 xx(c_mmot_smallsendsize);
461 xx(c_mmot_oddsendsize);
462 xx(c_mmot_bigsendsize);
463 xx(c_mmot_copyinmsg_fail);
464 xx(c_mmot_g_slow_copyin3);
465 xx(c_mmot_cold_006);
466 xx(c_mmot_cold_007);
467 xx(c_mmot_cold_008);
468 xx(c_mmot_cold_009);
469 xx(c_mmot_cold_010);
470 xx(c_mmot_cold_012);
471 xx(c_mmot_cold_013);
472 xx(c_mmot_cold_014);
473 xx(c_mmot_cold_016);
474 xx(c_mmot_cold_018);
475 xx(c_mmot_cold_019);
476 xx(c_mmot_cold_020);
477 xx(c_mmot_cold_021);
478 xx(c_mmot_cold_022);
479 xx(c_mmot_cold_023);
480 xx(c_mmot_cold_024);
481 xx(c_mmot_cold_025);
482 xx(c_mmot_cold_026);
483 xx(c_mmot_cold_027);
484 xx(c_mmot_hot_fSR_ok);
485 xx(c_mmot_cold_029);
486 xx(c_mmot_cold_030);
487 xx(c_mmot_cold_031);
488 xx(c_mmot_cold_032);
489 xx(c_mmot_cold_033);
490 xx(c_mmot_bad_rcvr);
491 xx(c_mmot_rcvr_swapped);
492 xx(c_mmot_rcvr_locked);
493 xx(c_mmot_rcvr_tswapped);
494 xx(c_mmot_rcvr_freed);
495 xx(c_mmot_g_slow_copyout6);
496 xx(c_mmot_g_slow_copyout5);
497 xx(c_mmot_cold_037);
498 xx(c_mmot_cold_038);
499 xx(c_mmot_cold_039);
500 xx(c_mmot_g_slow_copyout4);
501 xx(c_mmot_g_slow_copyout3);
502 xx(c_mmot_g_slow_copyout1);
503 xx(c_mmot_hot_ok3);
504 xx(c_mmot_hot_ok2);
505 xx(c_mmot_hot_ok1);
506 xx(c_mmot_g_slow_copyout2);
507 xx(c_mmot_getback_fast_copyin);
508 xx(c_mmot_cold_048);
509 xx(c_mmot_getback_FastSR);
510 xx(c_mmot_cold_050);
511 xx(c_mmot_cold_051);
512 xx(c_mmot_cold_052);
513 xx(c_mmot_cold_053);
514 xx(c_mmot_fastkernelreply);
515 xx(c_mmot_cold_055);
516 xx(c_mmot_getback_fast_put);
517
518 #undef xx
519 }
520
521 #else /* !HOTPATH_DEBUG */
522
523 /*
524 * Duplicate just these few so we can always do a quick sanity check
525 */
526 unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
527 unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
528 unsigned int c_mmot_kernel_send = 0; /* kernel server calls */
529 #define HOT(expr) /* no optional counters */
530
531 #endif /* !HOTPATH_DEBUG */
532
533 boolean_t enable_hotpath = TRUE; /* Patchable, just in case ... */
534 #endif /* HOTPATH_ENABLE */
535
536 /*
537 * Routine: mach_msg_overwrite_trap [mach trap]
538 * Purpose:
539 * Possibly send a message; possibly receive a message.
540 * Conditions:
541 * Nothing locked.
542 * Returns:
543 * All of mach_msg_send and mach_msg_receive error codes.
544 */
545
546 mach_msg_return_t
547 mach_msg_overwrite_trap(
548 mach_msg_header_t *msg,
549 mach_msg_option_t option,
550 mach_msg_size_t send_size,
551 mach_msg_size_t rcv_size,
552 mach_port_name_t rcv_name,
553 mach_msg_timeout_t timeout,
554 mach_port_name_t notify,
555 mach_msg_header_t *rcv_msg,
556 mach_msg_size_t scatter_list_size)
557 {
558 register mach_msg_header_t *hdr;
559 mach_msg_return_t mr = MACH_MSG_SUCCESS;
560 /* mask out some of the options before entering the hot path */
561 mach_msg_option_t masked_option =
562 option & ~(MACH_SEND_TRAILER|MACH_RCV_TRAILER_MASK|MACH_RCV_LARGE);
563 int i;
564
565 #if ENABLE_HOTPATH
566 /* BEGINNING OF HOT PATH */
567 if ((masked_option == (MACH_SEND_MSG|MACH_RCV_MSG)) && enable_hotpath) {
568 register thread_t self = current_thread();
569 register mach_msg_format_0_trailer_t *trailer;
570
571 ipc_space_t space = current_act()->task->itk_space;
572 ipc_kmsg_t kmsg;
573 register ipc_port_t dest_port;
574 ipc_object_t rcv_object;
575 register ipc_mqueue_t rcv_mqueue;
576 mach_msg_size_t reply_size;
577 ipc_kmsg_t rcv_kmsg;
578
579 c_mmot_combined_S_R++;
580
581 /*
582 * This case is divided into ten sections, each
583 * with a label. There are five optimized
584 * sections and six unoptimized sections, which
585 * do the same thing but handle all possible
586 * cases and are slower.
587 *
588 * The five sections for an RPC are
589 * 1) Get request message into a buffer.
590 * 2) Copyin request message and rcv_name.
591 * (fast_copyin or slow_copyin)
592 * 3) Enqueue request and dequeue reply.
593 * (fast_send_receive or
594 * slow_send and slow_receive)
595 * 4) Copyout reply message.
596 * (fast_copyout or slow_copyout)
597 * 5) Put reply message to user's buffer.
598 *
599 * Keep the locking hierarchy firmly in mind.
600 * (First spaces, then ports, then port sets,
601 * then message queues.) Only a non-blocking
602 * attempt can be made to acquire locks out of
603 * order, or acquire two locks on the same level.
604 * Acquiring two locks on the same level will
605 * fail if the objects are really the same,
606 * unless simple locking is disabled. This is OK,
607 * because then the extra unlock does nothing.
608 *
609 * There are two major reasons these RPCs can't use
610 * ipc_thread_switch, and use slow_send/slow_receive:
611 * 1) Kernel RPCs.
612 * 2) Servers fall behind clients, so
613 * client doesn't find a blocked server thread and
614 * server finds waiting messages and can't block.
615 */
616
617 mr = ipc_kmsg_get(msg, send_size, &kmsg);
618 if (mr != KERN_SUCCESS) {
619 return mr;
620 }
621 hdr = &kmsg->ikm_header;
622 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
623 send_size);
624
625 fast_copyin:
626 /*
627 * optimized ipc_kmsg_copyin/ipc_mqueue_copyin
628 *
629 * We have the request message data in kmsg.
630 * Must still do copyin, send, receive, etc.
631 *
632 * If the message isn't simple, we can't combine
633 * ipc_kmsg_copyin_header and ipc_mqueue_copyin,
634 * because copyin of the message body might
635 * affect rcv_name.
636 */
637
638 switch (hdr->msgh_bits) {
639 case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
640 MACH_MSG_TYPE_MAKE_SEND_ONCE): {
641 register ipc_entry_t table;
642 register ipc_entry_num_t size;
643 register ipc_port_t reply_port;
644
645 /* sending a request message */
646
647 {
648 register mach_port_index_t index;
649 register mach_port_gen_t gen;
650
651 {
652 register mach_port_name_t reply_name =
653 (mach_port_name_t)hdr->msgh_local_port;
654
655 if (reply_name != rcv_name) {
656 HOT(c_mmot_g_slow_copyin3++);
657 goto slow_copyin;
658 }
659
660 /* optimized ipc_entry_lookup of reply_name */
661
662 index = MACH_PORT_INDEX(reply_name);
663 gen = MACH_PORT_GEN(reply_name);
664
665 is_read_lock(space);
666 assert(space->is_active);
667
668 size = space->is_table_size;
669 table = space->is_table;
670
671 {
672 register ipc_entry_t entry;
673 register ipc_entry_bits_t bits;
674
675 if (index < size) {
676 entry = &table[index];
677 bits = entry->ie_bits;
678 if (IE_BITS_GEN(bits) != gen ||
679 (bits & IE_BITS_COLLISION)) {
680 entry = IE_NULL;
681 }
682 } else {
683 entry = IE_NULL;
684 }
685 if (entry == IE_NULL) {
686 entry = ipc_entry_lookup(space, reply_name);
687 if (entry == IE_NULL) {
688 HOT(c_mmot_cold_006++);
689 goto abort_request_copyin;
690 }
691 bits = entry->ie_bits;
692 }
693
694 /* check type bit */
695
696 if (! (bits & MACH_PORT_TYPE_RECEIVE)) {
697 HOT(c_mmot_cold_007++);
698 goto abort_request_copyin;
699 }
700
701 reply_port = (ipc_port_t) entry->ie_object;
702 assert(reply_port != IP_NULL);
703 }
704 }
705 }
706
707 /* optimized ipc_entry_lookup of dest_name */
708
709 {
710 register mach_port_index_t index;
711 register mach_port_gen_t gen;
712
713 {
714 register mach_port_name_t dest_name =
715 (mach_port_name_t)hdr->msgh_remote_port;
716
717 index = MACH_PORT_INDEX(dest_name);
718 gen = MACH_PORT_GEN(dest_name);
719
720 {
721 register ipc_entry_t entry;
722 register ipc_entry_bits_t bits;
723
724 if (index < size) {
725 entry = &table[index];
726 bits = entry->ie_bits;
727 if (IE_BITS_GEN(bits) != gen ||
728 (bits & IE_BITS_COLLISION)) {
729 entry = IE_NULL;
730 }
731 } else {
732 entry = IE_NULL;
733 }
734 if (entry == IE_NULL) {
735 entry = ipc_entry_lookup(space, dest_name);
736 if (entry == IE_NULL) {
737 HOT(c_mmot_cold_008++);
738 goto abort_request_copyin;
739 }
740 bits = entry->ie_bits;
741 }
742
743 /* check type bit */
744
745 if (! (bits & MACH_PORT_TYPE_SEND)) {
746 HOT(c_mmot_cold_009++);
747 goto abort_request_copyin;
748 }
749
750 assert(IE_BITS_UREFS(bits) > 0);
751
752 dest_port = (ipc_port_t) entry->ie_object;
753 assert(dest_port != IP_NULL);
754 }
755 }
756 }
757
758 /*
759 * To do an atomic copyin, need simultaneous
760 * locks on both ports and the space. If
761 * dest_port == reply_port, and simple locking is
762 * enabled, then we will abort. Otherwise it's
763 * OK to unlock twice.
764 */
765
766 ip_lock(dest_port);
767 if (!ip_active(dest_port) ||
768 !ip_lock_try(reply_port)) {
769 ip_unlock(dest_port);
770 HOT(c_mmot_cold_010++);
771 goto abort_request_copyin;
772 }
773 is_read_unlock(space);
774
775 assert(dest_port->ip_srights > 0);
776 dest_port->ip_srights++;
777 ip_reference(dest_port);
778
779 assert(ip_active(reply_port));
780 assert(reply_port->ip_receiver_name ==
781 (mach_port_name_t)hdr->msgh_local_port);
782 assert(reply_port->ip_receiver == space);
783
784 reply_port->ip_sorights++;
785 ip_reference(reply_port);
786
787 hdr->msgh_bits =
788 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
789 MACH_MSG_TYPE_PORT_SEND_ONCE);
790 hdr->msgh_remote_port = dest_port;
791 hdr->msgh_local_port = reply_port;
792
793 /* make sure we can queue to the destination */
794
795 if (dest_port->ip_receiver == ipc_space_kernel) {
796 /*
797 * The kernel server has a reference to
798 * the reply port, which it hands back
799 * to us in the reply message. We do
800 * not need to keep another reference to
801 * it.
802 */
803 ip_unlock(reply_port);
804
805 assert(ip_active(dest_port));
806 dest_port->ip_messages.imq_seqno++;
807 ip_unlock(dest_port);
808 goto kernel_send;
809 }
810
811 if (imq_full(&dest_port->ip_messages)) {
812 HOT(c_mmot_cold_013++);
813 goto abort_request_send_receive;
814 }
815
816 /* optimized ipc_mqueue_copyin */
817
818 rcv_object = (ipc_object_t) reply_port;
819 io_reference(rcv_object);
820 rcv_mqueue = &reply_port->ip_messages;
821 io_unlock(rcv_object);
822 HOT(c_mmot_hot_fSR_ok++);
823 goto fast_send_receive;
824
825 abort_request_copyin:
826 is_read_unlock(space);
827 goto slow_copyin;
828
829 abort_request_send_receive:
830 ip_unlock(dest_port);
831 ip_unlock(reply_port);
832 goto slow_send;
833 }
834
835 case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
836 register ipc_entry_num_t size;
837 register ipc_entry_t table;
838
839 /* sending a reply message */
840
841 {
842 register mach_port_name_t reply_name =
843 (mach_port_name_t)hdr->msgh_local_port;
844
845 if (reply_name != MACH_PORT_NULL) {
846 HOT(c_mmot_cold_018++);
847 goto slow_copyin;
848 }
849 }
850
851 is_write_lock(space);
852 assert(space->is_active);
853
854 /* optimized ipc_entry_lookup */
855
856 size = space->is_table_size;
857 table = space->is_table;
858
859 {
860 register ipc_entry_t entry;
861 register mach_port_gen_t gen;
862 register mach_port_index_t index;
863 ipc_table_index_t *requests;
864
865 {
866 register mach_port_name_t dest_name =
867 (mach_port_name_t)hdr->msgh_remote_port;
868
869 index = MACH_PORT_INDEX(dest_name);
870 gen = MACH_PORT_GEN(dest_name);
871 }
872
873 if (index >= size) {
874 HOT(c_mmot_cold_019++);
875 goto abort_reply_dest_copyin;
876 }
877
878 entry = &table[index];
879
880 /* check generation, collision bit, and type bit */
881
882 if ((entry->ie_bits & (IE_BITS_GEN_MASK|
883 IE_BITS_COLLISION|
884 MACH_PORT_TYPE_SEND_ONCE)) !=
885 (gen | MACH_PORT_TYPE_SEND_ONCE)) {
886 HOT(c_mmot_cold_020++);
887 goto abort_reply_dest_copyin;
888 }
889
890 /* optimized ipc_right_copyin */
891
892 assert(IE_BITS_TYPE(entry->ie_bits) ==
893 MACH_PORT_TYPE_SEND_ONCE);
894 assert(IE_BITS_UREFS(entry->ie_bits) == 1);
895
896 if (entry->ie_request != 0) {
897 HOT(c_mmot_cold_021++);
898 goto abort_reply_dest_copyin;
899 }
900
901 dest_port = (ipc_port_t) entry->ie_object;
902 assert(dest_port != IP_NULL);
903
904 ip_lock(dest_port);
905 if (!ip_active(dest_port)) {
906 ip_unlock(dest_port);
907 HOT(c_mmot_cold_022++);
908 goto abort_reply_dest_copyin;
909 }
910
911 assert(dest_port->ip_sorights > 0);
912
913 /* optimized ipc_entry_dealloc */
914
915
916 entry->ie_bits = gen;
917 entry->ie_next = table->ie_next;
918 table->ie_next = index;
919 entry->ie_object = IO_NULL;
920 }
921
922 hdr->msgh_bits =
923 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
924 0);
925 hdr->msgh_remote_port = dest_port;
926
927 /* make sure we can queue to the destination */
928
929 assert(dest_port->ip_receiver != ipc_space_kernel);
930
931 /* optimized ipc_entry_lookup/ipc_mqueue_copyin */
932
933 {
934 register ipc_entry_t entry;
935 register ipc_entry_bits_t bits;
936
937 {
938 register mach_port_index_t index;
939 register mach_port_gen_t gen;
940
941 index = MACH_PORT_INDEX(rcv_name);
942 gen = MACH_PORT_GEN(rcv_name);
943
944 if (index < size) {
945 entry = &table[index];
946 bits = entry->ie_bits;
947 if (IE_BITS_GEN(bits) != gen ||
948 (bits & IE_BITS_COLLISION)) {
949 entry = IE_NULL;
950 }
951 } else {
952 entry = IE_NULL;
953 }
954 if (entry == IE_NULL) {
955 entry = ipc_entry_lookup(space, rcv_name);
956 if (entry == IE_NULL) {
957 HOT(c_mmot_cold_024++);
958 goto abort_reply_rcv_copyin;
959 }
960 bits = entry->ie_bits;
961 }
962
963 }
964
965 /* check type bits; looking for receive or set */
966 #if 0
967 /*
968 * JMM - The check below for messages in the receive
969 * mqueue is insufficient to work with port sets, since
970 * the messages stay in the port queues. For now, don't
971 * allow portsets (but receiving on portsets when sending
972 * a message to a send-once right is actually a very
973 * common case (so we should re-enable).
974 */
975 if (bits & MACH_PORT_TYPE_PORT_SET) {
976 register ipc_pset_t rcv_pset;
977
978 rcv_pset = (ipc_pset_t) entry->ie_object;
979 assert(rcv_pset != IPS_NULL);
980
981 ips_lock(rcv_pset);
982 assert(ips_active(rcv_pset));
983
984 rcv_object = (ipc_object_t) rcv_pset;
985 rcv_mqueue = &rcv_pset->ips_messages;
986 } else
987 #endif /* 0 */
988 if (bits & MACH_PORT_TYPE_RECEIVE) {
989 register ipc_port_t rcv_port;
990
991 rcv_port = (ipc_port_t) entry->ie_object;
992 assert(rcv_port != IP_NULL);
993
994 if (!ip_lock_try(rcv_port)) {
995 HOT(c_mmot_cold_025++);
996 goto abort_reply_rcv_copyin;
997 }
998 assert(ip_active(rcv_port));
999
1000 if (rcv_port->ip_pset_count != 0) {
1001 ip_unlock(rcv_port);
1002 HOT(c_mmot_cold_026++);
1003 goto abort_reply_rcv_copyin;
1004 }
1005
1006 rcv_object = (ipc_object_t) rcv_port;
1007 rcv_mqueue = &rcv_port->ip_messages;
1008 } else {
1009 HOT(c_mmot_cold_027++);
1010 goto abort_reply_rcv_copyin;
1011 }
1012 }
1013
1014 is_write_unlock(space);
1015 io_reference(rcv_object);
1016 io_unlock(rcv_object);
1017 HOT(c_mmot_hot_fSR_ok++);
1018 goto fast_send_receive;
1019
1020 abort_reply_dest_copyin:
1021 is_write_unlock(space);
1022 HOT(c_mmot_cold_029++);
1023 goto slow_copyin;
1024
1025 abort_reply_rcv_copyin:
1026 ip_unlock(dest_port);
1027 is_write_unlock(space);
1028 HOT(c_mmot_cold_030++);
1029 goto slow_send;
1030 }
1031
1032 default:
1033 HOT(c_mmot_cold_031++);
1034 goto slow_copyin;
1035 }
1036 /*NOTREACHED*/
1037
1038 fast_send_receive:
1039 /*
1040 * optimized ipc_mqueue_send/ipc_mqueue_receive
1041 *
1042 * Finished get/copyin of kmsg and copyin of rcv_name.
1043 * space is unlocked, dest_port is locked,
1044 * we can queue kmsg to dest_port,
1045 * rcv_mqueue is set, and rcv_object holds a ref
1046 * so the mqueue cannot go away.
1047 *
1048 * JMM - For now, rcv_object is just a port. Portsets
1049 * are disabled for the time being.
1050 */
1051
1052 assert(ip_active(dest_port));
1053 assert(dest_port->ip_receiver != ipc_space_kernel);
1054 assert(!imq_full(&dest_port->ip_messages) ||
1055 (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1056 MACH_MSG_TYPE_PORT_SEND_ONCE));
1057 assert((hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR) == 0);
1058
1059 {
1060 register ipc_mqueue_t dest_mqueue;
1061 wait_queue_t waitq;
1062 thread_t receiver;
1063 #if THREAD_SWAPPER
1064 thread_act_t rcv_act;
1065 #endif
1066 spl_t s;
1067
1068 s = splsched();
1069 dest_mqueue = &dest_port->ip_messages;
1070 waitq = &dest_mqueue->imq_wait_queue;
1071 imq_lock(dest_mqueue);
1072
1073 wait_queue_peek_locked(waitq, IPC_MQUEUE_RECEIVE, &receiver, &waitq);
1074 /* queue still locked, thread locked - but still on q */
1075
1076 if (receiver == THREAD_NULL) {
1077 abort_send_receive:
1078 imq_unlock(dest_mqueue);
1079 splx(s);
1080 ip_unlock(dest_port);
1081 ipc_object_release(rcv_object);
1082 HOT(c_mmot_cold_032++);
1083 goto slow_send;
1084 }
1085
1086 assert(receiver->wait_queue == waitq);
1087 assert(receiver->wait_event == IPC_MQUEUE_RECEIVE);
1088
1089 /*
1090 * See if it is still running on another processor (trying to
1091 * block itself). If so, fall off.
1092 *
1093 * JMM - We have an opportunity here. Since the thread is locked
1094 * and we find it runnable, it must still be trying to get into
1095 * thread_block on itself. We could just "hand him the message"
1096 * and let him go (thread_go_locked()) and then fall down into a
1097 * slow receive for ourselves. Only his RECEIVE_TOO_LARGE handling
1098 * runs afoul of that. Clean this up!
1099 */
1100 if ((receiver->state & TH_RUN|TH_WAIT) != TH_WAIT) {
1101 assert(NCPUS > 1);
1102 HOT(c_mmot_cold_033++);
1103 fall_off:
1104 thread_unlock(receiver);
1105 if (waitq != &dest_mqueue->imq_wait_queue)
1106 wait_queue_unlock(waitq);
1107 goto abort_send_receive;
1108 }
1109
1110 /*
1111 * Check that the receiver can stay on the hot path.
1112 */
1113 if (send_size + REQUESTED_TRAILER_SIZE(receiver->ith_option) >
1114 receiver->ith_msize) {
1115 /*
1116 * The receiver can't accept the message.
1117 */
1118 HOT(c_mmot_bad_rcvr++);
1119 goto fall_off;
1120 }
1121
1122 #if THREAD_SWAPPER
1123 /*
1124 * Receiver looks okay -- is it swapped in?
1125 */
1126 rpc_lock(receiver);
1127 rcv_act = receiver->top_act;
1128 if (rcv_act->swap_state != TH_SW_IN &&
1129 rcv_act->swap_state != TH_SW_UNSWAPPABLE) {
1130 rpc_unlock(receiver);
1131 HOT(c_mmot_rcvr_swapped++);
1132 goto fall_off;
1133 }
1134
1135 /*
1136 * Make sure receiver stays swapped in (if we can).
1137 */
1138 if (!act_lock_try(rcv_act)) { /* out of order! */
1139 rpc_unlock(receiver);
1140 HOT(c_mmot_rcvr_locked++);
1141 goto fall_off;
1142 }
1143
1144 /*
1145 * Check for task swapping in progress affecting
1146 * receiver. Since rcv_act is attached to a shuttle,
1147 * its swap_state is covered by shuttle's thread_lock()
1148 * (sigh).
1149 */
1150 if ((rcv_act->swap_state != TH_SW_IN &&
1151 rcv_act->swap_state != TH_SW_UNSWAPPABLE) ||
1152 rcv_act->ast & AST_SWAPOUT) {
1153 act_unlock(rcv_act);
1154 rpc_unlock(receiver);
1155 HOT(c_mmot_rcvr_tswapped++);
1156 goto fall_off;
1157 }
1158
1159 /*
1160 * We don't need to make receiver unswappable here -- holding
1161 * act_lock() of rcv_act is sufficient to prevent either thread
1162 * or task swapping from changing its state (see swapout_scan(),
1163 * task_swapout()). Don't release lock till receiver's state
1164 * is consistent. Its task may then be marked for swapout,
1165 * but that's life.
1166 */
1167 rpc_unlock(receiver);
1168 /*
1169 * NB: act_lock(rcv_act) still held
1170 */
1171 #endif /* THREAD_SWAPPER */
1172
1173 /*
1174 * Before committing to the handoff, make sure that we are
1175 * really going to block (i.e. there are no messages already
1176 * queued for us. This violates lock ordering, so make sure
1177 * we don't deadlock. After the trylock succeeds below, we
1178 * may have up to 3 message queues locked:
1179 * - the dest port mqueue
1180 * - a portset mqueue (where waiting receiver was found)
1181 * - finally our own rcv_mqueue
1182 *
1183 * JMM - Need to make this check appropriate for portsets as
1184 * well before re-enabling them.
1185 */
1186 if (!imq_lock_try(rcv_mqueue)) {
1187 goto fall_off;
1188 }
1189 if (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL) {
1190 imq_unlock(rcv_mqueue);
1191 HOT(c_mmot_cold_033++);
1192 goto fall_off;
1193 }
1194
1195 /* At this point we are committed to do the "handoff". */
1196 c_mach_msg_trap_switch_fast++;
1197
1198 /*
1199 * JMM - Go ahead and pull the receiver from the runq. If the
1200 * runq wasn't the one for the mqueue, unlock it.
1201 */
1202 wait_queue_pull_thread_locked(waitq,
1203 receiver,
1204 (waitq != &dest_mqueue->imq_wait_queue));
1205
1206 /*
1207 * Store the kmsg and seqno where the receiver can pick it up.
1208 */
1209 receiver->ith_state = MACH_MSG_SUCCESS;
1210 receiver->ith_kmsg = kmsg;
1211 receiver->ith_seqno = dest_mqueue->imq_seqno++;
1212
1213 /*
1214 * Inline thread_go_locked
1215 *
1216 * JMM - Including hacked in version of setrun scheduler op
1217 * that doesn't try to put thread on a runq.
1218 */
1219 {
1220 receiver->state &= ~(TH_WAIT|TH_UNINT);
1221 receiver->state |= TH_RUN;
1222 receiver->wait_result = THREAD_AWAKENED;
1223
1224 receiver->metered_computation = 0;
1225 }
1226
1227 thread_unlock(receiver);
1228 #if THREAD_SWAPPER
1229 act_unlock(rcv_act);
1230 #endif /* THREAD_SWAPPER */
1231
1232 imq_unlock(dest_mqueue);
1233 ip_unlock(dest_port);
1234 current_task()->messages_sent++;
1235
1236
1237 /*
1238 * Put self on receive port's queue.
1239 * Also save state that the sender of
1240 * our reply message needs to determine if it
1241 * can hand off directly back to us.
1242 */
1243 self->ith_msg = (rcv_msg) ? rcv_msg : msg;
1244 self->ith_object = rcv_object; /* still holds reference */
1245 self->ith_msize = rcv_size;
1246 self->ith_option = option;
1247 self->ith_scatter_list_size = scatter_list_size;
1248 self->ith_continuation = thread_syscall_return;
1249
1250 waitq = &rcv_mqueue->imq_wait_queue;
1251 (void)wait_queue_assert_wait_locked(waitq,
1252 IPC_MQUEUE_RECEIVE,
1253 THREAD_ABORTSAFE,
1254 TRUE); /* unlock? */
1255 /* rcv_mqueue is unlocked */
1256
1257 /* Inline thread_block_reason (except don't select a new
1258 * new thread (we already have one), and don't turn off ASTs
1259 * (we don't want two threads to hog all the CPU by handing
1260 * off to each other).
1261 */
1262 {
1263 if (self->funnel_state & TH_FN_OWNED) {
1264 self->funnel_state = TH_FN_REFUNNEL;
1265 KERNEL_DEBUG(0x603242c | DBG_FUNC_NONE, self->funnel_lock, 3, 0, 0, 0);
1266 funnel_unlock(self->funnel_lock);
1267
1268 }
1269
1270 machine_clock_assist();
1271
1272 thread_lock(self);
1273 if (self->state & TH_ABORT)
1274 clear_wait_internal(self, THREAD_INTERRUPTED);
1275 thread_unlock(self);
1276
1277 /*
1278 * Switch directly to receiving thread, and block
1279 * this thread as though it had called ipc_mqueue_receive.
1280 */
1281 #if defined (__i386__)
1282 thread_run(self, (void (*)(void))0, receiver);
1283 #else
1284 thread_run(self, ipc_mqueue_receive_continue, receiver);
1285 #endif
1286
1287 /* if we fell thru */
1288 if (self->funnel_state & TH_FN_REFUNNEL) {
1289 kern_return_t wait_result2;
1290
1291 wait_result2 = self->wait_result;
1292 self->funnel_state = 0;
1293 KERNEL_DEBUG(0x6032428 | DBG_FUNC_NONE, self->funnel_lock, 6, 0, 0, 0);
1294 funnel_lock(self->funnel_lock);
1295 KERNEL_DEBUG(0x6032430 | DBG_FUNC_NONE, self->funnel_lock, 6, 0, 0, 0);
1296 self->funnel_state = TH_FN_OWNED;
1297 self->wait_result = wait_result2;
1298 }
1299 splx(s);
1300 }
1301
1302 ipc_mqueue_receive_continue();
1303 /* NOTREACHED */
1304 }
1305
1306 fast_copyout:
1307 /*
1308 * Nothing locked and no references held, except
1309 * we have kmsg with msgh_seqno filled in. Must
1310 * still check against rcv_size and do
1311 * ipc_kmsg_copyout/ipc_kmsg_put.
1312 */
1313
1314 reply_size = send_size + trailer->msgh_trailer_size;
1315 if (rcv_size < reply_size) {
1316 HOT(c_mmot_g_slow_copyout6++);
1317 goto slow_copyout;
1318 }
1319
1320 /* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */
1321
1322 switch (hdr->msgh_bits) {
1323 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
1324 MACH_MSG_TYPE_PORT_SEND_ONCE): {
1325 ipc_port_t reply_port =
1326 (ipc_port_t) hdr->msgh_local_port;
1327 mach_port_name_t dest_name, reply_name;
1328
1329 /* receiving a request message */
1330
1331 if (!IP_VALID(reply_port)) {
1332 HOT(c_mmot_g_slow_copyout5++);
1333 goto slow_copyout;
1334 }
1335
1336 is_write_lock(space);
1337 assert(space->is_active);
1338
1339 /*
1340 * To do an atomic copyout, need simultaneous
1341 * locks on both ports and the space. If
1342 * dest_port == reply_port, and simple locking is
1343 * enabled, then we will abort. Otherwise it's
1344 * OK to unlock twice.
1345 */
1346
1347 ip_lock(dest_port);
1348 if (!ip_active(dest_port) ||
1349 !ip_lock_try(reply_port)) {
1350 HOT(c_mmot_cold_037++);
1351 goto abort_request_copyout;
1352 }
1353
1354 if (!ip_active(reply_port)) {
1355 ip_unlock(reply_port);
1356 HOT(c_mmot_cold_038++);
1357 goto abort_request_copyout;
1358 }
1359
1360 assert(reply_port->ip_sorights > 0);
1361 ip_unlock(reply_port);
1362
1363 {
1364 register ipc_entry_t table;
1365 register ipc_entry_t entry;
1366 register mach_port_index_t index;
1367
1368 /* optimized ipc_entry_get */
1369
1370 table = space->is_table;
1371 index = table->ie_next;
1372
1373 if (index == 0) {
1374 HOT(c_mmot_cold_039++);
1375 goto abort_request_copyout;
1376 }
1377
1378 entry = &table[index];
1379 table->ie_next = entry->ie_next;
1380 entry->ie_request = 0;
1381
1382 {
1383 register mach_port_gen_t gen;
1384
1385 assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
1386 gen = IE_BITS_NEW_GEN(entry->ie_bits);
1387
1388 reply_name = MACH_PORT_MAKE(index, gen);
1389
1390 /* optimized ipc_right_copyout */
1391
1392 entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
1393 }
1394
1395 assert(MACH_PORT_VALID(reply_name));
1396 entry->ie_object = (ipc_object_t) reply_port;
1397 is_write_unlock(space);
1398 }
1399
1400 /* optimized ipc_object_copyout_dest */
1401
1402 assert(dest_port->ip_srights > 0);
1403 ip_release(dest_port);
1404
1405 if (dest_port->ip_receiver == space)
1406 dest_name = dest_port->ip_receiver_name;
1407 else
1408 dest_name = MACH_PORT_NULL;
1409
1410 if ((--dest_port->ip_srights == 0) &&
1411 (dest_port->ip_nsrequest != IP_NULL)) {
1412 ipc_port_t nsrequest;
1413 mach_port_mscount_t mscount;
1414
1415 /* a rather rare case */
1416
1417 nsrequest = dest_port->ip_nsrequest;
1418 mscount = dest_port->ip_mscount;
1419 dest_port->ip_nsrequest = IP_NULL;
1420 ip_unlock(dest_port);
1421 ipc_notify_no_senders(nsrequest, mscount);
1422 } else
1423 ip_unlock(dest_port);
1424
1425 hdr->msgh_bits =
1426 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
1427 MACH_MSG_TYPE_PORT_SEND);
1428 hdr->msgh_remote_port = (mach_port_t)reply_name;
1429 hdr->msgh_local_port = (mach_port_t)dest_name;
1430 HOT(c_mmot_hot_ok1++);
1431 goto fast_put;
1432
1433 abort_request_copyout:
1434 ip_unlock(dest_port);
1435 is_write_unlock(space);
1436 HOT(c_mmot_g_slow_copyout4++);
1437 goto slow_copyout;
1438 }
1439
1440 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1441 register mach_port_name_t dest_name;
1442
1443 /* receiving a reply message */
1444
1445 ip_lock(dest_port);
1446 if (!ip_active(dest_port)) {
1447 ip_unlock(dest_port);
1448 HOT(c_mmot_g_slow_copyout3++);
1449 goto slow_copyout;
1450 }
1451
1452 /* optimized ipc_object_copyout_dest */
1453
1454 assert(dest_port->ip_sorights > 0);
1455
1456 if (dest_port->ip_receiver == space) {
1457 ip_release(dest_port);
1458 dest_port->ip_sorights--;
1459 dest_name = dest_port->ip_receiver_name;
1460 ip_unlock(dest_port);
1461 } else {
1462 ip_unlock(dest_port);
1463
1464 ipc_notify_send_once(dest_port);
1465 dest_name = MACH_PORT_NULL;
1466 }
1467
1468 hdr->msgh_bits = MACH_MSGH_BITS(0,
1469 MACH_MSG_TYPE_PORT_SEND_ONCE);
1470 hdr->msgh_remote_port = MACH_PORT_NULL;
1471 hdr->msgh_local_port = (ipc_port_t)dest_name;
1472 HOT(c_mmot_hot_ok2++);
1473 goto fast_put;
1474 }
1475
1476 case MACH_MSGH_BITS_COMPLEX|
1477 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1478 register mach_port_name_t dest_name;
1479
1480 /* receiving a complex reply message */
1481
1482 ip_lock(dest_port);
1483 if (!ip_active(dest_port)) {
1484 ip_unlock(dest_port);
1485 HOT(c_mmot_g_slow_copyout1++);
1486 goto slow_copyout;
1487 }
1488
1489 /* optimized ipc_object_copyout_dest */
1490
1491 assert(dest_port->ip_sorights > 0);
1492
1493 if (dest_port->ip_receiver == space) {
1494 ip_release(dest_port);
1495 dest_port->ip_sorights--;
1496 dest_name = dest_port->ip_receiver_name;
1497 ip_unlock(dest_port);
1498 } else {
1499 ip_unlock(dest_port);
1500
1501 ipc_notify_send_once(dest_port);
1502 dest_name = MACH_PORT_NULL;
1503 }
1504
1505 hdr->msgh_bits =
1506 MACH_MSGH_BITS_COMPLEX |
1507 MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND_ONCE);
1508 hdr->msgh_remote_port = MACH_PORT_NULL;
1509 hdr->msgh_local_port = (mach_port_t)dest_name;
1510
1511 mr = ipc_kmsg_copyout_body(kmsg, space,
1512 current_map(),
1513 MACH_MSG_BODY_NULL);
1514 if (mr != MACH_MSG_SUCCESS) {
1515 if (ipc_kmsg_put(msg, kmsg, hdr->msgh_size +
1516 trailer->msgh_trailer_size) ==
1517 MACH_RCV_INVALID_DATA)
1518 return MACH_RCV_INVALID_DATA;
1519 else
1520 return mr | MACH_RCV_BODY_ERROR;
1521 }
1522 HOT(c_mmot_hot_ok3++);
1523 goto fast_put;
1524 }
1525
1526 default:
1527 HOT(c_mmot_g_slow_copyout2++);
1528 goto slow_copyout;
1529 }
1530 /*NOTREACHED*/
1531
1532 fast_put:
1533 mr = ipc_kmsg_put(rcv_msg ? rcv_msg : msg,
1534 kmsg,
1535 hdr->msgh_size + trailer->msgh_trailer_size);
1536 if (mr != MACH_MSG_SUCCESS) {
1537 return MACH_RCV_INVALID_DATA;
1538 }
1539 current_task()->messages_received++;
1540 return mr;
1541
1542
1543 /* BEGINNING OF WARM PATH */
1544
1545 /*
1546 * The slow path has a few non-register temporary
1547 * variables used only for call-by-reference.
1548 */
1549
1550 slow_copyin:
1551 {
1552 ipc_kmsg_t temp_kmsg;
1553 mach_port_seqno_t temp_seqno;
1554 ipc_object_t temp_rcv_object;
1555 ipc_mqueue_t temp_rcv_mqueue;
1556 register mach_port_name_t reply_name =
1557 (mach_port_name_t)hdr->msgh_local_port;
1558
1559
1560 /*
1561 * We have the message data in kmsg, but
1562 * we still need to copyin, send it,
1563 * receive a reply, and do copyout.
1564 */
1565
1566 mr = ipc_kmsg_copyin(kmsg, space, current_map(),
1567 MACH_PORT_NULL);
1568 if (mr != MACH_MSG_SUCCESS) {
1569 ipc_kmsg_free(kmsg);
1570 return(mr);
1571 }
1572
1573 /* try to get back on optimized path */
1574
1575 if ((reply_name != rcv_name) ||
1576 (hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR)) {
1577 HOT(c_mmot_cold_048++);
1578 goto slow_send;
1579 }
1580
1581 dest_port = (ipc_port_t) hdr->msgh_remote_port;
1582 assert(IP_VALID(dest_port));
1583
1584 ip_lock(dest_port);
1585 if (!ip_active(dest_port)) {
1586 ip_unlock(dest_port);
1587 goto slow_send;
1588 }
1589
1590 if (dest_port->ip_receiver == ipc_space_kernel) {
1591 dest_port->ip_messages.imq_seqno++;
1592 ip_unlock(dest_port);
1593 goto kernel_send;
1594 }
1595
1596 if (!imq_full(&dest_port->ip_messages) ||
1597 (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1598 MACH_MSG_TYPE_PORT_SEND_ONCE))
1599 {
1600 /*
1601 * Try an optimized ipc_mqueue_copyin.
1602 * It will work if this is a request message.
1603 */
1604
1605 register ipc_port_t reply_port;
1606
1607 reply_port = (ipc_port_t) hdr->msgh_local_port;
1608 if (IP_VALID(reply_port)) {
1609 if (ip_lock_try(reply_port)) {
1610 if (ip_active(reply_port) &&
1611 reply_port->ip_receiver == space &&
1612 reply_port->ip_receiver_name == rcv_name &&
1613 reply_port->ip_pset_count == 0)
1614 {
1615 /* Grab a reference to the reply port. */
1616 rcv_object = (ipc_object_t) reply_port;
1617 io_reference(rcv_object);
1618 rcv_mqueue = &reply_port->ip_messages;
1619 io_unlock(rcv_object);
1620 HOT(c_mmot_getback_FastSR++);
1621 goto fast_send_receive;
1622 }
1623 ip_unlock(reply_port);
1624 }
1625 }
1626 }
1627
1628 ip_unlock(dest_port);
1629 HOT(c_mmot_cold_050++);
1630 goto slow_send;
1631
1632 kernel_send:
1633 /*
1634 * Special case: send message to kernel services.
1635 * The request message has been copied into the
1636 * kmsg. Nothing is locked.
1637 */
1638
1639 {
1640 register ipc_port_t reply_port;
1641 mach_port_seqno_t local_seqno;
1642 spl_t s;
1643
1644 /*
1645 * Perform the kernel function.
1646 */
1647 c_mmot_kernel_send++;
1648
1649 current_task()->messages_sent++;
1650
1651 kmsg = ipc_kobject_server(kmsg);
1652 if (kmsg == IKM_NULL) {
1653 /*
1654 * No reply. Take the
1655 * slow receive path.
1656 */
1657 HOT(c_mmot_cold_051++);
1658 goto slow_get_rcv_port;
1659 }
1660
1661 /*
1662 * Check that:
1663 * the reply port is alive
1664 * we hold the receive right
1665 * the name has not changed.
1666 * the port is not in a set
1667 * If any of these are not true,
1668 * we cannot directly receive the reply
1669 * message.
1670 */
1671 hdr = &kmsg->ikm_header;
1672 send_size = hdr->msgh_size;
1673 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1674 round_msg(send_size));
1675 reply_port = (ipc_port_t) hdr->msgh_remote_port;
1676 ip_lock(reply_port);
1677
1678 if ((!ip_active(reply_port)) ||
1679 (reply_port->ip_receiver != space) ||
1680 (reply_port->ip_receiver_name != rcv_name) ||
1681 (reply_port->ip_pset_count != 0))
1682 {
1683 ip_unlock(reply_port);
1684 ipc_kmsg_send_always(kmsg);
1685 HOT(c_mmot_cold_052++);
1686 goto slow_get_rcv_port;
1687 }
1688
1689 s = splsched();
1690 rcv_mqueue = &reply_port->ip_messages;
1691 imq_lock(rcv_mqueue);
1692
1693 /* keep port locked, and don`t change ref count yet */
1694
1695 /*
1696 * If there are messages on the port
1697 * or other threads waiting for a message,
1698 * we cannot directly receive the reply.
1699 */
1700 if (!wait_queue_empty(&rcv_mqueue->imq_wait_queue) ||
1701 (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL))
1702 {
1703 imq_unlock(rcv_mqueue);
1704 splx(s);
1705 ip_unlock(reply_port);
1706 ipc_kmsg_send_always(kmsg);
1707 HOT(c_mmot_cold_053++);
1708 goto slow_get_rcv_port;
1709 }
1710
1711 /*
1712 * We can directly receive this reply.
1713 * Since there were no messages queued
1714 * on the reply port, there should be
1715 * no threads blocked waiting to send.
1716 */
1717 dest_port = reply_port;
1718 local_seqno = rcv_mqueue->imq_seqno++;
1719 imq_unlock(rcv_mqueue);
1720 splx(s);
1721
1722 /*
1723 * inline ipc_object_release.
1724 * Port is still locked.
1725 * Reference count was not incremented.
1726 */
1727 ip_check_unlock(reply_port);
1728
1729 if (option & MACH_RCV_TRAILER_MASK) {
1730 trailer->msgh_seqno = local_seqno;
1731 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1732 }
1733 /* copy out the kernel reply */
1734 HOT(c_mmot_fastkernelreply++);
1735 goto fast_copyout;
1736 }
1737
1738 slow_send:
1739 /*
1740 * Nothing is locked. We have acquired kmsg, but
1741 * we still need to send it and receive a reply.
1742 */
1743
1744 mr = ipc_kmsg_send(kmsg, MACH_MSG_OPTION_NONE,
1745 MACH_MSG_TIMEOUT_NONE);
1746 if (mr != MACH_MSG_SUCCESS) {
1747 mr |= ipc_kmsg_copyout_pseudo(kmsg, space,
1748 current_map(),
1749 MACH_MSG_BODY_NULL);
1750
1751 (void) ipc_kmsg_put(msg, kmsg, hdr->msgh_size);
1752 return(mr);
1753 }
1754
1755 slow_get_rcv_port:
1756 /*
1757 * We have sent the message. Copy in the receive port.
1758 */
1759 mr = ipc_mqueue_copyin(space, rcv_name,
1760 &temp_rcv_mqueue, &temp_rcv_object);
1761 if (mr != MACH_MSG_SUCCESS) {
1762 return(mr);
1763 }
1764 rcv_mqueue = temp_rcv_mqueue;
1765 rcv_object = temp_rcv_object;
1766 /* hold ref for rcv_object */
1767
1768 slow_receive:
1769 /*
1770 * Now we have sent the request and copied in rcv_name,
1771 * and hold ref for rcv_object (to keep mqueue alive).
1772 * Just receive a reply and try to get back to fast path.
1773 */
1774
1775 self->ith_continuation = (void (*)(mach_msg_return_t))0;
1776 ipc_mqueue_receive(rcv_mqueue,
1777 MACH_MSG_OPTION_NONE,
1778 MACH_MSG_SIZE_MAX,
1779 MACH_MSG_TIMEOUT_NONE,
1780 THREAD_ABORTSAFE);
1781
1782 mr = self->ith_state;
1783 temp_kmsg = self->ith_kmsg;
1784 temp_seqno = self->ith_seqno;
1785
1786 ipc_object_release(rcv_object);
1787
1788 if (mr != MACH_MSG_SUCCESS) {
1789 return(mr);
1790 }
1791
1792 kmsg = temp_kmsg;
1793 hdr = &kmsg->ikm_header;
1794 send_size = hdr->msgh_size;
1795 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1796 round_msg(send_size));
1797 if (option & MACH_RCV_TRAILER_MASK) {
1798 trailer->msgh_seqno = temp_seqno;
1799 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1800 }
1801 dest_port = (ipc_port_t) hdr->msgh_remote_port;
1802 HOT(c_mmot_cold_055++);
1803 goto fast_copyout;
1804
1805 slow_copyout:
1806 /*
1807 * Nothing locked and no references held, except
1808 * we have kmsg with msgh_seqno filled in. Must
1809 * still check against rcv_size and do
1810 * ipc_kmsg_copyout/ipc_kmsg_put.
1811 */
1812
1813 reply_size = send_size + trailer->msgh_trailer_size;
1814 if (rcv_size < reply_size) {
1815 if (msg_receive_error(kmsg, msg, option, temp_seqno,
1816 space) == MACH_RCV_INVALID_DATA) {
1817 mr = MACH_RCV_INVALID_DATA;
1818 return(mr);
1819 }
1820 else {
1821 mr = MACH_RCV_TOO_LARGE;
1822 return(mr);
1823 }
1824 }
1825
1826 mr = ipc_kmsg_copyout(kmsg, space, current_map(),
1827 MACH_PORT_NULL, MACH_MSG_BODY_NULL);
1828 if (mr != MACH_MSG_SUCCESS) {
1829 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1830 if (ipc_kmsg_put(msg, kmsg, reply_size) ==
1831 MACH_RCV_INVALID_DATA)
1832 mr = MACH_RCV_INVALID_DATA;
1833 }
1834 else {
1835 if (msg_receive_error(kmsg, msg, option,
1836 temp_seqno, space) == MACH_RCV_INVALID_DATA)
1837 mr = MACH_RCV_INVALID_DATA;
1838 }
1839
1840 return(mr);
1841 }
1842
1843 /* try to get back on optimized path */
1844 HOT(c_mmot_getback_fast_put++);
1845 goto fast_put;
1846
1847 /*NOTREACHED*/
1848 }
1849 } /* END OF HOT PATH */
1850 #endif /* ENABLE_HOTPATH */
1851
1852 if (option & MACH_SEND_MSG) {
1853 mr = mach_msg_send(msg, option, send_size,
1854 timeout, notify);
1855 if (mr != MACH_MSG_SUCCESS) {
1856 return mr;
1857 }
1858 }
1859
1860 if (option & MACH_RCV_MSG) {
1861 mach_msg_header_t *rcv;
1862
1863 /*
1864 * 1. MACH_RCV_OVERWRITE is on, and rcv_msg is our scatter list
1865 * and receive buffer
1866 * 2. MACH_RCV_OVERWRITE is off, and rcv_msg might be the
1867 * alternate receive buffer (separate send and receive buffers).
1868 */
1869 if (option & MACH_RCV_OVERWRITE)
1870 rcv = rcv_msg;
1871 else if (rcv_msg != MACH_MSG_NULL)
1872 rcv = rcv_msg;
1873 else
1874 rcv = msg;
1875 mr = mach_msg_receive(rcv, option, rcv_size, rcv_name,
1876 timeout, thread_syscall_return, scatter_list_size);
1877 thread_syscall_return(mr);
1878 }
1879
1880 return MACH_MSG_SUCCESS;
1881 }
1882
1883 /*
1884 * Routine: msg_receive_error [internal]
1885 * Purpose:
1886 * Builds a minimal header/trailer and copies it to
1887 * the user message buffer. Invoked when in the case of a
1888 * MACH_RCV_TOO_LARGE or MACH_RCV_BODY_ERROR error.
1889 * Conditions:
1890 * Nothing locked.
1891 * Returns:
1892 * MACH_MSG_SUCCESS minimal header/trailer copied
1893 * MACH_RCV_INVALID_DATA copyout to user buffer failed
1894 */
1895
1896 mach_msg_return_t
1897 msg_receive_error(
1898 ipc_kmsg_t kmsg,
1899 mach_msg_header_t *msg,
1900 mach_msg_option_t option,
1901 mach_port_seqno_t seqno,
1902 ipc_space_t space)
1903 {
1904 mach_msg_format_0_trailer_t *trailer;
1905
1906 /*
1907 * Copy out the destination port in the message.
1908 * Destroy all other rights and memory in the message.
1909 */
1910 ipc_kmsg_copyout_dest(kmsg, space);
1911
1912 /*
1913 * Build a minimal message with the requested trailer.
1914 */
1915 trailer = (mach_msg_format_0_trailer_t *)
1916 ((vm_offset_t)&kmsg->ikm_header +
1917 round_msg(sizeof(mach_msg_header_t)));
1918 kmsg->ikm_header.msgh_size = sizeof(mach_msg_header_t);
1919 bcopy( (char *)&trailer_template,
1920 (char *)trailer,
1921 sizeof(trailer_template));
1922 if (option & MACH_RCV_TRAILER_MASK) {
1923 trailer->msgh_seqno = seqno;
1924 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1925 }
1926
1927 /*
1928 * Copy the message to user space
1929 */
1930 if (ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size +
1931 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
1932 return(MACH_RCV_INVALID_DATA);
1933 else
1934 return(MACH_MSG_SUCCESS);
1935 }
mirror of XNU