]> git.saurik.com Git - apple/xnu.git/blob - osfmk/ipc/mach_msg.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-124.13.tar.gz
[apple/xnu.git] / osfmk / ipc / mach_msg.c
1 /*
2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22 /*
23 * @OSF_COPYRIGHT@
24 */
25 /*
26 * Mach Operating System
27 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
28 * All Rights Reserved.
29 *
30 * Permission to use, copy, modify and distribute this software and its
31 * documentation is hereby granted, provided that both the copyright
32 * notice and this permission notice appear in all copies of the
33 * software, derivative works or modified versions, and any portions
34 * thereof, and that both notices appear in supporting documentation.
35 *
36 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
39 *
40 * Carnegie Mellon requests users of this software to return to
41 *
42 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
43 * School of Computer Science
44 * Carnegie Mellon University
45 * Pittsburgh PA 15213-3890
46 *
47 * any improvements or extensions that they make and grant Carnegie Mellon
48 * the rights to redistribute these changes.
49 */
50 /*
51 */
52 /*
53 * File: ipc/mach_msg.c
54 * Author: Rich Draves
55 * Date: 1989
56 *
57 * Exported message traps. See mach/message.h.
58 */
59
60 #include <cpus.h>
61
62 #include <mach/kern_return.h>
63 #include <mach/port.h>
64 #include <mach/message.h>
65 #include <mach/mig_errors.h>
66 #include <kern/assert.h>
67 #include <kern/counters.h>
68 #include <kern/cpu_number.h>
69 #include <kern/task.h>
70 #include <kern/thread.h>
71 #include <kern/lock.h>
72 #include <kern/sched_prim.h>
73 #include <kern/exception.h>
74 #include <kern/misc_protos.h>
75 #include <vm/vm_map.h>
76 #include <ipc/ipc_kmsg.h>
77 #include <ipc/ipc_mqueue.h>
78 #include <ipc/ipc_object.h>
79 #include <ipc/ipc_notify.h>
80 #include <ipc/ipc_port.h>
81 #include <ipc/ipc_pset.h>
82 #include <ipc/ipc_space.h>
83 #include <ipc/ipc_entry.h>
84 #include <kern/kalloc.h>
85 #include <kern/thread_swap.h>
86 #include <kern/processor.h>
87
88 #include <kern/sf.h>
89 #include <kern/mk_sp.h> /* JMM - to support handoff hack */
90 #include <machine/machine_routines.h>
91 #include <sys/kdebug.h>
92
93 /*
94 * Forward declarations
95 */
96
97 mach_msg_return_t mach_msg_send(
98 mach_msg_header_t *msg,
99 mach_msg_option_t option,
100 mach_msg_size_t send_size,
101 mach_msg_timeout_t timeout,
102 mach_port_name_t notify);
103
104 mach_msg_return_t mach_msg_receive(
105 mach_msg_header_t *msg,
106 mach_msg_option_t option,
107 mach_msg_size_t rcv_size,
108 mach_port_name_t rcv_name,
109 mach_msg_timeout_t timeout,
110 void (*continuation)(mach_msg_return_t),
111 mach_msg_size_t slist_size);
112
113
114 mach_msg_return_t msg_receive_error(
115 ipc_kmsg_t kmsg,
116 mach_msg_header_t *msg,
117 mach_msg_option_t option,
118 mach_port_seqno_t seqno,
119 ipc_space_t space);
120
121 /* the size of each trailer has to be listed here for copyout purposes */
122 mach_msg_trailer_size_t trailer_size[] = {
123 sizeof(mach_msg_trailer_t),
124 sizeof(mach_msg_seqno_trailer_t),
125 sizeof(mach_msg_security_trailer_t) };
126
127 security_token_t KERNEL_SECURITY_TOKEN = KERNEL_SECURITY_TOKEN_VALUE;
128
129 mach_msg_format_0_trailer_t trailer_template = {
130 /* mach_msg_trailer_type_t */ MACH_MSG_TRAILER_FORMAT_0,
131 /* mach_msg_trailer_size_t */ MACH_MSG_TRAILER_MINIMUM_SIZE,
132 /* mach_port_seqno_t */ 0,
133 /* security_token_t */ KERNEL_SECURITY_TOKEN_VALUE
134 };
135
136 /*
137 * Routine: mach_msg_send
138 * Purpose:
139 * Send a message.
140 * Conditions:
141 * Nothing locked.
142 * Returns:
143 * MACH_MSG_SUCCESS Sent the message.
144 * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
145 * MACH_SEND_NO_BUFFER Couldn't allocate buffer.
146 * MACH_SEND_INVALID_DATA Couldn't copy message data.
147 * MACH_SEND_INVALID_HEADER
148 * Illegal value in the message header bits.
149 * MACH_SEND_INVALID_DEST The space is dead.
150 * MACH_SEND_INVALID_NOTIFY Bad notify port.
151 * MACH_SEND_INVALID_DEST Can't copyin destination port.
152 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
153 * MACH_SEND_TIMED_OUT Timeout expired without delivery.
154 * MACH_SEND_INTERRUPTED Delivery interrupted.
155 * MACH_SEND_NO_NOTIFY Can't allocate a msg-accepted request.
156 * MACH_SEND_WILL_NOTIFY Msg-accepted notif. requested.
157 * MACH_SEND_NOTIFY_IN_PROGRESS
158 * This space has already forced a message to this port.
159 */
160
161 mach_msg_return_t
162 mach_msg_send(
163 mach_msg_header_t *msg,
164 mach_msg_option_t option,
165 mach_msg_size_t send_size,
166 mach_msg_timeout_t timeout,
167 mach_port_name_t notify)
168 {
169 ipc_space_t space = current_space();
170 vm_map_t map = current_map();
171 ipc_kmsg_t kmsg;
172 mach_msg_return_t mr;
173
174 mr = ipc_kmsg_get(msg, send_size, &kmsg);
175
176 if (mr != MACH_MSG_SUCCESS)
177 return mr;
178
179 if (option & MACH_SEND_CANCEL) {
180 if (notify == MACH_PORT_NULL)
181 mr = MACH_SEND_INVALID_NOTIFY;
182 else
183 mr = ipc_kmsg_copyin(kmsg, space, map, notify);
184 } else
185 mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
186 if (mr != MACH_MSG_SUCCESS) {
187 ipc_kmsg_free(kmsg);
188 return mr;
189 }
190
191 mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, timeout);
192
193 if (mr != MACH_MSG_SUCCESS) {
194 mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
195 (void) ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
196 }
197
198 return mr;
199 }
200
201 /*
202 * Routine: mach_msg_receive
203 * Purpose:
204 * Receive a message.
205 * Conditions:
206 * Nothing locked.
207 * Returns:
208 * MACH_MSG_SUCCESS Received a message.
209 * MACH_RCV_INVALID_NAME The name doesn't denote a right,
210 * or the denoted right is not receive or port set.
211 * MACH_RCV_IN_SET Receive right is a member of a set.
212 * MACH_RCV_TOO_LARGE Message wouldn't fit into buffer.
213 * MACH_RCV_TIMED_OUT Timeout expired without a message.
214 * MACH_RCV_INTERRUPTED Reception interrupted.
215 * MACH_RCV_PORT_DIED Port/set died while receiving.
216 * MACH_RCV_PORT_CHANGED Port moved into set while receiving.
217 * MACH_RCV_INVALID_DATA Couldn't copy to user buffer.
218 * MACH_RCV_INVALID_NOTIFY Bad notify port.
219 * MACH_RCV_HEADER_ERROR
220 */
221
222 mach_msg_return_t
223 mach_msg_receive_results(void)
224 {
225 thread_t self = current_thread();
226 ipc_space_t space = current_space();
227 vm_map_t map = current_map();
228
229 ipc_object_t object = self->ith_object;
230 mach_msg_return_t mr = self->ith_state;
231 mach_msg_header_t *msg = self->ith_msg;
232 mach_msg_option_t option = self->ith_option;
233 ipc_kmsg_t kmsg = self->ith_kmsg;
234 mach_port_seqno_t seqno = self->ith_seqno;
235 mach_msg_size_t slist_size = self->ith_scatter_list_size;
236
237 mach_msg_format_0_trailer_t *trailer;
238
239 ipc_object_release(object);
240
241 if (mr != MACH_MSG_SUCCESS) {
242
243 if (mr == MACH_RCV_TOO_LARGE ) {
244 if (option & MACH_RCV_LARGE) {
245 /*
246 * We need to inform the user-level code that it needs more
247 * space. The value for how much space was returned in the
248 * msize save area instead of the message (which was left on
249 * the queue).
250 */
251 if (copyout((char *) &self->ith_msize,
252 (char *) &msg->msgh_size,
253 sizeof(mach_msg_size_t)))
254 mr = MACH_RCV_INVALID_DATA;
255 goto out;
256 }
257
258 if (msg_receive_error(kmsg, msg, option, seqno, space)
259 == MACH_RCV_INVALID_DATA)
260 mr = MACH_RCV_INVALID_DATA;
261 }
262 goto out;
263 }
264
265 trailer = (mach_msg_format_0_trailer_t *)
266 ((vm_offset_t)&kmsg->ikm_header +
267 round_msg(kmsg->ikm_header.msgh_size));
268 if (option & MACH_RCV_TRAILER_MASK) {
269 trailer->msgh_seqno = seqno;
270 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
271 }
272
273 /*
274 * If MACH_RCV_OVERWRITE was specified, try to get the scatter
275 * list and verify it against the contents of the message. If
276 * there is any problem with it, we will continue without it as
277 * normal.
278 */
279 if (option & MACH_RCV_OVERWRITE) {
280 mach_msg_size_t slist_size = self->ith_scatter_list_size;
281 mach_msg_body_t *slist;
282
283 slist = ipc_kmsg_copyin_scatter(msg, slist_size, kmsg);
284 mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL, slist);
285 ipc_kmsg_free_scatter(slist, slist_size);
286 } else {
287 mr = ipc_kmsg_copyout(kmsg, space, map,
288 MACH_PORT_NULL, MACH_MSG_BODY_NULL);
289 }
290
291 if (mr != MACH_MSG_SUCCESS) {
292 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
293 if (ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size +
294 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
295 mr = MACH_RCV_INVALID_DATA;
296 }
297 else {
298 if (msg_receive_error(kmsg, msg, option, seqno, space)
299 == MACH_RCV_INVALID_DATA)
300 mr = MACH_RCV_INVALID_DATA;
301 }
302 goto out;
303 }
304 mr = ipc_kmsg_put(msg,
305 kmsg,
306 kmsg->ikm_header.msgh_size +
307 trailer->msgh_trailer_size);
308 out:
309 return mr;
310 }
311
312 mach_msg_return_t
313 mach_msg_receive(
314 mach_msg_header_t *msg,
315 mach_msg_option_t option,
316 mach_msg_size_t rcv_size,
317 mach_port_name_t rcv_name,
318 mach_msg_timeout_t timeout,
319 void (*continuation)(mach_msg_return_t),
320 mach_msg_size_t slist_size)
321 {
322 thread_t self = current_thread();
323 ipc_space_t space = current_space();
324 vm_map_t map = current_map();
325 ipc_object_t object;
326 ipc_mqueue_t mqueue;
327 ipc_kmsg_t kmsg;
328 mach_port_seqno_t seqno;
329 mach_msg_return_t mr;
330 mach_msg_body_t *slist;
331 mach_msg_format_0_trailer_t *trailer;
332
333 mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
334 if (mr != MACH_MSG_SUCCESS) {
335 return mr;
336 }
337 /* hold ref for object */
338
339 self->ith_msg = msg;
340 self->ith_object = object;
341 self->ith_msize = rcv_size;
342 self->ith_option = option;
343 self->ith_scatter_list_size = slist_size;
344 self->ith_continuation = continuation;
345
346 ipc_mqueue_receive(mqueue, option, rcv_size, timeout, THREAD_ABORTSAFE);
347 return mach_msg_receive_results();
348 }
349
350 void
351 mach_msg_receive_continue(void)
352 {
353 thread_t self = current_thread();
354
355 (*self->ith_continuation)(mach_msg_receive_results());
356 }
357
358 /*
359 * Toggle this to compile the hotpath in/out
360 * If compiled in, the run-time toggle "enable_hotpath" below
361 * eases testing & debugging
362 */
363 #define ENABLE_HOTPATH 1 /* Hacked on for now */
364
365 #if ENABLE_HOTPATH
366 /*
367 * These counters allow tracing of hotpath behavior under test loads.
368 * A couple key counters are unconditional (see below).
369 */
370 #define HOTPATH_DEBUG 0 /* Toggle to include lots of counters */
371 #if HOTPATH_DEBUG
372 #define HOT(expr) expr
373
374 unsigned int c_mmot_FIRST = 0; /* Unused First Counter */
375 unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
376 unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
377 unsigned int c_mmot_kernel_send = 0; /* kernel server */
378 unsigned int c_mmot_cold_000 = 0; /* see below ... */
379 unsigned int c_mmot_smallsendsize = 0;
380 unsigned int c_mmot_oddsendsize = 0;
381 unsigned int c_mmot_bigsendsize = 0;
382 unsigned int c_mmot_copyinmsg_fail = 0;
383 unsigned int c_mmot_g_slow_copyin3 = 0;
384 unsigned int c_mmot_cold_006 = 0;
385 unsigned int c_mmot_cold_007 = 0;
386 unsigned int c_mmot_cold_008 = 0;
387 unsigned int c_mmot_cold_009 = 0;
388 unsigned int c_mmot_cold_010 = 0;
389 unsigned int c_mmot_cold_012 = 0;
390 unsigned int c_mmot_cold_013 = 0;
391 unsigned int c_mmot_cold_014 = 0;
392 unsigned int c_mmot_cold_016 = 0;
393 unsigned int c_mmot_cold_018 = 0;
394 unsigned int c_mmot_cold_019 = 0;
395 unsigned int c_mmot_cold_020 = 0;
396 unsigned int c_mmot_cold_021 = 0;
397 unsigned int c_mmot_cold_022 = 0;
398 unsigned int c_mmot_cold_023 = 0;
399 unsigned int c_mmot_cold_024 = 0;
400 unsigned int c_mmot_cold_025 = 0;
401 unsigned int c_mmot_cold_026 = 0;
402 unsigned int c_mmot_cold_027 = 0;
403 unsigned int c_mmot_hot_fSR_ok = 0;
404 unsigned int c_mmot_cold_029 = 0;
405 unsigned int c_mmot_cold_030 = 0;
406 unsigned int c_mmot_cold_031 = 0;
407 unsigned int c_mmot_cold_032 = 0;
408 unsigned int c_mmot_cold_033 = 0;
409 unsigned int c_mmot_bad_rcvr = 0;
410 unsigned int c_mmot_rcvr_swapped = 0;
411 unsigned int c_mmot_rcvr_locked = 0;
412 unsigned int c_mmot_rcvr_tswapped = 0;
413 unsigned int c_mmot_rcvr_freed = 0;
414 unsigned int c_mmot_g_slow_copyout6 = 0;
415 unsigned int c_mmot_g_slow_copyout5 = 0;
416 unsigned int c_mmot_cold_037 = 0;
417 unsigned int c_mmot_cold_038 = 0;
418 unsigned int c_mmot_cold_039 = 0;
419 unsigned int c_mmot_g_slow_copyout4 = 0;
420 unsigned int c_mmot_g_slow_copyout3 = 0;
421 unsigned int c_mmot_hot_ok1 = 0;
422 unsigned int c_mmot_hot_ok2 = 0;
423 unsigned int c_mmot_hot_ok3 = 0;
424 unsigned int c_mmot_g_slow_copyout1 = 0;
425 unsigned int c_mmot_g_slow_copyout2 = 0;
426 unsigned int c_mmot_getback_fast_copyin = 0;
427 unsigned int c_mmot_cold_048 = 0;
428 unsigned int c_mmot_getback_FastSR = 0;
429 unsigned int c_mmot_cold_050 = 0;
430 unsigned int c_mmot_cold_051 = 0;
431 unsigned int c_mmot_cold_052 = 0;
432 unsigned int c_mmot_cold_053 = 0;
433 unsigned int c_mmot_fastkernelreply = 0;
434 unsigned int c_mmot_cold_055 = 0;
435 unsigned int c_mmot_getback_fast_put = 0;
436 unsigned int c_mmot_LAST = 0; /* End Marker - Unused */
437
438 void db_mmot_zero_counters(void); /* forward; */
439 void db_mmot_show_counters(void); /* forward; */
440
441 void /* Call from the debugger to clear all counters */
442 db_mmot_zero_counters(void)
443 {
444 register unsigned int *ip = &c_mmot_FIRST;
445 while (ip <= &c_mmot_LAST)
446 *ip++ = 0;
447 }
448
449 void /* Call from the debugger to show all counters */
450 db_mmot_show_counters(void)
451 {
452 #define xx(str) printf("%s: %d\n", # str, str);
453
454 xx(c_mmot_combined_S_R);
455 xx(c_mach_msg_trap_switch_fast);
456 xx(c_mmot_kernel_send);
457 xx(c_mmot_cold_000);
458 xx(c_mmot_smallsendsize);
459 xx(c_mmot_oddsendsize);
460 xx(c_mmot_bigsendsize);
461 xx(c_mmot_copyinmsg_fail);
462 xx(c_mmot_g_slow_copyin3);
463 xx(c_mmot_cold_006);
464 xx(c_mmot_cold_007);
465 xx(c_mmot_cold_008);
466 xx(c_mmot_cold_009);
467 xx(c_mmot_cold_010);
468 xx(c_mmot_cold_012);
469 xx(c_mmot_cold_013);
470 xx(c_mmot_cold_014);
471 xx(c_mmot_cold_016);
472 xx(c_mmot_cold_018);
473 xx(c_mmot_cold_019);
474 xx(c_mmot_cold_020);
475 xx(c_mmot_cold_021);
476 xx(c_mmot_cold_022);
477 xx(c_mmot_cold_023);
478 xx(c_mmot_cold_024);
479 xx(c_mmot_cold_025);
480 xx(c_mmot_cold_026);
481 xx(c_mmot_cold_027);
482 xx(c_mmot_hot_fSR_ok);
483 xx(c_mmot_cold_029);
484 xx(c_mmot_cold_030);
485 xx(c_mmot_cold_031);
486 xx(c_mmot_cold_032);
487 xx(c_mmot_cold_033);
488 xx(c_mmot_bad_rcvr);
489 xx(c_mmot_rcvr_swapped);
490 xx(c_mmot_rcvr_locked);
491 xx(c_mmot_rcvr_tswapped);
492 xx(c_mmot_rcvr_freed);
493 xx(c_mmot_g_slow_copyout6);
494 xx(c_mmot_g_slow_copyout5);
495 xx(c_mmot_cold_037);
496 xx(c_mmot_cold_038);
497 xx(c_mmot_cold_039);
498 xx(c_mmot_g_slow_copyout4);
499 xx(c_mmot_g_slow_copyout3);
500 xx(c_mmot_g_slow_copyout1);
501 xx(c_mmot_hot_ok3);
502 xx(c_mmot_hot_ok2);
503 xx(c_mmot_hot_ok1);
504 xx(c_mmot_g_slow_copyout2);
505 xx(c_mmot_getback_fast_copyin);
506 xx(c_mmot_cold_048);
507 xx(c_mmot_getback_FastSR);
508 xx(c_mmot_cold_050);
509 xx(c_mmot_cold_051);
510 xx(c_mmot_cold_052);
511 xx(c_mmot_cold_053);
512 xx(c_mmot_fastkernelreply);
513 xx(c_mmot_cold_055);
514 xx(c_mmot_getback_fast_put);
515
516 #undef xx
517 }
518
519 #else /* !HOTPATH_DEBUG */
520
521 /*
522 * Duplicate just these few so we can always do a quick sanity check
523 */
524 unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
525 unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
526 unsigned int c_mmot_kernel_send = 0; /* kernel server calls */
527 #define HOT(expr) /* no optional counters */
528
529 #endif /* !HOTPATH_DEBUG */
530
531 boolean_t enable_hotpath = TRUE; /* Patchable, just in case ... */
532 #endif /* HOTPATH_ENABLE */
533
534 /*
535 * Routine: mach_msg_overwrite_trap [mach trap]
536 * Purpose:
537 * Possibly send a message; possibly receive a message.
538 * Conditions:
539 * Nothing locked.
540 * Returns:
541 * All of mach_msg_send and mach_msg_receive error codes.
542 */
543
544 mach_msg_return_t
545 mach_msg_overwrite_trap(
546 mach_msg_header_t *msg,
547 mach_msg_option_t option,
548 mach_msg_size_t send_size,
549 mach_msg_size_t rcv_size,
550 mach_port_name_t rcv_name,
551 mach_msg_timeout_t timeout,
552 mach_port_name_t notify,
553 mach_msg_header_t *rcv_msg,
554 mach_msg_size_t scatter_list_size)
555 {
556 register mach_msg_header_t *hdr;
557 mach_msg_return_t mr = MACH_MSG_SUCCESS;
558 /* mask out some of the options before entering the hot path */
559 mach_msg_option_t masked_option =
560 option & ~(MACH_SEND_TRAILER|MACH_RCV_TRAILER_MASK|MACH_RCV_LARGE);
561 int i;
562
563 #if ENABLE_HOTPATH
564 /* BEGINNING OF HOT PATH */
565 if ((masked_option == (MACH_SEND_MSG|MACH_RCV_MSG)) && enable_hotpath) {
566 register thread_t self = current_thread();
567 register mach_msg_format_0_trailer_t *trailer;
568
569 ipc_space_t space = current_act()->task->itk_space;
570 ipc_kmsg_t kmsg;
571 register ipc_port_t dest_port;
572 ipc_object_t rcv_object;
573 register ipc_mqueue_t rcv_mqueue;
574 mach_msg_size_t reply_size;
575 ipc_kmsg_t rcv_kmsg;
576
577 c_mmot_combined_S_R++;
578
579 /*
580 * This case is divided into ten sections, each
581 * with a label. There are five optimized
582 * sections and six unoptimized sections, which
583 * do the same thing but handle all possible
584 * cases and are slower.
585 *
586 * The five sections for an RPC are
587 * 1) Get request message into a buffer.
588 * 2) Copyin request message and rcv_name.
589 * (fast_copyin or slow_copyin)
590 * 3) Enqueue request and dequeue reply.
591 * (fast_send_receive or
592 * slow_send and slow_receive)
593 * 4) Copyout reply message.
594 * (fast_copyout or slow_copyout)
595 * 5) Put reply message to user's buffer.
596 *
597 * Keep the locking hierarchy firmly in mind.
598 * (First spaces, then ports, then port sets,
599 * then message queues.) Only a non-blocking
600 * attempt can be made to acquire locks out of
601 * order, or acquire two locks on the same level.
602 * Acquiring two locks on the same level will
603 * fail if the objects are really the same,
604 * unless simple locking is disabled. This is OK,
605 * because then the extra unlock does nothing.
606 *
607 * There are two major reasons these RPCs can't use
608 * ipc_thread_switch, and use slow_send/slow_receive:
609 * 1) Kernel RPCs.
610 * 2) Servers fall behind clients, so
611 * client doesn't find a blocked server thread and
612 * server finds waiting messages and can't block.
613 */
614
615 mr = ipc_kmsg_get(msg, send_size, &kmsg);
616 if (mr != KERN_SUCCESS) {
617 return mr;
618 }
619 hdr = &kmsg->ikm_header;
620 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
621 send_size);
622
623 fast_copyin:
624 /*
625 * optimized ipc_kmsg_copyin/ipc_mqueue_copyin
626 *
627 * We have the request message data in kmsg.
628 * Must still do copyin, send, receive, etc.
629 *
630 * If the message isn't simple, we can't combine
631 * ipc_kmsg_copyin_header and ipc_mqueue_copyin,
632 * because copyin of the message body might
633 * affect rcv_name.
634 */
635
636 switch (hdr->msgh_bits) {
637 case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
638 MACH_MSG_TYPE_MAKE_SEND_ONCE): {
639 register ipc_entry_t table;
640 register ipc_entry_num_t size;
641 register ipc_port_t reply_port;
642
643 /* sending a request message */
644
645 {
646 register mach_port_index_t index;
647 register mach_port_gen_t gen;
648
649 {
650 register mach_port_name_t reply_name =
651 (mach_port_name_t)hdr->msgh_local_port;
652
653 if (reply_name != rcv_name) {
654 HOT(c_mmot_g_slow_copyin3++);
655 goto slow_copyin;
656 }
657
658 /* optimized ipc_entry_lookup of reply_name */
659
660 index = MACH_PORT_INDEX(reply_name);
661 gen = MACH_PORT_GEN(reply_name);
662
663 is_read_lock(space);
664 assert(space->is_active);
665
666 size = space->is_table_size;
667 table = space->is_table;
668
669 {
670 register ipc_entry_t entry;
671 register ipc_entry_bits_t bits;
672
673 if (index < size) {
674 entry = &table[index];
675 bits = entry->ie_bits;
676 if (IE_BITS_GEN(bits) != gen ||
677 (bits & IE_BITS_COLLISION)) {
678 entry = IE_NULL;
679 }
680 } else {
681 entry = IE_NULL;
682 }
683 if (entry == IE_NULL) {
684 entry = ipc_entry_lookup(space, reply_name);
685 if (entry == IE_NULL) {
686 HOT(c_mmot_cold_006++);
687 goto abort_request_copyin;
688 }
689 bits = entry->ie_bits;
690 }
691
692 /* check type bit */
693
694 if (! (bits & MACH_PORT_TYPE_RECEIVE)) {
695 HOT(c_mmot_cold_007++);
696 goto abort_request_copyin;
697 }
698
699 reply_port = (ipc_port_t) entry->ie_object;
700 assert(reply_port != IP_NULL);
701 }
702 }
703 }
704
705 /* optimized ipc_entry_lookup of dest_name */
706
707 {
708 register mach_port_index_t index;
709 register mach_port_gen_t gen;
710
711 {
712 register mach_port_name_t dest_name =
713 (mach_port_name_t)hdr->msgh_remote_port;
714
715 index = MACH_PORT_INDEX(dest_name);
716 gen = MACH_PORT_GEN(dest_name);
717
718 {
719 register ipc_entry_t entry;
720 register ipc_entry_bits_t bits;
721
722 if (index < size) {
723 entry = &table[index];
724 bits = entry->ie_bits;
725 if (IE_BITS_GEN(bits) != gen ||
726 (bits & IE_BITS_COLLISION)) {
727 entry = IE_NULL;
728 }
729 } else {
730 entry = IE_NULL;
731 }
732 if (entry == IE_NULL) {
733 entry = ipc_entry_lookup(space, dest_name);
734 if (entry == IE_NULL) {
735 HOT(c_mmot_cold_008++);
736 goto abort_request_copyin;
737 }
738 bits = entry->ie_bits;
739 }
740
741 /* check type bit */
742
743 if (! (bits & MACH_PORT_TYPE_SEND)) {
744 HOT(c_mmot_cold_009++);
745 goto abort_request_copyin;
746 }
747
748 assert(IE_BITS_UREFS(bits) > 0);
749
750 dest_port = (ipc_port_t) entry->ie_object;
751 assert(dest_port != IP_NULL);
752 }
753 }
754 }
755
756 /*
757 * To do an atomic copyin, need simultaneous
758 * locks on both ports and the space. If
759 * dest_port == reply_port, and simple locking is
760 * enabled, then we will abort. Otherwise it's
761 * OK to unlock twice.
762 */
763
764 ip_lock(dest_port);
765 if (!ip_active(dest_port) ||
766 !ip_lock_try(reply_port)) {
767 ip_unlock(dest_port);
768 HOT(c_mmot_cold_010++);
769 goto abort_request_copyin;
770 }
771 is_read_unlock(space);
772
773 assert(dest_port->ip_srights > 0);
774 dest_port->ip_srights++;
775 ip_reference(dest_port);
776
777 assert(ip_active(reply_port));
778 assert(reply_port->ip_receiver_name ==
779 (mach_port_name_t)hdr->msgh_local_port);
780 assert(reply_port->ip_receiver == space);
781
782 reply_port->ip_sorights++;
783 ip_reference(reply_port);
784
785 hdr->msgh_bits =
786 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
787 MACH_MSG_TYPE_PORT_SEND_ONCE);
788 hdr->msgh_remote_port = dest_port;
789 hdr->msgh_local_port = reply_port;
790
791 /* make sure we can queue to the destination */
792
793 if (dest_port->ip_receiver == ipc_space_kernel) {
794 /*
795 * The kernel server has a reference to
796 * the reply port, which it hands back
797 * to us in the reply message. We do
798 * not need to keep another reference to
799 * it.
800 */
801 ip_unlock(reply_port);
802
803 assert(ip_active(dest_port));
804 dest_port->ip_messages.imq_seqno++;
805 ip_unlock(dest_port);
806 goto kernel_send;
807 }
808
809 if (imq_full(&dest_port->ip_messages)) {
810 HOT(c_mmot_cold_013++);
811 goto abort_request_send_receive;
812 }
813
814 /* optimized ipc_mqueue_copyin */
815
816 rcv_object = (ipc_object_t) reply_port;
817 io_reference(rcv_object);
818 rcv_mqueue = &reply_port->ip_messages;
819 io_unlock(rcv_object);
820 HOT(c_mmot_hot_fSR_ok++);
821 goto fast_send_receive;
822
823 abort_request_copyin:
824 is_read_unlock(space);
825 goto slow_copyin;
826
827 abort_request_send_receive:
828 ip_unlock(dest_port);
829 ip_unlock(reply_port);
830 goto slow_send;
831 }
832
833 case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
834 register ipc_entry_num_t size;
835 register ipc_entry_t table;
836
837 /* sending a reply message */
838
839 {
840 register mach_port_name_t reply_name =
841 (mach_port_name_t)hdr->msgh_local_port;
842
843 if (reply_name != MACH_PORT_NULL) {
844 HOT(c_mmot_cold_018++);
845 goto slow_copyin;
846 }
847 }
848
849 is_write_lock(space);
850 assert(space->is_active);
851
852 /* optimized ipc_entry_lookup */
853
854 size = space->is_table_size;
855 table = space->is_table;
856
857 {
858 register ipc_entry_t entry;
859 register mach_port_gen_t gen;
860 register mach_port_index_t index;
861 ipc_table_index_t *requests;
862
863 {
864 register mach_port_name_t dest_name =
865 (mach_port_name_t)hdr->msgh_remote_port;
866
867 index = MACH_PORT_INDEX(dest_name);
868 gen = MACH_PORT_GEN(dest_name);
869 }
870
871 if (index >= size) {
872 HOT(c_mmot_cold_019++);
873 goto abort_reply_dest_copyin;
874 }
875
876 entry = &table[index];
877
878 /* check generation, collision bit, and type bit */
879
880 if ((entry->ie_bits & (IE_BITS_GEN_MASK|
881 IE_BITS_COLLISION|
882 MACH_PORT_TYPE_SEND_ONCE)) !=
883 (gen | MACH_PORT_TYPE_SEND_ONCE)) {
884 HOT(c_mmot_cold_020++);
885 goto abort_reply_dest_copyin;
886 }
887
888 /* optimized ipc_right_copyin */
889
890 assert(IE_BITS_TYPE(entry->ie_bits) ==
891 MACH_PORT_TYPE_SEND_ONCE);
892 assert(IE_BITS_UREFS(entry->ie_bits) == 1);
893
894 if (entry->ie_request != 0) {
895 HOT(c_mmot_cold_021++);
896 goto abort_reply_dest_copyin;
897 }
898
899 dest_port = (ipc_port_t) entry->ie_object;
900 assert(dest_port != IP_NULL);
901
902 ip_lock(dest_port);
903 if (!ip_active(dest_port)) {
904 ip_unlock(dest_port);
905 HOT(c_mmot_cold_022++);
906 goto abort_reply_dest_copyin;
907 }
908
909 assert(dest_port->ip_sorights > 0);
910
911 /* optimized ipc_entry_dealloc */
912
913
914 entry->ie_bits = gen;
915 entry->ie_next = table->ie_next;
916 table->ie_next = index;
917 entry->ie_object = IO_NULL;
918 }
919
920 hdr->msgh_bits =
921 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
922 0);
923 hdr->msgh_remote_port = dest_port;
924
925 /* make sure we can queue to the destination */
926
927 assert(dest_port->ip_receiver != ipc_space_kernel);
928
929 /* optimized ipc_entry_lookup/ipc_mqueue_copyin */
930
931 {
932 register ipc_entry_t entry;
933 register ipc_entry_bits_t bits;
934
935 {
936 register mach_port_index_t index;
937 register mach_port_gen_t gen;
938
939 index = MACH_PORT_INDEX(rcv_name);
940 gen = MACH_PORT_GEN(rcv_name);
941
942 if (index < size) {
943 entry = &table[index];
944 bits = entry->ie_bits;
945 if (IE_BITS_GEN(bits) != gen ||
946 (bits & IE_BITS_COLLISION)) {
947 entry = IE_NULL;
948 }
949 } else {
950 entry = IE_NULL;
951 }
952 if (entry == IE_NULL) {
953 entry = ipc_entry_lookup(space, rcv_name);
954 if (entry == IE_NULL) {
955 HOT(c_mmot_cold_024++);
956 goto abort_reply_rcv_copyin;
957 }
958 bits = entry->ie_bits;
959 }
960
961 }
962
963 /* check type bits; looking for receive or set */
964 #if 0
965 /*
966 * JMM - The check below for messages in the receive
967 * mqueue is insufficient to work with port sets, since
968 * the messages stay in the port queues. For now, don't
969 * allow portsets (but receiving on portsets when sending
970 * a message to a send-once right is actually a very
971 * common case (so we should re-enable).
972 */
973 if (bits & MACH_PORT_TYPE_PORT_SET) {
974 register ipc_pset_t rcv_pset;
975
976 rcv_pset = (ipc_pset_t) entry->ie_object;
977 assert(rcv_pset != IPS_NULL);
978
979 ips_lock(rcv_pset);
980 assert(ips_active(rcv_pset));
981
982 rcv_object = (ipc_object_t) rcv_pset;
983 rcv_mqueue = &rcv_pset->ips_messages;
984 } else
985 #endif /* 0 */
986 if (bits & MACH_PORT_TYPE_RECEIVE) {
987 register ipc_port_t rcv_port;
988
989 rcv_port = (ipc_port_t) entry->ie_object;
990 assert(rcv_port != IP_NULL);
991
992 if (!ip_lock_try(rcv_port)) {
993 HOT(c_mmot_cold_025++);
994 goto abort_reply_rcv_copyin;
995 }
996 assert(ip_active(rcv_port));
997
998 if (rcv_port->ip_pset_count != 0) {
999 ip_unlock(rcv_port);
1000 HOT(c_mmot_cold_026++);
1001 goto abort_reply_rcv_copyin;
1002 }
1003
1004 rcv_object = (ipc_object_t) rcv_port;
1005 rcv_mqueue = &rcv_port->ip_messages;
1006 } else {
1007 HOT(c_mmot_cold_027++);
1008 goto abort_reply_rcv_copyin;
1009 }
1010 }
1011
1012 is_write_unlock(space);
1013 io_reference(rcv_object);
1014 io_unlock(rcv_object);
1015 HOT(c_mmot_hot_fSR_ok++);
1016 goto fast_send_receive;
1017
1018 abort_reply_dest_copyin:
1019 is_write_unlock(space);
1020 HOT(c_mmot_cold_029++);
1021 goto slow_copyin;
1022
1023 abort_reply_rcv_copyin:
1024 ip_unlock(dest_port);
1025 is_write_unlock(space);
1026 HOT(c_mmot_cold_030++);
1027 goto slow_send;
1028 }
1029
1030 default:
1031 HOT(c_mmot_cold_031++);
1032 goto slow_copyin;
1033 }
1034 /*NOTREACHED*/
1035
1036 fast_send_receive:
1037 /*
1038 * optimized ipc_mqueue_send/ipc_mqueue_receive
1039 *
1040 * Finished get/copyin of kmsg and copyin of rcv_name.
1041 * space is unlocked, dest_port is locked,
1042 * we can queue kmsg to dest_port,
1043 * rcv_mqueue is set, and rcv_object holds a ref
1044 * so the mqueue cannot go away.
1045 *
1046 * JMM - For now, rcv_object is just a port. Portsets
1047 * are disabled for the time being.
1048 */
1049
1050 assert(ip_active(dest_port));
1051 assert(dest_port->ip_receiver != ipc_space_kernel);
1052 assert(!imq_full(&dest_port->ip_messages) ||
1053 (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1054 MACH_MSG_TYPE_PORT_SEND_ONCE));
1055 assert((hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR) == 0);
1056
1057 {
1058 register ipc_mqueue_t dest_mqueue;
1059 wait_queue_t waitq;
1060 thread_t receiver;
1061 #if THREAD_SWAPPER
1062 thread_act_t rcv_act;
1063 #endif
1064 spl_t s;
1065
1066 s = splsched();
1067 dest_mqueue = &dest_port->ip_messages;
1068 waitq = &dest_mqueue->imq_wait_queue;
1069 imq_lock(dest_mqueue);
1070
1071 wait_queue_peek_locked(waitq, IPC_MQUEUE_RECEIVE, &receiver, &waitq);
1072 /* queue still locked, thread locked - but still on q */
1073
1074 if (receiver == THREAD_NULL) {
1075 abort_send_receive:
1076 imq_unlock(dest_mqueue);
1077 splx(s);
1078 ip_unlock(dest_port);
1079 ipc_object_release(rcv_object);
1080 HOT(c_mmot_cold_032++);
1081 goto slow_send;
1082 }
1083
1084 assert(receiver->wait_queue == waitq);
1085 assert(receiver->wait_event == IPC_MQUEUE_RECEIVE);
1086
1087 /*
1088 * See if it is still running on another processor (trying to
1089 * block itself). If so, fall off.
1090 *
1091 * JMM - We have an opportunity here. Since the thread is locked
1092 * and we find it runnable, it must still be trying to get into
1093 * thread_block on itself. We could just "hand him the message"
1094 * and let him go (thread_go_locked()) and then fall down into a
1095 * slow receive for ourselves. Only his RECEIVE_TOO_LARGE handling
1096 * runs afoul of that. Clean this up!
1097 */
1098 if ((receiver->state & TH_RUN|TH_WAIT) != TH_WAIT) {
1099 assert(NCPUS > 1);
1100 HOT(c_mmot_cold_033++);
1101 fall_off:
1102 thread_unlock(receiver);
1103 if (waitq != &dest_mqueue->imq_wait_queue)
1104 wait_queue_unlock(waitq);
1105 goto abort_send_receive;
1106 }
1107
1108 /*
1109 * Check that the receiver can stay on the hot path.
1110 */
1111 if (send_size + REQUESTED_TRAILER_SIZE(receiver->ith_option) >
1112 receiver->ith_msize) {
1113 /*
1114 * The receiver can't accept the message.
1115 */
1116 HOT(c_mmot_bad_rcvr++);
1117 goto fall_off;
1118 }
1119
1120 #if THREAD_SWAPPER
1121 /*
1122 * Receiver looks okay -- is it swapped in?
1123 */
1124 rpc_lock(receiver);
1125 rcv_act = receiver->top_act;
1126 if (rcv_act->swap_state != TH_SW_IN &&
1127 rcv_act->swap_state != TH_SW_UNSWAPPABLE) {
1128 rpc_unlock(receiver);
1129 HOT(c_mmot_rcvr_swapped++);
1130 goto fall_off;
1131 }
1132
1133 /*
1134 * Make sure receiver stays swapped in (if we can).
1135 */
1136 if (!act_lock_try(rcv_act)) { /* out of order! */
1137 rpc_unlock(receiver);
1138 HOT(c_mmot_rcvr_locked++);
1139 goto fall_off;
1140 }
1141
1142 /*
1143 * Check for task swapping in progress affecting
1144 * receiver. Since rcv_act is attached to a shuttle,
1145 * its swap_state is covered by shuttle's thread_lock()
1146 * (sigh).
1147 */
1148 if ((rcv_act->swap_state != TH_SW_IN &&
1149 rcv_act->swap_state != TH_SW_UNSWAPPABLE) ||
1150 rcv_act->ast & AST_SWAPOUT) {
1151 act_unlock(rcv_act);
1152 rpc_unlock(receiver);
1153 HOT(c_mmot_rcvr_tswapped++);
1154 goto fall_off;
1155 }
1156
1157 /*
1158 * We don't need to make receiver unswappable here -- holding
1159 * act_lock() of rcv_act is sufficient to prevent either thread
1160 * or task swapping from changing its state (see swapout_scan(),
1161 * task_swapout()). Don't release lock till receiver's state
1162 * is consistent. Its task may then be marked for swapout,
1163 * but that's life.
1164 */
1165 rpc_unlock(receiver);
1166 /*
1167 * NB: act_lock(rcv_act) still held
1168 */
1169 #endif /* THREAD_SWAPPER */
1170
1171 /*
1172 * Before committing to the handoff, make sure that we are
1173 * really going to block (i.e. there are no messages already
1174 * queued for us. This violates lock ordering, so make sure
1175 * we don't deadlock. After the trylock succeeds below, we
1176 * may have up to 3 message queues locked:
1177 * - the dest port mqueue
1178 * - a portset mqueue (where waiting receiver was found)
1179 * - finally our own rcv_mqueue
1180 *
1181 * JMM - Need to make this check appropriate for portsets as
1182 * well before re-enabling them.
1183 */
1184 if (!imq_lock_try(rcv_mqueue)) {
1185 goto fall_off;
1186 }
1187 if (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL) {
1188 imq_unlock(rcv_mqueue);
1189 HOT(c_mmot_cold_033++);
1190 goto fall_off;
1191 }
1192
1193 /* At this point we are committed to do the "handoff". */
1194 c_mach_msg_trap_switch_fast++;
1195
1196 /*
1197 * JMM - Go ahead and pull the receiver from the runq. If the
1198 * runq wasn't the one for the mqueue, unlock it.
1199 */
1200 wait_queue_pull_thread_locked(waitq,
1201 receiver,
1202 (waitq != &dest_mqueue->imq_wait_queue));
1203
1204 /*
1205 * Store the kmsg and seqno where the receiver can pick it up.
1206 */
1207 receiver->ith_state = MACH_MSG_SUCCESS;
1208 receiver->ith_kmsg = kmsg;
1209 receiver->ith_seqno = dest_mqueue->imq_seqno++;
1210
1211 /*
1212 * Inline thread_go_locked
1213 *
1214 * JMM - Including hacked in version of setrun scheduler op
1215 * that doesn't try to put thread on a runq.
1216 */
1217 {
1218 receiver->sp_state = MK_SP_RUNNABLE;
1219
1220 receiver->state &= ~(TH_WAIT|TH_UNINT);
1221 receiver->state |= TH_RUN;
1222 receiver->wait_result = THREAD_AWAKENED;
1223 }
1224
1225 thread_unlock(receiver);
1226 #if THREAD_SWAPPER
1227 act_unlock(rcv_act);
1228 #endif /* THREAD_SWAPPER */
1229
1230 imq_unlock(dest_mqueue);
1231 ip_unlock(dest_port);
1232 current_task()->messages_sent++;
1233
1234
1235 /*
1236 * Put self on receive port's queue.
1237 * Also save state that the sender of
1238 * our reply message needs to determine if it
1239 * can hand off directly back to us.
1240 */
1241 self->ith_msg = (rcv_msg) ? rcv_msg : msg;
1242 self->ith_object = rcv_object; /* still holds reference */
1243 self->ith_msize = rcv_size;
1244 self->ith_option = option;
1245 self->ith_scatter_list_size = scatter_list_size;
1246 self->ith_continuation = thread_syscall_return;
1247
1248 waitq = &rcv_mqueue->imq_wait_queue;
1249 wait_queue_assert_wait_locked(waitq,
1250 IPC_MQUEUE_RECEIVE,
1251 THREAD_ABORTSAFE,
1252 TRUE); /* unlock? */
1253 /* rcv_mqueue is unlocked */
1254
1255 /* Inline thread_block_reason (except don't select a new
1256 * new thread (we already have one), and don't turn off ASTs
1257 * (we don't want two threads to hog all the CPU by handing
1258 * off to each other).
1259 */
1260 {
1261 if (self->funnel_state & TH_FN_OWNED) {
1262 self->funnel_state = TH_FN_REFUNNEL;
1263 KERNEL_DEBUG(0x603242c | DBG_FUNC_NONE, self->funnel_lock, 3, 0, 0, 0);
1264 funnel_unlock(self->funnel_lock);
1265
1266 }
1267
1268 machine_clock_assist();
1269
1270 thread_lock(self);
1271 if (self->state & TH_ABORT)
1272 clear_wait_internal(self, THREAD_INTERRUPTED);
1273 thread_unlock(self);
1274
1275 /*
1276 * Switch directly to receiving thread, and block
1277 * this thread as though it had called ipc_mqueue_receive.
1278 */
1279 #if defined (__i386__)
1280 thread_run(self, (void (*)(void))0, receiver);
1281 #else
1282 thread_run(self, ipc_mqueue_receive_continue, receiver);
1283 #endif
1284
1285 /* if we fell thru */
1286 if (self->funnel_state & TH_FN_REFUNNEL) {
1287 kern_return_t wait_result2;
1288
1289 wait_result2 = self->wait_result;
1290 self->funnel_state = 0;
1291 KERNEL_DEBUG(0x6032428 | DBG_FUNC_NONE, self->funnel_lock, 6, 0, 0, 0);
1292 funnel_lock(self->funnel_lock);
1293 KERNEL_DEBUG(0x6032430 | DBG_FUNC_NONE, self->funnel_lock, 6, 0, 0, 0);
1294 self->funnel_state = TH_FN_OWNED;
1295 self->wait_result = wait_result2;
1296 }
1297 splx(s);
1298 }
1299
1300 ipc_mqueue_receive_continue();
1301 /* NOTREACHED */
1302 }
1303
1304 fast_copyout:
1305 /*
1306 * Nothing locked and no references held, except
1307 * we have kmsg with msgh_seqno filled in. Must
1308 * still check against rcv_size and do
1309 * ipc_kmsg_copyout/ipc_kmsg_put.
1310 */
1311
1312 reply_size = send_size + trailer->msgh_trailer_size;
1313 if (rcv_size < reply_size) {
1314 HOT(c_mmot_g_slow_copyout6++);
1315 goto slow_copyout;
1316 }
1317
1318 /* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */
1319
1320 switch (hdr->msgh_bits) {
1321 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
1322 MACH_MSG_TYPE_PORT_SEND_ONCE): {
1323 ipc_port_t reply_port =
1324 (ipc_port_t) hdr->msgh_local_port;
1325 mach_port_name_t dest_name, reply_name;
1326
1327 /* receiving a request message */
1328
1329 if (!IP_VALID(reply_port)) {
1330 HOT(c_mmot_g_slow_copyout5++);
1331 goto slow_copyout;
1332 }
1333
1334 is_write_lock(space);
1335 assert(space->is_active);
1336
1337 /*
1338 * To do an atomic copyout, need simultaneous
1339 * locks on both ports and the space. If
1340 * dest_port == reply_port, and simple locking is
1341 * enabled, then we will abort. Otherwise it's
1342 * OK to unlock twice.
1343 */
1344
1345 ip_lock(dest_port);
1346 if (!ip_active(dest_port) ||
1347 !ip_lock_try(reply_port)) {
1348 HOT(c_mmot_cold_037++);
1349 goto abort_request_copyout;
1350 }
1351
1352 if (!ip_active(reply_port)) {
1353 ip_unlock(reply_port);
1354 HOT(c_mmot_cold_038++);
1355 goto abort_request_copyout;
1356 }
1357
1358 assert(reply_port->ip_sorights > 0);
1359 ip_unlock(reply_port);
1360
1361 {
1362 register ipc_entry_t table;
1363 register ipc_entry_t entry;
1364 register mach_port_index_t index;
1365
1366 /* optimized ipc_entry_get */
1367
1368 table = space->is_table;
1369 index = table->ie_next;
1370
1371 if (index == 0) {
1372 HOT(c_mmot_cold_039++);
1373 goto abort_request_copyout;
1374 }
1375
1376 entry = &table[index];
1377 table->ie_next = entry->ie_next;
1378 entry->ie_request = 0;
1379
1380 {
1381 register mach_port_gen_t gen;
1382
1383 assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
1384 gen = IE_BITS_NEW_GEN(entry->ie_bits);
1385
1386 reply_name = MACH_PORT_MAKE(index, gen);
1387
1388 /* optimized ipc_right_copyout */
1389
1390 entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
1391 }
1392
1393 assert(MACH_PORT_VALID(reply_name));
1394 entry->ie_object = (ipc_object_t) reply_port;
1395 is_write_unlock(space);
1396 }
1397
1398 /* optimized ipc_object_copyout_dest */
1399
1400 assert(dest_port->ip_srights > 0);
1401 ip_release(dest_port);
1402
1403 if (dest_port->ip_receiver == space)
1404 dest_name = dest_port->ip_receiver_name;
1405 else
1406 dest_name = MACH_PORT_NULL;
1407
1408 if ((--dest_port->ip_srights == 0) &&
1409 (dest_port->ip_nsrequest != IP_NULL)) {
1410 ipc_port_t nsrequest;
1411 mach_port_mscount_t mscount;
1412
1413 /* a rather rare case */
1414
1415 nsrequest = dest_port->ip_nsrequest;
1416 mscount = dest_port->ip_mscount;
1417 dest_port->ip_nsrequest = IP_NULL;
1418 ip_unlock(dest_port);
1419 ipc_notify_no_senders(nsrequest, mscount);
1420 } else
1421 ip_unlock(dest_port);
1422
1423 hdr->msgh_bits =
1424 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
1425 MACH_MSG_TYPE_PORT_SEND);
1426 hdr->msgh_remote_port = (mach_port_t)reply_name;
1427 hdr->msgh_local_port = (mach_port_t)dest_name;
1428 HOT(c_mmot_hot_ok1++);
1429 goto fast_put;
1430
1431 abort_request_copyout:
1432 ip_unlock(dest_port);
1433 is_write_unlock(space);
1434 HOT(c_mmot_g_slow_copyout4++);
1435 goto slow_copyout;
1436 }
1437
1438 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1439 register mach_port_name_t dest_name;
1440
1441 /* receiving a reply message */
1442
1443 ip_lock(dest_port);
1444 if (!ip_active(dest_port)) {
1445 ip_unlock(dest_port);
1446 HOT(c_mmot_g_slow_copyout3++);
1447 goto slow_copyout;
1448 }
1449
1450 /* optimized ipc_object_copyout_dest */
1451
1452 assert(dest_port->ip_sorights > 0);
1453
1454 if (dest_port->ip_receiver == space) {
1455 ip_release(dest_port);
1456 dest_port->ip_sorights--;
1457 dest_name = dest_port->ip_receiver_name;
1458 ip_unlock(dest_port);
1459 } else {
1460 ip_unlock(dest_port);
1461
1462 ipc_notify_send_once(dest_port);
1463 dest_name = MACH_PORT_NULL;
1464 }
1465
1466 hdr->msgh_bits = MACH_MSGH_BITS(0,
1467 MACH_MSG_TYPE_PORT_SEND_ONCE);
1468 hdr->msgh_remote_port = MACH_PORT_NULL;
1469 hdr->msgh_local_port = (ipc_port_t)dest_name;
1470 HOT(c_mmot_hot_ok2++);
1471 goto fast_put;
1472 }
1473
1474 case MACH_MSGH_BITS_COMPLEX|
1475 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1476 register mach_port_name_t dest_name;
1477
1478 /* receiving a complex reply message */
1479
1480 ip_lock(dest_port);
1481 if (!ip_active(dest_port)) {
1482 ip_unlock(dest_port);
1483 HOT(c_mmot_g_slow_copyout1++);
1484 goto slow_copyout;
1485 }
1486
1487 /* optimized ipc_object_copyout_dest */
1488
1489 assert(dest_port->ip_sorights > 0);
1490
1491 if (dest_port->ip_receiver == space) {
1492 ip_release(dest_port);
1493 dest_port->ip_sorights--;
1494 dest_name = dest_port->ip_receiver_name;
1495 ip_unlock(dest_port);
1496 } else {
1497 ip_unlock(dest_port);
1498
1499 ipc_notify_send_once(dest_port);
1500 dest_name = MACH_PORT_NULL;
1501 }
1502
1503 hdr->msgh_bits =
1504 MACH_MSGH_BITS_COMPLEX |
1505 MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND_ONCE);
1506 hdr->msgh_remote_port = MACH_PORT_NULL;
1507 hdr->msgh_local_port = (mach_port_t)dest_name;
1508
1509 mr = ipc_kmsg_copyout_body(kmsg, space,
1510 current_map(),
1511 MACH_MSG_BODY_NULL);
1512 if (mr != MACH_MSG_SUCCESS) {
1513 if (ipc_kmsg_put(msg, kmsg, hdr->msgh_size +
1514 trailer->msgh_trailer_size) ==
1515 MACH_RCV_INVALID_DATA)
1516 return MACH_RCV_INVALID_DATA;
1517 else
1518 return mr | MACH_RCV_BODY_ERROR;
1519 }
1520 HOT(c_mmot_hot_ok3++);
1521 goto fast_put;
1522 }
1523
1524 default:
1525 HOT(c_mmot_g_slow_copyout2++);
1526 goto slow_copyout;
1527 }
1528 /*NOTREACHED*/
1529
1530 fast_put:
1531 mr = ipc_kmsg_put(rcv_msg ? rcv_msg : msg,
1532 kmsg,
1533 hdr->msgh_size + trailer->msgh_trailer_size);
1534 if (mr != MACH_MSG_SUCCESS) {
1535 return MACH_RCV_INVALID_DATA;
1536 }
1537 current_task()->messages_received++;
1538 return mr;
1539
1540
1541 /* BEGINNING OF WARM PATH */
1542
1543 /*
1544 * The slow path has a few non-register temporary
1545 * variables used only for call-by-reference.
1546 */
1547
1548 slow_copyin:
1549 {
1550 ipc_kmsg_t temp_kmsg;
1551 mach_port_seqno_t temp_seqno;
1552 ipc_object_t temp_rcv_object;
1553 ipc_mqueue_t temp_rcv_mqueue;
1554 register mach_port_name_t reply_name =
1555 (mach_port_name_t)hdr->msgh_local_port;
1556
1557
1558 /*
1559 * We have the message data in kmsg, but
1560 * we still need to copyin, send it,
1561 * receive a reply, and do copyout.
1562 */
1563
1564 mr = ipc_kmsg_copyin(kmsg, space, current_map(),
1565 MACH_PORT_NULL);
1566 if (mr != MACH_MSG_SUCCESS) {
1567 ipc_kmsg_free(kmsg);
1568 return(mr);
1569 }
1570
1571 /* try to get back on optimized path */
1572
1573 if ((reply_name != rcv_name) ||
1574 (hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR)) {
1575 HOT(c_mmot_cold_048++);
1576 goto slow_send;
1577 }
1578
1579 dest_port = (ipc_port_t) hdr->msgh_remote_port;
1580 assert(IP_VALID(dest_port));
1581
1582 ip_lock(dest_port);
1583 if (!ip_active(dest_port)) {
1584 ip_unlock(dest_port);
1585 goto slow_send;
1586 }
1587
1588 if (dest_port->ip_receiver == ipc_space_kernel) {
1589 dest_port->ip_messages.imq_seqno++;
1590 ip_unlock(dest_port);
1591 goto kernel_send;
1592 }
1593
1594 if (!imq_full(&dest_port->ip_messages) ||
1595 (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1596 MACH_MSG_TYPE_PORT_SEND_ONCE))
1597 {
1598 /*
1599 * Try an optimized ipc_mqueue_copyin.
1600 * It will work if this is a request message.
1601 */
1602
1603 register ipc_port_t reply_port;
1604
1605 reply_port = (ipc_port_t) hdr->msgh_local_port;
1606 if (IP_VALID(reply_port)) {
1607 if (ip_lock_try(reply_port)) {
1608 if (ip_active(reply_port) &&
1609 reply_port->ip_receiver == space &&
1610 reply_port->ip_receiver_name == rcv_name &&
1611 reply_port->ip_pset_count == 0)
1612 {
1613 /* Grab a reference to the reply port. */
1614 rcv_object = (ipc_object_t) reply_port;
1615 io_reference(rcv_object);
1616 rcv_mqueue = &reply_port->ip_messages;
1617 io_unlock(rcv_object);
1618 HOT(c_mmot_getback_FastSR++);
1619 goto fast_send_receive;
1620 }
1621 ip_unlock(reply_port);
1622 }
1623 }
1624 }
1625
1626 ip_unlock(dest_port);
1627 HOT(c_mmot_cold_050++);
1628 goto slow_send;
1629
1630 kernel_send:
1631 /*
1632 * Special case: send message to kernel services.
1633 * The request message has been copied into the
1634 * kmsg. Nothing is locked.
1635 */
1636
1637 {
1638 register ipc_port_t reply_port;
1639 mach_port_seqno_t local_seqno;
1640 spl_t s;
1641
1642 /*
1643 * Perform the kernel function.
1644 */
1645 c_mmot_kernel_send++;
1646
1647 current_task()->messages_sent++;
1648
1649 kmsg = ipc_kobject_server(kmsg);
1650 if (kmsg == IKM_NULL) {
1651 /*
1652 * No reply. Take the
1653 * slow receive path.
1654 */
1655 HOT(c_mmot_cold_051++);
1656 goto slow_get_rcv_port;
1657 }
1658
1659 /*
1660 * Check that:
1661 * the reply port is alive
1662 * we hold the receive right
1663 * the name has not changed.
1664 * the port is not in a set
1665 * If any of these are not true,
1666 * we cannot directly receive the reply
1667 * message.
1668 */
1669 hdr = &kmsg->ikm_header;
1670 send_size = hdr->msgh_size;
1671 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1672 round_msg(send_size));
1673 reply_port = (ipc_port_t) hdr->msgh_remote_port;
1674 ip_lock(reply_port);
1675
1676 if ((!ip_active(reply_port)) ||
1677 (reply_port->ip_receiver != space) ||
1678 (reply_port->ip_receiver_name != rcv_name) ||
1679 (reply_port->ip_pset_count != 0))
1680 {
1681 ip_unlock(reply_port);
1682 ipc_kmsg_send_always(kmsg);
1683 HOT(c_mmot_cold_052++);
1684 goto slow_get_rcv_port;
1685 }
1686
1687 s = splsched();
1688 rcv_mqueue = &reply_port->ip_messages;
1689 imq_lock(rcv_mqueue);
1690
1691 /* keep port locked, and don`t change ref count yet */
1692
1693 /*
1694 * If there are messages on the port
1695 * or other threads waiting for a message,
1696 * we cannot directly receive the reply.
1697 */
1698 if (!wait_queue_empty(&rcv_mqueue->imq_wait_queue) ||
1699 (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL))
1700 {
1701 imq_unlock(rcv_mqueue);
1702 splx(s);
1703 ip_unlock(reply_port);
1704 ipc_kmsg_send_always(kmsg);
1705 HOT(c_mmot_cold_053++);
1706 goto slow_get_rcv_port;
1707 }
1708
1709 /*
1710 * We can directly receive this reply.
1711 * Since there were no messages queued
1712 * on the reply port, there should be
1713 * no threads blocked waiting to send.
1714 */
1715 dest_port = reply_port;
1716 local_seqno = rcv_mqueue->imq_seqno++;
1717 imq_unlock(rcv_mqueue);
1718 splx(s);
1719
1720 /*
1721 * inline ipc_object_release.
1722 * Port is still locked.
1723 * Reference count was not incremented.
1724 */
1725 ip_check_unlock(reply_port);
1726
1727 if (option & MACH_RCV_TRAILER_MASK) {
1728 trailer->msgh_seqno = local_seqno;
1729 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1730 }
1731 /* copy out the kernel reply */
1732 HOT(c_mmot_fastkernelreply++);
1733 goto fast_copyout;
1734 }
1735
1736 slow_send:
1737 /*
1738 * Nothing is locked. We have acquired kmsg, but
1739 * we still need to send it and receive a reply.
1740 */
1741
1742 mr = ipc_kmsg_send(kmsg, MACH_MSG_OPTION_NONE,
1743 MACH_MSG_TIMEOUT_NONE);
1744 if (mr != MACH_MSG_SUCCESS) {
1745 mr |= ipc_kmsg_copyout_pseudo(kmsg, space,
1746 current_map(),
1747 MACH_MSG_BODY_NULL);
1748
1749 (void) ipc_kmsg_put(msg, kmsg, hdr->msgh_size);
1750 return(mr);
1751 }
1752
1753 slow_get_rcv_port:
1754 /*
1755 * We have sent the message. Copy in the receive port.
1756 */
1757 mr = ipc_mqueue_copyin(space, rcv_name,
1758 &temp_rcv_mqueue, &temp_rcv_object);
1759 if (mr != MACH_MSG_SUCCESS) {
1760 return(mr);
1761 }
1762 rcv_mqueue = temp_rcv_mqueue;
1763 rcv_object = temp_rcv_object;
1764 /* hold ref for rcv_object */
1765
1766 slow_receive:
1767 /*
1768 * Now we have sent the request and copied in rcv_name,
1769 * and hold ref for rcv_object (to keep mqueue alive).
1770 * Just receive a reply and try to get back to fast path.
1771 */
1772
1773 self->ith_continuation = (void (*)(mach_msg_return_t))0;
1774 ipc_mqueue_receive(rcv_mqueue,
1775 MACH_MSG_OPTION_NONE,
1776 MACH_MSG_SIZE_MAX,
1777 MACH_MSG_TIMEOUT_NONE,
1778 THREAD_ABORTSAFE);
1779
1780 mr = self->ith_state;
1781 temp_kmsg = self->ith_kmsg;
1782 temp_seqno = self->ith_seqno;
1783
1784 ipc_object_release(rcv_object);
1785
1786 if (mr != MACH_MSG_SUCCESS) {
1787 return(mr);
1788 }
1789
1790 kmsg = temp_kmsg;
1791 hdr = &kmsg->ikm_header;
1792 send_size = hdr->msgh_size;
1793 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1794 round_msg(send_size));
1795 if (option & MACH_RCV_TRAILER_MASK) {
1796 trailer->msgh_seqno = temp_seqno;
1797 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1798 }
1799 dest_port = (ipc_port_t) hdr->msgh_remote_port;
1800 HOT(c_mmot_cold_055++);
1801 goto fast_copyout;
1802
1803 slow_copyout:
1804 /*
1805 * Nothing locked and no references held, except
1806 * we have kmsg with msgh_seqno filled in. Must
1807 * still check against rcv_size and do
1808 * ipc_kmsg_copyout/ipc_kmsg_put.
1809 */
1810
1811 reply_size = send_size + trailer->msgh_trailer_size;
1812 if (rcv_size < reply_size) {
1813 if (msg_receive_error(kmsg, msg, option, temp_seqno,
1814 space) == MACH_RCV_INVALID_DATA) {
1815 mr = MACH_RCV_INVALID_DATA;
1816 return(mr);
1817 }
1818 else {
1819 mr = MACH_RCV_TOO_LARGE;
1820 return(mr);
1821 }
1822 }
1823
1824 mr = ipc_kmsg_copyout(kmsg, space, current_map(),
1825 MACH_PORT_NULL, MACH_MSG_BODY_NULL);
1826 if (mr != MACH_MSG_SUCCESS) {
1827 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1828 if (ipc_kmsg_put(msg, kmsg, reply_size) ==
1829 MACH_RCV_INVALID_DATA)
1830 mr = MACH_RCV_INVALID_DATA;
1831 }
1832 else {
1833 if (msg_receive_error(kmsg, msg, option,
1834 temp_seqno, space) == MACH_RCV_INVALID_DATA)
1835 mr = MACH_RCV_INVALID_DATA;
1836 }
1837
1838 return(mr);
1839 }
1840
1841 /* try to get back on optimized path */
1842 HOT(c_mmot_getback_fast_put++);
1843 goto fast_put;
1844
1845 /*NOTREACHED*/
1846 }
1847 } /* END OF HOT PATH */
1848 #endif /* ENABLE_HOTPATH */
1849
1850 if (option & MACH_SEND_MSG) {
1851 mr = mach_msg_send(msg, option, send_size,
1852 timeout, notify);
1853 if (mr != MACH_MSG_SUCCESS) {
1854 return mr;
1855 }
1856 }
1857
1858 if (option & MACH_RCV_MSG) {
1859 mach_msg_header_t *rcv;
1860
1861 /*
1862 * 1. MACH_RCV_OVERWRITE is on, and rcv_msg is our scatter list
1863 * and receive buffer
1864 * 2. MACH_RCV_OVERWRITE is off, and rcv_msg might be the
1865 * alternate receive buffer (separate send and receive buffers).
1866 */
1867 if (option & MACH_RCV_OVERWRITE)
1868 rcv = rcv_msg;
1869 else if (rcv_msg != MACH_MSG_NULL)
1870 rcv = rcv_msg;
1871 else
1872 rcv = msg;
1873 mr = mach_msg_receive(rcv, option, rcv_size, rcv_name,
1874 timeout, thread_syscall_return, scatter_list_size);
1875 thread_syscall_return(mr);
1876 }
1877
1878 return MACH_MSG_SUCCESS;
1879 }
1880
1881 /*
1882 * Routine: msg_receive_error [internal]
1883 * Purpose:
1884 * Builds a minimal header/trailer and copies it to
1885 * the user message buffer. Invoked when in the case of a
1886 * MACH_RCV_TOO_LARGE or MACH_RCV_BODY_ERROR error.
1887 * Conditions:
1888 * Nothing locked.
1889 * Returns:
1890 * MACH_MSG_SUCCESS minimal header/trailer copied
1891 * MACH_RCV_INVALID_DATA copyout to user buffer failed
1892 */
1893
1894 mach_msg_return_t
1895 msg_receive_error(
1896 ipc_kmsg_t kmsg,
1897 mach_msg_header_t *msg,
1898 mach_msg_option_t option,
1899 mach_port_seqno_t seqno,
1900 ipc_space_t space)
1901 {
1902 mach_msg_format_0_trailer_t *trailer;
1903
1904 /*
1905 * Copy out the destination port in the message.
1906 * Destroy all other rights and memory in the message.
1907 */
1908 ipc_kmsg_copyout_dest(kmsg, space);
1909
1910 /*
1911 * Build a minimal message with the requested trailer.
1912 */
1913 trailer = (mach_msg_format_0_trailer_t *)
1914 ((vm_offset_t)&kmsg->ikm_header +
1915 round_msg(sizeof(mach_msg_header_t)));
1916 kmsg->ikm_header.msgh_size = sizeof(mach_msg_header_t);
1917 bcopy( (char *)&trailer_template,
1918 (char *)trailer,
1919 sizeof(trailer_template));
1920 if (option & MACH_RCV_TRAILER_MASK) {
1921 trailer->msgh_seqno = seqno;
1922 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1923 }
1924
1925 /*
1926 * Copy the message to user space
1927 */
1928 if (ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size +
1929 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
1930 return(MACH_RCV_INVALID_DATA);
1931 else
1932 return(MACH_MSG_SUCCESS);
1933 }
mirror of XNU