]> git.saurik.com Git - apple/xnu.git/blob - osfmk/ipc/mach_msg.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-517.7.21.tar.gz
[apple/xnu.git] / osfmk / ipc / mach_msg.c
1 /*
2 * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22 /*
23 * @OSF_COPYRIGHT@
24 */
25 /*
26 * Mach Operating System
27 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
28 * All Rights Reserved.
29 *
30 * Permission to use, copy, modify and distribute this software and its
31 * documentation is hereby granted, provided that both the copyright
32 * notice and this permission notice appear in all copies of the
33 * software, derivative works or modified versions, and any portions
34 * thereof, and that both notices appear in supporting documentation.
35 *
36 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
39 *
40 * Carnegie Mellon requests users of this software to return to
41 *
42 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
43 * School of Computer Science
44 * Carnegie Mellon University
45 * Pittsburgh PA 15213-3890
46 *
47 * any improvements or extensions that they make and grant Carnegie Mellon
48 * the rights to redistribute these changes.
49 */
50 /*
51 */
52 /*
53 * File: ipc/mach_msg.c
54 * Author: Rich Draves
55 * Date: 1989
56 *
57 * Exported message traps. See mach/message.h.
58 */
59
60 #include <cpus.h>
61
62 #include <mach/kern_return.h>
63 #include <mach/port.h>
64 #include <mach/message.h>
65 #include <mach/mig_errors.h>
66 #include <kern/assert.h>
67 #include <kern/counters.h>
68 #include <kern/cpu_number.h>
69 #include <kern/task.h>
70 #include <kern/thread.h>
71 #include <kern/lock.h>
72 #include <kern/sched_prim.h>
73 #include <kern/exception.h>
74 #include <kern/misc_protos.h>
75 #include <vm/vm_map.h>
76 #include <ipc/ipc_kmsg.h>
77 #include <ipc/ipc_mqueue.h>
78 #include <ipc/ipc_object.h>
79 #include <ipc/ipc_notify.h>
80 #include <ipc/ipc_port.h>
81 #include <ipc/ipc_pset.h>
82 #include <ipc/ipc_space.h>
83 #include <ipc/ipc_entry.h>
84 #include <kern/kalloc.h>
85 #include <kern/thread_swap.h>
86 #include <kern/processor.h>
87
88 #include <kern/mk_sp.h>
89
90 #include <machine/machine_routines.h>
91 #include <sys/kdebug.h>
92
93 /*
94 * Forward declarations
95 */
96
97 mach_msg_return_t mach_msg_send(
98 mach_msg_header_t *msg,
99 mach_msg_option_t option,
100 mach_msg_size_t send_size,
101 mach_msg_timeout_t timeout,
102 mach_port_name_t notify);
103
104 mach_msg_return_t mach_msg_receive(
105 mach_msg_header_t *msg,
106 mach_msg_option_t option,
107 mach_msg_size_t rcv_size,
108 mach_port_name_t rcv_name,
109 mach_msg_timeout_t timeout,
110 void (*continuation)(mach_msg_return_t),
111 mach_msg_size_t slist_size);
112
113
114 mach_msg_return_t msg_receive_error(
115 ipc_kmsg_t kmsg,
116 mach_msg_header_t *msg,
117 mach_msg_option_t option,
118 mach_port_seqno_t seqno,
119 ipc_space_t space);
120
121 security_token_t KERNEL_SECURITY_TOKEN = KERNEL_SECURITY_TOKEN_VALUE;
122 audit_token_t KERNEL_AUDIT_TOKEN = KERNEL_AUDIT_TOKEN_VALUE;
123
124 mach_msg_format_0_trailer_t trailer_template = {
125 /* mach_msg_trailer_type_t */ MACH_MSG_TRAILER_FORMAT_0,
126 /* mach_msg_trailer_size_t */ MACH_MSG_TRAILER_MINIMUM_SIZE,
127 /* mach_port_seqno_t */ 0,
128 /* security_token_t */ KERNEL_SECURITY_TOKEN_VALUE
129 };
130
131 /*
132 * Routine: mach_msg_send
133 * Purpose:
134 * Send a message.
135 * Conditions:
136 * Nothing locked.
137 * Returns:
138 * MACH_MSG_SUCCESS Sent the message.
139 * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
140 * MACH_SEND_NO_BUFFER Couldn't allocate buffer.
141 * MACH_SEND_INVALID_DATA Couldn't copy message data.
142 * MACH_SEND_INVALID_HEADER
143 * Illegal value in the message header bits.
144 * MACH_SEND_INVALID_DEST The space is dead.
145 * MACH_SEND_INVALID_NOTIFY Bad notify port.
146 * MACH_SEND_INVALID_DEST Can't copyin destination port.
147 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
148 * MACH_SEND_TIMED_OUT Timeout expired without delivery.
149 * MACH_SEND_INTERRUPTED Delivery interrupted.
150 * MACH_SEND_NO_NOTIFY Can't allocate a msg-accepted request.
151 * MACH_SEND_WILL_NOTIFY Msg-accepted notif. requested.
152 * MACH_SEND_NOTIFY_IN_PROGRESS
153 * This space has already forced a message to this port.
154 */
155
156 mach_msg_return_t
157 mach_msg_send(
158 mach_msg_header_t *msg,
159 mach_msg_option_t option,
160 mach_msg_size_t send_size,
161 mach_msg_timeout_t timeout,
162 mach_port_name_t notify)
163 {
164 ipc_space_t space = current_space();
165 vm_map_t map = current_map();
166 ipc_kmsg_t kmsg;
167 mach_msg_return_t mr;
168
169 mr = ipc_kmsg_get(msg, send_size, &kmsg);
170
171 if (mr != MACH_MSG_SUCCESS)
172 return mr;
173
174 if (option & MACH_SEND_CANCEL) {
175 if (notify == MACH_PORT_NULL)
176 mr = MACH_SEND_INVALID_NOTIFY;
177 else
178 mr = ipc_kmsg_copyin(kmsg, space, map, notify);
179 } else
180 mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
181 if (mr != MACH_MSG_SUCCESS) {
182 ipc_kmsg_free(kmsg);
183 return mr;
184 }
185
186 mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, timeout);
187
188 if (mr != MACH_MSG_SUCCESS) {
189 mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
190 (void) ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size);
191 }
192
193 return mr;
194 }
195
196 /*
197 * Routine: mach_msg_receive
198 * Purpose:
199 * Receive a message.
200 * Conditions:
201 * Nothing locked.
202 * Returns:
203 * MACH_MSG_SUCCESS Received a message.
204 * MACH_RCV_INVALID_NAME The name doesn't denote a right,
205 * or the denoted right is not receive or port set.
206 * MACH_RCV_IN_SET Receive right is a member of a set.
207 * MACH_RCV_TOO_LARGE Message wouldn't fit into buffer.
208 * MACH_RCV_TIMED_OUT Timeout expired without a message.
209 * MACH_RCV_INTERRUPTED Reception interrupted.
210 * MACH_RCV_PORT_DIED Port/set died while receiving.
211 * MACH_RCV_PORT_CHANGED Port moved into set while receiving.
212 * MACH_RCV_INVALID_DATA Couldn't copy to user buffer.
213 * MACH_RCV_INVALID_NOTIFY Bad notify port.
214 * MACH_RCV_HEADER_ERROR
215 */
216
217 mach_msg_return_t
218 mach_msg_receive_results(void)
219 {
220 thread_t self = current_thread();
221 ipc_space_t space = current_space();
222 vm_map_t map = current_map();
223
224 ipc_object_t object = self->ith_object;
225 mach_msg_return_t mr = self->ith_state;
226 mach_msg_header_t *msg = self->ith_msg;
227 mach_msg_option_t option = self->ith_option;
228 ipc_kmsg_t kmsg = self->ith_kmsg;
229 mach_port_seqno_t seqno = self->ith_seqno;
230 mach_msg_size_t slist_size = self->ith_scatter_list_size;
231
232 mach_msg_format_0_trailer_t *trailer;
233
234 ipc_object_release(object);
235
236 if (mr != MACH_MSG_SUCCESS) {
237
238 if (mr == MACH_RCV_TOO_LARGE ) {
239 if (option & MACH_RCV_LARGE) {
240 /*
241 * We need to inform the user-level code that it needs more
242 * space. The value for how much space was returned in the
243 * msize save area instead of the message (which was left on
244 * the queue).
245 */
246 if (copyout((char *) &self->ith_msize,
247 (char *) &msg->msgh_size,
248 sizeof(mach_msg_size_t)))
249 mr = MACH_RCV_INVALID_DATA;
250 goto out;
251 }
252
253 if (msg_receive_error(kmsg, msg, option, seqno, space)
254 == MACH_RCV_INVALID_DATA)
255 mr = MACH_RCV_INVALID_DATA;
256 }
257 goto out;
258 }
259
260 trailer = (mach_msg_format_0_trailer_t *)
261 ((vm_offset_t)&kmsg->ikm_header +
262 round_msg(kmsg->ikm_header.msgh_size));
263 if (option & MACH_RCV_TRAILER_MASK) {
264 trailer->msgh_seqno = seqno;
265 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
266 }
267
268 /*
269 * If MACH_RCV_OVERWRITE was specified, try to get the scatter
270 * list and verify it against the contents of the message. If
271 * there is any problem with it, we will continue without it as
272 * normal.
273 */
274 if (option & MACH_RCV_OVERWRITE) {
275 mach_msg_size_t slist_size = self->ith_scatter_list_size;
276 mach_msg_body_t *slist;
277
278 slist = ipc_kmsg_copyin_scatter(msg, slist_size, kmsg);
279 mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL, slist);
280 ipc_kmsg_free_scatter(slist, slist_size);
281 } else {
282 mr = ipc_kmsg_copyout(kmsg, space, map,
283 MACH_PORT_NULL, MACH_MSG_BODY_NULL);
284 }
285
286 if (mr != MACH_MSG_SUCCESS) {
287 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
288 if (ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size +
289 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
290 mr = MACH_RCV_INVALID_DATA;
291 }
292 else {
293 if (msg_receive_error(kmsg, msg, option, seqno, space)
294 == MACH_RCV_INVALID_DATA)
295 mr = MACH_RCV_INVALID_DATA;
296 }
297 goto out;
298 }
299 mr = ipc_kmsg_put(msg,
300 kmsg,
301 kmsg->ikm_header.msgh_size +
302 trailer->msgh_trailer_size);
303 out:
304 return mr;
305 }
306
307 mach_msg_return_t
308 mach_msg_receive(
309 mach_msg_header_t *msg,
310 mach_msg_option_t option,
311 mach_msg_size_t rcv_size,
312 mach_port_name_t rcv_name,
313 mach_msg_timeout_t timeout,
314 void (*continuation)(mach_msg_return_t),
315 mach_msg_size_t slist_size)
316 {
317 thread_t self = current_thread();
318 ipc_space_t space = current_space();
319 vm_map_t map = current_map();
320 ipc_object_t object;
321 ipc_mqueue_t mqueue;
322 ipc_kmsg_t kmsg;
323 mach_port_seqno_t seqno;
324 mach_msg_return_t mr;
325 mach_msg_body_t *slist;
326 mach_msg_format_0_trailer_t *trailer;
327
328 mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
329 if (mr != MACH_MSG_SUCCESS) {
330 return mr;
331 }
332 /* hold ref for object */
333
334 self->ith_msg = msg;
335 self->ith_object = object;
336 self->ith_msize = rcv_size;
337 self->ith_option = option;
338 self->ith_scatter_list_size = slist_size;
339 self->ith_continuation = continuation;
340
341 ipc_mqueue_receive(mqueue, option, rcv_size, timeout, THREAD_ABORTSAFE);
342 if ((option & MACH_RCV_TIMEOUT) && timeout == 0)
343 _mk_sp_thread_perhaps_yield(self);
344 return mach_msg_receive_results();
345 }
346
347 void
348 mach_msg_receive_continue(void)
349 {
350 thread_t self = current_thread();
351
352 (*self->ith_continuation)(mach_msg_receive_results());
353 }
354
355 /*
356 * Toggle this to compile the hotpath in/out
357 * If compiled in, the run-time toggle "enable_hotpath" below
358 * eases testing & debugging
359 */
360 #define ENABLE_HOTPATH 1 /* Hacked on for now */
361
362 #if ENABLE_HOTPATH
363 /*
364 * These counters allow tracing of hotpath behavior under test loads.
365 * A couple key counters are unconditional (see below).
366 */
367 #define HOTPATH_DEBUG 0 /* Toggle to include lots of counters */
368 #if HOTPATH_DEBUG
369 #define HOT(expr) expr
370
371 unsigned int c_mmot_FIRST = 0; /* Unused First Counter */
372 unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
373 unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
374 unsigned int c_mmot_kernel_send = 0; /* kernel server */
375 unsigned int c_mmot_cold_000 = 0; /* see below ... */
376 unsigned int c_mmot_smallsendsize = 0;
377 unsigned int c_mmot_oddsendsize = 0;
378 unsigned int c_mmot_bigsendsize = 0;
379 unsigned int c_mmot_copyinmsg_fail = 0;
380 unsigned int c_mmot_g_slow_copyin3 = 0;
381 unsigned int c_mmot_cold_006 = 0;
382 unsigned int c_mmot_cold_007 = 0;
383 unsigned int c_mmot_cold_008 = 0;
384 unsigned int c_mmot_cold_009 = 0;
385 unsigned int c_mmot_cold_010 = 0;
386 unsigned int c_mmot_cold_012 = 0;
387 unsigned int c_mmot_cold_013 = 0;
388 unsigned int c_mmot_cold_014 = 0;
389 unsigned int c_mmot_cold_016 = 0;
390 unsigned int c_mmot_cold_018 = 0;
391 unsigned int c_mmot_cold_019 = 0;
392 unsigned int c_mmot_cold_020 = 0;
393 unsigned int c_mmot_cold_021 = 0;
394 unsigned int c_mmot_cold_022 = 0;
395 unsigned int c_mmot_cold_023 = 0;
396 unsigned int c_mmot_cold_024 = 0;
397 unsigned int c_mmot_cold_025 = 0;
398 unsigned int c_mmot_cold_026 = 0;
399 unsigned int c_mmot_cold_027 = 0;
400 unsigned int c_mmot_hot_fSR_ok = 0;
401 unsigned int c_mmot_cold_029 = 0;
402 unsigned int c_mmot_cold_030 = 0;
403 unsigned int c_mmot_cold_031 = 0;
404 unsigned int c_mmot_cold_032 = 0;
405 unsigned int c_mmot_cold_033 = 0;
406 unsigned int c_mmot_bad_rcvr = 0;
407 unsigned int c_mmot_rcvr_swapped = 0;
408 unsigned int c_mmot_rcvr_locked = 0;
409 unsigned int c_mmot_rcvr_tswapped = 0;
410 unsigned int c_mmot_rcvr_freed = 0;
411 unsigned int c_mmot_g_slow_copyout6 = 0;
412 unsigned int c_mmot_g_slow_copyout5 = 0;
413 unsigned int c_mmot_cold_037 = 0;
414 unsigned int c_mmot_cold_038 = 0;
415 unsigned int c_mmot_cold_039 = 0;
416 unsigned int c_mmot_g_slow_copyout4 = 0;
417 unsigned int c_mmot_g_slow_copyout3 = 0;
418 unsigned int c_mmot_hot_ok1 = 0;
419 unsigned int c_mmot_hot_ok2 = 0;
420 unsigned int c_mmot_hot_ok3 = 0;
421 unsigned int c_mmot_g_slow_copyout1 = 0;
422 unsigned int c_mmot_g_slow_copyout2 = 0;
423 unsigned int c_mmot_getback_fast_copyin = 0;
424 unsigned int c_mmot_cold_048 = 0;
425 unsigned int c_mmot_getback_FastSR = 0;
426 unsigned int c_mmot_cold_050 = 0;
427 unsigned int c_mmot_cold_051 = 0;
428 unsigned int c_mmot_cold_052 = 0;
429 unsigned int c_mmot_cold_053 = 0;
430 unsigned int c_mmot_fastkernelreply = 0;
431 unsigned int c_mmot_cold_055 = 0;
432 unsigned int c_mmot_getback_fast_put = 0;
433 unsigned int c_mmot_LAST = 0; /* End Marker - Unused */
434
435 void db_mmot_zero_counters(void); /* forward; */
436 void db_mmot_show_counters(void); /* forward; */
437
438 void /* Call from the debugger to clear all counters */
439 db_mmot_zero_counters(void)
440 {
441 register unsigned int *ip = &c_mmot_FIRST;
442 while (ip <= &c_mmot_LAST)
443 *ip++ = 0;
444 }
445
446 void /* Call from the debugger to show all counters */
447 db_mmot_show_counters(void)
448 {
449 #define xx(str) printf("%s: %d\n", # str, str);
450
451 xx(c_mmot_combined_S_R);
452 xx(c_mach_msg_trap_switch_fast);
453 xx(c_mmot_kernel_send);
454 xx(c_mmot_cold_000);
455 xx(c_mmot_smallsendsize);
456 xx(c_mmot_oddsendsize);
457 xx(c_mmot_bigsendsize);
458 xx(c_mmot_copyinmsg_fail);
459 xx(c_mmot_g_slow_copyin3);
460 xx(c_mmot_cold_006);
461 xx(c_mmot_cold_007);
462 xx(c_mmot_cold_008);
463 xx(c_mmot_cold_009);
464 xx(c_mmot_cold_010);
465 xx(c_mmot_cold_012);
466 xx(c_mmot_cold_013);
467 xx(c_mmot_cold_014);
468 xx(c_mmot_cold_016);
469 xx(c_mmot_cold_018);
470 xx(c_mmot_cold_019);
471 xx(c_mmot_cold_020);
472 xx(c_mmot_cold_021);
473 xx(c_mmot_cold_022);
474 xx(c_mmot_cold_023);
475 xx(c_mmot_cold_024);
476 xx(c_mmot_cold_025);
477 xx(c_mmot_cold_026);
478 xx(c_mmot_cold_027);
479 xx(c_mmot_hot_fSR_ok);
480 xx(c_mmot_cold_029);
481 xx(c_mmot_cold_030);
482 xx(c_mmot_cold_031);
483 xx(c_mmot_cold_032);
484 xx(c_mmot_cold_033);
485 xx(c_mmot_bad_rcvr);
486 xx(c_mmot_rcvr_swapped);
487 xx(c_mmot_rcvr_locked);
488 xx(c_mmot_rcvr_tswapped);
489 xx(c_mmot_rcvr_freed);
490 xx(c_mmot_g_slow_copyout6);
491 xx(c_mmot_g_slow_copyout5);
492 xx(c_mmot_cold_037);
493 xx(c_mmot_cold_038);
494 xx(c_mmot_cold_039);
495 xx(c_mmot_g_slow_copyout4);
496 xx(c_mmot_g_slow_copyout3);
497 xx(c_mmot_g_slow_copyout1);
498 xx(c_mmot_hot_ok3);
499 xx(c_mmot_hot_ok2);
500 xx(c_mmot_hot_ok1);
501 xx(c_mmot_g_slow_copyout2);
502 xx(c_mmot_getback_fast_copyin);
503 xx(c_mmot_cold_048);
504 xx(c_mmot_getback_FastSR);
505 xx(c_mmot_cold_050);
506 xx(c_mmot_cold_051);
507 xx(c_mmot_cold_052);
508 xx(c_mmot_cold_053);
509 xx(c_mmot_fastkernelreply);
510 xx(c_mmot_cold_055);
511 xx(c_mmot_getback_fast_put);
512
513 #undef xx
514 }
515
516 #else /* !HOTPATH_DEBUG */
517
518 /*
519 * Duplicate just these few so we can always do a quick sanity check
520 */
521 unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
522 unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
523 unsigned int c_mmot_kernel_send = 0; /* kernel server calls */
524 #define HOT(expr) /* no optional counters */
525
526 #endif /* !HOTPATH_DEBUG */
527
528 boolean_t enable_hotpath = TRUE; /* Patchable, just in case ... */
529 #endif /* HOTPATH_ENABLE */
530
531 /*
532 * Routine: mach_msg_overwrite_trap [mach trap]
533 * Purpose:
534 * Possibly send a message; possibly receive a message.
535 * Conditions:
536 * Nothing locked.
537 * Returns:
538 * All of mach_msg_send and mach_msg_receive error codes.
539 */
540
541 mach_msg_return_t
542 mach_msg_overwrite_trap(
543 mach_msg_header_t *msg,
544 mach_msg_option_t option,
545 mach_msg_size_t send_size,
546 mach_msg_size_t rcv_size,
547 mach_port_name_t rcv_name,
548 mach_msg_timeout_t timeout,
549 mach_port_name_t notify,
550 mach_msg_header_t *rcv_msg,
551 mach_msg_size_t scatter_list_size)
552 {
553 register mach_msg_header_t *hdr;
554 mach_msg_return_t mr = MACH_MSG_SUCCESS;
555 /* mask out some of the options before entering the hot path */
556 mach_msg_option_t masked_option =
557 option & ~(MACH_SEND_TRAILER|MACH_RCV_TRAILER_MASK|MACH_RCV_LARGE);
558 int i;
559
560 #if ENABLE_HOTPATH
561 /* BEGINNING OF HOT PATH */
562 if ((masked_option == (MACH_SEND_MSG|MACH_RCV_MSG)) && enable_hotpath) {
563 register thread_t self = current_thread();
564 register mach_msg_format_0_trailer_t *trailer;
565
566 ipc_space_t space = current_act()->task->itk_space;
567 ipc_kmsg_t kmsg;
568 register ipc_port_t dest_port;
569 ipc_object_t rcv_object;
570 register ipc_mqueue_t rcv_mqueue;
571 mach_msg_size_t reply_size;
572 ipc_kmsg_t rcv_kmsg;
573
574 c_mmot_combined_S_R++;
575
576 /*
577 * This case is divided into ten sections, each
578 * with a label. There are five optimized
579 * sections and six unoptimized sections, which
580 * do the same thing but handle all possible
581 * cases and are slower.
582 *
583 * The five sections for an RPC are
584 * 1) Get request message into a buffer.
585 * 2) Copyin request message and rcv_name.
586 * (fast_copyin or slow_copyin)
587 * 3) Enqueue request and dequeue reply.
588 * (fast_send_receive or
589 * slow_send and slow_receive)
590 * 4) Copyout reply message.
591 * (fast_copyout or slow_copyout)
592 * 5) Put reply message to user's buffer.
593 *
594 * Keep the locking hierarchy firmly in mind.
595 * (First spaces, then ports, then port sets,
596 * then message queues.) Only a non-blocking
597 * attempt can be made to acquire locks out of
598 * order, or acquire two locks on the same level.
599 * Acquiring two locks on the same level will
600 * fail if the objects are really the same,
601 * unless simple locking is disabled. This is OK,
602 * because then the extra unlock does nothing.
603 *
604 * There are two major reasons these RPCs can't use
605 * ipc_thread_switch, and use slow_send/slow_receive:
606 * 1) Kernel RPCs.
607 * 2) Servers fall behind clients, so
608 * client doesn't find a blocked server thread and
609 * server finds waiting messages and can't block.
610 */
611
612 mr = ipc_kmsg_get(msg, send_size, &kmsg);
613 if (mr != KERN_SUCCESS) {
614 return mr;
615 }
616 hdr = &kmsg->ikm_header;
617 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
618 send_size);
619
620 fast_copyin:
621 /*
622 * optimized ipc_kmsg_copyin/ipc_mqueue_copyin
623 *
624 * We have the request message data in kmsg.
625 * Must still do copyin, send, receive, etc.
626 *
627 * If the message isn't simple, we can't combine
628 * ipc_kmsg_copyin_header and ipc_mqueue_copyin,
629 * because copyin of the message body might
630 * affect rcv_name.
631 */
632
633 switch (hdr->msgh_bits) {
634 case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
635 MACH_MSG_TYPE_MAKE_SEND_ONCE): {
636 register ipc_entry_t table;
637 register ipc_entry_num_t size;
638 register ipc_port_t reply_port;
639
640 /* sending a request message */
641
642 {
643 register mach_port_index_t index;
644 register mach_port_gen_t gen;
645
646 {
647 register mach_port_name_t reply_name =
648 (mach_port_name_t)hdr->msgh_local_port;
649
650 if (reply_name != rcv_name) {
651 HOT(c_mmot_g_slow_copyin3++);
652 goto slow_copyin;
653 }
654
655 /* optimized ipc_entry_lookup of reply_name */
656
657 index = MACH_PORT_INDEX(reply_name);
658 gen = MACH_PORT_GEN(reply_name);
659
660 is_read_lock(space);
661 assert(space->is_active);
662
663 size = space->is_table_size;
664 table = space->is_table;
665
666 {
667 register ipc_entry_t entry;
668 register ipc_entry_bits_t bits;
669
670 if (index < size) {
671 entry = &table[index];
672 bits = entry->ie_bits;
673 if (IE_BITS_GEN(bits) != gen ||
674 (bits & IE_BITS_COLLISION)) {
675 entry = IE_NULL;
676 }
677 } else {
678 entry = IE_NULL;
679 }
680 if (entry == IE_NULL) {
681 entry = ipc_entry_lookup(space, reply_name);
682 if (entry == IE_NULL) {
683 HOT(c_mmot_cold_006++);
684 goto abort_request_copyin;
685 }
686 bits = entry->ie_bits;
687 }
688
689 /* check type bit */
690
691 if (! (bits & MACH_PORT_TYPE_RECEIVE)) {
692 HOT(c_mmot_cold_007++);
693 goto abort_request_copyin;
694 }
695
696 reply_port = (ipc_port_t) entry->ie_object;
697 assert(reply_port != IP_NULL);
698 }
699 }
700 }
701
702 /* optimized ipc_entry_lookup of dest_name */
703
704 {
705 register mach_port_index_t index;
706 register mach_port_gen_t gen;
707
708 {
709 register mach_port_name_t dest_name =
710 (mach_port_name_t)hdr->msgh_remote_port;
711
712 index = MACH_PORT_INDEX(dest_name);
713 gen = MACH_PORT_GEN(dest_name);
714
715 {
716 register ipc_entry_t entry;
717 register ipc_entry_bits_t bits;
718
719 if (index < size) {
720 entry = &table[index];
721 bits = entry->ie_bits;
722 if (IE_BITS_GEN(bits) != gen ||
723 (bits & IE_BITS_COLLISION)) {
724 entry = IE_NULL;
725 }
726 } else {
727 entry = IE_NULL;
728 }
729 if (entry == IE_NULL) {
730 entry = ipc_entry_lookup(space, dest_name);
731 if (entry == IE_NULL) {
732 HOT(c_mmot_cold_008++);
733 goto abort_request_copyin;
734 }
735 bits = entry->ie_bits;
736 }
737
738 /* check type bit */
739
740 if (! (bits & MACH_PORT_TYPE_SEND)) {
741 HOT(c_mmot_cold_009++);
742 goto abort_request_copyin;
743 }
744
745 assert(IE_BITS_UREFS(bits) > 0);
746
747 dest_port = (ipc_port_t) entry->ie_object;
748 assert(dest_port != IP_NULL);
749 }
750 }
751 }
752
753 /*
754 * To do an atomic copyin, need simultaneous
755 * locks on both ports and the space. If
756 * dest_port == reply_port, and simple locking is
757 * enabled, then we will abort. Otherwise it's
758 * OK to unlock twice.
759 */
760
761 ip_lock(dest_port);
762 if (!ip_active(dest_port) ||
763 !ip_lock_try(reply_port)) {
764 ip_unlock(dest_port);
765 HOT(c_mmot_cold_010++);
766 goto abort_request_copyin;
767 }
768 is_read_unlock(space);
769
770 assert(dest_port->ip_srights > 0);
771 dest_port->ip_srights++;
772 ip_reference(dest_port);
773
774 assert(ip_active(reply_port));
775 assert(reply_port->ip_receiver_name ==
776 (mach_port_name_t)hdr->msgh_local_port);
777 assert(reply_port->ip_receiver == space);
778
779 reply_port->ip_sorights++;
780 ip_reference(reply_port);
781
782 hdr->msgh_bits =
783 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
784 MACH_MSG_TYPE_PORT_SEND_ONCE);
785 hdr->msgh_remote_port = dest_port;
786 hdr->msgh_local_port = reply_port;
787
788 /* make sure we can queue to the destination */
789
790 if (dest_port->ip_receiver == ipc_space_kernel) {
791 /*
792 * The kernel server has a reference to
793 * the reply port, which it hands back
794 * to us in the reply message. We do
795 * not need to keep another reference to
796 * it.
797 */
798 ip_unlock(reply_port);
799
800 assert(ip_active(dest_port));
801 dest_port->ip_messages.imq_seqno++;
802 ip_unlock(dest_port);
803 goto kernel_send;
804 }
805
806 if (imq_full(&dest_port->ip_messages)) {
807 HOT(c_mmot_cold_013++);
808 goto abort_request_send_receive;
809 }
810
811 /* optimized ipc_mqueue_copyin */
812
813 rcv_object = (ipc_object_t) reply_port;
814 io_reference(rcv_object);
815 rcv_mqueue = &reply_port->ip_messages;
816 io_unlock(rcv_object);
817 HOT(c_mmot_hot_fSR_ok++);
818 goto fast_send_receive;
819
820 abort_request_copyin:
821 is_read_unlock(space);
822 goto slow_copyin;
823
824 abort_request_send_receive:
825 ip_unlock(dest_port);
826 ip_unlock(reply_port);
827 goto slow_send;
828 }
829
830 case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
831 register ipc_entry_num_t size;
832 register ipc_entry_t table;
833
834 /* sending a reply message */
835
836 {
837 register mach_port_name_t reply_name =
838 (mach_port_name_t)hdr->msgh_local_port;
839
840 if (reply_name != MACH_PORT_NULL) {
841 HOT(c_mmot_cold_018++);
842 goto slow_copyin;
843 }
844 }
845
846 is_write_lock(space);
847 assert(space->is_active);
848
849 /* optimized ipc_entry_lookup */
850
851 size = space->is_table_size;
852 table = space->is_table;
853
854 {
855 register ipc_entry_t entry;
856 register mach_port_gen_t gen;
857 register mach_port_index_t index;
858 ipc_table_index_t *requests;
859
860 {
861 register mach_port_name_t dest_name =
862 (mach_port_name_t)hdr->msgh_remote_port;
863
864 index = MACH_PORT_INDEX(dest_name);
865 gen = MACH_PORT_GEN(dest_name);
866 }
867
868 if (index >= size) {
869 HOT(c_mmot_cold_019++);
870 goto abort_reply_dest_copyin;
871 }
872
873 entry = &table[index];
874
875 /* check generation, collision bit, and type bit */
876
877 if ((entry->ie_bits & (IE_BITS_GEN_MASK|
878 IE_BITS_COLLISION|
879 MACH_PORT_TYPE_SEND_ONCE)) !=
880 (gen | MACH_PORT_TYPE_SEND_ONCE)) {
881 HOT(c_mmot_cold_020++);
882 goto abort_reply_dest_copyin;
883 }
884
885 /* optimized ipc_right_copyin */
886
887 assert(IE_BITS_TYPE(entry->ie_bits) ==
888 MACH_PORT_TYPE_SEND_ONCE);
889 assert(IE_BITS_UREFS(entry->ie_bits) == 1);
890
891 if (entry->ie_request != 0) {
892 HOT(c_mmot_cold_021++);
893 goto abort_reply_dest_copyin;
894 }
895
896 dest_port = (ipc_port_t) entry->ie_object;
897 assert(dest_port != IP_NULL);
898
899 ip_lock(dest_port);
900 if (!ip_active(dest_port)) {
901 ip_unlock(dest_port);
902 HOT(c_mmot_cold_022++);
903 goto abort_reply_dest_copyin;
904 }
905
906 assert(dest_port->ip_sorights > 0);
907
908 /* optimized ipc_entry_dealloc */
909
910
911 entry->ie_bits = gen;
912 entry->ie_next = table->ie_next;
913 table->ie_next = index;
914 entry->ie_object = IO_NULL;
915 }
916
917 hdr->msgh_bits =
918 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
919 0);
920 hdr->msgh_remote_port = dest_port;
921
922 /* make sure we can queue to the destination */
923
924 assert(dest_port->ip_receiver != ipc_space_kernel);
925
926 /* optimized ipc_entry_lookup/ipc_mqueue_copyin */
927
928 {
929 register ipc_entry_t entry;
930 register ipc_entry_bits_t bits;
931
932 {
933 register mach_port_index_t index;
934 register mach_port_gen_t gen;
935
936 index = MACH_PORT_INDEX(rcv_name);
937 gen = MACH_PORT_GEN(rcv_name);
938
939 if (index < size) {
940 entry = &table[index];
941 bits = entry->ie_bits;
942 if (IE_BITS_GEN(bits) != gen ||
943 (bits & IE_BITS_COLLISION)) {
944 entry = IE_NULL;
945 }
946 } else {
947 entry = IE_NULL;
948 }
949 if (entry == IE_NULL) {
950 entry = ipc_entry_lookup(space, rcv_name);
951 if (entry == IE_NULL) {
952 HOT(c_mmot_cold_024++);
953 goto abort_reply_rcv_copyin;
954 }
955 bits = entry->ie_bits;
956 }
957
958 }
959
960 /* check type bits; looking for receive or set */
961 #if 0
962 /*
963 * JMM - The check below for messages in the receive
964 * mqueue is insufficient to work with port sets, since
965 * the messages stay in the port queues. For now, don't
966 * allow portsets (but receiving on portsets when sending
967 * a message to a send-once right is actually a very
968 * common case (so we should re-enable).
969 */
970 if (bits & MACH_PORT_TYPE_PORT_SET) {
971 register ipc_pset_t rcv_pset;
972
973 rcv_pset = (ipc_pset_t) entry->ie_object;
974 assert(rcv_pset != IPS_NULL);
975
976 ips_lock(rcv_pset);
977 assert(ips_active(rcv_pset));
978
979 rcv_object = (ipc_object_t) rcv_pset;
980 rcv_mqueue = &rcv_pset->ips_messages;
981 } else
982 #endif /* 0 */
983 if (bits & MACH_PORT_TYPE_RECEIVE) {
984 register ipc_port_t rcv_port;
985
986 rcv_port = (ipc_port_t) entry->ie_object;
987 assert(rcv_port != IP_NULL);
988
989 if (!ip_lock_try(rcv_port)) {
990 HOT(c_mmot_cold_025++);
991 goto abort_reply_rcv_copyin;
992 }
993 assert(ip_active(rcv_port));
994
995 if (rcv_port->ip_pset_count != 0) {
996 ip_unlock(rcv_port);
997 HOT(c_mmot_cold_026++);
998 goto abort_reply_rcv_copyin;
999 }
1000
1001 rcv_object = (ipc_object_t) rcv_port;
1002 rcv_mqueue = &rcv_port->ip_messages;
1003 } else {
1004 HOT(c_mmot_cold_027++);
1005 goto abort_reply_rcv_copyin;
1006 }
1007 }
1008
1009 is_write_unlock(space);
1010 io_reference(rcv_object);
1011 io_unlock(rcv_object);
1012 HOT(c_mmot_hot_fSR_ok++);
1013 goto fast_send_receive;
1014
1015 abort_reply_dest_copyin:
1016 is_write_unlock(space);
1017 HOT(c_mmot_cold_029++);
1018 goto slow_copyin;
1019
1020 abort_reply_rcv_copyin:
1021 ip_unlock(dest_port);
1022 is_write_unlock(space);
1023 HOT(c_mmot_cold_030++);
1024 goto slow_send;
1025 }
1026
1027 default:
1028 HOT(c_mmot_cold_031++);
1029 goto slow_copyin;
1030 }
1031 /*NOTREACHED*/
1032
1033 fast_send_receive:
1034 /*
1035 * optimized ipc_mqueue_send/ipc_mqueue_receive
1036 *
1037 * Finished get/copyin of kmsg and copyin of rcv_name.
1038 * space is unlocked, dest_port is locked,
1039 * we can queue kmsg to dest_port,
1040 * rcv_mqueue is set, and rcv_object holds a ref
1041 * so the mqueue cannot go away.
1042 *
1043 * JMM - For now, rcv_object is just a port. Portsets
1044 * are disabled for the time being.
1045 */
1046
1047 assert(ip_active(dest_port));
1048 assert(dest_port->ip_receiver != ipc_space_kernel);
1049 // assert(!imq_full(&dest_port->ip_messages) ||
1050 // (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1051 // MACH_MSG_TYPE_PORT_SEND_ONCE));
1052 assert((hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR) == 0);
1053
1054 {
1055 register ipc_mqueue_t dest_mqueue;
1056 wait_queue_t waitq;
1057 thread_t receiver;
1058 processor_t processor;
1059 spl_t s;
1060
1061 s = splsched();
1062 processor = current_processor();
1063 if (processor->current_pri >= BASEPRI_RTQUEUES)
1064 goto abort_send_receive1;
1065
1066 dest_mqueue = &dest_port->ip_messages;
1067 waitq = &dest_mqueue->imq_wait_queue;
1068 imq_lock(dest_mqueue);
1069
1070 wait_queue_peek64_locked(waitq, IPC_MQUEUE_RECEIVE, &receiver, &waitq);
1071 /* queue still locked, thread locked - but still on q */
1072
1073 if ( receiver == THREAD_NULL ) {
1074 abort_send_receive:
1075 imq_unlock(dest_mqueue);
1076 abort_send_receive1:
1077 splx(s);
1078 ip_unlock(dest_port);
1079 ipc_object_release(rcv_object);
1080 HOT(c_mmot_cold_032++);
1081 goto slow_send;
1082 }
1083
1084 assert(receiver->wait_queue == waitq);
1085 assert(receiver->wait_event == IPC_MQUEUE_RECEIVE);
1086
1087 /*
1088 * Make sure that the scheduling state of the receiver is such
1089 * that we can handoff to it here. If not, fall off.
1090 *
1091 * JMM - We have an opportunity here. If the thread is locked
1092 * and we find it runnable, it may still be trying to get into
1093 * thread_block on itself. We could just "hand him the message"
1094 * and let him go (thread_go_locked()) and then fall down into a
1095 * slow receive for ourselves. Only his RECEIVE_TOO_LARGE handling
1096 * runs afoul of that. Clean this up!
1097 */
1098 if ((receiver->state & (TH_RUN|TH_WAIT)) != TH_WAIT ||
1099 receiver->sched_pri >= BASEPRI_RTQUEUES ||
1100 receiver->processor_set != processor->processor_set ||
1101 (receiver->bound_processor != PROCESSOR_NULL &&
1102 receiver->bound_processor != processor)) {
1103 HOT(c_mmot_cold_033++);
1104 fall_off:
1105 thread_unlock(receiver);
1106 if (waitq != &dest_mqueue->imq_wait_queue)
1107 wait_queue_unlock(waitq);
1108 goto abort_send_receive;
1109 }
1110
1111 /*
1112 * Check that the receiver can stay on the hot path.
1113 */
1114 if (send_size + REQUESTED_TRAILER_SIZE(receiver->ith_option) >
1115 receiver->ith_msize) {
1116 /*
1117 * The receiver can't accept the message.
1118 */
1119 HOT(c_mmot_bad_rcvr++);
1120 goto fall_off;
1121 }
1122
1123 /*
1124 * Before committing to the handoff, make sure that we are
1125 * really going to block (i.e. there are no messages already
1126 * queued for us. This violates lock ordering, so make sure
1127 * we don't deadlock. After the trylock succeeds below, we
1128 * may have up to 3 message queues locked:
1129 * - the dest port mqueue
1130 * - a portset mqueue (where waiting receiver was found)
1131 * - finally our own rcv_mqueue
1132 *
1133 * JMM - Need to make this check appropriate for portsets as
1134 * well before re-enabling them.
1135 */
1136 if (!imq_lock_try(rcv_mqueue)) {
1137 goto fall_off;
1138 }
1139 if (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL) {
1140 imq_unlock(rcv_mqueue);
1141 HOT(c_mmot_cold_033++);
1142 goto fall_off;
1143 }
1144
1145 /* At this point we are committed to do the "handoff". */
1146 c_mach_msg_trap_switch_fast++;
1147
1148 /*
1149 * JMM - Go ahead and pull the receiver from the runq. If the
1150 * runq wasn't the one for the mqueue, unlock it.
1151 */
1152 wait_queue_pull_thread_locked(waitq,
1153 receiver,
1154 (waitq != &dest_mqueue->imq_wait_queue));
1155
1156 /*
1157 * Store the kmsg and seqno where the receiver can pick it up.
1158 */
1159 receiver->ith_state = MACH_MSG_SUCCESS;
1160 receiver->ith_kmsg = kmsg;
1161 receiver->ith_seqno = dest_mqueue->imq_seqno++;
1162
1163 /*
1164 * Update the scheduling state for the handoff.
1165 */
1166 receiver->state &= ~(TH_WAIT|TH_UNINT);
1167 receiver->state |= TH_RUN;
1168
1169 pset_run_incr(receiver->processor_set);
1170 if (receiver->sched_mode & TH_MODE_TIMESHARE)
1171 pset_share_incr(receiver->processor_set);
1172
1173 receiver->wait_result = THREAD_AWAKENED;
1174
1175 receiver->computation_metered = 0;
1176 receiver->reason = AST_NONE;
1177
1178 thread_unlock(receiver);
1179
1180 imq_unlock(dest_mqueue);
1181 ip_unlock(dest_port);
1182 current_task()->messages_sent++;
1183
1184
1185 /*
1186 * Put self on receive port's queue.
1187 * Also save state that the sender of
1188 * our reply message needs to determine if it
1189 * can hand off directly back to us.
1190 */
1191 thread_lock(self);
1192 self->ith_msg = (rcv_msg) ? rcv_msg : msg;
1193 self->ith_object = rcv_object; /* still holds reference */
1194 self->ith_msize = rcv_size;
1195 self->ith_option = option;
1196 self->ith_scatter_list_size = scatter_list_size;
1197 self->ith_continuation = thread_syscall_return;
1198
1199 waitq = &rcv_mqueue->imq_wait_queue;
1200 (void)wait_queue_assert_wait64_locked(waitq,
1201 IPC_MQUEUE_RECEIVE,
1202 THREAD_ABORTSAFE,
1203 self);
1204 thread_unlock(self);
1205 imq_unlock(rcv_mqueue);
1206
1207 /*
1208 * Switch directly to receiving thread, and block
1209 * this thread as though it had called ipc_mqueue_receive.
1210 */
1211 thread_run(self, ipc_mqueue_receive_continue, receiver);
1212 /* NOTREACHED */
1213 }
1214
1215 fast_copyout:
1216 /*
1217 * Nothing locked and no references held, except
1218 * we have kmsg with msgh_seqno filled in. Must
1219 * still check against rcv_size and do
1220 * ipc_kmsg_copyout/ipc_kmsg_put.
1221 */
1222
1223 reply_size = send_size + trailer->msgh_trailer_size;
1224 if (rcv_size < reply_size) {
1225 HOT(c_mmot_g_slow_copyout6++);
1226 goto slow_copyout;
1227 }
1228
1229 /* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */
1230
1231 switch (hdr->msgh_bits) {
1232 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
1233 MACH_MSG_TYPE_PORT_SEND_ONCE): {
1234 ipc_port_t reply_port =
1235 (ipc_port_t) hdr->msgh_local_port;
1236 mach_port_name_t dest_name, reply_name;
1237
1238 /* receiving a request message */
1239
1240 if (!IP_VALID(reply_port)) {
1241 HOT(c_mmot_g_slow_copyout5++);
1242 goto slow_copyout;
1243 }
1244
1245 is_write_lock(space);
1246 assert(space->is_active);
1247
1248 /*
1249 * To do an atomic copyout, need simultaneous
1250 * locks on both ports and the space. If
1251 * dest_port == reply_port, and simple locking is
1252 * enabled, then we will abort. Otherwise it's
1253 * OK to unlock twice.
1254 */
1255
1256 ip_lock(dest_port);
1257 if (!ip_active(dest_port) ||
1258 !ip_lock_try(reply_port)) {
1259 HOT(c_mmot_cold_037++);
1260 goto abort_request_copyout;
1261 }
1262
1263 if (!ip_active(reply_port)) {
1264 ip_unlock(reply_port);
1265 HOT(c_mmot_cold_038++);
1266 goto abort_request_copyout;
1267 }
1268
1269 assert(reply_port->ip_sorights > 0);
1270 ip_unlock(reply_port);
1271
1272 {
1273 register ipc_entry_t table;
1274 register ipc_entry_t entry;
1275 register mach_port_index_t index;
1276
1277 /* optimized ipc_entry_get */
1278
1279 table = space->is_table;
1280 index = table->ie_next;
1281
1282 if (index == 0) {
1283 HOT(c_mmot_cold_039++);
1284 goto abort_request_copyout;
1285 }
1286
1287 entry = &table[index];
1288 table->ie_next = entry->ie_next;
1289 entry->ie_request = 0;
1290
1291 {
1292 register mach_port_gen_t gen;
1293
1294 assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
1295 gen = IE_BITS_NEW_GEN(entry->ie_bits);
1296
1297 reply_name = MACH_PORT_MAKE(index, gen);
1298
1299 /* optimized ipc_right_copyout */
1300
1301 entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
1302 }
1303
1304 assert(MACH_PORT_VALID(reply_name));
1305 entry->ie_object = (ipc_object_t) reply_port;
1306 is_write_unlock(space);
1307 }
1308
1309 /* optimized ipc_object_copyout_dest */
1310
1311 assert(dest_port->ip_srights > 0);
1312 ip_release(dest_port);
1313
1314 if (dest_port->ip_receiver == space)
1315 dest_name = dest_port->ip_receiver_name;
1316 else
1317 dest_name = MACH_PORT_NULL;
1318
1319 if ((--dest_port->ip_srights == 0) &&
1320 (dest_port->ip_nsrequest != IP_NULL)) {
1321 ipc_port_t nsrequest;
1322 mach_port_mscount_t mscount;
1323
1324 /* a rather rare case */
1325
1326 nsrequest = dest_port->ip_nsrequest;
1327 mscount = dest_port->ip_mscount;
1328 dest_port->ip_nsrequest = IP_NULL;
1329 ip_unlock(dest_port);
1330 ipc_notify_no_senders(nsrequest, mscount);
1331 } else
1332 ip_unlock(dest_port);
1333
1334 hdr->msgh_bits =
1335 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
1336 MACH_MSG_TYPE_PORT_SEND);
1337 hdr->msgh_remote_port = (mach_port_t)reply_name;
1338 hdr->msgh_local_port = (mach_port_t)dest_name;
1339 HOT(c_mmot_hot_ok1++);
1340 goto fast_put;
1341
1342 abort_request_copyout:
1343 ip_unlock(dest_port);
1344 is_write_unlock(space);
1345 HOT(c_mmot_g_slow_copyout4++);
1346 goto slow_copyout;
1347 }
1348
1349 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1350 register mach_port_name_t dest_name;
1351
1352 /* receiving a reply message */
1353
1354 ip_lock(dest_port);
1355 if (!ip_active(dest_port)) {
1356 ip_unlock(dest_port);
1357 HOT(c_mmot_g_slow_copyout3++);
1358 goto slow_copyout;
1359 }
1360
1361 /* optimized ipc_object_copyout_dest */
1362
1363 assert(dest_port->ip_sorights > 0);
1364
1365 if (dest_port->ip_receiver == space) {
1366 ip_release(dest_port);
1367 dest_port->ip_sorights--;
1368 dest_name = dest_port->ip_receiver_name;
1369 ip_unlock(dest_port);
1370 } else {
1371 ip_unlock(dest_port);
1372
1373 ipc_notify_send_once(dest_port);
1374 dest_name = MACH_PORT_NULL;
1375 }
1376
1377 hdr->msgh_bits = MACH_MSGH_BITS(0,
1378 MACH_MSG_TYPE_PORT_SEND_ONCE);
1379 hdr->msgh_remote_port = MACH_PORT_NULL;
1380 hdr->msgh_local_port = (ipc_port_t)dest_name;
1381 HOT(c_mmot_hot_ok2++);
1382 goto fast_put;
1383 }
1384
1385 case MACH_MSGH_BITS_COMPLEX|
1386 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1387 register mach_port_name_t dest_name;
1388
1389 /* receiving a complex reply message */
1390
1391 ip_lock(dest_port);
1392 if (!ip_active(dest_port)) {
1393 ip_unlock(dest_port);
1394 HOT(c_mmot_g_slow_copyout1++);
1395 goto slow_copyout;
1396 }
1397
1398 /* optimized ipc_object_copyout_dest */
1399
1400 assert(dest_port->ip_sorights > 0);
1401
1402 if (dest_port->ip_receiver == space) {
1403 ip_release(dest_port);
1404 dest_port->ip_sorights--;
1405 dest_name = dest_port->ip_receiver_name;
1406 ip_unlock(dest_port);
1407 } else {
1408 ip_unlock(dest_port);
1409
1410 ipc_notify_send_once(dest_port);
1411 dest_name = MACH_PORT_NULL;
1412 }
1413
1414 hdr->msgh_bits =
1415 MACH_MSGH_BITS_COMPLEX |
1416 MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND_ONCE);
1417 hdr->msgh_remote_port = MACH_PORT_NULL;
1418 hdr->msgh_local_port = (mach_port_t)dest_name;
1419
1420 mr = ipc_kmsg_copyout_body(kmsg, space,
1421 current_map(),
1422 MACH_MSG_BODY_NULL);
1423 if (mr != MACH_MSG_SUCCESS) {
1424 if (ipc_kmsg_put(msg, kmsg, hdr->msgh_size +
1425 trailer->msgh_trailer_size) ==
1426 MACH_RCV_INVALID_DATA)
1427 return MACH_RCV_INVALID_DATA;
1428 else
1429 return mr | MACH_RCV_BODY_ERROR;
1430 }
1431 HOT(c_mmot_hot_ok3++);
1432 goto fast_put;
1433 }
1434
1435 default:
1436 HOT(c_mmot_g_slow_copyout2++);
1437 goto slow_copyout;
1438 }
1439 /*NOTREACHED*/
1440
1441 fast_put:
1442 mr = ipc_kmsg_put(rcv_msg ? rcv_msg : msg,
1443 kmsg,
1444 hdr->msgh_size + trailer->msgh_trailer_size);
1445 if (mr != MACH_MSG_SUCCESS) {
1446 return MACH_RCV_INVALID_DATA;
1447 }
1448 current_task()->messages_received++;
1449 return mr;
1450
1451
1452 /* BEGINNING OF WARM PATH */
1453
1454 /*
1455 * The slow path has a few non-register temporary
1456 * variables used only for call-by-reference.
1457 */
1458
1459 slow_copyin:
1460 {
1461 ipc_kmsg_t temp_kmsg;
1462 mach_port_seqno_t temp_seqno;
1463 ipc_object_t temp_rcv_object;
1464 ipc_mqueue_t temp_rcv_mqueue;
1465 register mach_port_name_t reply_name =
1466 (mach_port_name_t)hdr->msgh_local_port;
1467
1468
1469 /*
1470 * We have the message data in kmsg, but
1471 * we still need to copyin, send it,
1472 * receive a reply, and do copyout.
1473 */
1474
1475 mr = ipc_kmsg_copyin(kmsg, space, current_map(),
1476 MACH_PORT_NULL);
1477 if (mr != MACH_MSG_SUCCESS) {
1478 ipc_kmsg_free(kmsg);
1479 return(mr);
1480 }
1481
1482 /* try to get back on optimized path */
1483
1484 if ((reply_name != rcv_name) ||
1485 (hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR)) {
1486 HOT(c_mmot_cold_048++);
1487 goto slow_send;
1488 }
1489
1490 dest_port = (ipc_port_t) hdr->msgh_remote_port;
1491 assert(IP_VALID(dest_port));
1492
1493 ip_lock(dest_port);
1494 if (!ip_active(dest_port)) {
1495 ip_unlock(dest_port);
1496 goto slow_send;
1497 }
1498
1499 if (dest_port->ip_receiver == ipc_space_kernel) {
1500 dest_port->ip_messages.imq_seqno++;
1501 ip_unlock(dest_port);
1502 goto kernel_send;
1503 }
1504
1505 if (!imq_full(&dest_port->ip_messages) ||
1506 (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1507 MACH_MSG_TYPE_PORT_SEND_ONCE))
1508 {
1509 /*
1510 * Try an optimized ipc_mqueue_copyin.
1511 * It will work if this is a request message.
1512 */
1513
1514 register ipc_port_t reply_port;
1515
1516 reply_port = (ipc_port_t) hdr->msgh_local_port;
1517 if (IP_VALID(reply_port)) {
1518 if (ip_lock_try(reply_port)) {
1519 if (ip_active(reply_port) &&
1520 reply_port->ip_receiver == space &&
1521 reply_port->ip_receiver_name == rcv_name &&
1522 reply_port->ip_pset_count == 0)
1523 {
1524 /* Grab a reference to the reply port. */
1525 rcv_object = (ipc_object_t) reply_port;
1526 io_reference(rcv_object);
1527 rcv_mqueue = &reply_port->ip_messages;
1528 io_unlock(rcv_object);
1529 HOT(c_mmot_getback_FastSR++);
1530 goto fast_send_receive;
1531 }
1532 ip_unlock(reply_port);
1533 }
1534 }
1535 }
1536
1537 ip_unlock(dest_port);
1538 HOT(c_mmot_cold_050++);
1539 goto slow_send;
1540
1541 kernel_send:
1542 /*
1543 * Special case: send message to kernel services.
1544 * The request message has been copied into the
1545 * kmsg. Nothing is locked.
1546 */
1547
1548 {
1549 register ipc_port_t reply_port;
1550 mach_port_seqno_t local_seqno;
1551 spl_t s;
1552
1553 /*
1554 * Perform the kernel function.
1555 */
1556 c_mmot_kernel_send++;
1557
1558 current_task()->messages_sent++;
1559
1560 kmsg = ipc_kobject_server(kmsg);
1561 if (kmsg == IKM_NULL) {
1562 /*
1563 * No reply. Take the
1564 * slow receive path.
1565 */
1566 HOT(c_mmot_cold_051++);
1567 goto slow_get_rcv_port;
1568 }
1569
1570 /*
1571 * Check that:
1572 * the reply port is alive
1573 * we hold the receive right
1574 * the name has not changed.
1575 * the port is not in a set
1576 * If any of these are not true,
1577 * we cannot directly receive the reply
1578 * message.
1579 */
1580 hdr = &kmsg->ikm_header;
1581 send_size = hdr->msgh_size;
1582 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1583 round_msg(send_size));
1584 reply_port = (ipc_port_t) hdr->msgh_remote_port;
1585 ip_lock(reply_port);
1586
1587 if ((!ip_active(reply_port)) ||
1588 (reply_port->ip_receiver != space) ||
1589 (reply_port->ip_receiver_name != rcv_name) ||
1590 (reply_port->ip_pset_count != 0))
1591 {
1592 ip_unlock(reply_port);
1593 ipc_kmsg_send_always(kmsg);
1594 HOT(c_mmot_cold_052++);
1595 goto slow_get_rcv_port;
1596 }
1597
1598 s = splsched();
1599 rcv_mqueue = &reply_port->ip_messages;
1600 imq_lock(rcv_mqueue);
1601
1602 /* keep port locked, and don`t change ref count yet */
1603
1604 /*
1605 * If there are messages on the port
1606 * or other threads waiting for a message,
1607 * we cannot directly receive the reply.
1608 */
1609 if (!wait_queue_empty(&rcv_mqueue->imq_wait_queue) ||
1610 (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL))
1611 {
1612 imq_unlock(rcv_mqueue);
1613 splx(s);
1614 ip_unlock(reply_port);
1615 ipc_kmsg_send_always(kmsg);
1616 HOT(c_mmot_cold_053++);
1617 goto slow_get_rcv_port;
1618 }
1619
1620 /*
1621 * We can directly receive this reply.
1622 * Since there were no messages queued
1623 * on the reply port, there should be
1624 * no threads blocked waiting to send.
1625 */
1626 dest_port = reply_port;
1627 local_seqno = rcv_mqueue->imq_seqno++;
1628 imq_unlock(rcv_mqueue);
1629 splx(s);
1630
1631 /*
1632 * inline ipc_object_release.
1633 * Port is still locked.
1634 * Reference count was not incremented.
1635 */
1636 ip_check_unlock(reply_port);
1637
1638 if (option & MACH_RCV_TRAILER_MASK) {
1639 trailer->msgh_seqno = local_seqno;
1640 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1641 }
1642 /* copy out the kernel reply */
1643 HOT(c_mmot_fastkernelreply++);
1644 goto fast_copyout;
1645 }
1646
1647 slow_send:
1648 /*
1649 * Nothing is locked. We have acquired kmsg, but
1650 * we still need to send it and receive a reply.
1651 */
1652
1653 mr = ipc_kmsg_send(kmsg, MACH_MSG_OPTION_NONE,
1654 MACH_MSG_TIMEOUT_NONE);
1655 if (mr != MACH_MSG_SUCCESS) {
1656 mr |= ipc_kmsg_copyout_pseudo(kmsg, space,
1657 current_map(),
1658 MACH_MSG_BODY_NULL);
1659
1660 (void) ipc_kmsg_put(msg, kmsg, hdr->msgh_size);
1661 return(mr);
1662 }
1663
1664 slow_get_rcv_port:
1665 /*
1666 * We have sent the message. Copy in the receive port.
1667 */
1668 mr = ipc_mqueue_copyin(space, rcv_name,
1669 &temp_rcv_mqueue, &temp_rcv_object);
1670 if (mr != MACH_MSG_SUCCESS) {
1671 return(mr);
1672 }
1673 rcv_mqueue = temp_rcv_mqueue;
1674 rcv_object = temp_rcv_object;
1675 /* hold ref for rcv_object */
1676
1677 slow_receive:
1678 /*
1679 * Now we have sent the request and copied in rcv_name,
1680 * and hold ref for rcv_object (to keep mqueue alive).
1681 * Just receive a reply and try to get back to fast path.
1682 */
1683
1684 self->ith_continuation = (void (*)(mach_msg_return_t))0;
1685 ipc_mqueue_receive(rcv_mqueue,
1686 MACH_MSG_OPTION_NONE,
1687 MACH_MSG_SIZE_MAX,
1688 MACH_MSG_TIMEOUT_NONE,
1689 THREAD_ABORTSAFE);
1690
1691 mr = self->ith_state;
1692 temp_kmsg = self->ith_kmsg;
1693 temp_seqno = self->ith_seqno;
1694
1695 ipc_object_release(rcv_object);
1696
1697 if (mr != MACH_MSG_SUCCESS) {
1698 return(mr);
1699 }
1700
1701 kmsg = temp_kmsg;
1702 hdr = &kmsg->ikm_header;
1703 send_size = hdr->msgh_size;
1704 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1705 round_msg(send_size));
1706 if (option & MACH_RCV_TRAILER_MASK) {
1707 trailer->msgh_seqno = temp_seqno;
1708 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1709 }
1710 dest_port = (ipc_port_t) hdr->msgh_remote_port;
1711 HOT(c_mmot_cold_055++);
1712 goto fast_copyout;
1713
1714 slow_copyout:
1715 /*
1716 * Nothing locked and no references held, except
1717 * we have kmsg with msgh_seqno filled in. Must
1718 * still check against rcv_size and do
1719 * ipc_kmsg_copyout/ipc_kmsg_put.
1720 */
1721
1722 reply_size = send_size + trailer->msgh_trailer_size;
1723 if (rcv_size < reply_size) {
1724 if (msg_receive_error(kmsg, msg, option, temp_seqno,
1725 space) == MACH_RCV_INVALID_DATA) {
1726 mr = MACH_RCV_INVALID_DATA;
1727 return(mr);
1728 }
1729 else {
1730 mr = MACH_RCV_TOO_LARGE;
1731 return(mr);
1732 }
1733 }
1734
1735 mr = ipc_kmsg_copyout(kmsg, space, current_map(),
1736 MACH_PORT_NULL, MACH_MSG_BODY_NULL);
1737 if (mr != MACH_MSG_SUCCESS) {
1738 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1739 if (ipc_kmsg_put(msg, kmsg, reply_size) ==
1740 MACH_RCV_INVALID_DATA)
1741 mr = MACH_RCV_INVALID_DATA;
1742 }
1743 else {
1744 if (msg_receive_error(kmsg, msg, option,
1745 temp_seqno, space) == MACH_RCV_INVALID_DATA)
1746 mr = MACH_RCV_INVALID_DATA;
1747 }
1748
1749 return(mr);
1750 }
1751
1752 /* try to get back on optimized path */
1753 HOT(c_mmot_getback_fast_put++);
1754 goto fast_put;
1755
1756 /*NOTREACHED*/
1757 }
1758 } /* END OF HOT PATH */
1759 #endif /* ENABLE_HOTPATH */
1760
1761 if (option & MACH_SEND_MSG) {
1762 mr = mach_msg_send(msg, option, send_size,
1763 timeout, notify);
1764 if (mr != MACH_MSG_SUCCESS) {
1765 return mr;
1766 }
1767 }
1768
1769 if (option & MACH_RCV_MSG) {
1770 mach_msg_header_t *rcv;
1771
1772 /*
1773 * 1. MACH_RCV_OVERWRITE is on, and rcv_msg is our scatter list
1774 * and receive buffer
1775 * 2. MACH_RCV_OVERWRITE is off, and rcv_msg might be the
1776 * alternate receive buffer (separate send and receive buffers).
1777 */
1778 if (option & MACH_RCV_OVERWRITE)
1779 rcv = rcv_msg;
1780 else if (rcv_msg != MACH_MSG_NULL)
1781 rcv = rcv_msg;
1782 else
1783 rcv = msg;
1784 mr = mach_msg_receive(rcv, option, rcv_size, rcv_name,
1785 timeout, thread_syscall_return, scatter_list_size);
1786 thread_syscall_return(mr);
1787 }
1788
1789 return MACH_MSG_SUCCESS;
1790 }
1791
1792 /*
1793 * Routine: mach_msg_trap [mach trap]
1794 * Purpose:
1795 * Possibly send a message; possibly receive a message.
1796 * Conditions:
1797 * Nothing locked.
1798 * Returns:
1799 * All of mach_msg_send and mach_msg_receive error codes.
1800 */
1801
1802 mach_msg_return_t
1803 mach_msg_trap(
1804 mach_msg_header_t *msg,
1805 mach_msg_option_t option,
1806 mach_msg_size_t send_size,
1807 mach_msg_size_t rcv_size,
1808 mach_port_name_t rcv_name,
1809 mach_msg_timeout_t timeout,
1810 mach_port_name_t notify)
1811 {
1812 return mach_msg_overwrite_trap(msg,
1813 option,
1814 send_size,
1815 rcv_size,
1816 rcv_name,
1817 timeout,
1818 notify,
1819 (mach_msg_header_t *)0,
1820 (mach_msg_size_t)0);
1821 }
1822
1823
1824 /*
1825 * Routine: msg_receive_error [internal]
1826 * Purpose:
1827 * Builds a minimal header/trailer and copies it to
1828 * the user message buffer. Invoked when in the case of a
1829 * MACH_RCV_TOO_LARGE or MACH_RCV_BODY_ERROR error.
1830 * Conditions:
1831 * Nothing locked.
1832 * Returns:
1833 * MACH_MSG_SUCCESS minimal header/trailer copied
1834 * MACH_RCV_INVALID_DATA copyout to user buffer failed
1835 */
1836
1837 mach_msg_return_t
1838 msg_receive_error(
1839 ipc_kmsg_t kmsg,
1840 mach_msg_header_t *msg,
1841 mach_msg_option_t option,
1842 mach_port_seqno_t seqno,
1843 ipc_space_t space)
1844 {
1845 mach_msg_format_0_trailer_t *trailer;
1846
1847 /*
1848 * Copy out the destination port in the message.
1849 * Destroy all other rights and memory in the message.
1850 */
1851 ipc_kmsg_copyout_dest(kmsg, space);
1852
1853 /*
1854 * Build a minimal message with the requested trailer.
1855 */
1856 trailer = (mach_msg_format_0_trailer_t *)
1857 ((vm_offset_t)&kmsg->ikm_header +
1858 round_msg(sizeof(mach_msg_header_t)));
1859 kmsg->ikm_header.msgh_size = sizeof(mach_msg_header_t);
1860 bcopy( (char *)&trailer_template,
1861 (char *)trailer,
1862 sizeof(trailer_template));
1863 if (option & MACH_RCV_TRAILER_MASK) {
1864 trailer->msgh_seqno = seqno;
1865 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1866 }
1867
1868 /*
1869 * Copy the message to user space
1870 */
1871 if (ipc_kmsg_put(msg, kmsg, kmsg->ikm_header.msgh_size +
1872 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
1873 return(MACH_RCV_INVALID_DATA);
1874 else
1875 return(MACH_MSG_SUCCESS);
1876 }
mirror of XNU