]> git.saurik.com Git - apple/xnu.git/blob - osfmk/ipc/mach_msg.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-792.6.61.tar.gz
[apple/xnu.git] / osfmk / ipc / mach_msg.c
1 /*
2 * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22 /*
23 * @OSF_COPYRIGHT@
24 */
25 /*
26 * Mach Operating System
27 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
28 * All Rights Reserved.
29 *
30 * Permission to use, copy, modify and distribute this software and its
31 * documentation is hereby granted, provided that both the copyright
32 * notice and this permission notice appear in all copies of the
33 * software, derivative works or modified versions, and any portions
34 * thereof, and that both notices appear in supporting documentation.
35 *
36 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
39 *
40 * Carnegie Mellon requests users of this software to return to
41 *
42 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
43 * School of Computer Science
44 * Carnegie Mellon University
45 * Pittsburgh PA 15213-3890
46 *
47 * any improvements or extensions that they make and grant Carnegie Mellon
48 * the rights to redistribute these changes.
49 */
50 /*
51 */
52 /*
53 * File: ipc/mach_msg.c
54 * Author: Rich Draves
55 * Date: 1989
56 *
57 * Exported message traps. See mach/message.h.
58 */
59
60 #include <mach/mach_types.h>
61 #include <mach/kern_return.h>
62 #include <mach/port.h>
63 #include <mach/message.h>
64 #include <mach/mig_errors.h>
65 #include <mach/mach_traps.h>
66
67 #include <kern/kern_types.h>
68 #include <kern/assert.h>
69 #include <kern/counters.h>
70 #include <kern/cpu_number.h>
71 #include <kern/ipc_kobject.h>
72 #include <kern/ipc_mig.h>
73 #include <kern/task.h>
74 #include <kern/thread.h>
75 #include <kern/lock.h>
76 #include <kern/sched_prim.h>
77 #include <kern/exception.h>
78 #include <kern/misc_protos.h>
79 #include <kern/kalloc.h>
80 #include <kern/processor.h>
81 #include <kern/syscall_subr.h>
82
83 #include <vm/vm_map.h>
84
85 #include <ipc/ipc_types.h>
86 #include <ipc/ipc_kmsg.h>
87 #include <ipc/ipc_mqueue.h>
88 #include <ipc/ipc_object.h>
89 #include <ipc/ipc_notify.h>
90 #include <ipc/ipc_port.h>
91 #include <ipc/ipc_pset.h>
92 #include <ipc/ipc_space.h>
93 #include <ipc/ipc_entry.h>
94
95 #include <machine/machine_routines.h>
96
97 #include <sys/kdebug.h>
98
99 #ifndef offsetof
100 #define offsetof(type, member) ((size_t)(&((type *)0)->member))
101 #endif /* offsetof */
102
103 /*
104 * Forward declarations - kernel internal routines
105 */
106
107 mach_msg_return_t mach_msg_send(
108 mach_msg_header_t *msg,
109 mach_msg_option_t option,
110 mach_msg_size_t send_size,
111 mach_msg_timeout_t send_timeout,
112 mach_port_name_t notify);
113
114 mach_msg_return_t mach_msg_receive(
115 mach_msg_header_t *msg,
116 mach_msg_option_t option,
117 mach_msg_size_t rcv_size,
118 mach_port_name_t rcv_name,
119 mach_msg_timeout_t rcv_timeout,
120 void (*continuation)(mach_msg_return_t),
121 mach_msg_size_t slist_size);
122
123
124 mach_msg_return_t mach_msg_receive_results(void);
125
126 mach_msg_return_t msg_receive_error(
127 ipc_kmsg_t kmsg,
128 mach_vm_address_t msg_addr,
129 mach_msg_option_t option,
130 mach_port_seqno_t seqno,
131 ipc_space_t space);
132
133 security_token_t KERNEL_SECURITY_TOKEN = KERNEL_SECURITY_TOKEN_VALUE;
134 audit_token_t KERNEL_AUDIT_TOKEN = KERNEL_AUDIT_TOKEN_VALUE;
135
136 mach_msg_format_0_trailer_t trailer_template = {
137 /* mach_msg_trailer_type_t */ MACH_MSG_TRAILER_FORMAT_0,
138 /* mach_msg_trailer_size_t */ MACH_MSG_TRAILER_MINIMUM_SIZE,
139 /* mach_port_seqno_t */ 0,
140 /* security_token_t */ KERNEL_SECURITY_TOKEN_VALUE
141 };
142
143 /*
144 * Routine: mach_msg_send
145 * Purpose:
146 * Send a message.
147 * Conditions:
148 * Nothing locked.
149 * Returns:
150 * MACH_MSG_SUCCESS Sent the message.
151 * MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
152 * MACH_SEND_NO_BUFFER Couldn't allocate buffer.
153 * MACH_SEND_INVALID_DATA Couldn't copy message data.
154 * MACH_SEND_INVALID_HEADER
155 * Illegal value in the message header bits.
156 * MACH_SEND_INVALID_DEST The space is dead.
157 * MACH_SEND_INVALID_NOTIFY Bad notify port.
158 * MACH_SEND_INVALID_DEST Can't copyin destination port.
159 * MACH_SEND_INVALID_REPLY Can't copyin reply port.
160 * MACH_SEND_TIMED_OUT Timeout expired without delivery.
161 * MACH_SEND_INTERRUPTED Delivery interrupted.
162 * MACH_SEND_NO_NOTIFY Can't allocate a msg-accepted request.
163 * MACH_SEND_WILL_NOTIFY Msg-accepted notif. requested.
164 * MACH_SEND_NOTIFY_IN_PROGRESS
165 * This space has already forced a message to this port.
166 */
167
168 mach_msg_return_t
169 mach_msg_send(
170 mach_msg_header_t *msg,
171 mach_msg_option_t option,
172 mach_msg_size_t send_size,
173 mach_msg_timeout_t send_timeout,
174 mach_port_name_t notify)
175 {
176 ipc_space_t space = current_space();
177 vm_map_t map = current_map();
178 ipc_kmsg_t kmsg;
179 mach_msg_return_t mr;
180 mach_msg_size_t msg_and_trailer_size;
181 mach_msg_max_trailer_t *trailer;
182
183 if ((send_size < sizeof(mach_msg_header_t)) || (send_size & 3))
184 return MACH_SEND_MSG_TOO_SMALL;
185
186 msg_and_trailer_size = send_size + MAX_TRAILER_SIZE;
187
188 kmsg = ipc_kmsg_alloc(msg_and_trailer_size);
189
190 if (kmsg == IKM_NULL)
191 return MACH_SEND_NO_BUFFER;
192
193 (void) memcpy((void *) kmsg->ikm_header, (const void *) msg, send_size);
194
195 kmsg->ikm_header->msgh_size = send_size;
196
197 /*
198 * reserve for the trailer the largest space (MAX_TRAILER_SIZE)
199 * However, the internal size field of the trailer (msgh_trailer_size)
200 * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to optimize
201 * the cases where no implicit data is requested.
202 */
203 trailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + send_size);
204 trailer->msgh_sender = current_thread()->task->sec_token;
205 trailer->msgh_audit = current_thread()->task->audit_token;
206 trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
207 trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
208
209 if (option & MACH_SEND_CANCEL) {
210 if (notify == MACH_PORT_NULL)
211 mr = MACH_SEND_INVALID_NOTIFY;
212 else
213 mr = ipc_kmsg_copyin(kmsg, space, map, notify);
214 } else
215 mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
216 if (mr != MACH_MSG_SUCCESS) {
217 ipc_kmsg_free(kmsg);
218 return mr;
219 }
220
221 mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, send_timeout);
222
223 if (mr != MACH_MSG_SUCCESS) {
224 mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
225 (void) memcpy((void *) msg, (const void *) kmsg->ikm_header,
226 kmsg->ikm_header->msgh_size);
227 ipc_kmsg_free(kmsg);
228 }
229
230 return mr;
231 }
232
233 /*
234 * Routine: mach_msg_receive
235 * Purpose:
236 * Receive a message.
237 * Conditions:
238 * Nothing locked.
239 * Returns:
240 * MACH_MSG_SUCCESS Received a message.
241 * MACH_RCV_INVALID_NAME The name doesn't denote a right,
242 * or the denoted right is not receive or port set.
243 * MACH_RCV_IN_SET Receive right is a member of a set.
244 * MACH_RCV_TOO_LARGE Message wouldn't fit into buffer.
245 * MACH_RCV_TIMED_OUT Timeout expired without a message.
246 * MACH_RCV_INTERRUPTED Reception interrupted.
247 * MACH_RCV_PORT_DIED Port/set died while receiving.
248 * MACH_RCV_PORT_CHANGED Port moved into set while receiving.
249 * MACH_RCV_INVALID_DATA Couldn't copy to user buffer.
250 * MACH_RCV_INVALID_NOTIFY Bad notify port.
251 * MACH_RCV_HEADER_ERROR
252 */
253
254 mach_msg_return_t
255 mach_msg_receive_results(void)
256 {
257 thread_t self = current_thread();
258 ipc_space_t space = current_space();
259 vm_map_t map = current_map();
260
261 ipc_object_t object = self->ith_object;
262 mach_msg_return_t mr = self->ith_state;
263 mach_vm_address_t msg_addr = self->ith_msg_addr;
264 mach_msg_option_t option = self->ith_option;
265 ipc_kmsg_t kmsg = self->ith_kmsg;
266 mach_port_seqno_t seqno = self->ith_seqno;
267
268 mach_msg_format_0_trailer_t *trailer;
269
270 ipc_object_release(object);
271
272 if (mr != MACH_MSG_SUCCESS) {
273
274 if (mr == MACH_RCV_TOO_LARGE ) {
275 if (option & MACH_RCV_LARGE) {
276 /*
277 * We need to inform the user-level code that it needs more
278 * space. The value for how much space was returned in the
279 * msize save area instead of the message (which was left on
280 * the queue).
281 */
282 if (copyout((char *) &self->ith_msize,
283 msg_addr + offsetof(mach_msg_header_t, msgh_size),
284 sizeof(mach_msg_size_t)))
285 mr = MACH_RCV_INVALID_DATA;
286 goto out;
287 }
288
289 if (msg_receive_error(kmsg, msg_addr, option, seqno, space)
290 == MACH_RCV_INVALID_DATA)
291 mr = MACH_RCV_INVALID_DATA;
292 }
293 goto out;
294 }
295
296 trailer = (mach_msg_format_0_trailer_t *)
297 ((vm_offset_t)kmsg->ikm_header +
298 round_msg(kmsg->ikm_header->msgh_size));
299 if (option & MACH_RCV_TRAILER_MASK) {
300 trailer->msgh_seqno = seqno;
301 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
302 }
303
304 /*
305 * If MACH_RCV_OVERWRITE was specified, try to get the scatter
306 * list and verify it against the contents of the message. If
307 * there is any problem with it, we will continue without it as
308 * normal.
309 */
310 if (option & MACH_RCV_OVERWRITE) {
311 mach_msg_size_t slist_size = self->ith_scatter_list_size;
312 mach_msg_body_t *slist;
313
314 slist = ipc_kmsg_get_scatter(msg_addr, slist_size, kmsg);
315 mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL, slist);
316 ipc_kmsg_free_scatter(slist, slist_size);
317 } else {
318 mr = ipc_kmsg_copyout(kmsg, space, map,
319 MACH_PORT_NULL, MACH_MSG_BODY_NULL);
320 }
321
322 if (mr != MACH_MSG_SUCCESS) {
323 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
324 if (ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size +
325 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
326 mr = MACH_RCV_INVALID_DATA;
327 }
328 else {
329 if (msg_receive_error(kmsg, msg_addr, option, seqno, space)
330 == MACH_RCV_INVALID_DATA)
331 mr = MACH_RCV_INVALID_DATA;
332 }
333 goto out;
334 }
335 mr = ipc_kmsg_put(msg_addr,
336 kmsg,
337 kmsg->ikm_header->msgh_size +
338 trailer->msgh_trailer_size);
339 out:
340 return mr;
341 }
342
343 mach_msg_return_t
344 mach_msg_receive(
345 mach_msg_header_t *msg,
346 mach_msg_option_t option,
347 mach_msg_size_t rcv_size,
348 mach_port_name_t rcv_name,
349 mach_msg_timeout_t rcv_timeout,
350 void (*continuation)(mach_msg_return_t),
351 mach_msg_size_t slist_size)
352 {
353 thread_t self = current_thread();
354 ipc_space_t space = current_space();
355 ipc_object_t object;
356 ipc_mqueue_t mqueue;
357 mach_msg_return_t mr;
358
359 mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
360 if (mr != MACH_MSG_SUCCESS) {
361 return mr;
362 }
363 /* hold ref for object */
364
365 self->ith_msg_addr = CAST_DOWN(mach_vm_address_t, msg);
366 self->ith_object = object;
367 self->ith_msize = rcv_size;
368 self->ith_option = option;
369 self->ith_scatter_list_size = slist_size;
370 self->ith_continuation = continuation;
371
372 ipc_mqueue_receive(mqueue, option, rcv_size, rcv_timeout, THREAD_ABORTSAFE);
373 if ((option & MACH_RCV_TIMEOUT) && rcv_timeout == 0)
374 thread_poll_yield(self);
375 return mach_msg_receive_results();
376 }
377
378 void
379 mach_msg_receive_continue(void)
380 {
381 thread_t self = current_thread();
382
383 (*self->ith_continuation)(mach_msg_receive_results());
384 }
385
386 /*
387 * Toggle this to compile the hotpath in/out
388 * If compiled in, the run-time toggle "enable_hotpath" below
389 * eases testing & debugging
390 */
391 #define ENABLE_HOTPATH 1 /* Hacked on for now */
392
393 #if ENABLE_HOTPATH
394 /*
395 * These counters allow tracing of hotpath behavior under test loads.
396 * A couple key counters are unconditional (see below).
397 */
398 #define HOTPATH_DEBUG 0 /* Toggle to include lots of counters */
399 #if HOTPATH_DEBUG
400 #define HOT(expr) expr
401
402 unsigned int c_mmot_FIRST = 0; /* Unused First Counter */
403 unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
404 unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
405 unsigned int c_mmot_kernel_send = 0; /* kernel server */
406 unsigned int c_mmot_cold_000 = 0; /* see below ... */
407 unsigned int c_mmot_smallsendsize = 0;
408 unsigned int c_mmot_oddsendsize = 0;
409 unsigned int c_mmot_bigsendsize = 0;
410 unsigned int c_mmot_copyinmsg_fail = 0;
411 unsigned int c_mmot_g_slow_copyin3 = 0;
412 unsigned int c_mmot_cold_006 = 0;
413 unsigned int c_mmot_cold_007 = 0;
414 unsigned int c_mmot_cold_008 = 0;
415 unsigned int c_mmot_cold_009 = 0;
416 unsigned int c_mmot_cold_010 = 0;
417 unsigned int c_mmot_cold_012 = 0;
418 unsigned int c_mmot_cold_013 = 0;
419 unsigned int c_mmot_cold_014 = 0;
420 unsigned int c_mmot_cold_016 = 0;
421 unsigned int c_mmot_cold_018 = 0;
422 unsigned int c_mmot_cold_019 = 0;
423 unsigned int c_mmot_cold_020 = 0;
424 unsigned int c_mmot_cold_021 = 0;
425 unsigned int c_mmot_cold_022 = 0;
426 unsigned int c_mmot_cold_023 = 0;
427 unsigned int c_mmot_cold_024 = 0;
428 unsigned int c_mmot_cold_025 = 0;
429 unsigned int c_mmot_cold_026 = 0;
430 unsigned int c_mmot_cold_027 = 0;
431 unsigned int c_mmot_hot_fSR_ok = 0;
432 unsigned int c_mmot_cold_029 = 0;
433 unsigned int c_mmot_cold_030 = 0;
434 unsigned int c_mmot_cold_031 = 0;
435 unsigned int c_mmot_cold_032 = 0;
436 unsigned int c_mmot_cold_033 = 0;
437 unsigned int c_mmot_bad_rcvr = 0;
438 unsigned int c_mmot_rcvr_swapped = 0;
439 unsigned int c_mmot_rcvr_locked = 0;
440 unsigned int c_mmot_rcvr_tswapped = 0;
441 unsigned int c_mmot_rcvr_freed = 0;
442 unsigned int c_mmot_g_slow_copyout6 = 0;
443 unsigned int c_mmot_g_slow_copyout5 = 0;
444 unsigned int c_mmot_cold_037 = 0;
445 unsigned int c_mmot_cold_038 = 0;
446 unsigned int c_mmot_cold_039 = 0;
447 unsigned int c_mmot_g_slow_copyout4 = 0;
448 unsigned int c_mmot_g_slow_copyout3 = 0;
449 unsigned int c_mmot_hot_ok1 = 0;
450 unsigned int c_mmot_hot_ok2 = 0;
451 unsigned int c_mmot_hot_ok3 = 0;
452 unsigned int c_mmot_g_slow_copyout1 = 0;
453 unsigned int c_mmot_g_slow_copyout2 = 0;
454 unsigned int c_mmot_getback_fast_copyin = 0;
455 unsigned int c_mmot_cold_048 = 0;
456 unsigned int c_mmot_getback_FastSR = 0;
457 unsigned int c_mmot_cold_050 = 0;
458 unsigned int c_mmot_cold_051 = 0;
459 unsigned int c_mmot_cold_052 = 0;
460 unsigned int c_mmot_cold_053 = 0;
461 unsigned int c_mmot_fastkernelreply = 0;
462 unsigned int c_mmot_cold_055 = 0;
463 unsigned int c_mmot_getback_fast_put = 0;
464 unsigned int c_mmot_LAST = 0; /* End Marker - Unused */
465
466 void db_mmot_zero_counters(void); /* forward; */
467 void db_mmot_show_counters(void); /* forward; */
468
469 void /* Call from the debugger to clear all counters */
470 db_mmot_zero_counters(void)
471 {
472 register unsigned int *ip = &c_mmot_FIRST;
473 while (ip <= &c_mmot_LAST)
474 *ip++ = 0;
475 }
476
477 void /* Call from the debugger to show all counters */
478 db_mmot_show_counters(void)
479 {
480 #define xx(str) printf("%s: %d\n", # str, str);
481
482 xx(c_mmot_combined_S_R);
483 xx(c_mach_msg_trap_switch_fast);
484 xx(c_mmot_kernel_send);
485 xx(c_mmot_cold_000);
486 xx(c_mmot_smallsendsize);
487 xx(c_mmot_oddsendsize);
488 xx(c_mmot_bigsendsize);
489 xx(c_mmot_copyinmsg_fail);
490 xx(c_mmot_g_slow_copyin3);
491 xx(c_mmot_cold_006);
492 xx(c_mmot_cold_007);
493 xx(c_mmot_cold_008);
494 xx(c_mmot_cold_009);
495 xx(c_mmot_cold_010);
496 xx(c_mmot_cold_012);
497 xx(c_mmot_cold_013);
498 xx(c_mmot_cold_014);
499 xx(c_mmot_cold_016);
500 xx(c_mmot_cold_018);
501 xx(c_mmot_cold_019);
502 xx(c_mmot_cold_020);
503 xx(c_mmot_cold_021);
504 xx(c_mmot_cold_022);
505 xx(c_mmot_cold_023);
506 xx(c_mmot_cold_024);
507 xx(c_mmot_cold_025);
508 xx(c_mmot_cold_026);
509 xx(c_mmot_cold_027);
510 xx(c_mmot_hot_fSR_ok);
511 xx(c_mmot_cold_029);
512 xx(c_mmot_cold_030);
513 xx(c_mmot_cold_031);
514 xx(c_mmot_cold_032);
515 xx(c_mmot_cold_033);
516 xx(c_mmot_bad_rcvr);
517 xx(c_mmot_rcvr_swapped);
518 xx(c_mmot_rcvr_locked);
519 xx(c_mmot_rcvr_tswapped);
520 xx(c_mmot_rcvr_freed);
521 xx(c_mmot_g_slow_copyout6);
522 xx(c_mmot_g_slow_copyout5);
523 xx(c_mmot_cold_037);
524 xx(c_mmot_cold_038);
525 xx(c_mmot_cold_039);
526 xx(c_mmot_g_slow_copyout4);
527 xx(c_mmot_g_slow_copyout3);
528 xx(c_mmot_g_slow_copyout1);
529 xx(c_mmot_hot_ok3);
530 xx(c_mmot_hot_ok2);
531 xx(c_mmot_hot_ok1);
532 xx(c_mmot_g_slow_copyout2);
533 xx(c_mmot_getback_fast_copyin);
534 xx(c_mmot_cold_048);
535 xx(c_mmot_getback_FastSR);
536 xx(c_mmot_cold_050);
537 xx(c_mmot_cold_051);
538 xx(c_mmot_cold_052);
539 xx(c_mmot_cold_053);
540 xx(c_mmot_fastkernelreply);
541 xx(c_mmot_cold_055);
542 xx(c_mmot_getback_fast_put);
543
544 #undef xx
545 }
546
547 #else /* !HOTPATH_DEBUG */
548
549 /*
550 * Duplicate just these few so we can always do a quick sanity check
551 */
552 unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */
553 unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */
554 unsigned int c_mmot_kernel_send = 0; /* kernel server calls */
555 #define HOT(expr) /* no optional counters */
556
557 #endif /* !HOTPATH_DEBUG */
558
559 boolean_t enable_hotpath = TRUE; /* Patchable, just in case ... */
560 #endif /* HOTPATH_ENABLE */
561
562 /*
563 * Routine: mach_msg_overwrite_trap [mach trap]
564 * Purpose:
565 * Possibly send a message; possibly receive a message.
566 * Conditions:
567 * Nothing locked.
568 * Returns:
569 * All of mach_msg_send and mach_msg_receive error codes.
570 */
571
572 mach_msg_return_t
573 mach_msg_overwrite_trap(
574 struct mach_msg_overwrite_trap_args *args)
575 {
576 mach_vm_address_t msg_addr = args->msg;
577 mach_msg_option_t option = args->option;
578 mach_msg_size_t send_size = args->send_size;
579 mach_msg_size_t rcv_size = args->rcv_size;
580 mach_port_name_t rcv_name = args->rcv_name;
581 mach_msg_timeout_t msg_timeout = args->timeout;
582 mach_port_name_t notify = args->notify;
583 mach_vm_address_t rcv_msg_addr = args->rcv_msg;
584 mach_msg_size_t scatter_list_size = 0; /* NOT INITIALIZED - but not used in pactice */
585
586 register mach_msg_header_t *hdr;
587 mach_msg_return_t mr = MACH_MSG_SUCCESS;
588 /* mask out some of the options before entering the hot path */
589 mach_msg_option_t masked_option =
590 option & ~(MACH_SEND_TRAILER|MACH_RCV_TRAILER_MASK|MACH_RCV_LARGE);
591
592 #if ENABLE_HOTPATH
593 /* BEGINNING OF HOT PATH */
594 if ((masked_option == (MACH_SEND_MSG|MACH_RCV_MSG)) && enable_hotpath) {
595 thread_t self = current_thread();
596 mach_msg_format_0_trailer_t *trailer;
597 ipc_space_t space = self->task->itk_space;
598 ipc_kmsg_t kmsg;
599 register ipc_port_t dest_port;
600 ipc_object_t rcv_object;
601 ipc_mqueue_t rcv_mqueue;
602 mach_msg_size_t reply_size;
603
604 c_mmot_combined_S_R++;
605
606 /*
607 * This case is divided into ten sections, each
608 * with a label. There are five optimized
609 * sections and six unoptimized sections, which
610 * do the same thing but handle all possible
611 * cases and are slower.
612 *
613 * The five sections for an RPC are
614 * 1) Get request message into a buffer.
615 * 2) Copyin request message and rcv_name.
616 * (fast_copyin or slow_copyin)
617 * 3) Enqueue request and dequeue reply.
618 * (fast_send_receive or
619 * slow_send and slow_receive)
620 * 4) Copyout reply message.
621 * (fast_copyout or slow_copyout)
622 * 5) Put reply message to user's buffer.
623 *
624 * Keep the locking hierarchy firmly in mind.
625 * (First spaces, then ports, then port sets,
626 * then message queues.) Only a non-blocking
627 * attempt can be made to acquire locks out of
628 * order, or acquire two locks on the same level.
629 * Acquiring two locks on the same level will
630 * fail if the objects are really the same,
631 * unless simple locking is disabled. This is OK,
632 * because then the extra unlock does nothing.
633 *
634 * There are two major reasons these RPCs can't use
635 * ipc_thread_switch, and use slow_send/slow_receive:
636 * 1) Kernel RPCs.
637 * 2) Servers fall behind clients, so
638 * client doesn't find a blocked server thread and
639 * server finds waiting messages and can't block.
640 */
641
642 mr = ipc_kmsg_get(msg_addr, send_size, &kmsg);
643 if (mr != KERN_SUCCESS) {
644 return mr;
645 }
646 hdr = kmsg->ikm_header;
647 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
648 send_size);
649
650 /*
651 * fast_copyin:
652 *
653 * optimized ipc_kmsg_copyin/ipc_mqueue_copyin
654 *
655 * We have the request message data in kmsg.
656 * Must still do copyin, send, receive, etc.
657 *
658 * If the message isn't simple, we can't combine
659 * ipc_kmsg_copyin_header and ipc_mqueue_copyin,
660 * because copyin of the message body might
661 * affect rcv_name.
662 */
663
664 switch (hdr->msgh_bits) {
665 case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
666 MACH_MSG_TYPE_MAKE_SEND_ONCE): {
667 register ipc_entry_t table;
668 register ipc_entry_num_t size;
669 register ipc_port_t reply_port;
670
671 /* sending a request message */
672
673 {
674 register mach_port_index_t index;
675 register mach_port_gen_t gen;
676
677 {
678 register mach_port_name_t reply_name =
679 (mach_port_name_t)hdr->msgh_local_port;
680
681 if (reply_name != rcv_name) {
682 HOT(c_mmot_g_slow_copyin3++);
683 goto slow_copyin;
684 }
685
686 /* optimized ipc_entry_lookup of reply_name */
687
688 index = MACH_PORT_INDEX(reply_name);
689 gen = MACH_PORT_GEN(reply_name);
690
691 is_read_lock(space);
692 assert(space->is_active);
693
694 size = space->is_table_size;
695 table = space->is_table;
696
697 {
698 register ipc_entry_t entry;
699 register ipc_entry_bits_t bits;
700
701 if (index < size) {
702 entry = &table[index];
703 bits = entry->ie_bits;
704 if (IE_BITS_GEN(bits) != gen ||
705 (bits & IE_BITS_COLLISION)) {
706 entry = IE_NULL;
707 }
708 } else {
709 entry = IE_NULL;
710 bits = 0;
711 }
712 if (entry == IE_NULL) {
713 entry = ipc_entry_lookup(space, reply_name);
714 if (entry == IE_NULL) {
715 HOT(c_mmot_cold_006++);
716 goto abort_request_copyin;
717 }
718 bits = entry->ie_bits;
719 }
720
721 /* check type bit */
722
723 if (! (bits & MACH_PORT_TYPE_RECEIVE)) {
724 HOT(c_mmot_cold_007++);
725 goto abort_request_copyin;
726 }
727
728 reply_port = (ipc_port_t) entry->ie_object;
729 assert(reply_port != IP_NULL);
730 }
731 }
732 }
733
734 /* optimized ipc_entry_lookup of dest_name */
735
736 {
737 register mach_port_index_t index;
738 register mach_port_gen_t gen;
739
740 {
741 register mach_port_name_t dest_name =
742 (mach_port_name_t)hdr->msgh_remote_port;
743
744 index = MACH_PORT_INDEX(dest_name);
745 gen = MACH_PORT_GEN(dest_name);
746
747 {
748 register ipc_entry_t entry;
749 register ipc_entry_bits_t bits;
750
751 if (index < size) {
752 entry = &table[index];
753 bits = entry->ie_bits;
754 if (IE_BITS_GEN(bits) != gen ||
755 (bits & IE_BITS_COLLISION)) {
756 entry = IE_NULL;
757 }
758 } else {
759 entry = IE_NULL;
760 bits = 0;
761 }
762 if (entry == IE_NULL) {
763 entry = ipc_entry_lookup(space, dest_name);
764 if (entry == IE_NULL) {
765 HOT(c_mmot_cold_008++);
766 goto abort_request_copyin;
767 }
768 bits = entry->ie_bits;
769 }
770
771 /* check type bit */
772
773 if (! (bits & MACH_PORT_TYPE_SEND)) {
774 HOT(c_mmot_cold_009++);
775 goto abort_request_copyin;
776 }
777
778 assert(IE_BITS_UREFS(bits) > 0);
779
780 dest_port = (ipc_port_t) entry->ie_object;
781 assert(dest_port != IP_NULL);
782 }
783 }
784 }
785
786 /*
787 * To do an atomic copyin, need simultaneous
788 * locks on both ports and the space. If
789 * dest_port == reply_port, and simple locking is
790 * enabled, then we will abort. Otherwise it's
791 * OK to unlock twice.
792 */
793
794 ip_lock(dest_port);
795 if (!ip_active(dest_port) ||
796 !ip_lock_try(reply_port)) {
797 ip_unlock(dest_port);
798 HOT(c_mmot_cold_010++);
799 goto abort_request_copyin;
800 }
801 is_read_unlock(space);
802
803 assert(dest_port->ip_srights > 0);
804 dest_port->ip_srights++;
805 ip_reference(dest_port);
806
807 assert(ip_active(reply_port));
808 assert(reply_port->ip_receiver_name ==
809 (mach_port_name_t)hdr->msgh_local_port);
810 assert(reply_port->ip_receiver == space);
811
812 reply_port->ip_sorights++;
813 ip_reference(reply_port);
814
815 hdr->msgh_bits =
816 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
817 MACH_MSG_TYPE_PORT_SEND_ONCE);
818 hdr->msgh_remote_port = dest_port;
819 hdr->msgh_local_port = reply_port;
820
821 /* make sure we can queue to the destination */
822
823 if (dest_port->ip_receiver == ipc_space_kernel) {
824 /*
825 * The kernel server has a reference to
826 * the reply port, which it hands back
827 * to us in the reply message. We do
828 * not need to keep another reference to
829 * it.
830 */
831 ip_unlock(reply_port);
832
833 assert(ip_active(dest_port));
834 dest_port->ip_messages.imq_seqno++;
835 ip_unlock(dest_port);
836 goto kernel_send;
837 }
838
839 if (imq_full(&dest_port->ip_messages)) {
840 HOT(c_mmot_cold_013++);
841 goto abort_request_send_receive;
842 }
843
844 /* optimized ipc_mqueue_copyin */
845
846 rcv_object = (ipc_object_t) reply_port;
847 io_reference(rcv_object);
848 rcv_mqueue = &reply_port->ip_messages;
849 io_unlock(rcv_object);
850 HOT(c_mmot_hot_fSR_ok++);
851 goto fast_send_receive;
852
853 abort_request_copyin:
854 is_read_unlock(space);
855 goto slow_copyin;
856
857 abort_request_send_receive:
858 ip_unlock(dest_port);
859 ip_unlock(reply_port);
860 goto slow_send;
861 }
862
863 case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
864 register ipc_entry_num_t size;
865 register ipc_entry_t table;
866
867 /* sending a reply message */
868
869 {
870 register mach_port_name_t reply_name =
871 (mach_port_name_t)hdr->msgh_local_port;
872
873 if (reply_name != MACH_PORT_NULL) {
874 HOT(c_mmot_cold_018++);
875 goto slow_copyin;
876 }
877 }
878
879 is_write_lock(space);
880 assert(space->is_active);
881
882 /* optimized ipc_entry_lookup */
883
884 size = space->is_table_size;
885 table = space->is_table;
886
887 {
888 register ipc_entry_t entry;
889 register mach_port_gen_t gen;
890 register mach_port_index_t index;
891
892 {
893 register mach_port_name_t dest_name =
894 (mach_port_name_t)hdr->msgh_remote_port;
895
896 index = MACH_PORT_INDEX(dest_name);
897 gen = MACH_PORT_GEN(dest_name);
898 }
899
900 if (index >= size) {
901 HOT(c_mmot_cold_019++);
902 goto abort_reply_dest_copyin;
903 }
904
905 entry = &table[index];
906
907 /* check generation, collision bit, and type bit */
908
909 if ((entry->ie_bits & (IE_BITS_GEN_MASK|
910 IE_BITS_COLLISION|
911 MACH_PORT_TYPE_SEND_ONCE)) !=
912 (gen | MACH_PORT_TYPE_SEND_ONCE)) {
913 HOT(c_mmot_cold_020++);
914 goto abort_reply_dest_copyin;
915 }
916
917 /* optimized ipc_right_copyin */
918
919 assert(IE_BITS_TYPE(entry->ie_bits) ==
920 MACH_PORT_TYPE_SEND_ONCE);
921 assert(IE_BITS_UREFS(entry->ie_bits) == 1);
922
923 if (entry->ie_request != 0) {
924 HOT(c_mmot_cold_021++);
925 goto abort_reply_dest_copyin;
926 }
927
928 dest_port = (ipc_port_t) entry->ie_object;
929 assert(dest_port != IP_NULL);
930
931 ip_lock(dest_port);
932 if (!ip_active(dest_port)) {
933 ip_unlock(dest_port);
934 HOT(c_mmot_cold_022++);
935 goto abort_reply_dest_copyin;
936 }
937
938 assert(dest_port->ip_sorights > 0);
939
940 /* optimized ipc_entry_dealloc */
941
942
943 entry->ie_bits = gen;
944 entry->ie_next = table->ie_next;
945 table->ie_next = index;
946 entry->ie_object = IO_NULL;
947 }
948
949 hdr->msgh_bits =
950 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
951 0);
952 hdr->msgh_remote_port = dest_port;
953
954 /* make sure we can queue to the destination */
955
956 assert(dest_port->ip_receiver != ipc_space_kernel);
957
958 /* optimized ipc_entry_lookup/ipc_mqueue_copyin */
959
960 {
961 register ipc_entry_t entry;
962 register ipc_entry_bits_t bits;
963
964 {
965 register mach_port_index_t index;
966 register mach_port_gen_t gen;
967
968 index = MACH_PORT_INDEX(rcv_name);
969 gen = MACH_PORT_GEN(rcv_name);
970
971 if (index < size) {
972 entry = &table[index];
973 bits = entry->ie_bits;
974 if (IE_BITS_GEN(bits) != gen ||
975 (bits & IE_BITS_COLLISION)) {
976 entry = IE_NULL;
977 }
978 } else {
979 entry = IE_NULL;
980 bits = 0;
981 }
982 if (entry == IE_NULL) {
983 entry = ipc_entry_lookup(space, rcv_name);
984 if (entry == IE_NULL) {
985 HOT(c_mmot_cold_024++);
986 goto abort_reply_rcv_copyin;
987 }
988 bits = entry->ie_bits;
989 }
990
991 }
992
993 /* check type bits; looking for receive or set */
994 #if 0
995 /*
996 * JMM - The check below for messages in the receive
997 * mqueue is insufficient to work with port sets, since
998 * the messages stay in the port queues. For now, don't
999 * allow portsets (but receiving on portsets when sending
1000 * a message to a send-once right is actually a very
1001 * common case (so we should re-enable).
1002 */
1003 if (bits & MACH_PORT_TYPE_PORT_SET) {
1004 register ipc_pset_t rcv_pset;
1005
1006 rcv_pset = (ipc_pset_t) entry->ie_object;
1007 assert(rcv_pset != IPS_NULL);
1008
1009 ips_lock(rcv_pset);
1010 assert(ips_active(rcv_pset));
1011
1012 rcv_object = (ipc_object_t) rcv_pset;
1013 rcv_mqueue = &rcv_pset->ips_messages;
1014 } else
1015 #endif /* 0 */
1016 if (bits & MACH_PORT_TYPE_RECEIVE) {
1017 register ipc_port_t rcv_port;
1018
1019 rcv_port = (ipc_port_t) entry->ie_object;
1020 assert(rcv_port != IP_NULL);
1021
1022 if (!ip_lock_try(rcv_port)) {
1023 HOT(c_mmot_cold_025++);
1024 goto abort_reply_rcv_copyin;
1025 }
1026 assert(ip_active(rcv_port));
1027
1028 if (rcv_port->ip_pset_count != 0) {
1029 ip_unlock(rcv_port);
1030 HOT(c_mmot_cold_026++);
1031 goto abort_reply_rcv_copyin;
1032 }
1033
1034 rcv_object = (ipc_object_t) rcv_port;
1035 rcv_mqueue = &rcv_port->ip_messages;
1036 } else {
1037 HOT(c_mmot_cold_027++);
1038 goto abort_reply_rcv_copyin;
1039 }
1040 }
1041
1042 is_write_unlock(space);
1043 io_reference(rcv_object);
1044 io_unlock(rcv_object);
1045 HOT(c_mmot_hot_fSR_ok++);
1046 goto fast_send_receive;
1047
1048 abort_reply_dest_copyin:
1049 is_write_unlock(space);
1050 HOT(c_mmot_cold_029++);
1051 goto slow_copyin;
1052
1053 abort_reply_rcv_copyin:
1054 ip_unlock(dest_port);
1055 is_write_unlock(space);
1056 HOT(c_mmot_cold_030++);
1057 goto slow_send;
1058 }
1059
1060 default:
1061 HOT(c_mmot_cold_031++);
1062 goto slow_copyin;
1063 }
1064 /*NOTREACHED*/
1065
1066 fast_send_receive:
1067 /*
1068 * optimized ipc_mqueue_send/ipc_mqueue_receive
1069 *
1070 * Finished get/copyin of kmsg and copyin of rcv_name.
1071 * space is unlocked, dest_port is locked,
1072 * we can queue kmsg to dest_port,
1073 * rcv_mqueue is set, and rcv_object holds a ref
1074 * so the mqueue cannot go away.
1075 *
1076 * JMM - For now, rcv_object is just a port. Portsets
1077 * are disabled for the time being.
1078 */
1079
1080 assert(ip_active(dest_port));
1081 assert(dest_port->ip_receiver != ipc_space_kernel);
1082 // assert(!imq_full(&dest_port->ip_messages) ||
1083 // (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1084 // MACH_MSG_TYPE_PORT_SEND_ONCE));
1085 assert((hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR) == 0);
1086
1087 {
1088 register ipc_mqueue_t dest_mqueue;
1089 wait_queue_t waitq;
1090 thread_t receiver;
1091 processor_t processor;
1092 boolean_t still_running;
1093 spl_t s;
1094
1095 s = splsched();
1096 processor = current_processor();
1097 if (processor->current_pri >= BASEPRI_RTQUEUES)
1098 goto abort_send_receive1;
1099
1100 dest_mqueue = &dest_port->ip_messages;
1101 waitq = &dest_mqueue->imq_wait_queue;
1102 imq_lock(dest_mqueue);
1103
1104 wait_queue_peek64_locked(waitq, IPC_MQUEUE_RECEIVE, &receiver, &waitq);
1105 /* queue still locked, thread locked - but still on q */
1106
1107 if ( receiver == THREAD_NULL ) {
1108 abort_send_receive:
1109 imq_unlock(dest_mqueue);
1110 abort_send_receive1:
1111 splx(s);
1112 ip_unlock(dest_port);
1113 ipc_object_release(rcv_object);
1114 HOT(c_mmot_cold_032++);
1115 goto slow_send;
1116 }
1117
1118 assert(receiver->state & TH_WAIT);
1119 assert(receiver->wait_queue == waitq);
1120 assert(receiver->wait_event == IPC_MQUEUE_RECEIVE);
1121
1122 /*
1123 * Make sure that the scheduling restrictions of the receiver
1124 * are consistent with a handoff here (if it comes down to that).
1125 */
1126 if ( receiver->sched_pri >= BASEPRI_RTQUEUES ||
1127 receiver->processor_set != processor->processor_set ||
1128 (receiver->bound_processor != PROCESSOR_NULL &&
1129 receiver->bound_processor != processor)) {
1130 HOT(c_mmot_cold_033++);
1131 fall_off:
1132 thread_unlock(receiver);
1133 if (waitq != &dest_mqueue->imq_wait_queue)
1134 wait_queue_unlock(waitq);
1135 goto abort_send_receive;
1136 }
1137
1138 /*
1139 * Check that the receiver can stay on the hot path.
1140 */
1141 if (ipc_kmsg_copyout_size(kmsg, receiver->map) +
1142 REQUESTED_TRAILER_SIZE(receiver->ith_option) > receiver->ith_msize) {
1143 /*
1144 * The receiver can't accept the message.
1145 */
1146 HOT(c_mmot_bad_rcvr++);
1147 goto fall_off;
1148 }
1149
1150 /*
1151 * Before committing to the handoff, make sure that we are
1152 * really going to block (i.e. there are no messages already
1153 * queued for us. This violates lock ordering, so make sure
1154 * we don't deadlock. After the trylock succeeds below, we
1155 * may have up to 3 message queues locked:
1156 * - the dest port mqueue
1157 * - a portset mqueue (where waiting receiver was found)
1158 * - finally our own rcv_mqueue
1159 *
1160 * JMM - Need to make this check appropriate for portsets as
1161 * well before re-enabling them.
1162 */
1163 if (!imq_lock_try(rcv_mqueue)) {
1164 goto fall_off;
1165 }
1166 if (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL) {
1167 imq_unlock(rcv_mqueue);
1168 HOT(c_mmot_cold_033++);
1169 goto fall_off;
1170 }
1171
1172 /* At this point we are committed to do the "handoff". */
1173 c_mach_msg_trap_switch_fast++;
1174
1175 /*
1176 * Go ahead and pull the receiver from the waitq. If the
1177 * waitq wasn't the one for the mqueue, unlock it.
1178 */
1179 wait_queue_pull_thread_locked(waitq,
1180 receiver,
1181 (waitq != &dest_mqueue->imq_wait_queue));
1182
1183 /*
1184 * Store the kmsg and seqno where the receiver can pick it up.
1185 */
1186 receiver->ith_state = MACH_MSG_SUCCESS;
1187 receiver->ith_kmsg = kmsg;
1188 receiver->ith_seqno = dest_mqueue->imq_seqno++;
1189
1190 /*
1191 * Unblock the receiver. If it was still running on another
1192 * CPU, we'll give it a chance to run with the message where
1193 * it is (and just select someother thread to run here).
1194 * Otherwise, we'll invoke it here as part of the handoff.
1195 */
1196 still_running = thread_unblock(receiver, THREAD_AWAKENED);
1197
1198 thread_unlock(receiver);
1199
1200 imq_unlock(dest_mqueue);
1201 ip_unlock(dest_port);
1202 current_task()->messages_sent++;
1203
1204
1205 /*
1206 * Put self on receive port's queue.
1207 * Also save state that the sender of
1208 * our reply message needs to determine if it
1209 * can hand off directly back to us.
1210 */
1211 thread_lock(self);
1212 self->ith_msg_addr = (rcv_msg_addr) ? rcv_msg_addr : msg_addr;
1213 self->ith_object = rcv_object; /* still holds reference */
1214 self->ith_msize = rcv_size;
1215 self->ith_option = option;
1216 self->ith_scatter_list_size = scatter_list_size;
1217 self->ith_continuation = thread_syscall_return;
1218
1219 waitq = &rcv_mqueue->imq_wait_queue;
1220 (void)wait_queue_assert_wait64_locked(waitq,
1221 IPC_MQUEUE_RECEIVE,
1222 THREAD_ABORTSAFE, 0,
1223 self);
1224 thread_unlock(self);
1225 imq_unlock(rcv_mqueue);
1226
1227 /*
1228 * If the receiving thread wasn't still running, we switch directly
1229 * to it here. Otherwise we let the scheduler pick something for
1230 * here. In either case, block this thread as though it had called
1231 * ipc_mqueue_receive.
1232 */
1233 if (still_running) {
1234 splx(s);
1235 thread_block(ipc_mqueue_receive_continue);
1236 } else {
1237 thread_run(self, ipc_mqueue_receive_continue, NULL, receiver);
1238 }
1239 /* NOTREACHED */
1240 }
1241
1242 fast_copyout:
1243 /*
1244 * Nothing locked and no references held, except
1245 * we have kmsg with msgh_seqno filled in. Must
1246 * still check against rcv_size and do
1247 * ipc_kmsg_copyout/ipc_kmsg_put.
1248 */
1249
1250 reply_size = send_size + trailer->msgh_trailer_size;
1251 if (rcv_size < reply_size) {
1252 HOT(c_mmot_g_slow_copyout6++);
1253 goto slow_copyout;
1254 }
1255
1256 /* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */
1257
1258 switch (hdr->msgh_bits) {
1259 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
1260 MACH_MSG_TYPE_PORT_SEND_ONCE): {
1261 ipc_port_t reply_port =
1262 (ipc_port_t) hdr->msgh_local_port;
1263 mach_port_name_t dest_name, reply_name;
1264
1265 /* receiving a request message */
1266
1267 if (!IP_VALID(reply_port)) {
1268 HOT(c_mmot_g_slow_copyout5++);
1269 goto slow_copyout;
1270 }
1271
1272 is_write_lock(space);
1273 assert(space->is_active);
1274
1275 /*
1276 * To do an atomic copyout, need simultaneous
1277 * locks on both ports and the space. If
1278 * dest_port == reply_port, and simple locking is
1279 * enabled, then we will abort. Otherwise it's
1280 * OK to unlock twice.
1281 */
1282
1283 ip_lock(dest_port);
1284 if (!ip_active(dest_port) ||
1285 !ip_lock_try(reply_port)) {
1286 HOT(c_mmot_cold_037++);
1287 goto abort_request_copyout;
1288 }
1289
1290 if (!ip_active(reply_port)) {
1291 ip_unlock(reply_port);
1292 HOT(c_mmot_cold_038++);
1293 goto abort_request_copyout;
1294 }
1295
1296 assert(reply_port->ip_sorights > 0);
1297 ip_unlock(reply_port);
1298
1299 {
1300 register ipc_entry_t table;
1301 register ipc_entry_t entry;
1302 register mach_port_index_t index;
1303
1304 /* optimized ipc_entry_get */
1305
1306 table = space->is_table;
1307 index = table->ie_next;
1308
1309 if (index == 0) {
1310 HOT(c_mmot_cold_039++);
1311 goto abort_request_copyout;
1312 }
1313
1314 entry = &table[index];
1315 table->ie_next = entry->ie_next;
1316 entry->ie_request = 0;
1317
1318 {
1319 register mach_port_gen_t gen;
1320
1321 assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
1322 gen = IE_BITS_NEW_GEN(entry->ie_bits);
1323
1324 reply_name = MACH_PORT_MAKE(index, gen);
1325
1326 /* optimized ipc_right_copyout */
1327
1328 entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
1329 }
1330
1331 assert(MACH_PORT_VALID(reply_name));
1332 entry->ie_object = (ipc_object_t) reply_port;
1333 is_write_unlock(space);
1334 }
1335
1336 /* optimized ipc_object_copyout_dest */
1337
1338 assert(dest_port->ip_srights > 0);
1339 ip_release(dest_port);
1340
1341 if (dest_port->ip_receiver == space)
1342 dest_name = dest_port->ip_receiver_name;
1343 else
1344 dest_name = MACH_PORT_NULL;
1345
1346 if ((--dest_port->ip_srights == 0) &&
1347 (dest_port->ip_nsrequest != IP_NULL)) {
1348 ipc_port_t nsrequest;
1349 mach_port_mscount_t mscount;
1350
1351 /* a rather rare case */
1352
1353 nsrequest = dest_port->ip_nsrequest;
1354 mscount = dest_port->ip_mscount;
1355 dest_port->ip_nsrequest = IP_NULL;
1356 ip_unlock(dest_port);
1357 ipc_notify_no_senders(nsrequest, mscount);
1358 } else
1359 ip_unlock(dest_port);
1360
1361 hdr->msgh_bits =
1362 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
1363 MACH_MSG_TYPE_PORT_SEND);
1364 hdr->msgh_remote_port = (mach_port_t)reply_name;
1365 hdr->msgh_local_port = (mach_port_t)dest_name;
1366 HOT(c_mmot_hot_ok1++);
1367 goto fast_put;
1368
1369 abort_request_copyout:
1370 ip_unlock(dest_port);
1371 is_write_unlock(space);
1372 HOT(c_mmot_g_slow_copyout4++);
1373 goto slow_copyout;
1374 }
1375
1376 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1377 register mach_port_name_t dest_name;
1378
1379 /* receiving a reply message */
1380
1381 ip_lock(dest_port);
1382 if (!ip_active(dest_port)) {
1383 ip_unlock(dest_port);
1384 HOT(c_mmot_g_slow_copyout3++);
1385 goto slow_copyout;
1386 }
1387
1388 /* optimized ipc_object_copyout_dest */
1389
1390 assert(dest_port->ip_sorights > 0);
1391
1392 if (dest_port->ip_receiver == space) {
1393 ip_release(dest_port);
1394 dest_port->ip_sorights--;
1395 dest_name = dest_port->ip_receiver_name;
1396 ip_unlock(dest_port);
1397 } else {
1398 ip_unlock(dest_port);
1399
1400 ipc_notify_send_once(dest_port);
1401 dest_name = MACH_PORT_NULL;
1402 }
1403
1404 hdr->msgh_bits = MACH_MSGH_BITS(0,
1405 MACH_MSG_TYPE_PORT_SEND_ONCE);
1406 hdr->msgh_remote_port = MACH_PORT_NULL;
1407 hdr->msgh_local_port = (ipc_port_t)dest_name;
1408 HOT(c_mmot_hot_ok2++);
1409 goto fast_put;
1410 }
1411
1412 case MACH_MSGH_BITS_COMPLEX|
1413 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
1414 register mach_port_name_t dest_name;
1415
1416 /* receiving a complex reply message */
1417
1418 ip_lock(dest_port);
1419 if (!ip_active(dest_port)) {
1420 ip_unlock(dest_port);
1421 HOT(c_mmot_g_slow_copyout1++);
1422 goto slow_copyout;
1423 }
1424
1425 /* optimized ipc_object_copyout_dest */
1426
1427 assert(dest_port->ip_sorights > 0);
1428
1429 if (dest_port->ip_receiver == space) {
1430 ip_release(dest_port);
1431 dest_port->ip_sorights--;
1432 dest_name = dest_port->ip_receiver_name;
1433 ip_unlock(dest_port);
1434 } else {
1435 ip_unlock(dest_port);
1436
1437 ipc_notify_send_once(dest_port);
1438 dest_name = MACH_PORT_NULL;
1439 }
1440
1441 hdr->msgh_bits =
1442 MACH_MSGH_BITS_COMPLEX |
1443 MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND_ONCE);
1444 hdr->msgh_remote_port = MACH_PORT_NULL;
1445 hdr->msgh_local_port = (mach_port_t)dest_name;
1446
1447 mr = ipc_kmsg_copyout_body(kmsg, space,
1448 current_map(),
1449 MACH_MSG_BODY_NULL);
1450 /* hdr and send_size may be invalid now - done use */
1451 if (mr != MACH_MSG_SUCCESS) {
1452 if (ipc_kmsg_put(msg_addr, kmsg,
1453 kmsg->ikm_header->msgh_size +
1454 trailer->msgh_trailer_size) ==
1455 MACH_RCV_INVALID_DATA)
1456 return MACH_RCV_INVALID_DATA;
1457 else
1458 return mr | MACH_RCV_BODY_ERROR;
1459 }
1460 HOT(c_mmot_hot_ok3++);
1461 goto fast_put;
1462 }
1463
1464 default:
1465 HOT(c_mmot_g_slow_copyout2++);
1466 goto slow_copyout;
1467 }
1468 /*NOTREACHED*/
1469
1470 fast_put:
1471 mr = ipc_kmsg_put(rcv_msg_addr ? rcv_msg_addr : msg_addr,
1472 kmsg,
1473 kmsg->ikm_header->msgh_size +
1474 trailer->msgh_trailer_size);
1475 if (mr != MACH_MSG_SUCCESS) {
1476 return MACH_RCV_INVALID_DATA;
1477 }
1478 current_task()->messages_received++;
1479 return mr;
1480
1481
1482 /* BEGINNING OF WARM PATH */
1483
1484 /*
1485 * The slow path has a few non-register temporary
1486 * variables used only for call-by-reference.
1487 */
1488
1489 slow_copyin:
1490 {
1491 mach_port_seqno_t temp_seqno = 0;
1492 register mach_port_name_t reply_name =
1493 (mach_port_name_t)hdr->msgh_local_port;
1494
1495
1496 /*
1497 * We have the message data in kmsg, but
1498 * we still need to copyin, send it,
1499 * receive a reply, and do copyout.
1500 */
1501
1502 mr = ipc_kmsg_copyin(kmsg, space, current_map(),
1503 MACH_PORT_NULL);
1504 if (mr != MACH_MSG_SUCCESS) {
1505 ipc_kmsg_free(kmsg);
1506 return(mr);
1507 }
1508
1509 /*
1510 * LP64support - We have to recompute the header pointer
1511 * and send_size - as they could have changed during the
1512 * complex copyin.
1513 */
1514 hdr = kmsg->ikm_header;
1515 send_size = hdr->msgh_size;
1516
1517 /* try to get back on optimized path */
1518 if ((reply_name != rcv_name) ||
1519 (hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR)) {
1520 HOT(c_mmot_cold_048++);
1521 goto slow_send;
1522 }
1523
1524 dest_port = (ipc_port_t) hdr->msgh_remote_port;
1525 assert(IP_VALID(dest_port));
1526
1527 ip_lock(dest_port);
1528 if (!ip_active(dest_port)) {
1529 ip_unlock(dest_port);
1530 goto slow_send;
1531 }
1532
1533 if (dest_port->ip_receiver == ipc_space_kernel) {
1534 dest_port->ip_messages.imq_seqno++;
1535 ip_unlock(dest_port);
1536 goto kernel_send;
1537 }
1538
1539 if (!imq_full(&dest_port->ip_messages) ||
1540 (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
1541 MACH_MSG_TYPE_PORT_SEND_ONCE))
1542 {
1543 /*
1544 * Try an optimized ipc_mqueue_copyin.
1545 * It will work if this is a request message.
1546 */
1547
1548 register ipc_port_t reply_port;
1549
1550 reply_port = (ipc_port_t) hdr->msgh_local_port;
1551 if (IP_VALID(reply_port)) {
1552 if (ip_lock_try(reply_port)) {
1553 if (ip_active(reply_port) &&
1554 reply_port->ip_receiver == space &&
1555 reply_port->ip_receiver_name == rcv_name &&
1556 reply_port->ip_pset_count == 0)
1557 {
1558 /* Grab a reference to the reply port. */
1559 rcv_object = (ipc_object_t) reply_port;
1560 io_reference(rcv_object);
1561 rcv_mqueue = &reply_port->ip_messages;
1562 io_unlock(rcv_object);
1563 HOT(c_mmot_getback_FastSR++);
1564 goto fast_send_receive;
1565 }
1566 ip_unlock(reply_port);
1567 }
1568 }
1569 }
1570
1571 ip_unlock(dest_port);
1572 HOT(c_mmot_cold_050++);
1573 goto slow_send;
1574
1575 kernel_send:
1576 /*
1577 * Special case: send message to kernel services.
1578 * The request message has been copied into the
1579 * kmsg. Nothing is locked.
1580 */
1581
1582 {
1583 register ipc_port_t reply_port;
1584 mach_port_seqno_t local_seqno;
1585 spl_t s;
1586
1587 /*
1588 * Perform the kernel function.
1589 */
1590 c_mmot_kernel_send++;
1591
1592 current_task()->messages_sent++;
1593
1594 kmsg = ipc_kobject_server(kmsg);
1595 if (kmsg == IKM_NULL) {
1596 /*
1597 * No reply. Take the
1598 * slow receive path.
1599 */
1600 HOT(c_mmot_cold_051++);
1601 goto slow_get_rcv_port;
1602 }
1603
1604 /*
1605 * Check that:
1606 * the reply port is alive
1607 * we hold the receive right
1608 * the name has not changed.
1609 * the port is not in a set
1610 * If any of these are not true,
1611 * we cannot directly receive the reply
1612 * message.
1613 */
1614 hdr = kmsg->ikm_header;
1615 send_size = hdr->msgh_size;
1616 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1617 round_msg(send_size));
1618 reply_port = (ipc_port_t) hdr->msgh_remote_port;
1619 ip_lock(reply_port);
1620
1621 if ((!ip_active(reply_port)) ||
1622 (reply_port->ip_receiver != space) ||
1623 (reply_port->ip_receiver_name != rcv_name) ||
1624 (reply_port->ip_pset_count != 0))
1625 {
1626 ip_unlock(reply_port);
1627 ipc_kmsg_send_always(kmsg);
1628 HOT(c_mmot_cold_052++);
1629 goto slow_get_rcv_port;
1630 }
1631
1632 s = splsched();
1633 rcv_mqueue = &reply_port->ip_messages;
1634 imq_lock(rcv_mqueue);
1635
1636 /* keep port locked, and don`t change ref count yet */
1637
1638 /*
1639 * If there are messages on the port
1640 * or other threads waiting for a message,
1641 * we cannot directly receive the reply.
1642 */
1643 if (!wait_queue_empty(&rcv_mqueue->imq_wait_queue) ||
1644 (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL))
1645 {
1646 imq_unlock(rcv_mqueue);
1647 splx(s);
1648 ip_unlock(reply_port);
1649 ipc_kmsg_send_always(kmsg);
1650 HOT(c_mmot_cold_053++);
1651 goto slow_get_rcv_port;
1652 }
1653
1654 /*
1655 * We can directly receive this reply.
1656 * Since there were no messages queued
1657 * on the reply port, there should be
1658 * no threads blocked waiting to send.
1659 */
1660 dest_port = reply_port;
1661 local_seqno = rcv_mqueue->imq_seqno++;
1662 imq_unlock(rcv_mqueue);
1663 splx(s);
1664
1665 /*
1666 * inline ipc_object_release.
1667 * Port is still locked.
1668 * Reference count was not incremented.
1669 */
1670 ip_check_unlock(reply_port);
1671
1672 if (option & MACH_RCV_TRAILER_MASK) {
1673 trailer->msgh_seqno = local_seqno;
1674 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1675 }
1676 /* copy out the kernel reply */
1677 HOT(c_mmot_fastkernelreply++);
1678 goto fast_copyout;
1679 }
1680
1681 slow_send:
1682 /*
1683 * Nothing is locked. We have acquired kmsg, but
1684 * we still need to send it and receive a reply.
1685 */
1686
1687 mr = ipc_kmsg_send(kmsg, MACH_MSG_OPTION_NONE,
1688 MACH_MSG_TIMEOUT_NONE);
1689 if (mr != MACH_MSG_SUCCESS) {
1690 mr |= ipc_kmsg_copyout_pseudo(kmsg, space,
1691 current_map(),
1692 MACH_MSG_BODY_NULL);
1693
1694 (void) ipc_kmsg_put(msg_addr, kmsg,
1695 kmsg->ikm_header->msgh_size);
1696 return(mr);
1697 }
1698
1699 slow_get_rcv_port:
1700 /*
1701 * We have sent the message. Copy in the receive port.
1702 */
1703 mr = ipc_mqueue_copyin(space, rcv_name,
1704 &rcv_mqueue, &rcv_object);
1705 if (mr != MACH_MSG_SUCCESS) {
1706 return(mr);
1707 }
1708 /* hold ref for rcv_object */
1709
1710 /*
1711 * slow_receive:
1712 *
1713 * Now we have sent the request and copied in rcv_name,
1714 * and hold ref for rcv_object (to keep mqueue alive).
1715 * Just receive a reply and try to get back to fast path.
1716 */
1717
1718 self->ith_continuation = (void (*)(mach_msg_return_t))0;
1719 ipc_mqueue_receive(rcv_mqueue,
1720 MACH_MSG_OPTION_NONE,
1721 MACH_MSG_SIZE_MAX,
1722 MACH_MSG_TIMEOUT_NONE,
1723 THREAD_ABORTSAFE);
1724
1725 mr = self->ith_state;
1726 temp_seqno = self->ith_seqno;
1727
1728 ipc_object_release(rcv_object);
1729
1730 if (mr != MACH_MSG_SUCCESS) {
1731 return(mr);
1732 }
1733
1734 kmsg = self->ith_kmsg;
1735 hdr = kmsg->ikm_header;
1736 send_size = hdr->msgh_size;
1737 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
1738 round_msg(send_size));
1739 if (option & MACH_RCV_TRAILER_MASK) {
1740 trailer->msgh_seqno = temp_seqno;
1741 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1742 }
1743 dest_port = (ipc_port_t) hdr->msgh_remote_port;
1744 HOT(c_mmot_cold_055++);
1745 goto fast_copyout;
1746
1747 slow_copyout:
1748 /*
1749 * Nothing locked and no references held, except
1750 * we have kmsg with msgh_seqno filled in. Must
1751 * still check against rcv_size and do
1752 * ipc_kmsg_copyout/ipc_kmsg_put.
1753 */
1754
1755 /* LP64support - have to compute real size as it would be received */
1756 reply_size = ipc_kmsg_copyout_size(kmsg, current_map()) +
1757 REQUESTED_TRAILER_SIZE(option);
1758 if (rcv_size < reply_size) {
1759 if (msg_receive_error(kmsg, msg_addr, option, temp_seqno,
1760 space) == MACH_RCV_INVALID_DATA) {
1761 mr = MACH_RCV_INVALID_DATA;
1762 return(mr);
1763 }
1764 else {
1765 mr = MACH_RCV_TOO_LARGE;
1766 return(mr);
1767 }
1768 }
1769
1770 mr = ipc_kmsg_copyout(kmsg, space, current_map(),
1771 MACH_PORT_NULL, MACH_MSG_BODY_NULL);
1772 if (mr != MACH_MSG_SUCCESS) {
1773 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
1774 if (ipc_kmsg_put(msg_addr, kmsg, reply_size) ==
1775 MACH_RCV_INVALID_DATA)
1776 mr = MACH_RCV_INVALID_DATA;
1777 }
1778 else {
1779 if (msg_receive_error(kmsg, msg_addr, option,
1780 temp_seqno, space) == MACH_RCV_INVALID_DATA)
1781 mr = MACH_RCV_INVALID_DATA;
1782 }
1783
1784 return(mr);
1785 }
1786
1787 /* try to get back on optimized path */
1788 HOT(c_mmot_getback_fast_put++);
1789 goto fast_put;
1790
1791 /*NOTREACHED*/
1792 }
1793 } /* END OF HOT PATH */
1794 #endif /* ENABLE_HOTPATH */
1795
1796 if (option & MACH_SEND_MSG) {
1797 ipc_space_t space = current_space();
1798 vm_map_t map = current_map();
1799 ipc_kmsg_t kmsg;
1800
1801 mr = ipc_kmsg_get(msg_addr, send_size, &kmsg);
1802
1803 if (mr != MACH_MSG_SUCCESS)
1804 return mr;
1805
1806 if (option & MACH_SEND_CANCEL) {
1807 if (notify == MACH_PORT_NULL)
1808 mr = MACH_SEND_INVALID_NOTIFY;
1809 else
1810 mr = ipc_kmsg_copyin(kmsg, space, map, notify);
1811 } else
1812 mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
1813 if (mr != MACH_MSG_SUCCESS) {
1814 ipc_kmsg_free(kmsg);
1815 return mr;
1816 }
1817
1818 mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, msg_timeout);
1819
1820 if (mr != MACH_MSG_SUCCESS) {
1821 mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
1822 (void) ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size);
1823 return mr;
1824 }
1825
1826 }
1827
1828 if (option & MACH_RCV_MSG) {
1829 thread_t self = current_thread();
1830 ipc_space_t space = current_space();
1831 ipc_object_t object;
1832 ipc_mqueue_t mqueue;
1833
1834 mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
1835 if (mr != MACH_MSG_SUCCESS) {
1836 return mr;
1837 }
1838 /* hold ref for object */
1839
1840 /*
1841 * 1. MACH_RCV_OVERWRITE is on, and rcv_msg is our scatter list
1842 * and receive buffer
1843 * 2. MACH_RCV_OVERWRITE is off, and rcv_msg might be the
1844 * alternate receive buffer (separate send and receive buffers).
1845 */
1846 if (option & MACH_RCV_OVERWRITE)
1847 self->ith_msg_addr = rcv_msg_addr;
1848 else if (rcv_msg_addr != (mach_vm_address_t)0)
1849 self->ith_msg_addr = rcv_msg_addr;
1850 else
1851 self->ith_msg_addr = msg_addr;
1852 self->ith_object = object;
1853 self->ith_msize = rcv_size;
1854 self->ith_option = option;
1855 self->ith_scatter_list_size = scatter_list_size;
1856 self->ith_continuation = thread_syscall_return;
1857
1858 ipc_mqueue_receive(mqueue, option, rcv_size, msg_timeout, THREAD_ABORTSAFE);
1859 if ((option & MACH_RCV_TIMEOUT) && msg_timeout == 0)
1860 thread_poll_yield(self);
1861 return mach_msg_receive_results();
1862 }
1863
1864 return MACH_MSG_SUCCESS;
1865 }
1866
1867 /*
1868 * Routine: mach_msg_trap [mach trap]
1869 * Purpose:
1870 * Possibly send a message; possibly receive a message.
1871 * Conditions:
1872 * Nothing locked.
1873 * Returns:
1874 * All of mach_msg_send and mach_msg_receive error codes.
1875 */
1876
1877 mach_msg_return_t
1878 mach_msg_trap(
1879 struct mach_msg_overwrite_trap_args *args)
1880 {
1881 kern_return_t kr;
1882 args->rcv_msg = (mach_vm_address_t)0;
1883
1884 kr = mach_msg_overwrite_trap(args);
1885 return kr;
1886 }
1887
1888
1889 /*
1890 * Routine: msg_receive_error [internal]
1891 * Purpose:
1892 * Builds a minimal header/trailer and copies it to
1893 * the user message buffer. Invoked when in the case of a
1894 * MACH_RCV_TOO_LARGE or MACH_RCV_BODY_ERROR error.
1895 * Conditions:
1896 * Nothing locked.
1897 * Returns:
1898 * MACH_MSG_SUCCESS minimal header/trailer copied
1899 * MACH_RCV_INVALID_DATA copyout to user buffer failed
1900 */
1901
1902 mach_msg_return_t
1903 msg_receive_error(
1904 ipc_kmsg_t kmsg,
1905 mach_vm_address_t msg_addr,
1906 mach_msg_option_t option,
1907 mach_port_seqno_t seqno,
1908 ipc_space_t space)
1909 {
1910 mach_msg_format_0_trailer_t *trailer;
1911
1912 /*
1913 * Copy out the destination port in the message.
1914 * Destroy all other rights and memory in the message.
1915 */
1916 ipc_kmsg_copyout_dest(kmsg, space);
1917
1918 /*
1919 * Build a minimal message with the requested trailer.
1920 */
1921 trailer = (mach_msg_format_0_trailer_t *)
1922 ((vm_offset_t)kmsg->ikm_header +
1923 round_msg(sizeof(mach_msg_header_t)));
1924 kmsg->ikm_header->msgh_size = sizeof(mach_msg_header_t);
1925 bcopy( (char *)&trailer_template,
1926 (char *)trailer,
1927 sizeof(trailer_template));
1928 if (option & MACH_RCV_TRAILER_MASK) {
1929 trailer->msgh_seqno = seqno;
1930 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
1931 }
1932
1933 /*
1934 * Copy the message to user space
1935 */
1936 if (ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size +
1937 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
1938 return(MACH_RCV_INVALID_DATA);
1939 else
1940 return(MACH_MSG_SUCCESS);
1941 }
mirror of XNU