osfmk/ipc/mach_msg.c

   1 /*
   2  * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  * NOTICE: This file was modified by McAfee Research in 2004 to introduce
  58  * support for mandatory and extensible security protections.  This notice
  59  * is included in support of clause 2.2 (b) of the Apple Public License,
  60  * Version 2.0.
  61  * Copyright (c) 2005 SPARTA, Inc.
  62  */
  63 /*
  64  */
  65 /*
  66  *      File:   ipc/mach_msg.c
  67  *      Author: Rich Draves
  68  *      Date:   1989
  69  *
  70  *      Exported message traps.  See mach/message.h.
  71  */
  72
  73 #include <mach/mach_types.h>
  74 #include <mach/kern_return.h>
  75 #include <mach/port.h>
  76 #include <mach/message.h>
  77 #include <mach/mig_errors.h>
  78 #include <mach/mach_traps.h>
  79
  80 #include <kern/kern_types.h>
  81 #include <kern/assert.h>
  82 #include <kern/counters.h>
  83 #include <kern/cpu_number.h>
  84 #include <kern/ipc_kobject.h>
  85 #include <kern/ipc_mig.h>
  86 #include <kern/task.h>
  87 #include <kern/thread.h>
  88 #include <kern/sched_prim.h>
  89 #include <kern/exception.h>
  90 #include <kern/misc_protos.h>
  91 #include <kern/kalloc.h>
  92 #include <kern/processor.h>
  93 #include <kern/syscall_subr.h>
  94
  95 #include <vm/vm_map.h>
  96
  97 #include <ipc/ipc_types.h>
  98 #include <ipc/ipc_kmsg.h>
  99 #include <ipc/ipc_mqueue.h>
 100 #include <ipc/ipc_object.h>
 101 #include <ipc/ipc_notify.h>
 102 #include <ipc/ipc_port.h>
 103 #include <ipc/ipc_pset.h>
 104 #include <ipc/ipc_space.h>
 105 #include <ipc/ipc_entry.h>
 106 #include <ipc/ipc_importance.h>
 107 #include <ipc/ipc_voucher.h>
 108
 109 #include <machine/machine_routines.h>
 110 #include <security/mac_mach_internal.h>
 111
 112 #include <sys/kdebug.h>
 113
 114 #ifndef offsetof
 115 #define offsetof(type, member)  ((size_t)(&((type *)0)->member))
 116 #endif /* offsetof */
 117
 118 /*
 119  * Forward declarations - kernel internal routines
 120  */
 121
 122 mach_msg_return_t mach_msg_send(
 123         mach_msg_header_t       *msg,
 124         mach_msg_option_t       option,
 125         mach_msg_size_t         send_size,
 126         mach_msg_timeout_t      send_timeout,
 127         mach_port_name_t        notify);
 128
 129 mach_msg_return_t mach_msg_receive(
 130         mach_msg_header_t       *msg,
 131         mach_msg_option_t       option,
 132         mach_msg_size_t         rcv_size,
 133         mach_port_name_t        rcv_name,
 134         mach_msg_timeout_t      rcv_timeout,
 135         void                    (*continuation)(mach_msg_return_t),
 136         mach_msg_size_t         slist_size);
 137
 138
 139 mach_msg_return_t msg_receive_error(
 140         ipc_kmsg_t              kmsg,
 141         mach_msg_option_t       option,
 142         mach_vm_address_t       rcv_addr,
 143         mach_msg_size_t         rcv_size,
 144         mach_port_seqno_t       seqno,
 145         ipc_space_t             space,
 146         mach_msg_size_t         *out_size);
 147
 148 security_token_t KERNEL_SECURITY_TOKEN = KERNEL_SECURITY_TOKEN_VALUE;
 149 audit_token_t KERNEL_AUDIT_TOKEN = KERNEL_AUDIT_TOKEN_VALUE;
 150
 151 mach_msg_format_0_trailer_t trailer_template = {
 152         /* mach_msg_trailer_type_t */ MACH_MSG_TRAILER_FORMAT_0,
 153         /* mach_msg_trailer_size_t */ MACH_MSG_TRAILER_MINIMUM_SIZE,
 154         /* mach_port_seqno_t */       0,
 155         /* security_token_t */        KERNEL_SECURITY_TOKEN_VALUE
 156 };
 157
 158 /*
 159  *      Routine:        mach_msg_send [Kernel Internal]
 160  *      Purpose:
 161  *              Routine for kernel-task threads to send a message.
 162  *
 163  *              Unlike mach_msg_send_from_kernel(), this routine
 164  *              looks port names up in the kernel's port namespace
 165  *              and copies in the kernel virtual memory (instead
 166  *              of taking a vm_map_copy_t pointer for OOL descriptors).
 167  *      Conditions:
 168  *              Nothing locked.
 169  *      Returns:
 170  *              MACH_MSG_SUCCESS        Sent the message.
 171  *              MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
 172  *              MACH_SEND_NO_BUFFER     Couldn't allocate buffer.
 173  *              MACH_SEND_INVALID_DATA  Couldn't copy message data.
 174  *              MACH_SEND_INVALID_HEADER
 175  *                      Illegal value in the message header bits.
 176  *              MACH_SEND_INVALID_DEST  The space is dead.
 177  *              MACH_SEND_INVALID_NOTIFY        Bad notify port.
 178  *              MACH_SEND_INVALID_DEST  Can't copyin destination port.
 179  *              MACH_SEND_INVALID_REPLY Can't copyin reply port.
 180  *              MACH_SEND_TIMED_OUT     Timeout expired without delivery.
 181  *              MACH_SEND_INTERRUPTED   Delivery interrupted.
 182  */
 183
 184 mach_msg_return_t
 185 mach_msg_send(
 186         mach_msg_header_t       *msg,
 187         mach_msg_option_t       option,
 188         mach_msg_size_t         send_size,
 189         mach_msg_timeout_t      send_timeout,
 190         mach_msg_priority_t     override)
 191 {
 192         ipc_space_t space = current_space();
 193         vm_map_t map = current_map();
 194         ipc_kmsg_t kmsg;
 195         mach_msg_return_t mr;
 196         mach_msg_size_t msg_and_trailer_size;
 197         mach_msg_max_trailer_t  *trailer;
 198
 199         if ((send_size < sizeof(mach_msg_header_t)) || (send_size & 3))
 200                 return MACH_SEND_MSG_TOO_SMALL;
 201
 202         if (send_size > MACH_MSG_SIZE_MAX - MAX_TRAILER_SIZE)
 203                 return MACH_SEND_TOO_LARGE;
 204
 205         KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_START);
 206
 207         msg_and_trailer_size = send_size + MAX_TRAILER_SIZE;
 208
 209         kmsg = ipc_kmsg_alloc(msg_and_trailer_size);
 210
 211         if (kmsg == IKM_NULL) {
 212                 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, MACH_SEND_NO_BUFFER);
 213                 return MACH_SEND_NO_BUFFER;
 214         }
 215
 216         KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_LINK) | DBG_FUNC_NONE,
 217                               (uintptr_t)0, /* this should only be called from the kernel! */
 218                               VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
 219                               0, 0,
 220                               0);
 221         (void) memcpy((void *) kmsg->ikm_header, (const void *) msg, send_size);
 222
 223         kmsg->ikm_header->msgh_size = send_size;
 224
 225         /*
 226          * reserve for the trailer the largest space (MAX_TRAILER_SIZE)
 227          * However, the internal size field of the trailer (msgh_trailer_size)
 228          * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to optimize
 229          * the cases where no implicit data is requested.
 230          */
 231         trailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + send_size);
 232         trailer->msgh_sender = current_thread()->task->sec_token;
 233         trailer->msgh_audit = current_thread()->task->audit_token;
 234         trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
 235         trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
 236
 237         mr = ipc_kmsg_copyin(kmsg, space, map, override, &option);
 238
 239         if (mr != MACH_MSG_SUCCESS) {
 240                 ipc_kmsg_free(kmsg);
 241                 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, mr);
 242                 return mr;
 243         }
 244
 245         mr = ipc_kmsg_send(kmsg, option, send_timeout);
 246
 247         if (mr != MACH_MSG_SUCCESS) {
 248             mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
 249             (void) memcpy((void *) msg, (const void *) kmsg->ikm_header,
 250                           kmsg->ikm_header->msgh_size);
 251             ipc_kmsg_free(kmsg);
 252             KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, mr);
 253         }
 254
 255         return mr;
 256 }
 257
 258 /*
 259  * message header as seen at user-space
 260  * (for MACH_RCV_LARGE/IDENTITY updating)
 261  */
 262 typedef struct
 263 {
 264   mach_msg_bits_t       msgh_bits;
 265   mach_msg_size_t       msgh_size;
 266   mach_port_name_t      msgh_remote_port;
 267   mach_port_name_t      msgh_local_port;
 268   mach_msg_size_t       msgh_reserved;
 269   mach_msg_id_t         msgh_id;
 270 } mach_msg_user_header_t;
 271
 272 /*
 273  *      Routine:        mach_msg_receive_results
 274  *      Purpose:
 275  *              Receive a message.
 276  *      Conditions:
 277  *              Nothing locked.
 278  *      Returns:
 279  *              MACH_MSG_SUCCESS        Received a message.
 280  *              MACH_RCV_INVALID_NAME   The name doesn't denote a right,
 281  *                      or the denoted right is not receive or port set.
 282  *              MACH_RCV_IN_SET         Receive right is a member of a set.
 283  *              MACH_RCV_TOO_LARGE      Message wouldn't fit into buffer.
 284  *              MACH_RCV_TIMED_OUT      Timeout expired without a message.
 285  *              MACH_RCV_INTERRUPTED    Reception interrupted.
 286  *              MACH_RCV_PORT_DIED      Port/set died while receiving.
 287  *              MACH_RCV_PORT_CHANGED   Port moved into set while receiving.
 288  *              MACH_RCV_INVALID_DATA   Couldn't copy to user buffer.
 289  *              MACH_RCV_INVALID_NOTIFY Bad notify port.
 290  *              MACH_RCV_HEADER_ERROR
 291  */
 292
 293 mach_msg_return_t
 294 mach_msg_receive_results(
 295         mach_msg_size_t *sizep)
 296 {
 297         thread_t          self = current_thread();
 298         ipc_space_t       space = current_space();
 299         vm_map_t          map = current_map();
 300
 301         ipc_object_t      object = self->ith_object;
 302         mach_msg_return_t mr = self->ith_state;
 303         mach_vm_address_t rcv_addr = self->ith_msg_addr;
 304         mach_msg_size_t   rcv_size = self->ith_rsize;
 305         mach_msg_option_t option = self->ith_option;
 306         ipc_kmsg_t        kmsg = self->ith_kmsg;
 307         mach_port_seqno_t seqno = self->ith_seqno;
 308
 309         mach_msg_trailer_size_t trailer_size;
 310         mach_msg_size_t   size = 0;
 311
 312         io_release(object);
 313
 314         if (mr != MACH_MSG_SUCCESS) {
 315
 316                 if (mr == MACH_RCV_TOO_LARGE) {
 317
 318                         /*
 319                          * If the receive operation occurs with MACH_RCV_LARGE set
 320                          * then no message was extracted from the queue, and the size
 321                          * and (optionally) receiver names were the only thing captured.
 322                          * Just copyout the size (and optional port name) in a fake
 323                          * header.
 324                          */
 325                         if (option & MACH_RCV_LARGE) {
 326
 327                                 if ((option & MACH_RCV_STACK) == 0 &&
 328                                     rcv_size >= offsetof(mach_msg_user_header_t, msgh_reserved)) {
 329
 330                                         /*
 331                                          * We need to inform the user-level code that it needs more
 332                                          * space.  The value for how much space was returned in the
 333                                          * msize save area instead of the message (which was left on
 334                                          * the queue).
 335                                          */
 336                                         if (option & MACH_RCV_LARGE_IDENTITY) {
 337                                                 if (copyout((char *) &self->ith_receiver_name,
 338                                                             rcv_addr + offsetof(mach_msg_user_header_t, msgh_local_port),
 339                                                             sizeof(mach_port_name_t)))
 340                                                         mr = MACH_RCV_INVALID_DATA;
 341                                         }
 342                                         if (copyout((char *) &self->ith_msize,
 343                                                     rcv_addr + offsetof(mach_msg_user_header_t, msgh_size),
 344                                                     sizeof(mach_msg_size_t)))
 345                                                 mr = MACH_RCV_INVALID_DATA;
 346                                 }
 347                         } else {
 348
 349                                 /* discard importance in message */
 350                                 ipc_importance_clean(kmsg);
 351
 352                                 if (msg_receive_error(kmsg, option, rcv_addr, rcv_size, seqno, space, &size)
 353                                     == MACH_RCV_INVALID_DATA)
 354                                         mr = MACH_RCV_INVALID_DATA;
 355                         }
 356                 }
 357
 358                 if (sizep)
 359                         *sizep = size;
 360                 return mr;
 361         }
 362
 363         /* MACH_MSG_SUCCESS */
 364
 365 #if IMPORTANCE_INHERITANCE
 366
 367         /* adopt/transform any importance attributes carried in the message */
 368         ipc_importance_receive(kmsg, option);
 369
 370 #endif  /* IMPORTANCE_INHERITANCE */
 371
 372         /* auto redeem the voucher in the message */
 373         ipc_voucher_receive_postprocessing(kmsg, option);
 374
 375         trailer_size = ipc_kmsg_add_trailer(kmsg, space, option, self, seqno, FALSE,
 376                         kmsg->ikm_header->msgh_remote_port->ip_context);
 377
 378         mr = ipc_kmsg_copyout(kmsg, space, map, MACH_MSG_BODY_NULL, option);
 379
 380         if (mr != MACH_MSG_SUCCESS) {
 381
 382                 /* already received importance, so have to undo that here */
 383                 ipc_importance_unreceive(kmsg, option);
 384
 385                 /* if we had a body error copyout what we have, otherwise a simple header/trailer */
 386                 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
 387                         if (ipc_kmsg_put(kmsg, option, rcv_addr, rcv_size, trailer_size, &size) == MACH_RCV_INVALID_DATA)
 388                                 mr = MACH_RCV_INVALID_DATA;
 389                 } else {
 390                         if (msg_receive_error(kmsg, option, rcv_addr, rcv_size, seqno, space, &size)
 391                                                 == MACH_RCV_INVALID_DATA)
 392                                 mr = MACH_RCV_INVALID_DATA;
 393                 }
 394         } else {
 395                 /* capture ksmg QoS values to the thread continuation state */
 396                 self->ith_qos = kmsg->ikm_qos;
 397                 self->ith_qos_override = kmsg->ikm_qos_override;
 398                 mr = ipc_kmsg_put(kmsg, option, rcv_addr, rcv_size, trailer_size, &size);
 399         }
 400
 401         if (sizep)
 402                 *sizep = size;
 403         return mr;
 404 }
 405 #ifndef _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG
 406 #define _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG 0x02000000 /* pthread event manager bit */
 407 #endif
 408 #ifndef _PTHREAD_PRIORITY_OVERCOMMIT_FLAG
 409 #define _PTHREAD_PRIORITY_OVERCOMMIT_FLAG    0x80000000 /* request overcommit threads */
 410 #endif
 411 #ifndef _PTHREAD_PRIORITY_QOS_CLASS_MASK
 412 #define _PTHREAD_PRIORITY_QOS_CLASS_MASK    0x003fff00  /* QoS class mask */
 413 #endif
 414
 415 /* JMM - this needs to invoke a pthread function to compute this */
 416 mach_msg_priority_t
 417 mach_msg_priority_combine(mach_msg_priority_t msg_qos,
 418                           mach_msg_priority_t recv_qos)
 419 {
 420     mach_msg_priority_t overcommit;
 421         mach_msg_priority_t no_oc_qos;
 422         mach_msg_priority_t res;
 423
 424         assert(msg_qos < _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG);
 425         overcommit = recv_qos & _PTHREAD_PRIORITY_OVERCOMMIT_FLAG;
 426         no_oc_qos =  recv_qos & ~overcommit;
 427         res = (no_oc_qos > msg_qos) ? no_oc_qos : msg_qos;
 428         res |= overcommit;
 429
 430         return res;
 431 }
 432
 433 /*
 434  *      Routine:        mach_msg_receive [Kernel Internal]
 435  *      Purpose:
 436  *              Routine for kernel-task threads to actively receive a message.
 437  *
 438  *              Unlike being dispatched to by ipc_kobject_server() or the
 439  *              reply part of mach_msg_rpc_from_kernel(), this routine
 440  *              looks up the receive port name in the kernel's port
 441  *              namespace and copies out received port rights to that namespace
 442  *              as well.  Out-of-line memory is copied out the kernel's
 443  *              address space (rather than just providing the vm_map_copy_t).
 444  *      Conditions:
 445  *              Nothing locked.
 446  *      Returns:
 447  *              MACH_MSG_SUCCESS        Received a message.
 448  *              See <mach/message.h> for list of MACH_RCV_XXX errors.
 449  */
 450 mach_msg_return_t
 451 mach_msg_receive(
 452         mach_msg_header_t       *msg,
 453         mach_msg_option_t       option,
 454         mach_msg_size_t         rcv_size,
 455         mach_port_name_t        rcv_name,
 456         mach_msg_timeout_t      rcv_timeout,
 457         void                    (*continuation)(mach_msg_return_t),
 458         __unused mach_msg_size_t slist_size)
 459 {
 460         thread_t self = current_thread();
 461         ipc_space_t space = current_space();
 462         ipc_object_t object;
 463         ipc_mqueue_t mqueue;
 464         mach_msg_return_t mr;
 465
 466         mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
 467         if (mr != MACH_MSG_SUCCESS) {
 468                 return mr;
 469         }
 470         /* hold ref for object */
 471
 472         self->ith_msg_addr = CAST_DOWN(mach_vm_address_t, msg);
 473         self->ith_object = object;
 474         self->ith_rsize = rcv_size;
 475         self->ith_msize = 0;
 476         self->ith_option = option;
 477         self->ith_continuation = continuation;
 478
 479         ipc_mqueue_receive(mqueue, option, rcv_size, rcv_timeout, THREAD_ABORTSAFE);
 480         if ((option & MACH_RCV_TIMEOUT) && rcv_timeout == 0)
 481                 thread_poll_yield(self);
 482         return mach_msg_receive_results(NULL);
 483 }
 484
 485 void
 486 mach_msg_receive_continue(void)
 487 {
 488         mach_msg_return_t mr;
 489         thread_t self = current_thread();
 490
 491         if (self->ith_state == MACH_PEEK_READY)
 492                 mr = MACH_PEEK_READY;
 493         else
 494                 mr = mach_msg_receive_results(NULL);
 495         (*self->ith_continuation)(mr);
 496 }
 497
 498
 499 /*
 500  *      Routine:        mach_msg_overwrite_trap [mach trap]
 501  *      Purpose:
 502  *              Possibly send a message; possibly receive a message.
 503  *      Conditions:
 504  *              Nothing locked.
 505  *      Returns:
 506  *              All of mach_msg_send and mach_msg_receive error codes.
 507  */
 508
 509 mach_msg_return_t
 510 mach_msg_overwrite_trap(
 511         struct mach_msg_overwrite_trap_args *args)
 512 {
 513         mach_vm_address_t       msg_addr = args->msg;
 514         mach_msg_option_t       option = args->option;
 515         mach_msg_size_t         send_size = args->send_size;
 516         mach_msg_size_t         rcv_size = args->rcv_size;
 517         mach_port_name_t        rcv_name = args->rcv_name;
 518         mach_msg_timeout_t      msg_timeout = args->timeout;
 519         mach_msg_priority_t override = args->override;
 520         mach_vm_address_t       rcv_msg_addr = args->rcv_msg;
 521         __unused mach_port_seqno_t temp_seqno = 0;
 522
 523         mach_msg_return_t  mr = MACH_MSG_SUCCESS;
 524         vm_map_t map = current_map();
 525
 526         /* Only accept options allowed by the user */
 527         option &= MACH_MSG_OPTION_USER;
 528
 529         if (option & MACH_SEND_MSG) {
 530                 ipc_space_t space = current_space();
 531                 ipc_kmsg_t kmsg;
 532
 533                 KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_START);
 534
 535                 mr = ipc_kmsg_get(msg_addr, send_size, &kmsg);
 536
 537                 if (mr != MACH_MSG_SUCCESS) {
 538                         KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, mr);
 539                         return mr;
 540                 }
 541
 542                 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_LINK) | DBG_FUNC_NONE,
 543                                       (uintptr_t)msg_addr,
 544                                       VM_KERNEL_ADDRPERM((uintptr_t)kmsg),
 545                                       0, 0,
 546                                       0);
 547
 548                 mr = ipc_kmsg_copyin(kmsg, space, map, override, &option);
 549
 550                 if (mr != MACH_MSG_SUCCESS) {
 551                         ipc_kmsg_free(kmsg);
 552                         KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, mr);
 553                         return mr;
 554                 }
 555
 556                 mr = ipc_kmsg_send(kmsg, option, msg_timeout);
 557
 558                 if (mr != MACH_MSG_SUCCESS) {
 559                         mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
 560                         (void) ipc_kmsg_put(kmsg, option, msg_addr, send_size, 0, NULL);
 561                         KDBG(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_INFO) | DBG_FUNC_END, mr);
 562                         return mr;
 563                 }
 564
 565         }
 566
 567         if (option & MACH_RCV_MSG) {
 568                 thread_t self = current_thread();
 569                 ipc_space_t space = current_space();
 570                 ipc_object_t object;
 571                 ipc_mqueue_t mqueue;
 572
 573                 mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
 574                 if (mr != MACH_MSG_SUCCESS) {
 575                         return mr;
 576                 }
 577                 /* hold ref for object */
 578
 579                 if (rcv_msg_addr != (mach_vm_address_t)0)
 580                         self->ith_msg_addr = rcv_msg_addr;
 581                 else
 582                         self->ith_msg_addr = msg_addr;
 583                 self->ith_object = object;
 584                 self->ith_rsize = rcv_size;
 585                 self->ith_msize = 0;
 586                 self->ith_option = option;
 587                 self->ith_receiver_name = MACH_PORT_NULL;
 588                 self->ith_continuation = thread_syscall_return;
 589
 590                 ipc_mqueue_receive(mqueue, option, rcv_size, msg_timeout, THREAD_ABORTSAFE);
 591                 if ((option & MACH_RCV_TIMEOUT) && msg_timeout == 0)
 592                         thread_poll_yield(self);
 593                 return mach_msg_receive_results(NULL);
 594         }
 595
 596         return MACH_MSG_SUCCESS;
 597 }
 598
 599 /*
 600  *      Routine:        mach_msg_trap [mach trap]
 601  *      Purpose:
 602  *              Possibly send a message; possibly receive a message.
 603  *      Conditions:
 604  *              Nothing locked.
 605  *      Returns:
 606  *              All of mach_msg_send and mach_msg_receive error codes.
 607  */
 608
 609 mach_msg_return_t
 610 mach_msg_trap(
 611         struct mach_msg_overwrite_trap_args *args)
 612 {
 613         kern_return_t kr;
 614         args->rcv_msg = (mach_vm_address_t)0;
 615
 616         kr = mach_msg_overwrite_trap(args);
 617         return kr;
 618 }
 619
 620
 621 /*
 622  *      Routine:        msg_receive_error       [internal]
 623  *      Purpose:
 624  *              Builds a minimal header/trailer and copies it to
 625  *              the user message buffer.  Invoked when in the case of a
 626  *              MACH_RCV_TOO_LARGE or MACH_RCV_BODY_ERROR error.
 627  *      Conditions:
 628  *              Nothing locked.
 629  *              size - maximum buffer size on input,
 630  *                     actual copied-out size on output
 631  *      Returns:
 632  *              MACH_MSG_SUCCESS        minimal header/trailer copied
 633  *              MACH_RCV_INVALID_DATA   copyout to user buffer failed
 634  */
 635
 636 mach_msg_return_t
 637 msg_receive_error(
 638         ipc_kmsg_t              kmsg,
 639         mach_msg_option_t       option,
 640         mach_vm_address_t       rcv_addr,
 641         mach_msg_size_t         rcv_size,
 642         mach_port_seqno_t       seqno,
 643         ipc_space_t             space,
 644         mach_msg_size_t         *sizep)
 645 {
 646         mach_vm_address_t       context;
 647         mach_msg_trailer_size_t trailer_size;
 648         mach_msg_max_trailer_t  *trailer;
 649
 650         context = kmsg->ikm_header->msgh_remote_port->ip_context;
 651
 652         /*
 653          * Copy out the destination port in the message.
 654          * Destroy all other rights and memory in the message.
 655          */
 656         ipc_kmsg_copyout_dest(kmsg, space);
 657
 658         /*
 659          * Build a minimal message with the requested trailer.
 660          */
 661         trailer = (mach_msg_max_trailer_t *)
 662                         ((vm_offset_t)kmsg->ikm_header +
 663                         round_msg(sizeof(mach_msg_header_t)));
 664         kmsg->ikm_header->msgh_size = sizeof(mach_msg_header_t);
 665         bcopy(  (char *)&trailer_template,
 666                 (char *)trailer,
 667                 sizeof(trailer_template));
 668
 669         trailer_size = ipc_kmsg_add_trailer(kmsg, space,
 670                         option, current_thread(), seqno,
 671                         TRUE, context);
 672
 673         /*
 674          * Copy the message to user space and return the size
 675          * (note that ipc_kmsg_put may also adjust the actual
 676          * size copied out to user-space).
 677          */
 678         if (ipc_kmsg_put(kmsg, option, rcv_addr, rcv_size, trailer_size, sizep) == MACH_RCV_INVALID_DATA)
 679                 return(MACH_RCV_INVALID_DATA);
 680         else
 681                 return(MACH_MSG_SUCCESS);
 682 }