osfmk/i386/bsd_i386.c

   1 /*
   2  * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 #ifdef  MACH_BSD
  29 #include <mach_debug.h>
  30 #include <mach_ldebug.h>
  31
  32 #include <mach/kern_return.h>
  33 #include <mach/mach_traps.h>
  34 #include <mach/thread_status.h>
  35 #include <mach/vm_param.h>
  36
  37 #include <kern/cpu_data.h>
  38 #include <kern/mach_param.h>
  39 #include <kern/task.h>
  40 #include <kern/thread.h>
  41 #include <kern/sched_prim.h>
  42 #include <kern/misc_protos.h>
  43 #include <kern/assert.h>
  44 #include <kern/debug.h>
  45 #include <kern/spl.h>
  46 #include <kern/syscall_sw.h>
  47 #include <ipc/ipc_port.h>
  48 #include <vm/vm_kern.h>
  49 #include <vm/pmap.h>
  50
  51 #include <i386/cpu_number.h>
  52 #include <i386/eflags.h>
  53 #include <i386/proc_reg.h>
  54 #include <i386/tss.h>
  55 #include <i386/user_ldt.h>
  56 #include <i386/fpu.h>
  57 #include <i386/machdep_call.h>
  58 #include <i386/vmparam.h>
  59 #include <i386/mp_desc.h>
  60 #include <i386/misc_protos.h>
  61 #include <i386/thread.h>
  62 #include <i386/trap.h>
  63 #include <i386/seg.h>
  64 #include <mach/i386/syscall_sw.h>
  65 #include <sys/syscall.h>
  66 #include <sys/kdebug.h>
  67 #include <sys/errno.h>
  68 #include <../bsd/sys/sysent.h>
  69
  70 #ifdef MACH_BSD
  71 extern void     mach_kauth_cred_uthread_update(void);
  72 extern void throttle_lowpri_io(int);
  73 #endif
  74
  75 #if CONFIG_MACF
  76 #include <security/mac_mach_internal.h>
  77 #endif
  78
  79 void * find_user_regs(thread_t);
  80
  81 unsigned int get_msr_exportmask(void);
  82
  83 unsigned int get_msr_nbits(void);
  84
  85 unsigned int get_msr_rbits(void);
  86
  87 /*
  88  * thread_userstack:
  89  *
  90  * Return the user stack pointer from the machine
  91  * dependent thread state info.
  92  */
  93 kern_return_t
  94 thread_userstack(
  95         __unused thread_t   thread,
  96         int                 flavor,
  97         thread_state_t      tstate,
  98         unsigned int        count,
  99         mach_vm_offset_t    *user_stack,
 100         int                 *customstack,
 101         __unused boolean_t  is64bit
 102         )
 103 {
 104         if (customstack) {
 105                 *customstack = 0;
 106         }
 107
 108         switch (flavor) {
 109         case x86_THREAD_STATE32:
 110         {
 111                 x86_thread_state32_t *state25;
 112
 113                 if (__improbable(count != x86_THREAD_STATE32_COUNT)) {
 114                         return KERN_INVALID_ARGUMENT;
 115                 }
 116
 117                 state25 = (x86_thread_state32_t *) tstate;
 118
 119                 if (state25->esp) {
 120                         *user_stack = state25->esp;
 121                         if (customstack) {
 122                                 *customstack = 1;
 123                         }
 124                 } else {
 125                         *user_stack = VM_USRSTACK32;
 126                         if (customstack) {
 127                                 *customstack = 0;
 128                         }
 129                 }
 130                 break;
 131         }
 132
 133         case x86_THREAD_FULL_STATE64:
 134         {
 135                 x86_thread_full_state64_t *state25;
 136
 137                 if (__improbable(count != x86_THREAD_FULL_STATE64_COUNT)) {
 138                         return KERN_INVALID_ARGUMENT;
 139                 }
 140
 141                 state25 = (x86_thread_full_state64_t *) tstate;
 142
 143                 if (state25->ss64.rsp) {
 144                         *user_stack = state25->ss64.rsp;
 145                         if (customstack) {
 146                                 *customstack = 1;
 147                         }
 148                 } else {
 149                         *user_stack = VM_USRSTACK64;
 150                         if (customstack) {
 151                                 *customstack = 0;
 152                         }
 153                 }
 154                 break;
 155         }
 156
 157         case x86_THREAD_STATE64:
 158         {
 159                 x86_thread_state64_t *state25;
 160
 161                 if (__improbable(count != x86_THREAD_STATE64_COUNT)) {
 162                         return KERN_INVALID_ARGUMENT;
 163                 }
 164
 165                 state25 = (x86_thread_state64_t *) tstate;
 166
 167                 if (state25->rsp) {
 168                         *user_stack = state25->rsp;
 169                         if (customstack) {
 170                                 *customstack = 1;
 171                         }
 172                 } else {
 173                         *user_stack = VM_USRSTACK64;
 174                         if (customstack) {
 175                                 *customstack = 0;
 176                         }
 177                 }
 178                 break;
 179         }
 180
 181         default:
 182                 return KERN_INVALID_ARGUMENT;
 183         }
 184
 185         return KERN_SUCCESS;
 186 }
 187
 188 /*
 189  * thread_userstackdefault:
 190  *
 191  * Return the default stack location for the
 192  * thread, if otherwise unknown.
 193  */
 194 kern_return_t
 195 thread_userstackdefault(
 196         mach_vm_offset_t *default_user_stack,
 197         boolean_t is64bit)
 198 {
 199         if (is64bit) {
 200                 *default_user_stack = VM_USRSTACK64;
 201         } else {
 202                 *default_user_stack = VM_USRSTACK32;
 203         }
 204         return KERN_SUCCESS;
 205 }
 206
 207 kern_return_t
 208 thread_entrypoint(
 209         __unused thread_t   thread,
 210         int                 flavor,
 211         thread_state_t      tstate,
 212         unsigned int        count,
 213         mach_vm_offset_t    *entry_point
 214         )
 215 {
 216         /*
 217          * Set a default.
 218          */
 219         if (*entry_point == 0) {
 220                 *entry_point = VM_MIN_ADDRESS;
 221         }
 222
 223         switch (flavor) {
 224         case x86_THREAD_STATE32:
 225         {
 226                 x86_thread_state32_t *state25;
 227
 228                 if (count != x86_THREAD_STATE32_COUNT) {
 229                         return KERN_INVALID_ARGUMENT;
 230                 }
 231
 232                 state25 = (i386_thread_state_t *) tstate;
 233                 *entry_point = state25->eip ? state25->eip : VM_MIN_ADDRESS;
 234                 break;
 235         }
 236
 237         case x86_THREAD_STATE64:
 238         {
 239                 x86_thread_state64_t *state25;
 240
 241                 if (count != x86_THREAD_STATE64_COUNT) {
 242                         return KERN_INVALID_ARGUMENT;
 243                 }
 244
 245                 state25 = (x86_thread_state64_t *) tstate;
 246                 *entry_point = state25->rip ? state25->rip : VM_MIN_ADDRESS64;
 247                 break;
 248         }
 249         }
 250         return KERN_SUCCESS;
 251 }
 252
 253 /*
 254  * FIXME - thread_set_child
 255  */
 256
 257 void thread_set_child(thread_t child, int pid);
 258 void
 259 thread_set_child(thread_t child, int pid)
 260 {
 261         pal_register_cache_state(child, DIRTY);
 262
 263         if (thread_is_64bit_addr(child)) {
 264                 x86_saved_state64_t     *iss64;
 265
 266                 iss64 = USER_REGS64(child);
 267
 268                 iss64->rax = pid;
 269                 iss64->rdx = 1;
 270                 iss64->isf.rflags &= ~EFL_CF;
 271         } else {
 272                 x86_saved_state32_t     *iss32;
 273
 274                 iss32 = USER_REGS32(child);
 275
 276                 iss32->eax = pid;
 277                 iss32->edx = 1;
 278                 iss32->efl &= ~EFL_CF;
 279         }
 280 }
 281
 282
 283
 284 /*
 285  * System Call handling code
 286  */
 287
 288 extern long fuword(vm_offset_t);
 289
 290 __attribute__((noreturn))
 291 void
 292 machdep_syscall(x86_saved_state_t *state)
 293 {
 294         int                     args[machdep_call_count];
 295         int                     trapno;
 296         int                     nargs;
 297         const machdep_call_t    *entry;
 298         x86_saved_state32_t     *regs;
 299
 300         assert(is_saved_state32(state));
 301         regs = saved_state32(state);
 302
 303         trapno = regs->eax;
 304 #if DEBUG_TRACE
 305         kprintf("machdep_syscall(0x%08x) code=%d\n", regs, trapno);
 306 #endif
 307
 308         DEBUG_KPRINT_SYSCALL_MDEP(
 309                 "machdep_syscall: trapno=%d\n", trapno);
 310
 311         if (trapno < 0 || trapno >= machdep_call_count) {
 312                 regs->eax = (unsigned int)kern_invalid(NULL);
 313
 314                 thread_exception_return();
 315                 /* NOTREACHED */
 316         }
 317         entry = &machdep_call_table[trapno];
 318         nargs = entry->nargs;
 319
 320         if (nargs != 0) {
 321                 if (copyin((user_addr_t) regs->uesp + sizeof(int),
 322                     (char *) args, (nargs * sizeof(int)))) {
 323                         regs->eax = KERN_INVALID_ADDRESS;
 324
 325                         thread_exception_return();
 326                         /* NOTREACHED */
 327                 }
 328         }
 329         switch (nargs) {
 330         case 0:
 331                 regs->eax = (*entry->routine.args_0)();
 332                 break;
 333         case 1:
 334                 regs->eax = (*entry->routine.args_1)(args[0]);
 335                 break;
 336         case 2:
 337                 regs->eax = (*entry->routine.args_2)(args[0], args[1]);
 338                 break;
 339         case 3:
 340                 if (!entry->bsd_style) {
 341                         regs->eax = (*entry->routine.args_3)(args[0], args[1], args[2]);
 342                 } else {
 343                         int     error;
 344                         uint32_t        rval;
 345
 346                         error = (*entry->routine.args_bsd_3)(&rval, args[0], args[1], args[2]);
 347                         if (error) {
 348                                 regs->eax = error;
 349                                 regs->efl |= EFL_CF;    /* carry bit */
 350                         } else {
 351                                 regs->eax = rval;
 352                                 regs->efl &= ~EFL_CF;
 353                         }
 354                 }
 355                 break;
 356         case 4:
 357                 regs->eax = (*entry->routine.args_4)(args[0], args[1], args[2], args[3]);
 358                 break;
 359
 360         default:
 361                 panic("machdep_syscall: too many args");
 362         }
 363
 364         DEBUG_KPRINT_SYSCALL_MDEP("machdep_syscall: retval=%u\n", regs->eax);
 365
 366 #if DEBUG || DEVELOPMENT
 367         kern_allocation_name_t
 368         prior __assert_only = thread_get_kernel_state(current_thread())->allocation_name;
 369         assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
 370 #endif /* DEBUG || DEVELOPMENT */
 371
 372         throttle_lowpri_io(1);
 373
 374         thread_exception_return();
 375         /* NOTREACHED */
 376 }
 377
 378 __attribute__((noreturn))
 379 void
 380 machdep_syscall64(x86_saved_state_t *state)
 381 {
 382         int                     trapno;
 383         const machdep_call_t    *entry;
 384         x86_saved_state64_t     *regs;
 385
 386         assert(is_saved_state64(state));
 387         regs = saved_state64(state);
 388
 389         trapno = (int)(regs->rax & SYSCALL_NUMBER_MASK);
 390
 391         DEBUG_KPRINT_SYSCALL_MDEP(
 392                 "machdep_syscall64: trapno=%d\n", trapno);
 393
 394         if (trapno < 0 || trapno >= machdep_call_count) {
 395                 regs->rax = (unsigned int)kern_invalid(NULL);
 396
 397                 thread_exception_return();
 398                 /* NOTREACHED */
 399         }
 400         entry = &machdep_call_table64[trapno];
 401
 402         switch (entry->nargs) {
 403         case 0:
 404                 regs->rax = (*entry->routine.args_0)();
 405                 break;
 406         case 1:
 407                 regs->rax = (*entry->routine.args64_1)(regs->rdi);
 408                 break;
 409         case 2:
 410                 regs->rax = (*entry->routine.args64_2)(regs->rdi, regs->rsi);
 411                 break;
 412         case 3:
 413                 if (!entry->bsd_style) {
 414                         regs->rax = (*entry->routine.args64_3)(regs->rdi, regs->rsi, regs->rdx);
 415                 } else {
 416                         int             error;
 417                         uint32_t        rval;
 418
 419                         error = (*entry->routine.args64_bsd_3)(&rval, regs->rdi, regs->rsi, regs->rdx);
 420                         if (error) {
 421                                 regs->rax = (uint64_t)error;
 422                                 regs->isf.rflags |= EFL_CF;    /* carry bit */
 423                         } else {
 424                                 regs->rax = rval;
 425                                 regs->isf.rflags &= ~(uint64_t)EFL_CF;
 426                         }
 427                 }
 428                 break;
 429         default:
 430                 panic("machdep_syscall64: too many args");
 431         }
 432
 433         DEBUG_KPRINT_SYSCALL_MDEP("machdep_syscall: retval=%llu\n", regs->rax);
 434
 435 #if DEBUG || DEVELOPMENT
 436         kern_allocation_name_t
 437         prior __assert_only = thread_get_kernel_state(current_thread())->allocation_name;
 438         assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
 439 #endif /* DEBUG || DEVELOPMENT */
 440
 441         throttle_lowpri_io(1);
 442
 443         thread_exception_return();
 444         /* NOTREACHED */
 445 }
 446
 447 #endif  /* MACH_BSD */
 448
 449
 450 typedef kern_return_t (*mach_call_t)(void *);
 451
 452 struct mach_call_args {
 453         syscall_arg_t arg1;
 454         syscall_arg_t arg2;
 455         syscall_arg_t arg3;
 456         syscall_arg_t arg4;
 457         syscall_arg_t arg5;
 458         syscall_arg_t arg6;
 459         syscall_arg_t arg7;
 460         syscall_arg_t arg8;
 461         syscall_arg_t arg9;
 462 };
 463
 464 static kern_return_t
 465 mach_call_arg_munger32(uint32_t sp, struct mach_call_args *args, const mach_trap_t *trapp);
 466
 467
 468 static kern_return_t
 469 mach_call_arg_munger32(uint32_t sp, struct mach_call_args *args, const mach_trap_t *trapp)
 470 {
 471         if (copyin((user_addr_t)(sp + sizeof(int)), (char *)args, trapp->mach_trap_u32_words * sizeof(int))) {
 472                 return KERN_INVALID_ARGUMENT;
 473         }
 474 #if CONFIG_REQUIRES_U32_MUNGING
 475         trapp->mach_trap_arg_munge32(args);
 476 #else
 477 #error U32 mach traps on x86_64 kernel requires munging
 478 #endif
 479         return KERN_SUCCESS;
 480 }
 481
 482
 483 __private_extern__ void mach_call_munger(x86_saved_state_t *state);
 484
 485 extern const char *const mach_syscall_name_table[];
 486
 487 __attribute__((noreturn))
 488 void
 489 mach_call_munger(x86_saved_state_t *state)
 490 {
 491         int argc;
 492         int call_number;
 493         mach_call_t mach_call;
 494         kern_return_t retval;
 495         struct mach_call_args args = {
 496                 .arg1 = 0,
 497                 .arg2 = 0,
 498                 .arg3 = 0,
 499                 .arg4 = 0,
 500                 .arg5 = 0,
 501                 .arg6 = 0,
 502                 .arg7 = 0,
 503                 .arg8 = 0,
 504                 .arg9 = 0
 505         };
 506         x86_saved_state32_t     *regs;
 507
 508         struct uthread *ut = get_bsdthread_info(current_thread());
 509         uthread_reset_proc_refcount(ut);
 510
 511         assert(is_saved_state32(state));
 512         regs = saved_state32(state);
 513
 514         call_number = -(regs->eax);
 515
 516         DEBUG_KPRINT_SYSCALL_MACH(
 517                 "mach_call_munger: code=%d(%s)\n",
 518                 call_number, mach_syscall_name_table[call_number]);
 519 #if DEBUG_TRACE
 520         kprintf("mach_call_munger(0x%08x) code=%d\n", regs, call_number);
 521 #endif
 522
 523         if (call_number < 0 || call_number >= mach_trap_count) {
 524                 i386_exception(EXC_SYSCALL, call_number, 1);
 525                 /* NOTREACHED */
 526         }
 527         mach_call = (mach_call_t)mach_trap_table[call_number].mach_trap_function;
 528
 529         if (mach_call == (mach_call_t)kern_invalid) {
 530                 DEBUG_KPRINT_SYSCALL_MACH(
 531                         "mach_call_munger: kern_invalid 0x%x\n", regs->eax);
 532                 i386_exception(EXC_SYSCALL, call_number, 1);
 533                 /* NOTREACHED */
 534         }
 535
 536         argc = mach_trap_table[call_number].mach_trap_arg_count;
 537         if (argc) {
 538                 retval = mach_call_arg_munger32(regs->uesp, &args, &mach_trap_table[call_number]);
 539                 if (retval != KERN_SUCCESS) {
 540                         regs->eax = retval;
 541
 542                         DEBUG_KPRINT_SYSCALL_MACH(
 543                                 "mach_call_munger: retval=0x%x\n", retval);
 544
 545                         thread_exception_return();
 546                         /* NOTREACHED */
 547                 }
 548         }
 549
 550 #ifdef MACH_BSD
 551         mach_kauth_cred_uthread_update();
 552 #endif
 553
 554         KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
 555             MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | DBG_FUNC_START,
 556             args.arg1, args.arg2, args.arg3, args.arg4, 0);
 557
 558 #if CONFIG_MACF
 559         /* Check mach trap filter mask, if exists. */
 560         task_t task = current_task();
 561         uint8_t *filter_mask = task->mach_trap_filter_mask;
 562
 563         if (__improbable(filter_mask != NULL &&
 564             !bitstr_test(filter_mask, call_number))) {
 565                 /* Not in filter mask, evaluate policy. */
 566                 if (mac_task_mach_trap_evaluate != NULL) {
 567                         retval = mac_task_mach_trap_evaluate(get_bsdtask_info(task),
 568                             call_number);
 569                         if (retval) {
 570                                 goto skip_machcall;
 571                         }
 572                 }
 573         }
 574 #endif /* CONFIG_MACF */
 575
 576         retval = mach_call(&args);
 577
 578 #if CONFIG_MACF
 579 skip_machcall:
 580 #endif
 581
 582         DEBUG_KPRINT_SYSCALL_MACH("mach_call_munger: retval=0x%x\n", retval);
 583
 584         KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
 585             MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | DBG_FUNC_END,
 586             retval, 0, 0, 0, 0);
 587
 588         regs->eax = retval;
 589
 590 #if DEBUG || DEVELOPMENT
 591         kern_allocation_name_t
 592         prior __assert_only = thread_get_kernel_state(current_thread())->allocation_name;
 593         assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
 594 #endif /* DEBUG || DEVELOPMENT */
 595
 596         throttle_lowpri_io(1);
 597
 598 #if PROC_REF_DEBUG
 599         if (__improbable(uthread_get_proc_refcount(ut) != 0)) {
 600                 panic("system call returned with uu_proc_refcount != 0");
 601         }
 602 #endif
 603
 604         thread_exception_return();
 605         /* NOTREACHED */
 606 }
 607
 608
 609 __private_extern__ void mach_call_munger64(x86_saved_state_t *regs);
 610
 611 __attribute__((noreturn))
 612 void
 613 mach_call_munger64(x86_saved_state_t *state)
 614 {
 615         int call_number;
 616         int argc;
 617         mach_call_t mach_call;
 618         struct mach_call_args args = {
 619                 .arg1 = 0,
 620                 .arg2 = 0,
 621                 .arg3 = 0,
 622                 .arg4 = 0,
 623                 .arg5 = 0,
 624                 .arg6 = 0,
 625                 .arg7 = 0,
 626                 .arg8 = 0,
 627                 .arg9 = 0
 628         };
 629         x86_saved_state64_t     *regs;
 630
 631         struct uthread *ut = get_bsdthread_info(current_thread());
 632         uthread_reset_proc_refcount(ut);
 633
 634         assert(is_saved_state64(state));
 635         regs = saved_state64(state);
 636
 637         call_number = (int)(regs->rax & SYSCALL_NUMBER_MASK);
 638
 639         DEBUG_KPRINT_SYSCALL_MACH(
 640                 "mach_call_munger64: code=%d(%s)\n",
 641                 call_number, mach_syscall_name_table[call_number]);
 642
 643         KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
 644             MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | DBG_FUNC_START,
 645             regs->rdi, regs->rsi, regs->rdx, regs->r10, 0);
 646
 647         if (call_number < 0 || call_number >= mach_trap_count) {
 648                 i386_exception(EXC_SYSCALL, regs->rax, 1);
 649                 /* NOTREACHED */
 650         }
 651         mach_call = (mach_call_t)mach_trap_table[call_number].mach_trap_function;
 652
 653         if (mach_call == (mach_call_t)kern_invalid) {
 654                 i386_exception(EXC_SYSCALL, regs->rax, 1);
 655                 /* NOTREACHED */
 656         }
 657         argc = mach_trap_table[call_number].mach_trap_arg_count;
 658         if (argc) {
 659                 int args_in_regs = MIN(6, argc);
 660                 __nochk_memcpy(&args.arg1, &regs->rdi, args_in_regs * sizeof(syscall_arg_t));
 661
 662                 if (argc > 6) {
 663                         int copyin_count;
 664
 665                         assert(argc <= 9);
 666                         copyin_count = (argc - 6) * (int)sizeof(syscall_arg_t);
 667
 668                         if (copyin((user_addr_t)(regs->isf.rsp + sizeof(user_addr_t)), (char *)&args.arg7, copyin_count)) {
 669                                 regs->rax = KERN_INVALID_ARGUMENT;
 670
 671                                 thread_exception_return();
 672                                 /* NOTREACHED */
 673                         }
 674                 }
 675         }
 676
 677 #ifdef MACH_BSD
 678         mach_kauth_cred_uthread_update();
 679 #endif
 680
 681 #if CONFIG_MACF
 682         /* Check syscall filter mask, if exists. */
 683         task_t task = current_task();
 684         uint8_t *filter_mask = task->mach_trap_filter_mask;
 685
 686         if (__improbable(filter_mask != NULL &&
 687             !bitstr_test(filter_mask, call_number))) {
 688                 /* Not in filter mask, evaluate policy. */
 689                 if (mac_task_mach_trap_evaluate != NULL) {
 690                         regs->rax = mac_task_mach_trap_evaluate(get_bsdtask_info(task),
 691                             call_number);
 692                         if (regs->rax) {
 693                                 goto skip_machcall;
 694                         }
 695                 }
 696         }
 697 #endif /* CONFIG_MACF */
 698
 699         regs->rax = (uint64_t)mach_call((void *)&args);
 700
 701 #if CONFIG_MACF
 702 skip_machcall:
 703 #endif
 704
 705         DEBUG_KPRINT_SYSCALL_MACH( "mach_call_munger64: retval=0x%llx\n", regs->rax);
 706
 707         KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
 708             MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | DBG_FUNC_END,
 709             regs->rax, 0, 0, 0, 0);
 710
 711 #if DEBUG || DEVELOPMENT
 712         kern_allocation_name_t
 713         prior __assert_only = thread_get_kernel_state(current_thread())->allocation_name;
 714         assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
 715 #endif /* DEBUG || DEVELOPMENT */
 716
 717         throttle_lowpri_io(1);
 718
 719 #if PROC_REF_DEBUG
 720         if (__improbable(uthread_get_proc_refcount(ut) != 0)) {
 721                 panic("system call returned with uu_proc_refcount != 0");
 722         }
 723 #endif
 724
 725         thread_exception_return();
 726         /* NOTREACHED */
 727 }
 728
 729
 730 /*
 731  * thread_setuserstack:
 732  *
 733  * Sets the user stack pointer into the machine
 734  * dependent thread state info.
 735  */
 736 void
 737 thread_setuserstack(
 738         thread_t        thread,
 739         mach_vm_address_t       user_stack)
 740 {
 741         pal_register_cache_state(thread, DIRTY);
 742         if (thread_is_64bit_addr(thread)) {
 743                 x86_saved_state64_t     *iss64;
 744
 745                 iss64 = USER_REGS64(thread);
 746
 747                 iss64->isf.rsp = (uint64_t)user_stack;
 748         } else {
 749                 x86_saved_state32_t     *iss32;
 750
 751                 iss32 = USER_REGS32(thread);
 752
 753                 iss32->uesp = CAST_DOWN_EXPLICIT(unsigned int, user_stack);
 754         }
 755 }
 756
 757 /*
 758  * thread_adjuserstack:
 759  *
 760  * Returns the adjusted user stack pointer from the machine
 761  * dependent thread state info.  Used for small (<2G) deltas.
 762  */
 763 user_addr_t
 764 thread_adjuserstack(
 765         thread_t        thread,
 766         int             adjust)
 767 {
 768         pal_register_cache_state(thread, DIRTY);
 769         if (thread_is_64bit_addr(thread)) {
 770                 x86_saved_state64_t     *iss64;
 771
 772                 iss64 = USER_REGS64(thread);
 773
 774                 iss64->isf.rsp += adjust;
 775
 776                 return iss64->isf.rsp;
 777         } else {
 778                 x86_saved_state32_t     *iss32;
 779
 780                 iss32 = USER_REGS32(thread);
 781
 782                 iss32->uesp += adjust;
 783
 784                 return CAST_USER_ADDR_T(iss32->uesp);
 785         }
 786 }
 787
 788 /*
 789  * thread_setentrypoint:
 790  *
 791  * Sets the user PC into the machine
 792  * dependent thread state info.
 793  */
 794 void
 795 thread_setentrypoint(thread_t thread, mach_vm_address_t entry)
 796 {
 797         pal_register_cache_state(thread, DIRTY);
 798         if (thread_is_64bit_addr(thread)) {
 799                 x86_saved_state64_t     *iss64;
 800
 801                 iss64 = USER_REGS64(thread);
 802
 803                 iss64->isf.rip = (uint64_t)entry;
 804         } else {
 805                 x86_saved_state32_t     *iss32;
 806
 807                 iss32 = USER_REGS32(thread);
 808
 809                 iss32->eip = CAST_DOWN_EXPLICIT(unsigned int, entry);
 810         }
 811 }
 812
 813
 814 kern_return_t
 815 thread_setsinglestep(thread_t thread, int on)
 816 {
 817         pal_register_cache_state(thread, DIRTY);
 818         if (thread_is_64bit_addr(thread)) {
 819                 x86_saved_state64_t     *iss64;
 820
 821                 iss64 = USER_REGS64(thread);
 822
 823                 if (on) {
 824                         iss64->isf.rflags |= EFL_TF;
 825                 } else {
 826                         iss64->isf.rflags &= ~EFL_TF;
 827                 }
 828         } else {
 829                 x86_saved_state32_t     *iss32;
 830
 831                 iss32 = USER_REGS32(thread);
 832
 833                 if (on) {
 834                         iss32->efl |= EFL_TF;
 835                         /* Ensure IRET */
 836                         if (iss32->cs == SYSENTER_CS) {
 837                                 iss32->cs = SYSENTER_TF_CS;
 838                         }
 839                 } else {
 840                         iss32->efl &= ~EFL_TF;
 841                 }
 842         }
 843
 844         return KERN_SUCCESS;
 845 }
 846
 847 void *
 848 get_user_regs(thread_t th)
 849 {
 850         pal_register_cache_state(th, DIRTY);
 851         return USER_STATE(th);
 852 }
 853
 854 void *
 855 find_user_regs(thread_t thread)
 856 {
 857         return get_user_regs(thread);
 858 }
 859
 860 #if CONFIG_DTRACE
 861 /*
 862  * DTrace would like to have a peek at the kernel interrupt state, if available.
 863  */
 864 x86_saved_state_t *find_kern_regs(thread_t);
 865
 866 x86_saved_state_t *
 867 find_kern_regs(thread_t thread)
 868 {
 869         if (thread == current_thread() &&
 870             NULL != current_cpu_datap()->cpu_int_state &&
 871             !(USER_STATE(thread) == current_cpu_datap()->cpu_int_state &&
 872             current_cpu_datap()->cpu_interrupt_level == 1)) {
 873                 return current_cpu_datap()->cpu_int_state;
 874         } else {
 875                 return NULL;
 876         }
 877 }
 878
 879 vm_offset_t dtrace_get_cpu_int_stack_top(void);
 880
 881 vm_offset_t
 882 dtrace_get_cpu_int_stack_top(void)
 883 {
 884         return current_cpu_datap()->cpu_int_stack_top;
 885 }
 886 #endif