[apple/xnu.git] / bsd / dev / i386 / systemcalls.c

/*
 * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
#include <kern/task.h>
#include <kern/thread.h>
#include <kern/assert.h>
#include <kern/clock.h>
#include <kern/locks.h>
#include <kern/sched_prim.h>
#include <kern/debug.h>
#include <mach/machine/thread_status.h>
#include <mach/thread_act.h>
#include <mach/branch_predicates.h>

#include <sys/kernel.h>
#include <sys/vm.h>
#include <sys/proc_internal.h>
#include <sys/syscall.h>
#include <sys/systm.h>
#include <sys/user.h>
#include <sys/errno.h>
#include <sys/kdebug.h>
#include <sys/sysent.h>
#include <sys/sysproto.h>
#include <sys/kauth.h>
#include <sys/systm.h>

#include <security/audit/audit.h>

#include <i386/seg.h>
#include <i386/machine_routines.h>
#include <mach/i386/syscall_sw.h>

#include <machine/pal_routines.h>

#if CONFIG_DTRACE
extern int32_t dtrace_systrace_syscall(struct proc *, void *, int *);
extern void dtrace_systrace_syscall_return(unsigned short, int, int *);
#endif

extern void unix_syscall(x86_saved_state_t *);
extern void unix_syscall64(x86_saved_state_t *);
extern void *find_user_regs(thread_t);

/* dynamically generated at build time based on syscalls.master */
extern const char *syscallnames[];

#define code_is_kdebug_trace(code) (((code) == SYS_kdebug_trace) ||   \
                                    ((code) == SYS_kdebug_trace64) || \
                                    ((code) == SYS_kdebug_trace_string))

/*
 * Function:	unix_syscall
 *
 * Inputs:	regs	- pointer to i386 save area
 *
 * Outputs:	none
 */
__attribute__((noreturn))
void
unix_syscall(x86_saved_state_t *state)
{
	thread_t		thread;
	void			*vt;
	unsigned int		code;
	struct sysent		*callp;

	int			error;
	vm_offset_t		params;
	struct proc		*p;
	struct uthread		*uthread;
	x86_saved_state32_t	*regs;
	boolean_t		is_vfork;
	pid_t			pid;

	assert(is_saved_state32(state));
	regs = saved_state32(state);
#if DEBUG
	if (regs->eax == 0x800)
		thread_exception_return();
#endif
	thread = current_thread();
	uthread = get_bsdthread_info(thread);

	uthread_reset_proc_refcount(uthread);

	/* Get the approriate proc; may be different from task's for vfork() */
	is_vfork = uthread->uu_flag & UT_VFORK;
	if (__improbable(is_vfork != 0))
		p = current_proc();
	else 
		p = (struct proc *)get_bsdtask_info(current_task());

	code = regs->eax & I386_SYSCALL_NUMBER_MASK;
	DEBUG_KPRINT_SYSCALL_UNIX("unix_syscall: code=%d(%s) eip=%u\n",
							  code, syscallnames[code >= nsysent ? SYS_invalid : code], (uint32_t)regs->eip);
	params = (vm_offset_t) (regs->uesp + sizeof (int));

	regs->efl &= ~(EFL_CF);

	callp = (code >= nsysent) ? &sysent[SYS_invalid] : &sysent[code];

	if (__improbable(callp == sysent)) {
		code = fuword(params);
		params += sizeof(int);
		callp = (code >= nsysent) ? &sysent[SYS_invalid] : &sysent[code];
	}

	vt = (void *)uthread->uu_arg;

	if (callp->sy_arg_bytes != 0) {
#if CONFIG_REQUIRES_U32_MUNGING
		sy_munge_t	*mungerp;
#else
#error U32 syscalls on x86_64 kernel requires munging
#endif
		uint32_t	 nargs;

		assert((unsigned) callp->sy_arg_bytes <= sizeof (uthread->uu_arg));
		nargs = callp->sy_arg_bytes;
		error = copyin((user_addr_t) params, (char *) vt, nargs);
		if (error) {
			regs->eax = error;
			regs->efl |= EFL_CF;
			thread_exception_return();
			/* NOTREACHED */
		}

		if (__probable(!code_is_kdebug_trace(code))) {
			int *ip = (int *)vt;

			KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
				BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
				*ip, *(ip+1), *(ip+2), *(ip+3), 0);
		}

#if CONFIG_REQUIRES_U32_MUNGING
		mungerp = callp->sy_arg_munge32;

		if (mungerp != NULL)
			(*mungerp)(vt);
#endif
	} else
		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 
			BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
			0, 0, 0, 0, 0);

	/*
	 * Delayed binding of thread credential to process credential, if we
	 * are not running with an explicitly set thread credential.
	 */
	kauth_cred_uthread_update(uthread, p);

	uthread->uu_rval[0] = 0;
	uthread->uu_rval[1] = 0;
	uthread->uu_flag |= UT_NOTCANCELPT;
	uthread->syscall_code = code;
	pid = proc_pid(p);

#ifdef JOE_DEBUG
        uthread->uu_iocount = 0;
        uthread->uu_vpindex = 0;
#endif

	AUDIT_SYSCALL_ENTER(code, p, uthread);
	error = (*(callp->sy_call))((void *) p, (void *) vt, &(uthread->uu_rval[0]));
	AUDIT_SYSCALL_EXIT(code, p, uthread, error);

#ifdef JOE_DEBUG
        if (uthread->uu_iocount)
                printf("system call returned with uu_iocount != 0\n");
#endif
#if CONFIG_DTRACE
	uthread->t_dtrace_errno = error;
#endif /* CONFIG_DTRACE */

	if (__improbable(error == ERESTART)) {
		/*
		 * Move the user's pc back to repeat the syscall:
		 * 5 bytes for a sysenter, or 2 for an int 8x.
		 * The SYSENTER_TF_CS covers single-stepping over a sysenter
		 * - see debug trap handler in idt.s/idt64.s
		 */

		pal_syscall_restart(thread, state);
	}
	else if (error != EJUSTRETURN) {
		if (__improbable(error)) {
		    regs->eax = error;
		    regs->efl |= EFL_CF;	/* carry bit */
		} else { /* (not error) */
			/*
			 * We split retval across two registers, in case the
			 * syscall had a 64-bit return value, in which case
			 * eax/edx matches the function call ABI.
			 */
		    regs->eax = uthread->uu_rval[0];
		    regs->edx = uthread->uu_rval[1];
		} 
	}

	DEBUG_KPRINT_SYSCALL_UNIX(
		"unix_syscall: error=%d retval=(%u,%u)\n",
		error, regs->eax, regs->edx);

	uthread->uu_flag &= ~UT_NOTCANCELPT;

#if DEBUG || DEVELOPMENT
	kern_allocation_name_t
	prior __assert_only = thread_set_allocation_name(NULL);
	assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
#endif /* DEBUG || DEVELOPMENT */

	if (__improbable(uthread->uu_lowpri_window)) {
	        /*
		 * task is marked as a low priority I/O type
		 * and the I/O we issued while in this system call
		 * collided with normal I/O operations... we'll
		 * delay in order to mitigate the impact of this
		 * task on the normal operation of the system
		 */
		throttle_lowpri_io(1);
	}
	if (__probable(!code_is_kdebug_trace(code)))
		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 
			BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
			error, uthread->uu_rval[0], uthread->uu_rval[1], pid, 0);

	if (__improbable(!is_vfork && callp->sy_call == (sy_call_t *)execve && !error)) {
		pal_execve_return(thread);
	}

#if PROC_REF_DEBUG
	if (__improbable(uthread_get_proc_refcount(uthread) != 0)) {
		panic("system call returned with uu_proc_refcount != 0");
	}
#endif

	thread_exception_return();
	/* NOTREACHED */
}

__attribute__((noreturn))
void
unix_syscall64(x86_saved_state_t *state)
{
	thread_t	thread;
	void			*vt;
	unsigned int	code;
	struct sysent	*callp;
	int		args_in_regs;
	boolean_t	args_start_at_rdi;
	int		error;
	struct proc	*p;
	struct uthread	*uthread;
	x86_saved_state64_t *regs;
	pid_t		pid;

	assert(is_saved_state64(state));
	regs = saved_state64(state);
#if	DEBUG
	if (regs->rax == 0x2000800)
		thread_exception_return();
#endif
	thread = current_thread();
	uthread = get_bsdthread_info(thread);

	uthread_reset_proc_refcount(uthread);

	/* Get the approriate proc; may be different from task's for vfork() */
	if (__probable(!(uthread->uu_flag & UT_VFORK)))
		p = (struct proc *)get_bsdtask_info(current_task());
	else 
		p = current_proc();

	/* Verify that we are not being called from a task without a proc */
	if (__improbable(p == NULL)) {
		regs->rax = EPERM;
		regs->isf.rflags |= EFL_CF;
		task_terminate_internal(current_task());
		thread_exception_return();
		/* NOTREACHED */
	}

	code = regs->rax & SYSCALL_NUMBER_MASK;
	DEBUG_KPRINT_SYSCALL_UNIX(
		"unix_syscall64: code=%d(%s) rip=%llx\n",
		code, syscallnames[code >= nsysent ? SYS_invalid : code], regs->isf.rip);
	callp = (code >= nsysent) ? &sysent[SYS_invalid] : &sysent[code];

	vt = (void *)uthread->uu_arg;

	if (__improbable(callp == sysent)) {
	        /*
		 * indirect system call... system call number
		 * passed as 'arg0'
		 */
		code = regs->rdi;
		callp = (code >= nsysent) ? &sysent[SYS_invalid] : &sysent[code];
		args_start_at_rdi = FALSE;
		args_in_regs = 5;
	} else {
		args_start_at_rdi = TRUE;
		args_in_regs = 6;
	}

	if (callp->sy_narg != 0) {
		assert(callp->sy_narg <= 8); /* size of uu_arg */

		args_in_regs = MIN(args_in_regs, callp->sy_narg);
		memcpy(vt, args_start_at_rdi ? &regs->rdi : &regs->rsi, args_in_regs * sizeof(syscall_arg_t));


		if (!code_is_kdebug_trace(code)) {
			uint64_t *ip = (uint64_t *)vt;

			KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 
				BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
				(int)(*ip), (int)(*(ip+1)), (int)(*(ip+2)), (int)(*(ip+3)), 0);
		}

		if (__improbable(callp->sy_narg > args_in_regs)) {
			int copyin_count;

			copyin_count = (callp->sy_narg - args_in_regs) * sizeof(syscall_arg_t);

			error = copyin((user_addr_t)(regs->isf.rsp + sizeof(user_addr_t)), (char *)&uthread->uu_arg[args_in_regs], copyin_count);
			if (error) {
				regs->rax = error;
				regs->isf.rflags |= EFL_CF;
				thread_exception_return();
				/* NOTREACHED */
			}
		}
	} else
		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
			BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
			0, 0, 0, 0, 0);

	/*
	 * Delayed binding of thread credential to process credential, if we
	 * are not running with an explicitly set thread credential.
	 */
	kauth_cred_uthread_update(uthread, p);

	uthread->uu_rval[0] = 0;
	uthread->uu_rval[1] = 0;
	uthread->uu_flag |= UT_NOTCANCELPT;
	uthread->syscall_code = code;
	pid = proc_pid(p);

#ifdef JOE_DEBUG
        uthread->uu_iocount = 0;
        uthread->uu_vpindex = 0;
#endif

	AUDIT_SYSCALL_ENTER(code, p, uthread);
	error = (*(callp->sy_call))((void *) p, vt, &(uthread->uu_rval[0]));
	AUDIT_SYSCALL_EXIT(code, p, uthread, error);

#ifdef JOE_DEBUG
        if (uthread->uu_iocount)
               printf("system call returned with uu_iocount != 0\n");
#endif

#if CONFIG_DTRACE
	uthread->t_dtrace_errno = error;
#endif /* CONFIG_DTRACE */
	
	if (__improbable(error == ERESTART)) {
		/*
		 * all system calls come through via the syscall instruction
		 * in 64 bit mode... its 2 bytes in length
		 * move the user's pc back to repeat the syscall:
		 */
		pal_syscall_restart( thread, state );
	}
	else if (error != EJUSTRETURN) {
		if (__improbable(error)) {
			regs->rax = error;
			regs->isf.rflags |= EFL_CF;	/* carry bit */
		} else { /* (not error) */

			switch (callp->sy_return_type) {
			case _SYSCALL_RET_INT_T:
				regs->rax = uthread->uu_rval[0];
				regs->rdx = uthread->uu_rval[1];
				break;
			case _SYSCALL_RET_UINT_T:
				regs->rax = ((u_int)uthread->uu_rval[0]);
				regs->rdx = ((u_int)uthread->uu_rval[1]);
				break;
			case _SYSCALL_RET_OFF_T:
			case _SYSCALL_RET_ADDR_T:
			case _SYSCALL_RET_SIZE_T:
			case _SYSCALL_RET_SSIZE_T:
			case _SYSCALL_RET_UINT64_T:
			        regs->rax = *((uint64_t *)(&uthread->uu_rval[0]));
				regs->rdx = 0;
				break;
			case _SYSCALL_RET_NONE:
				break;
			default:
				panic("unix_syscall: unknown return type");
				break;
			}
			regs->isf.rflags &= ~EFL_CF;
		} 
	}

	DEBUG_KPRINT_SYSCALL_UNIX(
		"unix_syscall64: error=%d retval=(%llu,%llu)\n",
		error, regs->rax, regs->rdx);
	
	uthread->uu_flag &= ~UT_NOTCANCELPT;

#if DEBUG || DEVELOPMENT
	kern_allocation_name_t
	prior __assert_only = thread_set_allocation_name(NULL);
	assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
#endif /* DEBUG || DEVELOPMENT */

	if (__improbable(uthread->uu_lowpri_window)) {
	        /*
		 * task is marked as a low priority I/O type
		 * and the I/O we issued while in this system call
		 * collided with normal I/O operations... we'll
		 * delay in order to mitigate the impact of this
		 * task on the normal operation of the system
		 */
		throttle_lowpri_io(1);
	}
	if (__probable(!code_is_kdebug_trace(code)))
		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 
			BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
			error, uthread->uu_rval[0], uthread->uu_rval[1], pid, 0);

#if PROC_REF_DEBUG
	if (__improbable(uthread_get_proc_refcount(uthread))) {
		panic("system call returned with uu_proc_refcount != 0");
	}
#endif

	thread_exception_return();
	/* NOTREACHED */
}


void
unix_syscall_return(int error)
{
	thread_t		thread;
	struct uthread		*uthread;
	struct proc *p;
	unsigned int code;
	struct sysent *callp;

	thread = current_thread();
	uthread = get_bsdthread_info(thread);

	pal_register_cache_state(thread, DIRTY);

	p = current_proc();

	if (proc_is64bit(p)) {
		x86_saved_state64_t *regs;

		regs = saved_state64(find_user_regs(thread));

		code = uthread->syscall_code;
		callp = (code >= nsysent) ? &sysent[SYS_invalid] : &sysent[code];

#if CONFIG_DTRACE
		if (callp->sy_call == dtrace_systrace_syscall)
			dtrace_systrace_syscall_return( code, error, uthread->uu_rval );
#endif /* CONFIG_DTRACE */
		AUDIT_SYSCALL_EXIT(code, p, uthread, error);

		if (error == ERESTART) {
			/*
			 * repeat the syscall
			 */
			pal_syscall_restart( thread, find_user_regs(thread) );
		}
		else if (error != EJUSTRETURN) {
			if (error) {
				regs->rax = error;
				regs->isf.rflags |= EFL_CF;	/* carry bit */
			} else { /* (not error) */

				switch (callp->sy_return_type) {
				case _SYSCALL_RET_INT_T:
					regs->rax = uthread->uu_rval[0];
					regs->rdx = uthread->uu_rval[1];
					break;
				case _SYSCALL_RET_UINT_T:
					regs->rax = ((u_int)uthread->uu_rval[0]);
					regs->rdx = ((u_int)uthread->uu_rval[1]);
					break;
				case _SYSCALL_RET_OFF_T:
				case _SYSCALL_RET_ADDR_T:
				case _SYSCALL_RET_SIZE_T:
				case _SYSCALL_RET_SSIZE_T:
				case _SYSCALL_RET_UINT64_T:
					regs->rax = *((uint64_t *)(&uthread->uu_rval[0]));
					regs->rdx = 0;
					break;
				case _SYSCALL_RET_NONE:
					break;
				default:
					panic("unix_syscall: unknown return type");
					break;
				}
				regs->isf.rflags &= ~EFL_CF;
			} 
		}
		DEBUG_KPRINT_SYSCALL_UNIX(
			"unix_syscall_return: error=%d retval=(%llu,%llu)\n",
			error, regs->rax, regs->rdx);
	} else {
		x86_saved_state32_t	*regs;

		regs = saved_state32(find_user_regs(thread));

		regs->efl &= ~(EFL_CF);

		code = uthread->syscall_code;
		callp = (code >= nsysent) ? &sysent[SYS_invalid] : &sysent[code];

#if CONFIG_DTRACE
		if (callp->sy_call == dtrace_systrace_syscall)
			dtrace_systrace_syscall_return( code, error, uthread->uu_rval );
#endif /* CONFIG_DTRACE */
		AUDIT_SYSCALL_EXIT(code, p, uthread, error);

		if (error == ERESTART) {
			pal_syscall_restart( thread, find_user_regs(thread) );
		}
		else if (error != EJUSTRETURN) {
			if (error) {
				regs->eax = error;
				regs->efl |= EFL_CF;	/* carry bit */
			} else { /* (not error) */
				regs->eax = uthread->uu_rval[0];
				regs->edx = uthread->uu_rval[1];
			} 
		}
		DEBUG_KPRINT_SYSCALL_UNIX(
			"unix_syscall_return: error=%d retval=(%u,%u)\n",
			error, regs->eax, regs->edx);
	}


	uthread->uu_flag &= ~UT_NOTCANCELPT;

#if DEBUG || DEVELOPMENT
	kern_allocation_name_t
	prior __assert_only = thread_set_allocation_name(NULL);
	assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
#endif /* DEBUG || DEVELOPMENT */

	if (uthread->uu_lowpri_window) {
	        /*
		 * task is marked as a low priority I/O type
		 * and the I/O we issued while in this system call
		 * collided with normal I/O operations... we'll
		 * delay in order to mitigate the impact of this
		 * task on the normal operation of the system
		 */
		throttle_lowpri_io(1);
	}
	if (!code_is_kdebug_trace(code))
		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 
			BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
			error, uthread->uu_rval[0], uthread->uu_rval[1], p->p_pid, 0);

	thread_exception_return();
	/* NOTREACHED */
}
Commit	Line	Data
0c530ab8	1	/*
39037602	2	* Copyright (c) 2000-2016 Apple Inc. All rights reserved.
0c530ab8	3	*
2d21ac55	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0c530ab8	5	*
2d21ac55 A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
0c530ab8	14	*
2d21ac55 A	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
0c530ab8 A	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
0c530ab8 A	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55 A	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
0c530ab8	25	*
2d21ac55	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
0c530ab8 A	27	*/
	28	#include <kern/task.h>
	29	#include <kern/thread.h>
	30	#include <kern/assert.h>
	31	#include <kern/clock.h>
	32	#include <kern/locks.h>
	33	#include <kern/sched_prim.h>
b0d623f7	34	#include <kern/debug.h>
0c530ab8 A	35	#include <mach/machine/thread_status.h>
0c530ab8 A	36	#include <mach/thread_act.h>
6d2010ae	37	#include <mach/branch_predicates.h>
0c530ab8 A	38
	39	#include <sys/kernel.h>
	40	#include <sys/vm.h>
	41	#include <sys/proc_internal.h>
	42	#include <sys/syscall.h>
	43	#include <sys/systm.h>
	44	#include <sys/user.h>
	45	#include <sys/errno.h>
0c530ab8 A	46	#include <sys/kdebug.h>
	47	#include <sys/sysent.h>
	48	#include <sys/sysproto.h>
	49	#include <sys/kauth.h>
	50	#include <sys/systm.h>
	51
b0d623f7	52	#include <security/audit/audit.h>
0c530ab8 A	53
	54	#include <i386/seg.h>
	55	#include <i386/machine_routines.h>
	56	#include <mach/i386/syscall_sw.h>
	57
6d2010ae A	58	#include <machine/pal_routines.h>
6d2010ae A	59
2d21ac55 A	60	#if CONFIG_DTRACE
	61	extern int32_t dtrace_systrace_syscall(struct proc , void , int *);
	62	extern void dtrace_systrace_syscall_return(unsigned short, int, int *);
	63	#endif
	64
0c530ab8 A	65	extern void unix_syscall(x86_saved_state_t *);
0c530ab8 A	66	extern void unix_syscall64(x86_saved_state_t *);
0c530ab8	67	extern void *find_user_regs(thread_t);
0c530ab8	68
b0d623f7 A	69	/* dynamically generated at build time based on syscalls.master */
	70	extern const char *syscallnames[];
	71
3e170ce0 A	72	#define code_is_kdebug_trace(code) (((code) == SYS_kdebug_trace) \|\| \
	73	((code) == SYS_kdebug_trace64) \|\| \
	74	((code) == SYS_kdebug_trace_string))
a1c7dba1	75
0c530ab8 A	76	/*
	77	* Function: unix_syscall
	78	*
	79	* Inputs: regs - pointer to i386 save area
	80	*
	81	* Outputs: none
	82	*/
39037602	83	__attribute__((noreturn))
0c530ab8 A	84	void
	85	unix_syscall(x86_saved_state_t *state)
	86	{
2d21ac55 A	87	thread_t thread;
	88	void *vt;
	89	unsigned int code;
	90	struct sysent *callp;
	91
	92	int error;
	93	vm_offset_t params;
	94	struct proc *p;
	95	struct uthread *uthread;
0c530ab8	96	x86_saved_state32_t *regs;
6d2010ae	97	boolean_t is_vfork;
743345f9	98	pid_t pid;
0c530ab8 A	99
	100	assert(is_saved_state32(state));
	101	regs = saved_state32(state);
2d21ac55	102	#if DEBUG
0c530ab8 A	103	if (regs->eax == 0x800)
0c530ab8 A	104	thread_exception_return();
2d21ac55	105	#endif
0c530ab8 A	106	thread = current_thread();
	107	uthread = get_bsdthread_info(thread);
	108
3e170ce0	109	uthread_reset_proc_refcount(uthread);
3e170ce0	110
0c530ab8	111	/* Get the approriate proc; may be different from task's for vfork() */
6d2010ae A	112	is_vfork = uthread->uu_flag & UT_VFORK;
6d2010ae A	113	if (__improbable(is_vfork != 0))
0c530ab8	114	p = current_proc();
6d2010ae A	115	else
6d2010ae A	116	p = (struct proc *)get_bsdtask_info(current_task());
0c530ab8	117
2d21ac55	118	code = regs->eax & I386_SYSCALL_NUMBER_MASK;
b0d623f7	119	DEBUG_KPRINT_SYSCALL_UNIX("unix_syscall: code=%d(%s) eip=%u\n",
39037602	120	code, syscallnames[code >= nsysent ? SYS_invalid : code], (uint32_t)regs->eip);
b0d623f7	121	params = (vm_offset_t) (regs->uesp + sizeof (int));
2d21ac55 A	122
	123	regs->efl &= ~(EFL_CF);
	124
39037602	125	callp = (code >= nsysent) ? &sysent[SYS_invalid] : &sysent[code];
0c530ab8	126
6d2010ae	127	if (__improbable(callp == sysent)) {
0c530ab8	128	code = fuword(params);
2d21ac55	129	params += sizeof(int);
39037602	130	callp = (code >= nsysent) ? &sysent[SYS_invalid] : &sysent[code];
0c530ab8	131	}
2d21ac55	132
0c530ab8 A	133	vt = (void *)uthread->uu_arg;
0c530ab8 A	134
2d21ac55	135	if (callp->sy_arg_bytes != 0) {
fe8ab488	136	#if CONFIG_REQUIRES_U32_MUNGING
0c530ab8	137	sy_munge_t *mungerp;
fe8ab488 A	138	#else
	139	#error U32 syscalls on x86_64 kernel requires munging
	140	#endif
39236c6e	141	uint32_t nargs;
0c530ab8	142
2d21ac55	143	assert((unsigned) callp->sy_arg_bytes <= sizeof (uthread->uu_arg));
39236c6e A	144	nargs = callp->sy_arg_bytes;
	145	error = copyin((user_addr_t) params, (char *) vt, nargs);
	146	if (error) {
	147	regs->eax = error;
	148	regs->efl \|= EFL_CF;
	149	thread_exception_return();
	150	/* NOTREACHED */
0c530ab8	151	}
2d21ac55	152
a1c7dba1 A	153	if (__probable(!code_is_kdebug_trace(code))) {
a1c7dba1 A	154	int ip = (int )vt;
0c530ab8	155
316670eb A	156	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
	157	BSDDBG_CODE(DBG_BSD_EXCP_SC, code) \| DBG_FUNC_START,
	158	ip, (ip+1), (ip+2), (ip+3), 0);
0c530ab8	159	}
fe8ab488 A	160
fe8ab488 A	161	#if CONFIG_REQUIRES_U32_MUNGING
0c530ab8 A	162	mungerp = callp->sy_arg_munge32;
0c530ab8 A	163
0c530ab8	164	if (mungerp != NULL)
fe8ab488 A	165	(*mungerp)(vt);
fe8ab488 A	166	#endif
0c530ab8	167	} else
316670eb A	168	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
316670eb A	169	BSDDBG_CODE(DBG_BSD_EXCP_SC, code) \| DBG_FUNC_START,
2d21ac55 A	170	0, 0, 0, 0, 0);
2d21ac55 A	171
0c530ab8 A	172	/*
	173	* Delayed binding of thread credential to process credential, if we
	174	* are not running with an explicitly set thread credential.
	175	*/
2d21ac55	176	kauth_cred_uthread_update(uthread, p);
0c530ab8 A	177
0c530ab8 A	178	uthread->uu_rval[0] = 0;
fe8ab488	179	uthread->uu_rval[1] = 0;
2d21ac55	180	uthread->uu_flag \|= UT_NOTCANCELPT;
fe8ab488	181	uthread->syscall_code = code;
743345f9	182	pid = proc_pid(p);
0c530ab8	183
2d21ac55 A	184	#ifdef JOE_DEBUG
	185	uthread->uu_iocount = 0;
	186	uthread->uu_vpindex = 0;
	187	#endif
0c530ab8 A	188
	189	AUDIT_SYSCALL_ENTER(code, p, uthread);
	190	error = ((callp->sy_call))((void ) p, (void *) vt, &(uthread->uu_rval[0]));
fe8ab488	191	AUDIT_SYSCALL_EXIT(code, p, uthread, error);
2d21ac55 A	192
	193	#ifdef JOE_DEBUG
	194	if (uthread->uu_iocount)
b0d623f7	195	printf("system call returned with uu_iocount != 0\n");
2d21ac55 A	196	#endif
	197	#if CONFIG_DTRACE
	198	uthread->t_dtrace_errno = error;
	199	#endif /* CONFIG_DTRACE */
	200
6d2010ae	201	if (__improbable(error == ERESTART)) {
0c530ab8 A	202	/*
	203	* Move the user's pc back to repeat the syscall:
	204	* 5 bytes for a sysenter, or 2 for an int 8x.
	205	* The SYSENTER_TF_CS covers single-stepping over a sysenter
	206	* - see debug trap handler in idt.s/idt64.s
	207	*/
b0d623f7	208
6d2010ae	209	pal_syscall_restart(thread, state);
0c530ab8 A	210	}
0c530ab8 A	211	else if (error != EJUSTRETURN) {
6d2010ae	212	if (__improbable(error)) {
0c530ab8 A	213	regs->eax = error;
	214	regs->efl \|= EFL_CF; /* carry bit */
	215	} else { /* (not error) */
fe8ab488 A	216	/*
	217	* We split retval across two registers, in case the
	218	* syscall had a 64-bit return value, in which case
	219	* eax/edx matches the function call ABI.
	220	*/
0c530ab8 A	221	regs->eax = uthread->uu_rval[0];
0c530ab8 A	222	regs->edx = uthread->uu_rval[1];
0c530ab8 A	223	}
	224	}
	225
b0d623f7 A	226	DEBUG_KPRINT_SYSCALL_UNIX(
	227	"unix_syscall: error=%d retval=(%u,%u)\n",
	228	error, regs->eax, regs->edx);
	229
2d21ac55	230	uthread->uu_flag &= ~UT_NOTCANCELPT;
6d2010ae	231
5ba3f43e A	232	#if DEBUG \|\| DEVELOPMENT
	233	kern_allocation_name_t
	234	prior __assert_only = thread_set_allocation_name(NULL);
	235	assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
	236	#endif /* DEBUG \|\| DEVELOPMENT */
	237
6d2010ae	238	if (__improbable(uthread->uu_lowpri_window)) {
0c530ab8 A	239	/*
	240	* task is marked as a low priority I/O type
	241	* and the I/O we issued while in this system call
	242	* collided with normal I/O operations... we'll
	243	* delay in order to mitigate the impact of this
	244	* task on the normal operation of the system
	245	*/
39236c6e	246	throttle_lowpri_io(1);
0c530ab8	247	}
a1c7dba1	248	if (__probable(!code_is_kdebug_trace(code)))
316670eb A	249	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
316670eb A	250	BSDDBG_CODE(DBG_BSD_EXCP_SC, code) \| DBG_FUNC_END,
743345f9	251	error, uthread->uu_rval[0], uthread->uu_rval[1], pid, 0);
b0d623f7	252
6d2010ae A	253	if (__improbable(!is_vfork && callp->sy_call == (sy_call_t *)execve && !error)) {
	254	pal_execve_return(thread);
	255	}
0c530ab8	256
3e170ce0 A	257	#if PROC_REF_DEBUG
	258	if (__improbable(uthread_get_proc_refcount(uthread) != 0)) {
	259	panic("system call returned with uu_proc_refcount != 0");
	260	}
	261	#endif
	262
0c530ab8 A	263	thread_exception_return();
	264	/* NOTREACHED */
	265	}
	266
39037602	267	__attribute__((noreturn))
0c530ab8 A	268	void
	269	unix_syscall64(x86_saved_state_t *state)
	270	{
	271	thread_t thread;
fe8ab488	272	void *vt;
0c530ab8 A	273	unsigned int code;
0c530ab8 A	274	struct sysent *callp;
0c530ab8	275	int args_in_regs;
fe8ab488	276	boolean_t args_start_at_rdi;
0c530ab8	277	int error;
0c530ab8 A	278	struct proc *p;
0c530ab8 A	279	struct uthread *uthread;
0c530ab8	280	x86_saved_state64_t *regs;
743345f9	281	pid_t pid;
0c530ab8 A	282
	283	assert(is_saved_state64(state));
	284	regs = saved_state64(state);
6d2010ae	285	#if DEBUG
0c530ab8 A	286	if (regs->rax == 0x2000800)
0c530ab8 A	287	thread_exception_return();
6d2010ae	288	#endif
0c530ab8 A	289	thread = current_thread();
	290	uthread = get_bsdthread_info(thread);
	291
3e170ce0	292	uthread_reset_proc_refcount(uthread);
3e170ce0	293
0c530ab8	294	/* Get the approriate proc; may be different from task's for vfork() */
6d2010ae	295	if (__probable(!(uthread->uu_flag & UT_VFORK)))
0c530ab8 A	296	p = (struct proc *)get_bsdtask_info(current_task());
	297	else
	298	p = current_proc();
	299
	300	/* Verify that we are not being called from a task without a proc */
6d2010ae	301	if (__improbable(p == NULL)) {
0c530ab8 A	302	regs->rax = EPERM;
	303	regs->isf.rflags \|= EFL_CF;
	304	task_terminate_internal(current_task());
	305	thread_exception_return();
	306	/* NOTREACHED */
	307	}
0c530ab8 A	308
0c530ab8 A	309	code = regs->rax & SYSCALL_NUMBER_MASK;
b0d623f7 A	310	DEBUG_KPRINT_SYSCALL_UNIX(
b0d623f7 A	311	"unix_syscall64: code=%d(%s) rip=%llx\n",
39037602 A	312	code, syscallnames[code >= nsysent ? SYS_invalid : code], regs->isf.rip);
39037602 A	313	callp = (code >= nsysent) ? &sysent[SYS_invalid] : &sysent[code];
fe8ab488 A	314
fe8ab488 A	315	vt = (void *)uthread->uu_arg;
0c530ab8	316
6d2010ae	317	if (__improbable(callp == sysent)) {
0c530ab8 A	318	/*
	319	* indirect system call... system call number
	320	* passed as 'arg0'
	321	*/
fe8ab488	322	code = regs->rdi;
39037602	323	callp = (code >= nsysent) ? &sysent[SYS_invalid] : &sysent[code];
fe8ab488	324	args_start_at_rdi = FALSE;
0c530ab8	325	args_in_regs = 5;
fe8ab488 A	326	} else {
	327	args_start_at_rdi = TRUE;
	328	args_in_regs = 6;
0c530ab8 A	329	}
	330
	331	if (callp->sy_narg != 0) {
fe8ab488 A	332	assert(callp->sy_narg <= 8); /* size of uu_arg */
	333
	334	args_in_regs = MIN(args_in_regs, callp->sy_narg);
	335	memcpy(vt, args_start_at_rdi ? &regs->rdi : &regs->rsi, args_in_regs * sizeof(syscall_arg_t));
	336
	337
a1c7dba1	338	if (!code_is_kdebug_trace(code)) {
fe8ab488	339	uint64_t ip = (uint64_t )vt;
0c530ab8	340
316670eb A	341	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
	342	BSDDBG_CODE(DBG_BSD_EXCP_SC, code) \| DBG_FUNC_START,
	343	(int)(ip), (int)((ip+1)), (int)((ip+2)), (int)((ip+3)), 0);
0c530ab8	344	}
0c530ab8	345
6d2010ae	346	if (__improbable(callp->sy_narg > args_in_regs)) {
2d21ac55	347	int copyin_count;
0c530ab8	348
fe8ab488	349	copyin_count = (callp->sy_narg - args_in_regs) * sizeof(syscall_arg_t);
0c530ab8	350
fe8ab488	351	error = copyin((user_addr_t)(regs->isf.rsp + sizeof(user_addr_t)), (char *)&uthread->uu_arg[args_in_regs], copyin_count);
0c530ab8	352	if (error) {
2d21ac55	353	regs->rax = error;
0c530ab8 A	354	regs->isf.rflags \|= EFL_CF;
	355	thread_exception_return();
	356	/* NOTREACHED */
	357	}
	358	}
0c530ab8	359	} else
316670eb A	360	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
	361	BSDDBG_CODE(DBG_BSD_EXCP_SC, code) \| DBG_FUNC_START,
	362	0, 0, 0, 0, 0);
0c530ab8 A	363
	364	/*
	365	* Delayed binding of thread credential to process credential, if we
	366	* are not running with an explicitly set thread credential.
	367	*/
2d21ac55	368	kauth_cred_uthread_update(uthread, p);
0c530ab8 A	369
	370	uthread->uu_rval[0] = 0;
	371	uthread->uu_rval[1] = 0;
2d21ac55	372	uthread->uu_flag \|= UT_NOTCANCELPT;
fe8ab488	373	uthread->syscall_code = code;
743345f9	374	pid = proc_pid(p);
0c530ab8	375
6d2010ae A	376	#ifdef JOE_DEBUG
	377	uthread->uu_iocount = 0;
	378	uthread->uu_vpindex = 0;
	379	#endif
0c530ab8 A	380
0c530ab8 A	381	AUDIT_SYSCALL_ENTER(code, p, uthread);
fe8ab488 A	382	error = ((callp->sy_call))((void ) p, vt, &(uthread->uu_rval[0]));
fe8ab488 A	383	AUDIT_SYSCALL_EXIT(code, p, uthread, error);
2d21ac55	384
6d2010ae A	385	#ifdef JOE_DEBUG
	386	if (uthread->uu_iocount)
	387	printf("system call returned with uu_iocount != 0\n");
	388	#endif
	389
2d21ac55 A	390	#if CONFIG_DTRACE
	391	uthread->t_dtrace_errno = error;
	392	#endif /* CONFIG_DTRACE */
0c530ab8	393
6d2010ae	394	if (__improbable(error == ERESTART)) {
0c530ab8 A	395	/*
	396	* all system calls come through via the syscall instruction
	397	* in 64 bit mode... its 2 bytes in length
	398	* move the user's pc back to repeat the syscall:
	399	*/
6d2010ae	400	pal_syscall_restart( thread, state );
0c530ab8 A	401	}
0c530ab8 A	402	else if (error != EJUSTRETURN) {
6d2010ae	403	if (__improbable(error)) {
2d21ac55	404	regs->rax = error;
0c530ab8 A	405	regs->isf.rflags \|= EFL_CF; /* carry bit */
	406	} else { /* (not error) */
	407
	408	switch (callp->sy_return_type) {
	409	case _SYSCALL_RET_INT_T:
	410	regs->rax = uthread->uu_rval[0];
	411	regs->rdx = uthread->uu_rval[1];
	412	break;
	413	case _SYSCALL_RET_UINT_T:
	414	regs->rax = ((u_int)uthread->uu_rval[0]);
	415	regs->rdx = ((u_int)uthread->uu_rval[1]);
	416	break;
	417	case _SYSCALL_RET_OFF_T:
	418	case _SYSCALL_RET_ADDR_T:
	419	case _SYSCALL_RET_SIZE_T:
	420	case _SYSCALL_RET_SSIZE_T:
d1ecb069	421	case _SYSCALL_RET_UINT64_T:
0c530ab8 A	422	regs->rax = ((uint64_t )(&uthread->uu_rval[0]));
	423	regs->rdx = 0;
	424	break;
	425	case _SYSCALL_RET_NONE:
	426	break;
	427	default:
	428	panic("unix_syscall: unknown return type");
	429	break;
	430	}
	431	regs->isf.rflags &= ~EFL_CF;
	432	}
	433	}
	434
b0d623f7 A	435	DEBUG_KPRINT_SYSCALL_UNIX(
	436	"unix_syscall64: error=%d retval=(%llu,%llu)\n",
	437	error, regs->rax, regs->rdx);
	438
2d21ac55	439	uthread->uu_flag &= ~UT_NOTCANCELPT;
0c530ab8	440
5ba3f43e A	441	#if DEBUG \|\| DEVELOPMENT
	442	kern_allocation_name_t
	443	prior __assert_only = thread_set_allocation_name(NULL);
	444	assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
	445	#endif /* DEBUG \|\| DEVELOPMENT */
	446
6d2010ae	447	if (__improbable(uthread->uu_lowpri_window)) {
0c530ab8 A	448	/*
	449	* task is marked as a low priority I/O type
	450	* and the I/O we issued while in this system call
	451	* collided with normal I/O operations... we'll
	452	* delay in order to mitigate the impact of this
	453	* task on the normal operation of the system
	454	*/
39236c6e	455	throttle_lowpri_io(1);
0c530ab8	456	}
a1c7dba1	457	if (__probable(!code_is_kdebug_trace(code)))
316670eb A	458	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
316670eb A	459	BSDDBG_CODE(DBG_BSD_EXCP_SC, code) \| DBG_FUNC_END,
743345f9	460	error, uthread->uu_rval[0], uthread->uu_rval[1], pid, 0);
0c530ab8	461
3e170ce0 A	462	#if PROC_REF_DEBUG
	463	if (__improbable(uthread_get_proc_refcount(uthread))) {
	464	panic("system call returned with uu_proc_refcount != 0");
	465	}
	466	#endif
	467
0c530ab8 A	468	thread_exception_return();
	469	/* NOTREACHED */
	470	}
	471
	472
	473	void
	474	unix_syscall_return(int error)
	475	{
	476	thread_t thread;
	477	struct uthread *uthread;
	478	struct proc *p;
	479	unsigned int code;
0c530ab8	480	struct sysent *callp;
0c530ab8 A	481
	482	thread = current_thread();
	483	uthread = get_bsdthread_info(thread);
	484
6d2010ae	485	pal_register_cache_state(thread, DIRTY);
b0d623f7	486
0c530ab8 A	487	p = current_proc();
	488
	489	if (proc_is64bit(p)) {
2d21ac55	490	x86_saved_state64_t *regs;
0c530ab8 A	491
	492	regs = saved_state64(find_user_regs(thread));
	493
fe8ab488	494	code = uthread->syscall_code;
39037602	495	callp = (code >= nsysent) ? &sysent[SYS_invalid] : &sysent[code];
0c530ab8	496
2d21ac55 A	497	#if CONFIG_DTRACE
	498	if (callp->sy_call == dtrace_systrace_syscall)
	499	dtrace_systrace_syscall_return( code, error, uthread->uu_rval );
	500	#endif /* CONFIG_DTRACE */
b0d623f7	501	AUDIT_SYSCALL_EXIT(code, p, uthread, error);
0c530ab8 A	502
0c530ab8 A	503	if (error == ERESTART) {
2d21ac55	504	/*
6d2010ae	505	* repeat the syscall
0c530ab8	506	*/
6d2010ae	507	pal_syscall_restart( thread, find_user_regs(thread) );
0c530ab8 A	508	}
0c530ab8 A	509	else if (error != EJUSTRETURN) {
2d21ac55 A	510	if (error) {
2d21ac55 A	511	regs->rax = error;
0c530ab8 A	512	regs->isf.rflags \|= EFL_CF; /* carry bit */
	513	} else { /* (not error) */
	514
2d21ac55	515	switch (callp->sy_return_type) {
0c530ab8	516	case _SYSCALL_RET_INT_T:
2d21ac55	517	regs->rax = uthread->uu_rval[0];
0c530ab8 A	518	regs->rdx = uthread->uu_rval[1];
	519	break;
	520	case _SYSCALL_RET_UINT_T:
2d21ac55	521	regs->rax = ((u_int)uthread->uu_rval[0]);
0c530ab8 A	522	regs->rdx = ((u_int)uthread->uu_rval[1]);
	523	break;
	524	case _SYSCALL_RET_OFF_T:
	525	case _SYSCALL_RET_ADDR_T:
	526	case _SYSCALL_RET_SIZE_T:
	527	case _SYSCALL_RET_SSIZE_T:
d1ecb069	528	case _SYSCALL_RET_UINT64_T:
2d21ac55	529	regs->rax = ((uint64_t )(&uthread->uu_rval[0]));
0c530ab8 A	530	regs->rdx = 0;
	531	break;
	532	case _SYSCALL_RET_NONE:
2d21ac55	533	break;
0c530ab8	534	default:
2d21ac55	535	panic("unix_syscall: unknown return type");
0c530ab8 A	536	break;
	537	}
	538	regs->isf.rflags &= ~EFL_CF;
	539	}
	540	}
b0d623f7 A	541	DEBUG_KPRINT_SYSCALL_UNIX(
	542	"unix_syscall_return: error=%d retval=(%llu,%llu)\n",
	543	error, regs->rax, regs->rdx);
0c530ab8	544	} else {
2d21ac55	545	x86_saved_state32_t *regs;
0c530ab8 A	546
	547	regs = saved_state32(find_user_regs(thread));
	548
2d21ac55	549	regs->efl &= ~(EFL_CF);
fe8ab488 A	550
fe8ab488 A	551	code = uthread->syscall_code;
39037602	552	callp = (code >= nsysent) ? &sysent[SYS_invalid] : &sysent[code];
2d21ac55 A	553
	554	#if CONFIG_DTRACE
	555	if (callp->sy_call == dtrace_systrace_syscall)
	556	dtrace_systrace_syscall_return( code, error, uthread->uu_rval );
	557	#endif /* CONFIG_DTRACE */
b0d623f7	558	AUDIT_SYSCALL_EXIT(code, p, uthread, error);
0c530ab8	559
0c530ab8	560	if (error == ERESTART) {
6d2010ae	561	pal_syscall_restart( thread, find_user_regs(thread) );
0c530ab8 A	562	}
0c530ab8 A	563	else if (error != EJUSTRETURN) {
2d21ac55 A	564	if (error) {
2d21ac55 A	565	regs->eax = error;
0c530ab8 A	566	regs->efl \|= EFL_CF; /* carry bit */
0c530ab8 A	567	} else { /* (not error) */
2d21ac55	568	regs->eax = uthread->uu_rval[0];
0c530ab8	569	regs->edx = uthread->uu_rval[1];
0c530ab8 A	570	}
0c530ab8 A	571	}
b0d623f7 A	572	DEBUG_KPRINT_SYSCALL_UNIX(
	573	"unix_syscall_return: error=%d retval=(%u,%u)\n",
	574	error, regs->eax, regs->edx);
0c530ab8	575	}
0c530ab8	576
0c530ab8	577
2d21ac55	578	uthread->uu_flag &= ~UT_NOTCANCELPT;
0c530ab8	579
5ba3f43e A	580	#if DEBUG \|\| DEVELOPMENT
	581	kern_allocation_name_t
	582	prior __assert_only = thread_set_allocation_name(NULL);
	583	assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
	584	#endif /* DEBUG \|\| DEVELOPMENT */
	585
593a1d5f	586	if (uthread->uu_lowpri_window) {
0c530ab8 A	587	/*
	588	* task is marked as a low priority I/O type
	589	* and the I/O we issued while in this system call
	590	* collided with normal I/O operations... we'll
	591	* delay in order to mitigate the impact of this
	592	* task on the normal operation of the system
	593	*/
39236c6e	594	throttle_lowpri_io(1);
0c530ab8	595	}
a1c7dba1	596	if (!code_is_kdebug_trace(code))
316670eb A	597	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
	598	BSDDBG_CODE(DBG_BSD_EXCP_SC, code) \| DBG_FUNC_END,
	599	error, uthread->uu_rval[0], uthread->uu_rval[1], p->p_pid, 0);
0c530ab8 A	600
	601	thread_exception_return();
	602	/* NOTREACHED */
	603	}