]> git.saurik.com Git - apple/xnu.git/blobdiff - bsd/dev/i386/systemcalls.c
projects / apple / xnu.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
xnu-1228.tar.gz
[apple/xnu.git] / bsd / dev / i386 / systemcalls.c
diff --git a/bsd/dev/i386/systemcalls.c b/bsd/dev/i386/systemcalls.c
index 9f5e9d914d4e29cdd6a9ddf85ed9428b9637af42..fa6db0a3d5f9d64ff5dc92ea1321e01514b8c6b4 100644 (file)
--- a/bsd/dev/i386/systemcalls.c
+++ b/bsd/dev/i386/systemcalls.c
@@ -1,5 +1,5 @@
 /*
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -41,7 +41,6 @@
 #include <sys/systm.h>
 #include <sys/user.h>
 #include <sys/errno.h>
 #include <sys/systm.h>
 #include <sys/user.h>
 #include <sys/errno.h>
-#include <sys/ktrace.h>
 #include <sys/kdebug.h>
 #include <sys/sysent.h>
 #include <sys/sysproto.h>
 #include <sys/kdebug.h>
 #include <sys/sysent.h>
 #include <sys/sysproto.h>
@@ -54,15 +53,18 @@
 #include <i386/machine_routines.h>
 #include <mach/i386/syscall_sw.h>
 
 #include <i386/machine_routines.h>
 #include <mach/i386/syscall_sw.h>
 
+#if CONFIG_DTRACE
+extern int32_t dtrace_systrace_syscall(struct proc *, void *, int *);
+extern void dtrace_systrace_syscall_return(unsigned short, int, int *);
+#endif
+
 extern void unix_syscall(x86_saved_state_t *);
 extern void unix_syscall64(x86_saved_state_t *);
 extern void unix_syscall(x86_saved_state_t *);
 extern void unix_syscall64(x86_saved_state_t *);
-extern void unix_syscall_return(int);
 extern void *find_user_regs(thread_t);
 extern void *find_user_regs(thread_t);
-extern void IOSleep(int);
-extern void exit_funnel_section(void);
-
-extern void Debugger(const char      * message);
+extern void throttle_lowpri_io(int *lowpri_window, mount_t v_mount);
 
 
+extern void x86_toggle_sysenter_arg_store(thread_t thread, boolean_t valid);
+extern boolean_t x86_sysenter_arg_store_isvalid(thread_t thread);
 /*
  * Function:   unix_syscall
  *
 /*
  * Function:   unix_syscall
  *
@@ -73,25 +75,24 @@ extern void Debugger(const char      * message);
 void
 unix_syscall(x86_saved_state_t *state)
 {
 void
 unix_syscall(x86_saved_state_t *state)
 {
-       thread_t        thread;
-       void            *vt; 
-       unsigned short  code;
-       struct sysent   *callp;
-       int             nargs;
-       int             error;
-       int             funnel_type;
-       vm_offset_t     params;
-       struct proc     *p;
-       struct uthread  *uthread;
-       unsigned int cancel_enable;
+       thread_t                thread;
+       void                    *vt;
+       unsigned int            code;
+       struct sysent           *callp;
+
+       int                     error;
+       vm_offset_t             params;
+       struct proc             *p;
+       struct uthread          *uthread;
        x86_saved_state32_t     *regs;
        x86_saved_state32_t     *regs;
+       boolean_t               args_in_uthread;
 
        assert(is_saved_state32(state));
        regs = saved_state32(state);
 
        assert(is_saved_state32(state));
        regs = saved_state32(state);
-
+#if DEBUG
        if (regs->eax == 0x800)
                thread_exception_return();
        if (regs->eax == 0x800)
                thread_exception_return();
-
+#endif
        thread = current_thread();
        uthread = get_bsdthread_info(thread);
 
        thread = current_thread();
        uthread = get_bsdthread_info(thread);
 
@@ -110,36 +111,44 @@ unix_syscall(x86_saved_state_t *state)
                /* NOTREACHED */
        }
 
                /* NOTREACHED */
        }
 
-       //printf("[scall : eax %x]",  regs->eax);
-       code = regs->eax;
+       code = regs->eax & I386_SYSCALL_NUMBER_MASK;
+       args_in_uthread = ((regs->eax & I386_SYSCALL_ARG_BYTES_MASK) != 0) && x86_sysenter_arg_store_isvalid(thread);
        params = (vm_offset_t) ((caddr_t)regs->uesp + sizeof (int));
        params = (vm_offset_t) ((caddr_t)regs->uesp + sizeof (int));
-       callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
+
+       regs->efl &= ~(EFL_CF);
+
+       callp = (code >= NUM_SYSENT) ? &sysent[63] : &sysent[code];
 
        if (callp == sysent) {
                code = fuword(params);
 
        if (callp == sysent) {
                code = fuword(params);
-               params += sizeof (int);
-               callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
+               params += sizeof(int);
+               callp = (code >= NUM_SYSENT) ? &sysent[63] : &sysent[code];
        }
        }
+
        vt = (void *)uthread->uu_arg;
 
        vt = (void *)uthread->uu_arg;
 
-       nargs = callp->sy_narg * sizeof (syscall_arg_t);
-       if (nargs != 0) {
+       if (callp->sy_arg_bytes != 0) {
                sy_munge_t      *mungerp;
 
                sy_munge_t      *mungerp;
 
-               assert(nargs <= 8);
-
-               error = copyin((user_addr_t) params, (char *) vt, nargs);
-               if (error) {
-                       regs->eax = error;
-                       regs->efl |= EFL_CF;
-                       thread_exception_return();
-                       /* NOTREACHED */
+               assert((unsigned) callp->sy_arg_bytes <= sizeof (uthread->uu_arg));
+               if (!args_in_uthread)
+               {
+                       uint32_t nargs;
+                       nargs = callp->sy_arg_bytes;
+                       error = copyin((user_addr_t) params, (char *) vt, nargs);
+                       if (error) {
+                               regs->eax = error;
+                               regs->efl |= EFL_CF;
+                               thread_exception_return();
+                               /* NOTREACHED */
+                       }
                }
                }
+
                if (code != 180) {
                if (code != 180) {
-                       int *ip = (int *)vt;
+                       int *ip = (int *)vt;
 
                        KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
 
                        KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
-                                             *ip, *(ip+1), *(ip+2), *(ip+3), 0);
+                                     *ip, *(ip+1), *(ip+2), *(ip+3), 0);
                }
                mungerp = callp->sy_arg_munge32;
 
                }
                mungerp = callp->sy_arg_munge32;
 
@@ -153,52 +162,37 @@ unix_syscall(x86_saved_state_t *state)
                if (mungerp != NULL)
                        (*mungerp)(NULL, vt);
        } else
                if (mungerp != NULL)
                        (*mungerp)(NULL, vt);
        } else
-               KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
-                                     0, 0, 0, 0, 0);
+               KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
+                       0, 0, 0, 0, 0);
+
        /*
         * Delayed binding of thread credential to process credential, if we
         * are not running with an explicitly set thread credential.
         */
        /*
         * Delayed binding of thread credential to process credential, if we
         * are not running with an explicitly set thread credential.
         */
-       if (uthread->uu_ucred != p->p_ucred &&
-           (uthread->uu_flag & UT_SETUID) == 0) {
-               kauth_cred_t old = uthread->uu_ucred;
-               uthread->uu_ucred = kauth_cred_proc_ref(p);
-               if (IS_VALID_CRED(old))
-                       kauth_cred_unref(&old);
-       }
+       kauth_cred_uthread_update(uthread, p);
 
        uthread->uu_rval[0] = 0;
        uthread->uu_rval[1] = regs->edx;
 
        uthread->uu_rval[0] = 0;
        uthread->uu_rval[1] = regs->edx;
+       uthread->uu_flag |= UT_NOTCANCELPT;
 
 
-       cancel_enable = callp->sy_cancel;
-       
-       if (cancel_enable == _SYSCALL_CANCEL_NONE) {
-               uthread->uu_flag |= UT_NOTCANCELPT;
-       } else {
-               if ((uthread->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL) {
-                       if (cancel_enable == _SYSCALL_CANCEL_PRE) {
-                                       /* system call cancelled; return to handle cancellation */
-                                       regs->eax = (long long)EINTR;
-                                       regs->efl |= EFL_CF;
-                                       thread_exception_return();
-                                       /* NOTREACHED */
-                       } else {
-                               thread_abort_safely(thread);
-                       }
-               }
-       }
-
-       funnel_type = (callp->sy_funnel & FUNNEL_MASK);
-       if (funnel_type == KERNEL_FUNNEL)
-               thread_funnel_set(kernel_flock, TRUE);
 
 
-       if (KTRPOINT(p, KTR_SYSCALL))
-               ktrsyscall(p, code, callp->sy_narg, vt);
+#ifdef JOE_DEBUG
+        uthread->uu_iocount = 0;
+        uthread->uu_vpindex = 0;
+#endif
 
        AUDIT_SYSCALL_ENTER(code, p, uthread);
        error = (*(callp->sy_call))((void *) p, (void *) vt, &(uthread->uu_rval[0]));
 
        AUDIT_SYSCALL_ENTER(code, p, uthread);
        error = (*(callp->sy_call))((void *) p, (void *) vt, &(uthread->uu_rval[0]));
-       AUDIT_SYSCALL_EXIT(error, p, uthread);
-       
+        AUDIT_SYSCALL_EXIT(code, p, uthread, error);
+
+#ifdef JOE_DEBUG
+        if (uthread->uu_iocount)
+                joe_debug("system call returned with uu_iocount != 0");
+#endif
+#if CONFIG_DTRACE
+       uthread->t_dtrace_errno = error;
+#endif /* CONFIG_DTRACE */
+
        if (error == ERESTART) {
                /*
                 * Move the user's pc back to repeat the syscall:
        if (error == ERESTART) {
                /*
                 * Move the user's pc back to repeat the syscall:
@@ -206,8 +200,9 @@ unix_syscall(x86_saved_state_t *state)
                 * The SYSENTER_TF_CS covers single-stepping over a sysenter
                 * - see debug trap handler in idt.s/idt64.s
                 */
                 * The SYSENTER_TF_CS covers single-stepping over a sysenter
                 * - see debug trap handler in idt.s/idt64.s
                 */
-               if (regs->cs == SYSENTER_CS || regs->cs == SYSENTER_TF_CS)
+               if (regs->cs == SYSENTER_CS || regs->cs == SYSENTER_TF_CS) {
                        regs->eip -= 5;
                        regs->eip -= 5;
+               }
                else
                        regs->eip -= 2;
        }
                else
                        regs->eip -= 2;
        }
@@ -218,24 +213,17 @@ unix_syscall(x86_saved_state_t *state)
                } else { /* (not error) */
                    regs->eax = uthread->uu_rval[0];
                    regs->edx = uthread->uu_rval[1];
                } else { /* (not error) */
                    regs->eax = uthread->uu_rval[0];
                    regs->edx = uthread->uu_rval[1];
-                   regs->efl &= ~EFL_CF;
                } 
        }
 
                } 
        }
 
-       if (KTRPOINT(p, KTR_SYSRET))
-               ktrsysret(p, code, error, uthread->uu_rval[0]);
-
-       if (cancel_enable == _SYSCALL_CANCEL_NONE)
-                uthread->uu_flag &= ~UT_NOTCANCELPT;
-
+       uthread->uu_flag &= ~UT_NOTCANCELPT;
+#if DEBUG
        /*
        /*
-        * if we're holding the funnel
-        * than drop it regardless of whether
-        * we took it on system call entry
+        * if we're holding the funnel panic
         */
         */
-       exit_funnel_section();
-
-       if (uthread->uu_lowpri_delay) {
+       syscall_exit_funnelcheck();
+#endif /* DEBUG */
+       if (uthread->uu_lowpri_window && uthread->v_mount) {
                /*
                 * task is marked as a low priority I/O type
                 * and the I/O we issued while in this system call
                /*
                 * task is marked as a low priority I/O type
                 * and the I/O we issued while in this system call
@@ -243,8 +231,7 @@ unix_syscall(x86_saved_state_t *state)
                 * delay in order to mitigate the impact of this
                 * task on the normal operation of the system
                 */
                 * delay in order to mitigate the impact of this
                 * task on the normal operation of the system
                 */
-               IOSleep(uthread->uu_lowpri_delay);
-               uthread->uu_lowpri_delay = 0;
+               throttle_lowpri_io(&uthread->uu_lowpri_window,uthread->v_mount);
        }
        if (code != 180)
                KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
        }
        if (code != 180)
                KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
@@ -259,15 +246,13 @@ void
 unix_syscall64(x86_saved_state_t *state)
 {
        thread_t        thread;
 unix_syscall64(x86_saved_state_t *state)
 {
        thread_t        thread;
-       unsigned short  code;
+       unsigned int    code;
        struct sysent   *callp;
        void            *uargp;
        int             args_in_regs;
        int             error;
        struct sysent   *callp;
        void            *uargp;
        int             args_in_regs;
        int             error;
-       int             funnel_type;
        struct proc     *p;
        struct uthread  *uthread;
        struct proc     *p;
        struct uthread  *uthread;
-       unsigned int cancel_enable;
        x86_saved_state64_t *regs;
 
        assert(is_saved_state64(state));
        x86_saved_state64_t *regs;
 
        assert(is_saved_state64(state));
@@ -296,7 +281,7 @@ unix_syscall64(x86_saved_state_t *state)
        args_in_regs = 6;
 
        code = regs->rax & SYSCALL_NUMBER_MASK;
        args_in_regs = 6;
 
        code = regs->rax & SYSCALL_NUMBER_MASK;
-       callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
+       callp = (code >= NUM_SYSENT) ? &sysent[63] : &sysent[code];
        uargp = (void *)(&regs->rdi);
 
        if (callp == sysent) {
        uargp = (void *)(&regs->rdi);
 
        if (callp == sysent) {
@@ -305,28 +290,28 @@ unix_syscall64(x86_saved_state_t *state)
                 * passed as 'arg0'
                 */
                code = regs->rdi;
                 * passed as 'arg0'
                 */
                code = regs->rdi;
-               callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
+               callp = (code >= NUM_SYSENT) ? &sysent[63] : &sysent[code];
                uargp = (void *)(&regs->rsi);
                args_in_regs = 5;
        }
 
        if (callp->sy_narg != 0) {
                if (code != 180) {
                uargp = (void *)(&regs->rsi);
                args_in_regs = 5;
        }
 
        if (callp->sy_narg != 0) {
                if (code != 180) {
-                       uint64_t *ip = (uint64_t *)uargp;
+                       uint64_t *ip = (uint64_t *)uargp;
 
                        KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
 
                        KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
-                                             (int)(*ip), (int)(*(ip+1)), (int)(*(ip+2)), (int)(*(ip+3)), 0);
+                                       (int)(*ip), (int)(*(ip+1)), (int)(*(ip+2)), (int)(*(ip+3)), 0);
                }
                }
-               assert(callp->sy_narg <= 8);
+               assert(callp->sy_narg <= 8);
 
                if (callp->sy_narg > args_in_regs) {
 
                if (callp->sy_narg > args_in_regs) {
-                       int copyin_count;
+                       int copyin_count;
 
 
-                       copyin_count = (callp->sy_narg - args_in_regs) * sizeof(uint64_t);
+                       copyin_count = (callp->sy_narg - args_in_regs) * sizeof(uint64_t);
 
 
-                       error = copyin((user_addr_t)(regs->isf.rsp + sizeof(user_addr_t)), (char *)&regs->v_arg6, copyin_count);
+                       error = copyin((user_addr_t)(regs->isf.rsp + sizeof(user_addr_t)), (char *)&regs->v_arg6, copyin_count);
                        if (error) {
                        if (error) {
-                               regs->rax = error;
+                               regs->rax = error;
                                regs->isf.rflags |= EFL_CF;
                                thread_exception_return();
                                /* NOTREACHED */
                                regs->isf.rflags |= EFL_CF;
                                thread_exception_return();
                                /* NOTREACHED */
@@ -335,11 +320,10 @@ unix_syscall64(x86_saved_state_t *state)
                /*
                 * XXX Turn 64 bit unsafe calls into nosys()
                 */
                /*
                 * XXX Turn 64 bit unsafe calls into nosys()
                 */
-               if (callp->sy_funnel & UNSAFE_64BIT) {
-                       callp = &sysent[63];
+               if (callp->sy_flags & UNSAFE_64BIT) {
+                       callp = &sysent[63];
                        goto unsafe;
                }
                        goto unsafe;
                }
-
        } else
                KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
                                      0, 0, 0, 0, 0);
        } else
                KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
                                      0, 0, 0, 0, 0);
@@ -349,45 +333,22 @@ unsafe:
         * Delayed binding of thread credential to process credential, if we
         * are not running with an explicitly set thread credential.
         */
         * Delayed binding of thread credential to process credential, if we
         * are not running with an explicitly set thread credential.
         */
-       if (uthread->uu_ucred != p->p_ucred &&
-           (uthread->uu_flag & UT_SETUID) == 0) {
-               kauth_cred_t old = uthread->uu_ucred;
-               uthread->uu_ucred = kauth_cred_proc_ref(p);
-               if (IS_VALID_CRED(old))
-                       kauth_cred_unref(&old);
-       }
+       kauth_cred_uthread_update(uthread, p);
 
        uthread->uu_rval[0] = 0;
        uthread->uu_rval[1] = 0;
 
 
        uthread->uu_rval[0] = 0;
        uthread->uu_rval[1] = 0;
 
-       cancel_enable = callp->sy_cancel;
        
        
-       if (cancel_enable == _SYSCALL_CANCEL_NONE) {
-               uthread->uu_flag |= UT_NOTCANCELPT;
-       } else {
-               if ((uthread->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL) {
-                       if (cancel_enable == _SYSCALL_CANCEL_PRE) {
-                                       /* system call cancelled; return to handle cancellation */
-                                       regs->rax = EINTR;
-                                       regs->isf.rflags |= EFL_CF;
-                                       thread_exception_return();
-                                       /* NOTREACHED */
-                       } else {
-                               thread_abort_safely(thread);
-                       }
-               }
-       }
+       uthread->uu_flag |= UT_NOTCANCELPT;
 
 
-       funnel_type = (callp->sy_funnel & FUNNEL_MASK);
-       if (funnel_type == KERNEL_FUNNEL)
-               thread_funnel_set(kernel_flock, TRUE);
-
-       if (KTRPOINT(p, KTR_SYSCALL))
-               ktrsyscall(p, code, callp->sy_narg, uargp);
 
        AUDIT_SYSCALL_ENTER(code, p, uthread);
        error = (*(callp->sy_call))((void *) p, uargp, &(uthread->uu_rval[0]));
 
        AUDIT_SYSCALL_ENTER(code, p, uthread);
        error = (*(callp->sy_call))((void *) p, uargp, &(uthread->uu_rval[0]));
-       AUDIT_SYSCALL_EXIT(error, p, uthread);
+        AUDIT_SYSCALL_EXIT(code, p, uthread, error);
+
+#if CONFIG_DTRACE
+       uthread->t_dtrace_errno = error;
+#endif /* CONFIG_DTRACE */
        
        if (error == ERESTART) {
                /*
        
        if (error == ERESTART) {
                /*
@@ -399,7 +360,7 @@ unsafe:
        }
        else if (error != EJUSTRETURN) {
                if (error) {
        }
        else if (error != EJUSTRETURN) {
                if (error) {
-                       regs->rax = error;
+                       regs->rax = error;
                        regs->isf.rflags |= EFL_CF;     /* carry bit */
                } else { /* (not error) */
 
                        regs->isf.rflags |= EFL_CF;     /* carry bit */
                } else { /* (not error) */
 
@@ -429,20 +390,15 @@ unsafe:
                } 
        }
 
                } 
        }
 
-       if (KTRPOINT(p, KTR_SYSRET))
-               ktrsysret(p, code, error, uthread->uu_rval[0]);
 
 
-       if (cancel_enable == _SYSCALL_CANCEL_NONE)
-                uthread->uu_flag &= ~UT_NOTCANCELPT;
+       uthread->uu_flag &= ~UT_NOTCANCELPT;
 
        /*
 
        /*
-        * if we're holding the funnel
-        * than drop it regardless of whether
-        * we took it on system call entry
+        * if we're holding the funnel panic
         */
         */
-       exit_funnel_section();
+       syscall_exit_funnelcheck();
 
 
-       if (uthread->uu_lowpri_delay) {
+       if (uthread->uu_lowpri_window && uthread->v_mount) {
                /*
                 * task is marked as a low priority I/O type
                 * and the I/O we issued while in this system call
                /*
                 * task is marked as a low priority I/O type
                 * and the I/O we issued while in this system call
@@ -450,8 +406,7 @@ unsafe:
                 * delay in order to mitigate the impact of this
                 * task on the normal operation of the system
                 */
                 * delay in order to mitigate the impact of this
                 * task on the normal operation of the system
                 */
-               IOSleep(uthread->uu_lowpri_delay);
-               uthread->uu_lowpri_delay = 0;
+               throttle_lowpri_io(&uthread->uu_lowpri_window,uthread->v_mount);
        }
        if (code != 180)
                KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
        }
        if (code != 180)
                KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
@@ -468,10 +423,9 @@ unix_syscall_return(int error)
        thread_t                thread;
        struct uthread          *uthread;
        struct proc *p;
        thread_t                thread;
        struct uthread          *uthread;
        struct proc *p;
-       unsigned short code;
+       unsigned int code;
        vm_offset_t params;
        struct sysent *callp;
        vm_offset_t params;
        struct sysent *callp;
-       unsigned int cancel_enable;
 
        thread = current_thread();
        uthread = get_bsdthread_info(thread);
 
        thread = current_thread();
        uthread = get_bsdthread_info(thread);
@@ -479,103 +433,107 @@ unix_syscall_return(int error)
        p = current_proc();
 
        if (proc_is64bit(p)) {
        p = current_proc();
 
        if (proc_is64bit(p)) {
-               x86_saved_state64_t *regs;
+               x86_saved_state64_t *regs;
 
                regs = saved_state64(find_user_regs(thread));
 
 
                regs = saved_state64(find_user_regs(thread));
 
-               /* reconstruct code for tracing before blasting rax */
+               /* reconstruct code for tracing before blasting rax */
                code = regs->rax & SYSCALL_NUMBER_MASK;
                code = regs->rax & SYSCALL_NUMBER_MASK;
-               callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
+               callp = (code >= NUM_SYSENT) ? &sysent[63] : &sysent[code];
 
                if (callp == sysent)
 
                if (callp == sysent)
-                       /*
+                       /*
                         * indirect system call... system call number
                         * passed as 'arg0'
                         */
                         * indirect system call... system call number
                         * passed as 'arg0'
                         */
-                       code = regs->rdi;
+                       code = regs->rdi;
+
+#if CONFIG_DTRACE
+               if (callp->sy_call == dtrace_systrace_syscall)
+                       dtrace_systrace_syscall_return( code, error, uthread->uu_rval );
+#endif /* CONFIG_DTRACE */
 
                if (error == ERESTART) {
 
                if (error == ERESTART) {
-                       /*
+                       /*
                         * all system calls come through via the syscall instruction
                         * in 64 bit mode... its 2 bytes in length
                         * move the user's pc back to repeat the syscall:
                         */
                         * all system calls come through via the syscall instruction
                         * in 64 bit mode... its 2 bytes in length
                         * move the user's pc back to repeat the syscall:
                         */
-                       regs->isf.rip -= 2;
+                       regs->isf.rip -= 2;
                }
                else if (error != EJUSTRETURN) {
                }
                else if (error != EJUSTRETURN) {
-                       if (error) {
-                               regs->rax = error;
+                       if (error) {
+                               regs->rax = error;
                                regs->isf.rflags |= EFL_CF;     /* carry bit */
                        } else { /* (not error) */
 
                                regs->isf.rflags |= EFL_CF;     /* carry bit */
                        } else { /* (not error) */
 
-                               switch (callp->sy_return_type) {
+                               switch (callp->sy_return_type) {
                                case _SYSCALL_RET_INT_T:
                                case _SYSCALL_RET_INT_T:
-                                       regs->rax = uthread->uu_rval[0];
+                                       regs->rax = uthread->uu_rval[0];
                                        regs->rdx = uthread->uu_rval[1];
                                        break;
                                case _SYSCALL_RET_UINT_T:
                                        regs->rdx = uthread->uu_rval[1];
                                        break;
                                case _SYSCALL_RET_UINT_T:
-                                       regs->rax = ((u_int)uthread->uu_rval[0]);
+                                       regs->rax = ((u_int)uthread->uu_rval[0]);
                                        regs->rdx = ((u_int)uthread->uu_rval[1]);
                                        break;
                                case _SYSCALL_RET_OFF_T:
                                case _SYSCALL_RET_ADDR_T:
                                case _SYSCALL_RET_SIZE_T:
                                case _SYSCALL_RET_SSIZE_T:
                                        regs->rdx = ((u_int)uthread->uu_rval[1]);
                                        break;
                                case _SYSCALL_RET_OFF_T:
                                case _SYSCALL_RET_ADDR_T:
                                case _SYSCALL_RET_SIZE_T:
                                case _SYSCALL_RET_SSIZE_T:
-                                       regs->rax = *((uint64_t *)(&uthread->uu_rval[0]));
+                                       regs->rax = *((uint64_t *)(&uthread->uu_rval[0]));
                                        regs->rdx = 0;
                                        break;
                                case _SYSCALL_RET_NONE:
                                        regs->rdx = 0;
                                        break;
                                case _SYSCALL_RET_NONE:
-                                       break;
+                                       break;
                                default:
                                default:
-                                       panic("unix_syscall: unknown return type");
+                                       panic("unix_syscall: unknown return type");
                                        break;
                                }
                                regs->isf.rflags &= ~EFL_CF;
                        } 
                }
        } else {
                                        break;
                                }
                                regs->isf.rflags &= ~EFL_CF;
                        } 
                }
        } else {
-               x86_saved_state32_t     *regs;
+               x86_saved_state32_t     *regs;
 
                regs = saved_state32(find_user_regs(thread));
 
 
                regs = saved_state32(find_user_regs(thread));
 
+               regs->efl &= ~(EFL_CF);
                /* reconstruct code for tracing before blasting eax */
                /* reconstruct code for tracing before blasting eax */
-               code = regs->eax;
-               callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
+               code = regs->eax & I386_SYSCALL_NUMBER_MASK;
+               callp = (code >= NUM_SYSENT) ? &sysent[63] : &sysent[code];
+
+#if CONFIG_DTRACE
+               if (callp->sy_call == dtrace_systrace_syscall)
+                       dtrace_systrace_syscall_return( code, error, uthread->uu_rval );
+#endif /* CONFIG_DTRACE */
 
                if (callp == sysent) {
 
                if (callp == sysent) {
-                       params = (vm_offset_t) ((caddr_t)regs->uesp + sizeof (int));
-                       code = fuword(params);
+                       params = (vm_offset_t) ((caddr_t)regs->uesp + sizeof (int));
+                       code = fuword(params);
                }
                if (error == ERESTART) {
                }
                if (error == ERESTART) {
-                       regs->eip -= ((regs->cs & 0xffff) == SYSENTER_CS) ? 5 : 2;
+                       regs->eip -= ((regs->cs & 0xffff) == SYSENTER_CS) ? 5 : 2;
                }
                else if (error != EJUSTRETURN) {
                }
                else if (error != EJUSTRETURN) {
-                       if (error) {
-                               regs->eax = error;
+                       if (error) {
+                               regs->eax = error;
                                regs->efl |= EFL_CF;    /* carry bit */
                        } else { /* (not error) */
                                regs->efl |= EFL_CF;    /* carry bit */
                        } else { /* (not error) */
-                               regs->eax = uthread->uu_rval[0];
+                               regs->eax = uthread->uu_rval[0];
                                regs->edx = uthread->uu_rval[1];
                                regs->edx = uthread->uu_rval[1];
-                               regs->efl &= ~EFL_CF;
                        } 
                }
        }
                        } 
                }
        }
-       if (KTRPOINT(p, KTR_SYSRET))
-               ktrsysret(p, code, error, uthread->uu_rval[0]);
 
 
-       cancel_enable = callp->sy_cancel;
 
 
-       if (cancel_enable == _SYSCALL_CANCEL_NONE)
-                uthread->uu_flag &= ~UT_NOTCANCELPT;
+       uthread->uu_flag &= ~UT_NOTCANCELPT;
 
        /*
 
        /*
-        * if we're holding the funnel
-        * than drop it regardless of whether
-        * we took it on system call entry
+        * if we're holding the funnel panic
         */
         */
-       exit_funnel_section();
+       syscall_exit_funnelcheck();
 
 
-       if (uthread->uu_lowpri_delay) {
+       if (uthread->uu_lowpri_window && uthread->v_mount) {
                /*
                 * task is marked as a low priority I/O type
                 * and the I/O we issued while in this system call
                /*
                 * task is marked as a low priority I/O type
                 * and the I/O we issued while in this system call
@@ -583,8 +541,7 @@ unix_syscall_return(int error)
                 * delay in order to mitigate the impact of this
                 * task on the normal operation of the system
                 */
                 * delay in order to mitigate the impact of this
                 * task on the normal operation of the system
                 */
-               IOSleep(uthread->uu_lowpri_delay);
-               uthread->uu_lowpri_delay = 0;
+               throttle_lowpri_io(&uthread->uu_lowpri_window,uthread->v_mount);
        }
        if (code != 180)
                KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
        }
        if (code != 180)
                KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
@@ -614,3 +571,34 @@ munge_wwwlww(
        arg64[1] = arg32[1];    /* wWwlww */
        arg64[0] = arg32[0];    /* Wwwlww */
 }      
        arg64[1] = arg32[1];    /* wWwlww */
        arg64[0] = arg32[0];    /* Wwwlww */
 }      
+
+
+void
+munge_wwlwww(
+       __unused const void     *in32,
+       void                    *out64)
+{
+       uint32_t        *arg32;
+       uint64_t        *arg64;
+
+       /* we convert in place in out64 */
+       arg32 = (uint32_t *) out64;
+       arg64 = (uint64_t *) out64;
+
+       arg64[5] = arg32[6];    /* wwlwwW */
+       arg64[4] = arg32[5];    /* wwlwWw */
+       arg64[3] = arg32[4];    /* wwlWww  */
+       arg32[5] = arg32[3];    /* wwLwww (hi) */
+       arg32[4] = arg32[2];    /* wwLwww (lo) */
+       arg64[1] = arg32[1];    /* wWlwww */
+       arg64[0] = arg32[0];    /* Wwlwww */
+}      
+
+#ifdef JOE_DEBUG
+joe_debug(char *p) {
+
+        printf("%s\n", p);
+}
+#endif
+
+
mirror of XNU