xnu-3789.51.2.tar.gz

[apple/xnu.git] / osfmk / x86_64 / idt64.s

diff --git a/osfmk/x86_64/idt64.s b/osfmk/x86_64/idt64.s
index c6a38aef8235c6d4f7f04fdd7ae0755f3fe82dea..3ab3d89bc4f1bf3968e4f8d7b49a20b3a5f81e33 100644 (file)
--- a/osfmk/x86_64/idt64.s
+++ b/osfmk/x86_64/idt64.s
@@ -284,13 +284,15 @@ L_common_dispatch:
        clac            /* Clear EFLAGS.AC if SMAP is present/enabled */
 1:
        /*
        clac            /* Clear EFLAGS.AC if SMAP is present/enabled */
 1:
        /*

-        * On entering the kernel, we don't need to switch cr3
+        * On entering the kernel, we typically don't switch CR3

         * because the kernel shares the user's address space.
         * because the kernel shares the user's address space.

-        * But we mark the kernel's cr3 as "active".
-        * If, however, the invalid cr3 flag is set, we have to flush tlbs
-        * since the kernel's mapping was changed while we were in userspace.
+        * But we mark the kernel's cr3 as "active" for TLB coherency evaluation
+        * If, however, the CPU's invalid TLB flag is set, we have to invalidate the TLB
+        * since the kernel pagetables were changed while we were in userspace.

         *
         *

-        * But: if global no_shared_cr3 is TRUE we do switch to the kernel's cr3
+        * For threads with a mapped pagezero (some WINE games) on non-SMAP platforms,
+        * we switch to the kernel's address space on entry. Also, 
+        * if the global no_shared_cr3 is TRUE we do switch to the kernel's cr3

         * so that illicit accesses to userspace can be trapped.
         */
        mov     %gs:CPU_KERNEL_CR3, %rcx
         * so that illicit accesses to userspace can be trapped.
         */
        mov     %gs:CPU_KERNEL_CR3, %rcx


@@ -298,8 +300,14 @@ L_common_dispatch:
        test    $3, %esi                        /* user/kernel? */
        jz      2f                              /* skip cr3 reload from kernel */
        xor     %rbp, %rbp
        test    $3, %esi                        /* user/kernel? */
        jz      2f                              /* skip cr3 reload from kernel */
        xor     %rbp, %rbp

+       cmpl    $0, %gs:CPU_PAGEZERO_MAPPED
+       jnz     11f

        cmpl    $0, EXT(no_shared_cr3)(%rip)
        je      2f
        cmpl    $0, EXT(no_shared_cr3)(%rip)
        je      2f

+11:
+       xor     %eax, %eax
+       movw    %gs:CPU_KERNEL_PCID, %ax
+       or      %rax, %rcx

        mov     %rcx, %cr3                      /* load kernel cr3 */
        jmp     4f                              /* and skip tlb flush test */
 2:
        mov     %rcx, %cr3                      /* load kernel cr3 */
        jmp     4f                              /* and skip tlb flush test */
 2:


@@ -307,9 +315,9 @@ L_common_dispatch:
        shr     $32, %rcx
        testl   %ecx, %ecx
        jz      4f
        shr     $32, %rcx
        testl   %ecx, %ecx
        jz      4f

-       movl    $0, %gs:CPU_TLB_INVALID

        testl   $(1<<16), %ecx                  /* Global? */
        jz      3f
        testl   $(1<<16), %ecx                  /* Global? */
        jz      3f

+       movl    $0, %gs:CPU_TLB_INVALID

        mov     %cr4, %rcx      /* RMWW CR4, for lack of an alternative*/
        and     $(~CR4_PGE), %rcx
        mov     %rcx, %cr4
        mov     %cr4, %rcx      /* RMWW CR4, for lack of an alternative*/
        and     $(~CR4_PGE), %rcx
        mov     %rcx, %cr4


@@ -317,6 +325,7 @@ L_common_dispatch:
        mov     %rcx, %cr4
        jmp     4f
 3:
        mov     %rcx, %cr4
        jmp     4f
 3:

+       movb    $0, %gs:CPU_TLB_INVALID_LOCAL

        mov     %cr3, %rcx
        mov     %rcx, %cr3
 4:
        mov     %cr3, %rcx
        mov     %rcx, %cr3
 4:


@@ -370,18 +379,21 @@ Entry(ret_to_user)
        mov     %rcx, %gs:CPU_DR7
 2:
        /*
        mov     %rcx, %gs:CPU_DR7
 2:
        /*

-        * On exiting the kernel there's no need to switch cr3 since we're
+        * On exiting the kernel there's typically no need to switch cr3 since we're

         * already running in the user's address space which includes the
         * already running in the user's address space which includes the

-        * kernel. Nevertheless, we now mark the task's cr3 as active.
-        * But, if no_shared_cr3 is set, we do need to switch cr3 at this point.
+        * kernel. We now mark the task's cr3 as active, for TLB coherency.
+        * If the target address space has a pagezero mapping present, or
+        * if no_shared_cr3 is set, we do need to switch cr3 at this point.

         */
        mov     %gs:CPU_TASK_CR3, %rcx
        mov     %rcx, %gs:CPU_ACTIVE_CR3
         */
        mov     %gs:CPU_TASK_CR3, %rcx
        mov     %rcx, %gs:CPU_ACTIVE_CR3

+       cmpl    $0, %gs:CPU_PAGEZERO_MAPPED
+       jnz     L_cr3_switch_island

        movl    EXT(no_shared_cr3)(%rip), %eax
        test    %eax, %eax              /* -no_shared_cr3 */
        movl    EXT(no_shared_cr3)(%rip), %eax
        test    %eax, %eax              /* -no_shared_cr3 */

-       jz      3f
-       mov     %rcx, %cr3
-3:
+       jnz     L_cr3_switch_island
+
+L_cr3_switch_return:

        mov     %gs:CPU_DR7, %rax       /* Is there a debug control register?*/
        cmp     $0, %rax
        je      4f
        mov     %gs:CPU_DR7, %rax       /* Is there a debug control register?*/
        cmp     $0, %rax
        je      4f


@@ -450,6 +462,7 @@ EXT(ret32_set_gs):
 EXT(ret32_iret):
        iretq                           /* return from interrupt */
 
 EXT(ret32_iret):
        iretq                           /* return from interrupt */
 

+       

 L_fast_exit:
        pop     %rdx                    /* user return eip */
        pop     %rcx                    /* pop and toss cs */
 L_fast_exit:
        pop     %rdx                    /* user return eip */
        pop     %rcx                    /* pop and toss cs */


@@ -459,6 +472,13 @@ L_fast_exit:
        sti                             /* interrupts enabled after sysexit */
        sysexitl                        /* 32-bit sysexit */
 
        sti                             /* interrupts enabled after sysexit */
        sysexitl                        /* 32-bit sysexit */
 

+L_cr3_switch_island:
+       xor     %eax, %eax
+       movw    %gs:CPU_ACTIVE_PCID, %ax
+       or      %rax, %rcx
+       mov     %rcx, %cr3
+       jmp     L_cr3_switch_return
+

 ret_to_kernel:
 #if DEBUG_IDT64
        cmpl    $(SS_64), SS_FLAVOR(%r15)       /* 64-bit state? */
 ret_to_kernel:
 #if DEBUG_IDT64
        cmpl    $(SS_64), SS_FLAVOR(%r15)       /* 64-bit state? */


@@ -677,16 +697,8 @@ Entry(idt64_debug)
 Entry(idt64_double_fault)
        PUSH_FUNCTION(HNDL_DOUBLE_FAULT)
        pushq   $(T_DOUBLE_FAULT)
 Entry(idt64_double_fault)
        PUSH_FUNCTION(HNDL_DOUBLE_FAULT)
        pushq   $(T_DOUBLE_FAULT)

+       jmp     L_dispatch_kernel

 
 

-       push    %rax
-       leaq    EXT(idt64_syscall)(%rip), %rax
-       cmp     %rax, ISF64_RIP+8(%rsp)
-       pop     %rax
-       jne     L_dispatch_kernel
-
-       mov     ISF64_RSP(%rsp), %rsp
-       jmp     L_syscall_continue
-       

 
 /*
  * For GP/NP/SS faults, we use the IST1 stack.
 
 /*
  * For GP/NP/SS faults, we use the IST1 stack.


@@ -1028,8 +1040,6 @@ Entry(hndl_alltraps)
        cli                                     /* hold off intrs - critical section */
        xorl    %ecx, %ecx                      /* don't check if we're in the PFZ */
 
        cli                                     /* hold off intrs - critical section */
        xorl    %ecx, %ecx                      /* don't check if we're in the PFZ */
 

-#define CLI cli
-#define STI sti

 
 Entry(return_from_trap)
        movq    %gs:CPU_ACTIVE_THREAD,%r15      /* Get current thread */
 
 Entry(return_from_trap)
        movq    %gs:CPU_ACTIVE_THREAD,%r15      /* Get current thread */


@@ -1067,13 +1077,13 @@ L_return_from_trap_with_ast:
        movl    %eax, R64_RBX(%r15)     /* let the PFZ know we've pended an AST */
        jmp     EXT(return_to_user)
 2:     
        movl    %eax, R64_RBX(%r15)     /* let the PFZ know we've pended an AST */
        jmp     EXT(return_to_user)
 2:     

-       STI                             /* interrupts always enabled on return to user mode */
+       sti                             /* interrupts always enabled on return to user mode */

 
        xor     %edi, %edi              /* zero %rdi */
        xorq    %rbp, %rbp              /* clear framepointer */
        CCALL(i386_astintr)             /* take the AST */
 
 
        xor     %edi, %edi              /* zero %rdi */
        xorq    %rbp, %rbp              /* clear framepointer */
        CCALL(i386_astintr)             /* take the AST */
 

-       CLI
+       cli

        mov     %rsp, %r15              /* AST changes stack, saved state */
        xorl    %ecx, %ecx              /* don't check if we're in the PFZ */
        jmp     EXT(return_from_trap)   /* and check again (rare) */
        mov     %rsp, %r15              /* AST changes stack, saved state */
        xorl    %ecx, %ecx              /* don't check if we're in the PFZ */
        jmp     EXT(return_from_trap)   /* and check again (rare) */


@@ -1160,8 +1170,6 @@ Entry(hndl_allintrs)
 
        CCALL1(interrupt, %r15)         /* call generic interrupt routine */
 
 
        CCALL1(interrupt, %r15)         /* call generic interrupt routine */
 

-       cli                             /* just in case we returned with intrs enabled */
-

        .globl  EXT(return_to_iret)
 LEXT(return_to_iret)                   /* (label for kdb_kintr and hardclock) */
 
        .globl  EXT(return_to_iret)
 LEXT(return_to_iret)                   /* (label for kdb_kintr and hardclock) */
 


@@ -1417,7 +1425,6 @@ Entry(hndl_mdep_scall64)
 
 Entry(hndl_diag_scall64)
        CCALL1(diagCall64, %r15)        // Call diagnostics
 
 Entry(hndl_diag_scall64)
        CCALL1(diagCall64, %r15)        // Call diagnostics

-       cli                             // Disable interruptions just in case

        test    %eax, %eax              // What kind of return is this?
        je      1f                      // - branch if bad (zero)
        jmp     EXT(return_to_user)     // Normal return, do not check asts...
        test    %eax, %eax              // What kind of return is this?
        je      1f                      // - branch if bad (zero)
        jmp     EXT(return_to_user)     // Normal return, do not check asts...