xnu-7195.50.7.100.1.tar.gz

[apple/xnu.git] / osfmk / x86_64 / idt64.s

diff --git a/osfmk/x86_64/idt64.s b/osfmk/x86_64/idt64.s
index d54c1c095b99f6ed4fc4374cb746e1540cd491ac..2c2fc49260e69732e6c25cdb9670fa58d21dae31 100644 (file)
--- a/osfmk/x86_64/idt64.s
+++ b/osfmk/x86_64/idt64.s
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2010-2019 Apple Inc. All rights reserved.
+ * Copyright (c) 2010-2020 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -28,6 +28,7 @@
 #include <i386/asm.h>
 #include <assym.s>
 #include <debug.h>
 #include <i386/asm.h>
 #include <assym.s>
 #include <debug.h>

+#include "dwarf_unwind.h"

 #include <i386/eflags.h>
 #include <i386/rtclock_asm.h>
 #include <i386/trap.h>
 #include <i386/eflags.h>
 #include <i386/rtclock_asm.h>
 #include <i386/trap.h>


@@ -79,6 +80,7 @@
 #define        HNDL_DOUBLE_FAULT       7
 #define        HNDL_MACHINE_CHECK      8
 
 #define        HNDL_DOUBLE_FAULT       7
 #define        HNDL_MACHINE_CHECK      8
 

+       

 /* Begin double-mapped descriptor section */
        
 .section       __HIB, __desc
 /* Begin double-mapped descriptor section */
        
 .section       __HIB, __desc


@@ -159,8 +161,70 @@ EXT(idt64_hndl_table1):
 
 Entry(idt64_page_fault)
        pushq   $(HNDL_ALLTRAPS)
 
 Entry(idt64_page_fault)
        pushq   $(HNDL_ALLTRAPS)

-       push    $(T_PAGE_FAULT)
+#if !(DEVELOPMENT || DEBUG)
+       pushq   $(T_PAGE_FAULT)

        jmp     L_dispatch
        jmp     L_dispatch

+#else
+       pushq   $(T_PAGE_FAULT)
+
+       pushq   %rax
+       pushq   %rbx
+       pushq   %rcx
+       testb   $3, 8+8+8+ISF64_CS(%rsp)        /* Coming from userspace? */
+       jz      L_pfkern                /* No? (relatively uncommon), goto L_pfkern */
+
+       /*
+        * We faulted from the user; if the fault address is at the user's %rip,
+        * abort trying to save the cacheline since that adds another page fault's
+        * overhead when we recover, below.
+        */
+       movq    8+8+8+ISF64_RIP(%rsp), %rbx
+       movq    %cr2, %rcx
+       cmpq    %rbx, %rcx
+
+       /* note that the next 3 instructions do not affect RFLAGS */
+       swapgs
+       leaq    EXT(idt64_hndl_table0)(%rip), %rax
+       mov     16(%rax), %rax  /* Offset of per-CPU shadow */
+
+       jne     L_dispatch_from_user_with_rbx_rcx_pushes
+       jmp     abort_rip_cacheline_read
+
+L_pfkern:
+       /*
+        * Kernel page fault
+        * If the fault occurred on while reading from the user's code cache line, abort the cache line read;
+        * otherwise, treat this as a regular kernel fault
+        */
+       movq    8+8+8+ISF64_RIP(%rsp), %rbx
+       leaq    rip_cacheline_read(%rip), %rcx
+       cmpq    %rcx, %rbx
+       jb      regular_kernel_page_fault
+       leaq    rip_cacheline_read_end(%rip), %rcx
+       cmpq    %rcx, %rbx
+       jbe     L_pf_on_clread  /* Did we hit a #PF within the cacheline read? */
+
+regular_kernel_page_fault:
+       /* No, regular kernel #PF */
+       popq    %rcx
+       popq    %rbx
+       jmp     L_dispatch_from_kernel_no_push_rax
+
+L_pf_on_clread:
+       /*
+        * We faulted while trying to read user instruction memory at the parent fault's %rip; abort that action by
+        * changing the return address on the stack, restoring cr2 to its previous value, peeling off the pushes we
+        * added on entry to the page fault handler, then performing an iretq
+        */
+       popq    %rcx
+       movq    %rcx, %cr2
+       popq    %rbx
+       leaq    abort_rip_cacheline_read(%rip), %rax
+       movq    %rax, 8+ISF64_RIP(%rsp)
+       popq    %rax
+       addq    $24, %rsp       /* pop the 2 pushes + the error code */
+       iretq                   /* Resume previous trap/fault processing */
+#endif /* !(DEVELOPMENT || DEBUG) */

 
 /*
  * #DB handler, which runs on IST1, will treat as spurious any #DB received while executing in the
 
 /*
  * #DB handler, which runs on IST1, will treat as spurious any #DB received while executing in the


@@ -531,6 +595,85 @@ L_sysenter_continue:
        orl     $(EFL_IF), ISF64_RFLAGS(%rsp)
        jmp     L_u64bit_entry_check
 
        orl     $(EFL_IF), ISF64_RFLAGS(%rsp)
        jmp     L_u64bit_entry_check
 

+#if DEVELOPMENT || DEBUG
+do_cacheline_stash:
+       /*
+        * Copy the cache line that includes the user's EIP/RIP into the shadow cpu structure
+        * for later extraction/sanity-checking in user_trap().
+        */
+
+       pushq   %rbx
+       pushq   %rcx
+L_dispatch_from_user_with_rbx_rcx_pushes:
+       movq    8+8+8+ISF64_RIP(%rsp), %rbx
+       andq    $-64, %rbx      /* Round address to cacheline boundary */
+       pushf
+       /*
+        * disable SMAP, if it's enabled (note that CLAC is present in BDW and later only, so we're
+        * using generic instructions here without checking whether the CPU supports SMAP first)
+        */
+       orq     $(1 << 18), (%rsp)
+       popf
+       /*
+        * Note that we only check for a faulting read on the first read, since if the first read
+        * succeeds, the rest of the cache line should also be readible since we are running with
+        * interrupts disabled here and a TLB invalidation cannot sneak in and pull the rug out.
+        */
+       movq    %cr2, %rcx      /* stash the original %cr2 in case the first cacheline read triggers a #PF */
+                               /* This value of %cr2 is restored in the page fault handler if it detects */
+                               /* that the fault occurrent on the next instruction, so the original #PF can */
+                               /* continue to be handled without issue. */
+rip_cacheline_read:
+       mov     (%rbx), %rcx
+       /* Note that CPU_RTIMES in the shadow cpu struct was just a convenient place to stash the cacheline */
+       mov     %rcx, %gs:CPU_RTIMES(%rax)
+       movq    %cr2, %rcx
+       mov     8(%rbx), %rcx
+       mov     %rcx, %gs:8+CPU_RTIMES(%rax)
+       movq    %cr2, %rcx
+       mov     16(%rbx), %rcx
+       mov     %rcx, %gs:16+CPU_RTIMES(%rax)
+       movq    %cr2, %rcx
+       mov     24(%rbx), %rcx
+       mov     %rcx, %gs:24+CPU_RTIMES(%rax)
+       movq    %cr2, %rcx
+       mov     32(%rbx), %rcx
+       mov     %rcx, %gs:32+CPU_RTIMES(%rax)
+       movq    %cr2, %rcx
+       mov     40(%rbx), %rcx
+       mov     %rcx, %gs:40+CPU_RTIMES(%rax)
+       movq    %cr2, %rcx
+       mov     48(%rbx), %rcx
+       mov     %rcx, %gs:48+CPU_RTIMES(%rax)
+       movq    %cr2, %rcx
+rip_cacheline_read_end:
+       mov     56(%rbx), %rcx
+       mov     %rcx, %gs:56+CPU_RTIMES(%rax)
+
+       pushf
+       andq    $~(1 << 18), (%rsp)     /* reenable SMAP */
+       popf
+
+       jmp     cacheline_read_cleanup_stack
+
+abort_rip_cacheline_read:
+       pushf
+       andq    $~(1 << 18), (%rsp)     /* reenable SMAP */
+       popf
+abort_rip_cacheline_read_no_smap_reenable:
+       movl    $0xdeadc0de, %ecx                       /* Write a sentinel so higher-level code knows this was aborted */
+       shlq    $32, %rcx
+       movl    $0xbeefcafe, %ebx
+       orq     %rbx, %rcx
+       movq    %rcx, %gs:CPU_RTIMES(%rax)
+       movq    %rcx, %gs:8+CPU_RTIMES(%rax)
+
+cacheline_read_cleanup_stack:
+       popq    %rcx
+       popq    %rbx
+       jmp     L_dispatch_kgsb
+#endif /* if DEVELOPMENT || DEBUG */
+

 /*
  * Common dispatch point.
  * Determine what mode has been interrupted and save state accordingly.
 /*
  * Common dispatch point.
  * Determine what mode has been interrupted and save state accordingly.


@@ -546,9 +689,20 @@ L_dispatch:
        testb   $3, 8+ISF64_CS(%rsp)
        jz      1f
 L_dispatch_from_user_no_push_rax:
        testb   $3, 8+ISF64_CS(%rsp)
        jz      1f
 L_dispatch_from_user_no_push_rax:

-       swapgs
+       swapgs

        leaq    EXT(idt64_hndl_table0)(%rip), %rax
        leaq    EXT(idt64_hndl_table0)(%rip), %rax

-       mov     16(%rax), %rax
+       mov     16(%rax), %rax  /* Offset of per-CPU shadow */
+
+#if DEVELOPMENT || DEBUG
+       /* Stash the cacheline for #UD, #PF, and #GP */
+       cmpl    $(T_INVALID_OPCODE), 8+ISF64_TRAPNO(%rsp)
+       je      do_cacheline_stash
+       cmpl    $(T_PAGE_FAULT), 8+ISF64_TRAPNO(%rsp)
+       je      do_cacheline_stash
+       cmpl    $(T_GENERAL_PROTECTION), 8+ISF64_TRAPNO(%rsp)
+       je      do_cacheline_stash
+#endif
+

 L_dispatch_kgsb:
        mov     %gs:CPU_SHADOWTASK_CR3(%rax), %rax
        mov     %rax, %cr3
 L_dispatch_kgsb:
        mov     %gs:CPU_SHADOWTASK_CR3(%rax), %rax
        mov     %rax, %cr3


@@ -1379,13 +1533,22 @@ Entry(hndl_alltraps)
 Entry(return_from_trap)
        movq    %gs:CPU_ACTIVE_THREAD,%r15      /* Get current thread */
        movl    $-1, TH_IOTIER_OVERRIDE(%r15)   /* Reset IO tier override to -1 before returning to userspace */
 Entry(return_from_trap)
        movq    %gs:CPU_ACTIVE_THREAD,%r15      /* Get current thread */
        movl    $-1, TH_IOTIER_OVERRIDE(%r15)   /* Reset IO tier override to -1 before returning to userspace */

+

        cmpl    $0, TH_RWLOCK_COUNT(%r15)       /* Check if current thread has pending RW locks held */
        jz      1f
        cmpl    $0, TH_RWLOCK_COUNT(%r15)       /* Check if current thread has pending RW locks held */
        jz      1f

-       xorq    %rbp, %rbp              /* clear framepointer */
-       mov     %r15, %rdi              /* Set RDI to current thread */
+       xorq    %rbp, %rbp                      /* clear framepointer */
+       mov     %r15, %rdi                      /* Set RDI to current thread */

        CCALL(lck_rw_clear_promotions_x86)      /* Clear promotions if needed */
 1:     
        CCALL(lck_rw_clear_promotions_x86)      /* Clear promotions if needed */
 1:     

-       movq    TH_PCB_ISS(%r15), %r15          /* PCB stack */
+
+       cmpl    $0, TH_TMP_ALLOC_CNT(%r15)      /* Check if current thread has KHEAP_TEMP leaks */
+       jz      1f
+       xorq    %rbp, %rbp                      /* clear framepointer */
+       mov     %r15, %rdi                      /* Set RDI to current thread */
+       CCALL(kheap_temp_leak_panic)
+1:
+
+       movq    TH_PCB_ISS(%r15), %r15          /* PCB stack */

        movl    %gs:CPU_PENDING_AST,%eax
        testl   %eax,%eax
        je      EXT(return_to_user)             /* branch if no AST */
        movl    %gs:CPU_PENDING_AST,%eax
        testl   %eax,%eax
        je      EXT(return_to_user)             /* branch if no AST */


@@ -1428,7 +1591,13 @@ L_return_from_trap_with_ast:
  * 
  */
 trap_from_kernel:
  * 
  */
 trap_from_kernel:

+
+UNWIND_PROLOGUE        
+       

        movq    %r15, %rdi              /* saved state addr */
        movq    %r15, %rdi              /* saved state addr */

+
+UNWIND_DIRECTIVES      
+

        pushq   R64_RIP(%r15)           /* Simulate a CALL from fault point */
        pushq   %rbp                    /* Extend framepointer chain */
        movq    %rsp, %rbp
        pushq   R64_RIP(%r15)           /* Simulate a CALL from fault point */
        pushq   %rbp                    /* Extend framepointer chain */
        movq    %rsp, %rbp


@@ -1459,7 +1628,8 @@ trap_from_kernel:
        mov     %rsp, %r15              /* AST changes stack, saved state */
        jmp     ret_to_kernel
 
        mov     %rsp, %r15              /* AST changes stack, saved state */
        jmp     ret_to_kernel
 

-
+UNWIND_EPILOGUE
+       

 /*
  * All interrupts on all tasks enter here with:
  *     r15      x86_saved_state_t
 /*
  * All interrupts on all tasks enter here with:
  *     r15      x86_saved_state_t


@@ -1471,6 +1641,9 @@ trap_from_kernel:
  *     direction flag cleared
  */
 Entry(hndl_allintrs)
  *     direction flag cleared
  */
 Entry(hndl_allintrs)

+
+UNWIND_PROLOGUE        
+       

        /*
         * test whether already on interrupt stack
         */
        /*
         * test whether already on interrupt stack
         */


@@ -1490,6 +1663,8 @@ Entry(hndl_allintrs)
        pushq   %rcx                    /* save pointer to old stack */
        pushq   %gs:CPU_INT_STATE       /* save previous intr state */
        movq    %r15,%gs:CPU_INT_STATE  /* set intr state */
        pushq   %rcx                    /* save pointer to old stack */
        pushq   %gs:CPU_INT_STATE       /* save previous intr state */
        movq    %r15,%gs:CPU_INT_STATE  /* set intr state */

+
+UNWIND_DIRECTIVES      

        
        TIME_INT_ENTRY                  /* do timing */
 
        
        TIME_INT_ENTRY                  /* do timing */
 


@@ -1502,6 +1677,8 @@ Entry(hndl_allintrs)
        incl    %gs:CPU_INTERRUPT_LEVEL
 
        CCALL1(interrupt, %r15)         /* call generic interrupt routine */
        incl    %gs:CPU_INTERRUPT_LEVEL
 
        CCALL1(interrupt, %r15)         /* call generic interrupt routine */

+       
+UNWIND_EPILOGUE

 
 .globl EXT(return_to_iret)
 LEXT(return_to_iret)                   /* (label for kdb_kintr and hardclock) */
 
 .globl EXT(return_to_iret)
 LEXT(return_to_iret)                   /* (label for kdb_kintr and hardclock) */


@@ -1529,21 +1706,20 @@ LEXT(return_to_iret)                    /* (label for kdb_kintr and hardclock) */
        mov     %rax,%cr0               /* set cr0 */
 2:
        /* Load interrupted code segment into %eax */
        mov     %rax,%cr0               /* set cr0 */
 2:
        /* Load interrupted code segment into %eax */

-       movl    R32_CS(%r15),%eax       /* assume 32-bit state */
-       cmpl    $(SS_64),SS_FLAVOR(%r15)/* 64-bit? */   
+       movl    R64_CS(%r15),%eax       /* assume 64-bit state */
+       cmpl    $(SS_32),SS_FLAVOR(%r15)/* 32-bit? */

 #if DEBUG_IDT64
 #if DEBUG_IDT64

-       jne     4f
-       movl    R64_CS(%r15),%eax       /* 64-bit user mode */
+       jne     5f
+       movl    R32_CS(%r15),%eax       /* 32-bit user mode */

        jmp     3f
        jmp     3f

-4:
-       cmpl    $(SS_32),SS_FLAVOR(%r15)
+5:
+       cmpl    $(SS_64),SS_FLAVOR(%r15)

        je      3f
        POSTCODE2(0x6431)
        CCALL1(panic_idt64, %r15)
        hlt
 #else
        je      3f
        POSTCODE2(0x6431)
        CCALL1(panic_idt64, %r15)
        hlt
 #else

-       jne     3f
-       movl    R64_CS(%r15),%eax       /* 64-bit user mode */
+       je      4f

 #endif
 3:
        testb   $3,%al                  /* user mode, */
 #endif
 3:
        testb   $3,%al                  /* user mode, */


@@ -1569,6 +1745,9 @@ LEXT(return_to_iret)                      /* (label for kdb_kintr and hardclock) */
 
        mov     %rsp, %r15              /* AST changes stack, saved state */
        jmp     ret_to_kernel
 
        mov     %rsp, %r15              /* AST changes stack, saved state */
        jmp     ret_to_kernel

+4:
+       movl    R32_CS(%r15),%eax       /* 32-bit user mode */
+       jmp     3b

 
 
 /*
 
 
 /*