X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/008676633c2ad2c325837c2b64915f7ded690a8f..527f99514973766e9c0382a4d8550dfb00f54939:/osfmk/x86_64/idt64.s?ds=sidebyside diff --git a/osfmk/x86_64/idt64.s b/osfmk/x86_64/idt64.s index 78b07486b..13f3f0218 100644 --- a/osfmk/x86_64/idt64.s +++ b/osfmk/x86_64/idt64.s @@ -66,31 +66,41 @@ * syscall - synchronous system call request * fatal - fatal traps */ - /* - * Handlers: + * Indices of handlers for each exception type. */ -#define HNDL_ALLINTRS EXT(hndl_allintrs) -#define HNDL_ALLTRAPS EXT(hndl_alltraps) -#define HNDL_SYSENTER EXT(hndl_sysenter) -#define HNDL_SYSCALL EXT(hndl_syscall) -#define HNDL_UNIX_SCALL EXT(hndl_unix_scall) -#define HNDL_MACH_SCALL EXT(hndl_mach_scall) -#define HNDL_MDEP_SCALL EXT(hndl_mdep_scall) -#define HNDL_DOUBLE_FAULT EXT(hndl_double_fault) -#define HNDL_MACHINE_CHECK EXT(hndl_machine_check) - - -#if 1 -#define PUSH_FUNCTION(func) \ - sub $8, %rsp ;\ - push %rax ;\ - leaq func(%rip), %rax ;\ - movq %rax, 8(%rsp) ;\ - pop %rax -#else -#define PUSH_FUNCTION(func) pushq func -#endif +#define HNDL_ALLINTRS 0 +#define HNDL_ALLTRAPS 1 +#define HNDL_SYSENTER 2 +#define HNDL_SYSCALL 3 +#define HNDL_UNIX_SCALL 4 +#define HNDL_MACH_SCALL 5 +#define HNDL_MDEP_SCALL 6 +#define HNDL_DOUBLE_FAULT 7 +#define HNDL_MACHINE_CHECK 8 + +/* Begin double-mapped descriptor section */ + +.section __HIB, __desc +.globl EXT(idt64_hndl_table0) +EXT(idt64_hndl_table0): + .quad EXT(ks_dispatch) + .quad EXT(ks_64bit_return) + .quad 0 /* Populated with CPU shadow displacement*/ + .quad EXT(ks_return) + +EXT(idt64_hndl_table1): + .quad EXT(hndl_allintrs) + .quad EXT(hndl_alltraps) + .quad EXT(hndl_sysenter) + .quad EXT(hndl_syscall) + .quad EXT(hndl_unix_scall) + .quad EXT(hndl_mach_scall) + .quad EXT(hndl_mdep_scall) + .quad EXT(hndl_double_fault) + .quad EXT(hndl_machine_check) +.text + /* The wrapper for all non-special traps/interrupts */ /* Everything up to PUSH_FUNCTION is just to output @@ -101,12 +111,12 @@ push %rax ;\ POSTCODE2(0x6400+n) ;\ pop %rax ;\ - PUSH_FUNCTION(f) ;\ + pushq $(f) ;\ pushq $(n) ;\ jmp L_dispatch #else #define IDT_ENTRY_WRAPPER(n, f) \ - PUSH_FUNCTION(f) ;\ + pushq $(f) ;\ pushq $(n) ;\ jmp L_dispatch #endif @@ -136,9 +146,175 @@ #define TRAP_IST2(n, f) #define USER_TRAP_SPC(n, f) +/* Begin double-mapped text section */ +.section __HIB, __text /* Generate all the stubs */ #include "idt_table.h" +Entry(idt64_page_fault) + pushq $(HNDL_ALLTRAPS) + push $(T_PAGE_FAULT) + jmp L_dispatch + +Entry(idt64_debug) + push $0 /* error code */ + pushq $(HNDL_ALLTRAPS) + pushq $(T_DEBUG) + jmp L_dispatch +/* + * Legacy interrupt gate System call handlers. + * These are entered via a syscall interrupt. The system call number in %rax + * is saved to the error code slot in the stack frame. We then branch to the + * common state saving code. + */ + +#ifndef UNIX_INT +#error NO UNIX INT!!! +#endif +Entry(idt64_unix_scall) + pushq %rax /* save system call number */ + pushq $(HNDL_UNIX_SCALL) + pushq $(UNIX_INT) + jmp L_dispatch + +Entry(idt64_mach_scall) + pushq %rax /* save system call number */ + pushq $(HNDL_MACH_SCALL) + pushq $(MACH_INT) + jmp L_dispatch + +Entry(idt64_mdep_scall) + pushq %rax /* save system call number */ + pushq $(HNDL_MDEP_SCALL) + pushq $(MACHDEP_INT) + jmp L_dispatch + +/* + * For GP/NP/SS faults, we use the IST1 stack. + * For faults from user-space, we have to copy the machine state to the + * PCB stack and then dispatch as normal. + * For faults in kernel-space, we need to scrub for kernel exit faults and + * treat these as user-space faults. But for all other kernel-space faults + * we continue to run on the IST1 stack and we dispatch to handle the fault + * as fatal. + */ +Entry(idt64_gen_prot) + pushq $(HNDL_ALLTRAPS) + pushq $(T_GENERAL_PROTECTION) + jmp L_dispatch + +Entry(idt64_stack_fault) + pushq $(HNDL_ALLTRAPS) + pushq $(T_STACK_FAULT) + jmp L_dispatch + +Entry(idt64_segnp) + pushq $(HNDL_ALLTRAPS) + pushq $(T_SEGMENT_NOT_PRESENT) + jmp L_dispatch + +/* + * Fatal exception handlers: + */ +Entry(idt64_db_task_dbl_fault) + pushq $(HNDL_DOUBLE_FAULT) + pushq $(T_DOUBLE_FAULT) + jmp L_dispatch + +Entry(idt64_db_task_stk_fault) + pushq $(HNDL_DOUBLE_FAULT) + pushq $(T_STACK_FAULT) + jmp L_dispatch + +Entry(idt64_mc) + push $(0) /* Error */ + pushq $(HNDL_MACHINE_CHECK) + pushq $(T_MACHINE_CHECK) + jmp L_dispatch + +/* + * NMI + * This may or may not be fatal but extreme care is required + * because it may fall when control was already in another trampoline. + * + * We get here on IST2 stack which is used for NMIs only. + */ +Entry(idt64_nmi) + push %rax /* save RAX to ISF64_ERR */ + push %rcx /* save RCX to ISF64_TRAPFN */ + push %rdx /* save RDX to ISF64_TRAPNO */ + jmp L_dispatch + +Entry(idt64_double_fault) + pushq $(HNDL_DOUBLE_FAULT) + pushq $(T_DOUBLE_FAULT) + jmp L_dispatch + +Entry(hi64_syscall) +Entry(idt64_syscall) + swapgs + /* Use RAX as a temporary by shifting its contents into R11[32:63] + * The systemcall number is defined to be a 32-bit quantity, as is + * RFLAGS. + */ + shlq $32, %rax + or %rax, %r11 +.globl EXT(dblsyscall_patch_point) +EXT(dblsyscall_patch_point): +// movabsq $0x12345678ABCDEFFFULL, %rax + /* Generate offset to the double-mapped per-CPU data shadow + * into RAX + */ + leaq EXT(idt64_hndl_table0)(%rip), %rax + mov 16(%rax), %rax + mov %rsp, %gs:CPU_UBER_TMP(%rax) /* save user stack */ + mov %gs:CPU_ESTACK(%rax), %rsp /* switch stack to per-cpu estack */ + sub $(ISF64_SIZE), %rsp + + /* + * Synthesize an ISF frame on the exception stack + */ + movl $(USER_DS), ISF64_SS(%rsp) + mov %rcx, ISF64_RIP(%rsp) /* rip */ + + mov %gs:CPU_UBER_TMP(%rax), %rcx + mov %rcx, ISF64_RSP(%rsp) /* user stack --changed */ + + mov %r11, %rax + shrq $32, %rax /* Restore RAX */ + mov %r11d, %r11d /* Clear r11[32:63] */ + + mov %r11, ISF64_RFLAGS(%rsp) /* rflags */ + movl $(SYSCALL_CS), ISF64_CS(%rsp) /* cs - a pseudo-segment */ + mov %rax, ISF64_ERR(%rsp) /* err/rax - syscall code */ + movq $(HNDL_SYSCALL), ISF64_TRAPFN(%rsp) + movq $(T_SYSCALL), ISF64_TRAPNO(%rsp) /* trapno */ + swapgs + jmp L_dispatch /* this can only be 64-bit */ + +Entry(hi64_sysenter) +Entry(idt64_sysenter) + /* Synthesize an interrupt stack frame onto the + * exception stack. + */ + push $(USER_DS) /* ss */ + push %rcx /* uesp */ + pushf /* flags */ + /* + * Clear, among others, the Nested Task (NT) flags bit; + * this is zeroed by INT, but not by SYSENTER. + */ + push $0 + popf + push $(SYSENTER_CS) /* cs */ +L_sysenter_continue: + push %rdx /* eip */ + push %rax /* err/eax - syscall code */ + pushq $(HNDL_SYSENTER) + pushq $(T_SYSENTER) + orl $(EFL_IF), ISF64_RFLAGS(%rsp) + jmp L_dispatch + /* * Common dispatch point. * Determine what mode has been interrupted and save state accordingly. @@ -148,13 +324,113 @@ * GSBASE from user-space: pthread area, or * from kernel-space: cpu_data */ + L_dispatch: + pushq %rax + testb $3, 8+ISF64_CS(%rsp) + jz 1f + swapgs + leaq EXT(idt64_hndl_table0)(%rip), %rax + mov 16(%rax), %rax + + mov %gs:CPU_TASK_CR3(%rax), %rax + mov %rax, %cr3 +#if DEBUG + mov %rax, %gs:CPU_ENTRY_CR3 +#endif +1: + /* The text/data relationship here must be preserved in the doublemap, and the contents must be remapped */ + leaq EXT(idt64_hndl_table0)(%rip), %rax + /* Indirect branch to non-doublemapped trampolines */ + jmp *(%rax) +/* User return: register restoration and address space switch sequence */ +Entry(ks_64bit_return) + mov R64_R14(%r15), %r14 + mov R64_R13(%r15), %r13 + mov R64_R12(%r15), %r12 + mov R64_R11(%r15), %r11 + mov R64_R10(%r15), %r10 + mov R64_R9(%r15), %r9 + mov R64_R8(%r15), %r8 + mov R64_RSI(%r15), %rsi + mov R64_RDI(%r15), %rdi + mov R64_RBP(%r15), %rbp + mov R64_RDX(%r15), %rdx + mov R64_RCX(%r15), %rcx + mov R64_RBX(%r15), %rbx + mov R64_RAX(%r15), %rax + /* Switch to per-CPU exception stack */ + mov %gs:CPU_ESTACK, %rsp + + /* Synthesize interrupt stack frame from PCB savearea to exception stack */ + push R64_SS(%r15) + push R64_RSP(%r15) + push R64_RFLAGS(%r15) + push R64_CS(%r15) + push R64_RIP(%r15) + + mov R64_R15(%r15), %r15 + cmpq $(KERNEL64_CS), 8(%rsp) + jz 1f + /* Discover user cr3/ASID */ + push %rax + mov %gs:CPU_UCR3, %rax +#if DEBUG + mov %rax, %gs:CPU_EXIT_CR3 +#endif + mov %rax, %cr3 + /* Continue execution on the shared/doublemapped trampoline */ + pop %rax + swapgs +1: + cmpl $(SYSCALL_CS), 8(%rsp) /* test for exit via SYSRET */ + je L_sysret +EXT(ret64_iret): + iretq /* return from interrupt */ +L_sysret: + /* + * Here to restore rcx/r11/rsp and perform the sysret back to user-space. + * rcx user rip + * r11 user rflags + * rsp user stack pointer + */ + pop %rcx + add $8, %rsp + pop %r11 + pop %rsp + sysretq /* return from system call */ +/* End of double-mapped TEXT */ +.text + +Entry(ks_dispatch) + popq %rax cmpl $(KERNEL64_CS), ISF64_CS(%rsp) - je L_dispatch_kernel + je EXT(ks_dispatch_kernel) - swapgs + mov %rax, %gs:CPU_UBER_TMP + mov %gs:CPU_UBER_ISF, %rax + add $(ISF64_SIZE), %rax + + xchg %rsp, %rax +/* Memory to memory moves (aint x86 wonderful): + * Transfer the exception frame from the per-CPU exception stack to the + * 'PCB' stack programmed at cswitch. + */ + push ISF64_SS(%rax) + push ISF64_RSP(%rax) + push ISF64_RFLAGS(%rax) + push ISF64_CS(%rax) + push ISF64_RIP(%rax) + push ISF64_ERR(%rax) + push ISF64_TRAPFN(%rax) + push ISF64_TRAPNO(%rax) + mov %gs:CPU_UBER_TMP, %rax + jmp EXT(ks_dispatch_user) -L_dispatch_user: +Entry (ks_return) + jmp . + +Entry(ks_dispatch_user) cmpl $(TASK_MAP_32BIT), %gs:CPU_TASK_MAP je L_dispatch_U32 /* 32-bit user task */ @@ -165,7 +441,7 @@ L_dispatch_U64: mov %gs:CPU_KERNEL_STACK, %rsp jmp L_dispatch_64bit -L_dispatch_kernel: +Entry(ks_dispatch_kernel) subq $(ISS64_OFFSET), %rsp mov %r15, R64_R15(%rsp) mov %rsp, %r15 @@ -212,14 +488,11 @@ L_64bit_entry_reject: /* * Here for a 64-bit user attempting an invalid kernel entry. */ - pushq %rax - leaq HNDL_ALLTRAPS(%rip), %rax - movq %rax, ISF64_TRAPFN+8(%rsp) - popq %rax + movq $(HNDL_ALLTRAPS), ISF64_TRAPFN(%rsp) movq $(T_INVALID_OPCODE), ISF64_TRAPNO(%rsp) jmp L_dispatch_U64 -L_32bit_entry_check: +Entry(ks_32bit_entry_check) /* * Check we're not a confused 64-bit user. */ @@ -315,21 +588,16 @@ L_common_dispatch: shr $32, %rcx testl %ecx, %ecx jz 4f - testl $(1<<16), %ecx /* Global? */ - jz 3f movl $0, %gs:CPU_TLB_INVALID mov %cr4, %rcx /* RMWW CR4, for lack of an alternative*/ and $(~CR4_PGE), %rcx mov %rcx, %cr4 or $(CR4_PGE), %rcx mov %rcx, %cr4 - jmp 4f -3: - movb $0, %gs:CPU_TLB_INVALID_LOCAL - mov %cr3, %rcx - mov %rcx, %cr3 4: mov %gs:CPU_ACTIVE_THREAD, %rcx /* Get the active thread */ + testq %rcx, %rcx + je 5f movl $-1, TH_IOTIER_OVERRIDE(%rcx) /* Reset IO tier override to -1 before handling trap */ cmpq $0, TH_PCB_IDS(%rcx) /* Is there a debug register state? */ je 5f @@ -338,7 +606,8 @@ L_common_dispatch: 5: incl %gs:hwIntCnt(,%ebx,4) // Bump the trap/intr count /* Dispatch the designated handler */ - jmp *%rdx + leaq EXT(idt64_hndl_table1)(%rip), %rax + jmp *(%rax, %rdx, 8) /* * Control is passed here to return to user. @@ -428,6 +697,40 @@ L_32bit_return: movl R32_SS(%r15), %eax movl %eax, R64_SS(%r15) + /* Validate DS/ES/FS/GS segment selectors with the Load Access Rights instruction prior to restoration */ + /* Exempt "known good" statically configured selectors, e.g. USER_DS and 0 */ + cmpl $(USER_DS), R32_DS(%r15) + jz 22f + cmpl $0, R32_DS(%r15) + jz 22f + larw R32_DS(%r15), %ax + jz 22f + movl $(USER_DS), R32_DS(%r15) +22: + cmpl $(USER_DS), R32_ES(%r15) + jz 33f + cmpl $0, R32_ES(%r15) + jz 33f + larw R32_ES(%r15), %ax + jz 33f + movl $(USER_DS), R32_ES(%r15) +33: + cmpl $(USER_DS), R32_FS(%r15) + jz 44f + cmpl $0, R32_FS(%r15) + jz 44f + larw R32_FS(%r15), %ax + jz 44f + movl $(USER_DS), R32_FS(%r15) +44: + cmpl $(USER_CTHREAD), R32_GS(%r15) + jz 55f + cmpl $0, R32_GS(%r15) + jz 55f + larw R32_GS(%r15), %ax + jz 55f + movl $(USER_CTHREAD), R32_GS(%r15) +55: /* * Restore general 32-bit registers */ @@ -456,13 +759,13 @@ L_32bit_return: xor %r15, %r15 EXT(ret32_set_ds): - movl R32_DS(%rsp), %ds + movw R32_DS(%rsp), %ds EXT(ret32_set_es): - movl R32_ES(%rsp), %es + movw R32_ES(%rsp), %es EXT(ret32_set_fs): - movl R32_FS(%rsp), %fs + movw R32_FS(%rsp), %fs EXT(ret32_set_gs): - movl R32_GS(%rsp), %gs + movw R32_GS(%rsp), %gs /* pop compat frame + trapno, trapfn and error */ add $(ISS64_OFFSET)+8+8+8, %rsp @@ -472,7 +775,6 @@ EXT(ret32_set_gs): EXT(ret32_iret): iretq /* return from interrupt */ - L_fast_exit: pop %rdx /* user return eip */ pop %rcx /* pop and toss cs */ @@ -510,176 +812,10 @@ L_64bit_return: * Restore general 64-bit registers. * Here on fault stack and PCB address in R15. */ - mov R64_R14(%r15), %r14 - mov R64_R13(%r15), %r13 - mov R64_R12(%r15), %r12 - mov R64_R11(%r15), %r11 - mov R64_R10(%r15), %r10 - mov R64_R9(%r15), %r9 - mov R64_R8(%r15), %r8 - mov R64_RSI(%r15), %rsi - mov R64_RDI(%r15), %rdi - mov R64_RBP(%r15), %rbp - mov R64_RDX(%r15), %rdx - mov R64_RCX(%r15), %rcx - mov R64_RBX(%r15), %rbx - mov R64_RAX(%r15), %rax - - /* - * We must swap GS base if we're returning to user-space, - * or we're returning from an NMI that occurred in a trampoline - * before the user GS had been swapped. In the latter case, the NMI - * handler will have flagged the high-order 32-bits of the CS. - */ - cmpq $(KERNEL64_CS), R64_CS(%r15) - jz 1f - swapgs -1: - mov R64_R15(%r15), %rsp - xchg %r15, %rsp - add $(ISS64_OFFSET)+24, %rsp /* pop saved state */ - /* + trapno/trapfn/error */ - cmpl $(SYSCALL_CS),ISF64_CS-24(%rsp) - /* test for fast entry/exit */ - je L_sysret -.globl _dump_iretq -EXT(ret64_iret): - iretq /* return from interrupt */ - -L_sysret: - /* - * Here to load rcx/r11/rsp and perform the sysret back to user-space. - * rcx user rip - * r11 user rflags - * rsp user stack pointer - */ - mov ISF64_RIP-24(%rsp), %rcx - mov ISF64_RFLAGS-24(%rsp), %r11 - mov ISF64_RSP-24(%rsp), %rsp - sysretq /* return from systen call */ - - - -/* - * System call handlers. - * These are entered via a syscall interrupt. The system call number in %rax - * is saved to the error code slot in the stack frame. We then branch to the - * common state saving code. - */ - -#ifndef UNIX_INT -#error NO UNIX INT!!! -#endif -Entry(idt64_unix_scall) - swapgs /* switch to kernel gs (cpu_data) */ - pushq %rax /* save system call number */ - PUSH_FUNCTION(HNDL_UNIX_SCALL) - pushq $(UNIX_INT) - jmp L_32bit_entry_check - - -Entry(idt64_mach_scall) - swapgs /* switch to kernel gs (cpu_data) */ - pushq %rax /* save system call number */ - PUSH_FUNCTION(HNDL_MACH_SCALL) - pushq $(MACH_INT) - jmp L_32bit_entry_check - - -Entry(idt64_mdep_scall) - swapgs /* switch to kernel gs (cpu_data) */ - pushq %rax /* save system call number */ - PUSH_FUNCTION(HNDL_MDEP_SCALL) - pushq $(MACHDEP_INT) - jmp L_32bit_entry_check - -/* Programmed into MSR_IA32_LSTAR by mp_desc.c */ -Entry(hi64_syscall) -Entry(idt64_syscall) -L_syscall_continue: - swapgs /* Kapow! get per-cpu data area */ - mov %rsp, %gs:CPU_UBER_TMP /* save user stack */ - mov %gs:CPU_UBER_ISF, %rsp /* switch stack to pcb */ - - /* - * Save values in the ISF frame in the PCB - * to cons up the saved machine state. - */ - movl $(USER_DS), ISF64_SS(%rsp) - movl $(SYSCALL_CS), ISF64_CS(%rsp) /* cs - a pseudo-segment */ - mov %r11, ISF64_RFLAGS(%rsp) /* rflags */ - mov %rcx, ISF64_RIP(%rsp) /* rip */ - mov %gs:CPU_UBER_TMP, %rcx - mov %rcx, ISF64_RSP(%rsp) /* user stack */ - mov %rax, ISF64_ERR(%rsp) /* err/rax - syscall code */ - movq $(T_SYSCALL), ISF64_TRAPNO(%rsp) /* trapno */ - leaq HNDL_SYSCALL(%rip), %r11; - movq %r11, ISF64_TRAPFN(%rsp) - mov ISF64_RFLAGS(%rsp), %r11 /* Avoid leak, restore R11 */ - jmp L_dispatch_U64 /* this can only be 64-bit */ - -/* - * sysenter entry point - * Requires user code to set up: - * edx: user instruction pointer (return address) - * ecx: user stack pointer - * on which is pushed stub ret addr and saved ebx - * Return to user-space is made using sysexit. - * Note: sysenter/sysexit cannot be used for calls returning a value in edx, - * or requiring ecx to be preserved. - */ -Entry(hi64_sysenter) -Entry(idt64_sysenter) - movq (%rsp), %rsp - /* - * Push values on to the PCB stack - * to cons up the saved machine state. - */ - push $(USER_DS) /* ss */ - push %rcx /* uesp */ - pushf /* flags */ - /* - * Clear, among others, the Nested Task (NT) flags bit; - * this is zeroed by INT, but not by SYSENTER. - */ - push $0 - popf - push $(SYSENTER_CS) /* cs */ -L_sysenter_continue: - swapgs /* switch to kernel gs (cpu_data) */ - push %rdx /* eip */ - push %rax /* err/eax - syscall code */ - PUSH_FUNCTION(HNDL_SYSENTER) - pushq $(T_SYSENTER) - orl $(EFL_IF), ISF64_RFLAGS(%rsp) - jmp L_32bit_entry_check - - -Entry(idt64_page_fault) - PUSH_FUNCTION(HNDL_ALLTRAPS) - push $(T_PAGE_FAULT) - push %rax /* save %rax temporarily */ - testb $3, 8+ISF64_CS(%rsp) /* was trap from kernel? */ - jz L_kernel_trap /* - yes, handle with care */ - pop %rax /* restore %rax, swapgs, and continue */ - swapgs - jmp L_dispatch_user - - -/* - * Debug trap. Check for single-stepping across system call into - * kernel. If this is the case, taking the debug trap has turned - * off single-stepping - save the flags register with the trace - * bit set. - */ -Entry(idt64_debug) - push $0 /* error code */ - PUSH_FUNCTION(HNDL_ALLTRAPS) - pushq $(T_DEBUG) - - testb $3, ISF64_CS(%rsp) - jnz L_dispatch + leaq EXT(idt64_hndl_table0)(%rip), %rax + jmp *8(%rax) +Entry(ks_idt64_debug_kernel) /* * trap came from kernel mode */ @@ -688,7 +824,7 @@ Entry(idt64_debug) lea EXT(idt64_sysenter)(%rip), %rax cmp %rax, ISF64_RIP+8(%rsp) pop %rax - jne L_dispatch + jne EXT(ks_dispatch_kernel) /* * Interrupt stack frame has been pushed on the temporary stack. * We have to switch to pcb stack and patch up the saved state. @@ -703,37 +839,7 @@ Entry(idt64_debug) mov ISF64_ERR(%rcx),%rcx /* restore %rcx */ jmp L_sysenter_continue /* continue sysenter entry */ - -Entry(idt64_double_fault) - PUSH_FUNCTION(HNDL_DOUBLE_FAULT) - pushq $(T_DOUBLE_FAULT) - jmp L_dispatch_kernel - - -/* - * For GP/NP/SS faults, we use the IST1 stack. - * For faults from user-space, we have to copy the machine state to the - * PCB stack and then dispatch as normal. - * For faults in kernel-space, we need to scrub for kernel exit faults and - * treat these as user-space faults. But for all other kernel-space faults - * we continue to run on the IST1 stack and we dispatch to handle the fault - * as fatal. - */ -Entry(idt64_gen_prot) - PUSH_FUNCTION(HNDL_ALLTRAPS) - pushq $(T_GENERAL_PROTECTION) - jmp trap_check_kernel_exit /* check for kernel exit sequence */ - -Entry(idt64_stack_fault) - PUSH_FUNCTION(HNDL_ALLTRAPS) - pushq $(T_STACK_FAULT) - jmp trap_check_kernel_exit /* check for kernel exit sequence */ - -Entry(idt64_segnp) - PUSH_FUNCTION(HNDL_ALLTRAPS) - pushq $(T_SEGMENT_NOT_PRESENT) - /* indicate fault type */ -trap_check_kernel_exit: +Entry(ks_trap_check_kernel_exit) testb $3,ISF64_CS(%rsp) jz L_kernel_gpf @@ -761,7 +867,7 @@ trap_check_kernel_exit: pop %rax mov %gs:CPU_UBER_TMP, %rsp /* user RCX into RSP */ xchg %rcx, %rsp /* to PCB stack with user RCX */ - jmp L_dispatch_user + jmp EXT(ks_dispatch_user) L_kernel_gpf: /* Here for GPF from kernel_space. Check for recoverable cases. */ @@ -784,10 +890,10 @@ L_kernel_gpf: leaq EXT(ret32_set_gs)(%rip), %rax cmp %rax, 8+ISF64_RIP(%rsp) je L_32bit_fault_set_seg - + jmp EXT(ks_kernel_trap) /* Fall through */ -L_kernel_trap: +Entry(ks_kernel_trap) /* * Here after taking an unexpected trap from kernel mode - perhaps * while running in the trampolines hereabouts. @@ -812,7 +918,7 @@ L_kernel_trap: cmpq $(PAGE_SIZE), %rax /* current stack in PCB? */ jb 2f /* - yes, deal with it */ pop %rax /* - no, restore %rax */ - jmp L_dispatch_kernel + jmp EXT(ks_dispatch_kernel) 2: /* * Here if %rsp is in the PCB @@ -829,7 +935,7 @@ L_kernel_trap: pushq 8+ISF64_TRAPFN(%rax) pushq 8+ISF64_TRAPNO(%rax) movq (%rax), %rax - jmp L_dispatch_kernel + jmp EXT(ks_dispatch_kernel) /* @@ -915,46 +1021,8 @@ L_32bit_fault_set_seg: /* the compatibility frame */ jmp L_dispatch_U32_after_fault -/* - * Fatal exception handlers: - */ -Entry(idt64_db_task_dbl_fault) - PUSH_FUNCTION(HNDL_DOUBLE_FAULT) - pushq $(T_DOUBLE_FAULT) - jmp L_dispatch - -Entry(idt64_db_task_stk_fault) - PUSH_FUNCTION(HNDL_DOUBLE_FAULT) - pushq $(T_STACK_FAULT) - jmp L_dispatch - -Entry(idt64_mc) - push $(0) /* Error */ - PUSH_FUNCTION(HNDL_MACHINE_CHECK) - pushq $(T_MACHINE_CHECK) - jmp L_dispatch - -/* - * NMI - * This may or may not be fatal but extreme care is required - * because it may fall when control was already in another trampoline. - * - * We get here on IST2 stack which is used for NMIs only. - * We must be aware of the interrupted state: - * - from user-space, we - * - copy state to the PCB and continue; - * - from kernel-space, we - * - copy state to the kernel stack and continue, but - * - check what GSBASE was active, set the kernel base and - * - ensure that the active state is restored when the NMI is dismissed. - */ -Entry(idt64_nmi) - push %rax /* save RAX to ISF64_ERR */ - push %rcx /* save RCX to ISF64_TRAPFN */ - push %rdx /* save RDX to ISF64_TRAPNO */ - testb $3, ISF64_CS(%rsp) /* NMI from user-space? */ - je 1f +Entry(ks_idt64_nmi_kernel) /* From user-space: copy interrupt state to user PCB */ swapgs mov %gs:CPU_UBER_ISF, %rcx /* PCB stack addr */ @@ -1013,8 +1081,7 @@ Entry(idt64_nmi) push ISF64_CS(%rcx) push ISF64_RIP(%rcx) push $(0) /* error code 0 */ - lea HNDL_ALLINTRS(%rip), %rax - push %rax /* trapfn allintrs */ + push $(HNDL_ALLINTRS) /* trapfn allintrs */ push $(T_NMI) /* trapno T_NMI */ mov ISF64_ERR(%rcx), %rax mov ISF64_TRAPNO(%rcx), %rdx @@ -1086,12 +1153,10 @@ L_return_from_trap_with_ast: je 2f /* not in the PFZ... go service AST */ movl %eax, R64_RBX(%r15) /* let the PFZ know we've pended an AST */ jmp EXT(return_to_user) -2: - sti /* interrupts always enabled on return to user mode */ +2: - xor %edi, %edi /* zero %rdi */ xorq %rbp, %rbp /* clear framepointer */ - CCALL(i386_astintr) /* take the AST */ + CCALL(ast_taken_user) /* handle all ASTs (enables interrupts, may return via continuation) */ cli mov %rsp, %r15 /* AST changes stack, saved state */ @@ -1131,7 +1196,7 @@ trap_from_kernel: testq %rcx,%rcx /* are we on the kernel stack? */ jne ret_to_kernel /* no, skip it */ - CCALL1(i386_astintr, $1) /* take the AST */ + CCALL(ast_taken_kernel) /* take the AST */ mov %rsp, %r15 /* AST changes stack, saved state */ jmp ret_to_kernel @@ -1180,7 +1245,7 @@ Entry(hndl_allintrs) CCALL1(interrupt, %r15) /* call generic interrupt routine */ - .globl EXT(return_to_iret) +.globl EXT(return_to_iret) LEXT(return_to_iret) /* (label for kdb_kintr and hardclock) */ decl %gs:CPU_INTERRUPT_LEVEL @@ -1242,7 +1307,7 @@ LEXT(return_to_iret) /* (label for kdb_kintr and hardclock) */ * to do as much as the case where the interrupt came from user * space. */ - CCALL1(i386_astintr, $1) + CCALL(ast_taken_kernel) mov %rsp, %r15 /* AST changes stack, saved state */ jmp ret_to_kernel @@ -1442,11 +1507,16 @@ Entry(hndl_diag_scall64) sti CCALL3(i386_exception, $EXC_SYSCALL, $0x6000, $1) /* no return */ - +/* TODO assert at all 'C' entry points that we're never operating on the fault stack's alias mapping */ Entry(hndl_machine_check) + /* Adjust SP and savearea to their canonical, non-aliased addresses */ + subq EXT(dblmap_dist)(%rip), %rsp + subq EXT(dblmap_dist)(%rip), %r15 CCALL1(panic_machine_check64, %r15) hlt Entry(hndl_double_fault) + subq EXT(dblmap_dist)(%rip), %rsp + subq EXT(dblmap_dist)(%rip), %r15 CCALL1(panic_double_fault64, %r15) hlt