2 * Copyright (c) 2010 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
29 #include <i386/asm64.h>
31 #include <i386/eflags.h>
32 #include <i386/trap.h>
33 #include <i386/rtclock_asm.h>
34 #define _ARCH_I386_ASM_HELP_H_ /* Prevent inclusion of user header */
35 #include <mach/i386/syscall_sw.h>
36 #include <i386/postcode.h>
37 #include <i386/proc_reg.h>
38 #include <mach/exception_types.h>
42 * Low-memory compability-mode handlers.
44 #define LO_ALLINTRS EXT(lo_allintrs)
45 #define LO_ALLTRAPS EXT(lo_alltraps)
46 #define LO_SYSCALL EXT(lo_syscall)
47 #define LO_UNIX_SCALL EXT(lo_unix_scall)
48 #define LO_MACH_SCALL EXT(lo_mach_scall)
49 #define LO_MDEP_SCALL EXT(lo_mdep_scall)
50 #define LO_DOUBLE_FAULT EXT(lo_df64)
51 #define LO_MACHINE_CHECK EXT(lo_mc64)
54 * Interrupt descriptor table and code vectors for it.
56 * The IDT64_BASE_ENTRY macro lays down a fake descriptor that must be
57 * reformatted ("fixed") before use.
58 * All vector are rebased in uber-space.
59 * Special vectors (e.g. double-fault) use a non-0 IST.
61 #define IDT64_BASE_ENTRY(vec,seg,ist,type) \
64 .long KERNEL_UBER_BASE_HI32 ;\
71 #define IDT64_ENTRY(vec,ist,type) \
72 IDT64_BASE_ENTRY(EXT(vec),KERNEL64_CS,ist,type)
73 #define IDT64_ENTRY_LOCAL(vec,ist,type) \
74 IDT64_BASE_ENTRY(vec,KERNEL64_CS,ist,type)
77 * Push trap number and address of compatibility mode handler,
78 * then branch to common trampoline. Error already pushed.
80 #define EXCEP64_ERR(n,name) \
81 IDT64_ENTRY(name,0,K_INTR_GATE) ;\
83 push $(LO_ALLTRAPS) ;\
89 * Push error(0), trap number and address of compatibility mode handler,
90 * then branch to common trampoline.
92 #define EXCEPTION64(n,name) \
93 IDT64_ENTRY(name,0,K_INTR_GATE) ;\
96 push $(LO_ALLTRAPS) ;\
102 * Interrupt from user.
103 * Push error (0), trap number and address of compatibility mode handler,
104 * then branch to common trampoline.
106 #define EXCEP64_USR(n,name) \
107 IDT64_ENTRY(name,0,U_INTR_GATE) ;\
110 push $(LO_ALLTRAPS) ;\
112 jmp L_enter_lohandler
116 * Special interrupt code from user.
118 #define EXCEP64_SPC_USR(n,name) \
119 IDT64_ENTRY(name,0,U_INTR_GATE)
123 * Special interrupt code.
124 * In 64-bit mode we may use an IST slot instead of task gates.
126 #define EXCEP64_IST(n,name,ist) \
127 IDT64_ENTRY(name,ist,K_INTR_GATE)
128 #define EXCEP64_SPC(n,name) \
129 IDT64_ENTRY(name,0,K_INTR_GATE)
134 * Push zero err, interrupt vector and address of compatibility mode handler,
135 * then branch to common trampoline.
137 #define INTERRUPT64(n) \
138 IDT64_ENTRY_LOCAL(L_ ## n,0,K_INTR_GATE) ;\
142 push $(LO_ALLINTRS) ;\
144 jmp L_enter_lohandler
150 Entry(hi64_data_base)
153 Entry(hi64_text_base)
155 EXCEPTION64(0x00,t64_zero_div)
156 EXCEP64_SPC(0x01,hi64_debug)
157 INTERRUPT64(0x02) /* NMI */
158 EXCEP64_USR(0x03,t64_int3)
159 EXCEP64_USR(0x04,t64_into)
160 EXCEP64_USR(0x05,t64_bounds)
161 EXCEPTION64(0x06,t64_invop)
162 EXCEPTION64(0x07,t64_nofpu)
163 EXCEP64_IST(0x08,hi64_double_fault,1)
164 EXCEPTION64(0x09,a64_fpu_over)
165 EXCEPTION64(0x0a,a64_inv_tss)
166 EXCEP64_SPC(0x0b,hi64_segnp)
167 EXCEP64_SPC(0x0c,hi64_stack_fault)
168 EXCEP64_SPC(0x0d,hi64_gen_prot)
169 EXCEP64_SPC(0x0e, hi64_page_fault)
170 EXCEPTION64(0x0f,t64_trap_0f)
171 EXCEPTION64(0x10,t64_fpu_err)
172 EXCEPTION64(0x11,t64_trap_11)
173 EXCEP64_IST(0x12,mc64,1)
174 EXCEPTION64(0x13,t64_sse_err)
175 EXCEPTION64(0x14,t64_trap_14)
176 EXCEPTION64(0x15,t64_trap_15)
177 EXCEPTION64(0x16,t64_trap_16)
178 EXCEPTION64(0x17,t64_trap_17)
179 EXCEPTION64(0x18,t64_trap_18)
180 EXCEPTION64(0x19,t64_trap_19)
181 EXCEPTION64(0x1a,t64_trap_1a)
182 EXCEPTION64(0x1b,t64_trap_1b)
183 EXCEPTION64(0x1c,t64_trap_1c)
184 EXCEPTION64(0x1d,t64_trap_1d)
185 EXCEPTION64(0x1e,t64_trap_1e)
186 EXCEPTION64(0x1f,t64_trap_1f)
288 EXCEP64_USR(0x7f, t64_dtrace_ret)
290 EXCEP64_SPC_USR(0x80,hi64_unix_scall)
291 EXCEP64_SPC_USR(0x81,hi64_mach_scall)
292 EXCEP64_SPC_USR(0x82,hi64_mdep_scall)
424 EXCEPTION64(0xff,t64_preempt)
430 * Trap/interrupt entry points.
432 * All traps must create the following 32-bit save area on the PCB "stack"
433 * - this is identical to the legacy mode 32-bit case:
442 * cr2 (defined only for page fault)
452 * user esp - if from user
453 * user ss - if from user
455 * Above this is the trap number and compatibility mode handler address
456 * (packed into an 8-byte stack entry) and the 64-bit interrupt stack frame:
471 * Control is passed here to return to the compatibility mode user.
472 * At this stage we're in kernel space in compatibility mode
473 * but we need to switch into 64-bit mode in the 4G-based trampoline
474 * space before performing the iret.
477 movl %gs:CPU_ACTIVE_THREAD,%ecx
479 movl TH_PCB_IDS(%ecx),%eax /* Obtain this thread's debug state */
480 cmpl $0,%eax /* Is there a debug register context? */
481 je 2f /* branch if not */
482 cmpl $(TASK_MAP_32BIT), %gs:CPU_TASK_MAP /* Are we a 32-bit task? */
484 movl DS_DR0(%eax), %ecx /* If so, load the 32 bit DRs */
486 movl DS_DR1(%eax), %ecx
488 movl DS_DR2(%eax), %ecx
490 movl DS_DR3(%eax), %ecx
492 movl DS_DR7(%eax), %ecx
493 movl %ecx, %gs:CPU_DR7
494 movl $0, %gs:CPU_DR7 + 4
497 ENTER_64BIT_MODE() /* Enter long mode */
498 mov DS64_DR0(%eax), %rcx /* Load the full width DRs*/
500 mov DS64_DR1(%eax), %rcx
502 mov DS64_DR2(%eax), %rcx
504 mov DS64_DR3(%eax), %rcx
506 mov DS64_DR7(%eax), %rcx
507 mov %rcx, %gs:CPU_DR7
508 jmp 3f /* Enter uberspace */
515 * Now switch %cr3, if necessary.
517 swapgs /* switch back to uber-kernel gs base */
518 mov %gs:CPU_TASK_CR3,%rcx
519 mov %rcx,%gs:CPU_ACTIVE_CR3
523 /* flag the copyio engine state as WINDOWS_CLEAN */
524 mov %gs:CPU_ACTIVE_THREAD,%eax
525 movl $(WINDOWS_CLEAN),TH_COPYIO_STATE(%eax)
526 mov %rcx,%cr3 /* switch to user's address space */
529 mov %gs:CPU_DR7, %rax /* Is there a debug control register?*/
532 mov %rax, %dr7 /* Set DR7 */
537 * Adjust stack to use uber-space.
539 mov $(KERNEL_UBER_BASE_HI32), %rax
541 shrd $32, %rax, %rsp /* relocate into uber-space */
543 cmpl $(SS_32), SS_FLAVOR(%rsp) /* 32-bit state? */
551 swapgs /* switch back to uber-kernel gs base */
554 * Adjust stack to use uber-space.
556 mov $(KERNEL_UBER_BASE_HI32), %rax
558 shrd $32, %rax, %rsp /* relocate into uber-space */
560 /* Check for return to 64-bit kernel space (EFI today) */
561 cmpl $(SS_32), SS_FLAVOR(%rsp) /* 32-bit state? */
563 /* fall through for 32-bit return */
567 * Restore registers into the machine state for iret.
569 movl R32_EIP(%rsp), %eax
570 movl %eax, ISC32_RIP(%rsp)
571 movl R32_EFLAGS(%rsp), %eax
572 movl %eax, ISC32_RFLAGS(%rsp)
573 movl R32_CS(%rsp), %eax
574 movl %eax, ISC32_CS(%rsp)
575 movl R32_UESP(%rsp), %eax
576 movl %eax, ISC32_RSP(%rsp)
577 movl R32_SS(%rsp), %eax
578 movl %eax, ISC32_SS(%rsp)
581 * Restore general 32-bit registers
583 movl R32_EAX(%rsp), %eax
584 movl R32_EBX(%rsp), %ebx
585 movl R32_ECX(%rsp), %ecx
586 movl R32_EDX(%rsp), %edx
587 movl R32_EBP(%rsp), %ebp
588 movl R32_ESI(%rsp), %esi
589 movl R32_EDI(%rsp), %edi
592 * Restore segment registers. We make take an exception here but
593 * we've got enough space left in the save frame area to absorb
594 * a hardware frame plus the trapfn and trapno
598 movw R32_DS(%rsp), %ds
600 movw R32_ES(%rsp), %es
602 movw R32_FS(%rsp), %fs
604 movw R32_GS(%rsp), %gs
606 add $(ISC32_OFFSET)+8+8+8, %rsp /* pop compat frame +
607 trapno, trapfn and error */
608 cmpl $(SYSENTER_CS),ISF64_CS-8-8-8(%rsp)
609 /* test for fast entry/exit */
612 iretq /* return from interrupt */
615 pop %rdx /* user return eip */
616 pop %rcx /* pop and toss cs */
617 andl $(~EFL_IF), (%rsp) /* clear interrupts enable, sti below */
618 popf /* flags - carry denotes failure */
619 pop %rcx /* user return esp */
621 sti /* interrupts enabled after sysexit */
622 .byte 0x0f,0x35 /* 32-bit sysexit */
627 * Set the GS Base MSR with the user's gs base.
629 movl %gs:CPU_UBER_USER_GS_BASE, %eax
630 movl %gs:CPU_UBER_USER_GS_BASE+4, %edx
631 movl $(MSR_IA32_GS_BASE), %ecx
633 testb $3, R64_CS(%rsp) /* returning to user-space? */
635 wrmsr /* set 64-bit base */
639 * Restore general 64-bit registers
641 mov R64_R15(%rsp), %r15
642 mov R64_R14(%rsp), %r14
643 mov R64_R13(%rsp), %r13
644 mov R64_R12(%rsp), %r12
645 mov R64_R11(%rsp), %r11
646 mov R64_R10(%rsp), %r10
647 mov R64_R9(%rsp), %r9
648 mov R64_R8(%rsp), %r8
649 mov R64_RSI(%rsp), %rsi
650 mov R64_RDI(%rsp), %rdi
651 mov R64_RBP(%rsp), %rbp
652 mov R64_RDX(%rsp), %rdx
653 mov R64_RBX(%rsp), %rbx
654 mov R64_RCX(%rsp), %rcx
655 mov R64_RAX(%rsp), %rax
657 add $(ISS64_OFFSET)+8+8+8, %rsp /* pop saved state frame +
658 trapno, trapfn and error */
659 cmpl $(SYSCALL_CS),ISF64_CS-8-8-8(%rsp)
660 /* test for fast entry/exit */
663 iretq /* return from interrupt */
667 * Here to load rcx/r11/rsp and perform the sysret back to user-space.
670 * rsp user stack pointer
672 mov ISF64_RIP-8-8-8(%rsp), %rcx
673 mov ISF64_RFLAGS-8-8-8(%rsp), %r11
674 mov ISF64_RSP-8-8-8(%rsp), %rsp
675 sysretq /* return from system call */
678 * Common path to enter locore handlers.
681 swapgs /* switch to kernel gs (cpu_data) */
682 L_enter_lohandler_continue:
683 cmpl $(USER64_CS), ISF64_CS(%rsp)
684 je L_64bit_enter /* this is a 64-bit user task */
685 cmpl $(KERNEL64_CS), ISF64_CS(%rsp)
686 je L_64bit_enter /* we're in 64-bit (EFI) code */
690 * System call handlers.
691 * These are entered via a syscall interrupt. The system call number in %rax
692 * is saved to the error code slot in the stack frame. We then branch to the
693 * common state saving code.
696 Entry(hi64_unix_scall)
697 swapgs /* switch to kernel gs (cpu_data) */
698 L_unix_scall_continue:
699 push %rax /* save system call number */
700 push $(LO_UNIX_SCALL)
702 jmp L_32bit_enter_check
705 Entry(hi64_mach_scall)
706 swapgs /* switch to kernel gs (cpu_data) */
707 L_mach_scall_continue:
708 push %rax /* save system call number */
709 push $(LO_MACH_SCALL)
711 jmp L_32bit_enter_check
714 Entry(hi64_mdep_scall)
715 swapgs /* switch to kernel gs (cpu_data) */
716 L_mdep_scall_continue:
717 push %rax /* save system call number */
718 push $(LO_MDEP_SCALL)
720 jmp L_32bit_enter_check
724 swapgs /* Kapow! get per-cpu data area */
726 mov %rsp, %gs:CPU_UBER_TMP /* save user stack */
727 mov %gs:CPU_UBER_ISF, %rsp /* switch stack to pcb */
730 * Save values in the ISF frame in the PCB
731 * to cons up the saved machine state.
733 movl $(USER_DS), ISF64_SS(%rsp)
734 movl $(SYSCALL_CS), ISF64_CS(%rsp) /* cs - a pseudo-segment */
735 mov %r11, ISF64_RFLAGS(%rsp) /* rflags */
736 mov %rcx, ISF64_RIP(%rsp) /* rip */
737 mov %gs:CPU_UBER_TMP, %rcx
738 mov %rcx, ISF64_RSP(%rsp) /* user stack */
739 mov %rax, ISF64_ERR(%rsp) /* err/rax - syscall code */
740 movl $(T_SYSCALL), ISF64_TRAPNO(%rsp) /* trapno */
741 movl $(LO_SYSCALL), ISF64_TRAPFN(%rsp)
742 jmp L_64bit_enter /* this can only be a 64-bit task */
747 * Check we're not a confused 64-bit user.
749 cmpl $(TASK_MAP_32BIT), %gs:CPU_TASK_MAP
750 jne L_64bit_entry_reject
753 * sysenter entry point
754 * Requires user code to set up:
755 * edx: user instruction pointer (return address)
756 * ecx: user stack pointer
757 * on which is pushed stub ret addr and saved ebx
758 * Return to user-space is made using sysexit.
759 * Note: sysenter/sysexit cannot be used for calls returning a value in edx,
760 * or requiring ecx to be preserved.
763 mov (%rsp), %rsp /* switch from temporary stack to pcb */
765 * Push values on to the PCB stack
766 * to cons up the saved machine state.
768 push $(USER_DS) /* ss */
772 * Clear, among others, the Nested Task (NT) flags bit;
773 * this is zeroed by INT, but not by SYSENTER.
777 push $(SYSENTER_CS) /* cs */
778 swapgs /* switch to kernel gs (cpu_data) */
781 push %rax /* err/eax - syscall code */
784 orl $(EFL_IF), ISF64_RFLAGS(%rsp)
785 movl $(LO_MACH_SCALL), ISF64_TRAPFN(%rsp)
787 js L_32bit_enter_check
788 movl $(LO_UNIX_SCALL), ISF64_TRAPFN(%rsp)
789 cmpl $(TASK_MAP_32BIT), %gs:CPU_TASK_MAP
790 jne L_64bit_entry_reject
791 /* If the caller (typically LibSystem) has recorded the cumulative size of
792 * the arguments in EAX, copy them over from the user stack directly.
793 * We recover from exceptions inline--if the copy loop doesn't complete
794 * due to an exception, we fall back to copyin from compatibility mode.
795 * We can potentially extend this mechanism to mach traps as well (DRK).
797 L_sysenter_copy_args:
798 testl $(I386_SYSCALL_ARG_BYTES_MASK), %eax
801 mov %gs:CPU_UBER_ARG_STORE, %r8
803 mov %gs:CPU_UBER_ARG_STORE_VALID, %r12
805 shrl $(I386_SYSCALL_ARG_DWORDS_SHIFT), %r9d
806 andl $(I386_SYSCALL_ARG_DWORDS_MASK), %r9d
808 EXT(hi64_sysenter_user_arg_copy):
810 movl 4(%rcx, %r10, 4), %r11d
811 movl %r11d, (%r8, %r10, 4)
816 /* Fall through to 32-bit handler */
821 * Make space for the compatibility save area.
823 sub $(ISC32_OFFSET), %rsp
824 movl $(SS_32), SS_FLAVOR(%rsp)
829 mov %ds, R32_DS(%rsp)
830 mov %es, R32_ES(%rsp)
831 mov %fs, R32_FS(%rsp)
832 mov %gs, R32_GS(%rsp)
835 * Save general 32-bit registers
837 mov %eax, R32_EAX(%rsp)
838 mov %ebx, R32_EBX(%rsp)
839 mov %ecx, R32_ECX(%rsp)
840 mov %edx, R32_EDX(%rsp)
841 mov %ebp, R32_EBP(%rsp)
842 mov %esi, R32_ESI(%rsp)
843 mov %edi, R32_EDI(%rsp)
845 /* Unconditionally save cr2; only meaningful on page faults */
847 mov %eax, R32_CR2(%rsp)
850 * Copy registers already saved in the machine state
851 * (in the interrupt stack frame) into the compat save area.
853 mov ISC32_RIP(%rsp), %eax
854 mov %eax, R32_EIP(%rsp)
855 mov ISC32_RFLAGS(%rsp), %eax
856 mov %eax, R32_EFLAGS(%rsp)
857 mov ISC32_CS(%rsp), %eax
858 mov %eax, R32_CS(%rsp)
863 mov ISC32_RSP(%rsp), %eax
864 mov %eax, R32_UESP(%rsp)
865 mov ISC32_SS(%rsp), %eax
866 mov %eax, R32_SS(%rsp)
867 L_32bit_enter_after_fault:
868 mov ISC32_TRAPNO(%rsp), %ebx /* %ebx := trapno for later */
869 mov %ebx, R32_TRAPNO(%rsp)
870 mov ISC32_ERR(%rsp), %eax
871 mov %eax, R32_ERR(%rsp)
872 mov ISC32_TRAPFN(%rsp), %edx
875 * Common point to enter lo_handler in compatibilty mode:
877 * %edx locore handler address
881 * Switch address space to kernel
882 * if not shared space and not already mapped.
883 * Note: cpu_task_map is valid only if cpu_task_cr3 is loaded in cr3.
886 mov %gs:CPU_TASK_CR3, %rcx
887 cmp %rax, %rcx /* is the task's cr3 loaded? */
889 cmpl $(TASK_MAP_64BIT_SHARED), %gs:CPU_TASK_MAP
892 mov %gs:CPU_KERNEL_CR3, %rcx
896 mov %rcx, %gs:CPU_ACTIVE_CR3
898 movl %gs:CPU_ACTIVE_THREAD,%ecx /* Get the active thread */
899 cmpl $0, TH_PCB_IDS(%ecx) /* Is there a debug register state? */
901 xor %ecx, %ecx /* If so, reset DR7 (the control) */
905 * Switch to compatibility mode.
906 * Then establish kernel segments.
908 swapgs /* Done with uber-kernel gs */
912 * Now in compatibility mode and running in compatibility space
913 * prepare to enter the locore handler.
915 * %edx lo_handler pointer
916 * Note: the stack pointer (now 32-bit) is now directly addressing the
917 * the kernel below 4G and therefore is automagically re-based.
919 mov $(KERNEL_DS), %eax
924 mov $(CPU_DATA_GS), %eax
927 incl %gs:hwIntCnt(,%ebx,4) /* Bump the trap/intr count */
929 /* Dispatch the designated lo handler */
933 L_64bit_entry_reject:
935 * Here for a 64-bit user attempting an invalid kernel entry.
937 movl $(LO_ALLTRAPS), ISF64_TRAPFN(%rsp)
938 movl $(T_INVALID_OPCODE), ISF64_TRAPNO(%rsp)
939 /* Fall through... */
943 * Here for a 64-bit user task, or special 64-bit kernel code.
944 * Make space for the save area.
946 sub $(ISS64_OFFSET), %rsp
947 movl $(SS_64), SS_FLAVOR(%rsp)
953 mov %fs, R64_FS(%rsp)
954 mov %gs, R64_GS(%rsp)
956 /* Save general-purpose registers */
957 mov %rax, R64_RAX(%rsp)
958 mov %rcx, R64_RCX(%rsp)
959 mov %rbx, R64_RBX(%rsp)
960 mov %rbp, R64_RBP(%rsp)
961 mov %r11, R64_R11(%rsp)
962 mov %r12, R64_R12(%rsp)
963 mov %r13, R64_R13(%rsp)
964 mov %r14, R64_R14(%rsp)
965 mov %r15, R64_R15(%rsp)
967 /* cr2 is significant only for page-faults */
969 mov %rax, R64_CR2(%rsp)
971 /* Other registers (which may contain syscall args) */
972 mov %rdi, R64_RDI(%rsp) /* arg0 .. */
973 mov %rsi, R64_RSI(%rsp)
974 mov %rdx, R64_RDX(%rsp)
975 mov %r10, R64_R10(%rsp)
976 mov %r8, R64_R8(%rsp)
977 mov %r9, R64_R9(%rsp) /* .. arg5 */
979 L_64bit_enter_after_fault:
981 * At this point we're almost ready to join the common lo-entry code.
983 mov R64_TRAPNO(%rsp), %ebx
984 mov R64_TRAPFN(%rsp), %edx
986 testb $3, ISF64_CS+ISS64_OFFSET(%rsp)
990 jmp L_enter_lohandler2
992 Entry(hi64_page_fault)
995 cmpl $(KERNEL_UBER_BASE_HI32), ISF64_RIP+4(%rsp)
996 jne L_enter_lohandler
997 cmpl $(EXT(hi64_sysenter_user_arg_copy)), ISF64_RIP(%rsp)
999 mov ISF64_RSP(%rsp), %rsp
1003 * Debug trap. Check for single-stepping across system call into
1004 * kernel. If this is the case, taking the debug trap has turned
1005 * off single-stepping - save the flags register with the trace
1009 swapgs /* set %gs for cpu data */
1010 push $0 /* error code */
1014 testb $3, ISF64_CS(%rsp)
1015 jnz L_enter_lohandler_continue
1018 * trap came from kernel mode
1020 cmpl $(KERNEL_UBER_BASE_HI32), ISF64_RIP+4(%rsp)
1021 jne L_enter_lohandler_continue /* trap not in uber-space */
1023 cmpl $(EXT(hi64_mach_scall)), ISF64_RIP(%rsp)
1025 add $(ISF64_SIZE),%rsp /* remove entire intr stack frame */
1026 jmp L_mach_scall_continue /* continue system call entry */
1028 cmpl $(EXT(hi64_mdep_scall)), ISF64_RIP(%rsp)
1030 add $(ISF64_SIZE),%rsp /* remove entire intr stack frame */
1031 jmp L_mdep_scall_continue /* continue system call entry */
1033 cmpl $(EXT(hi64_unix_scall)), ISF64_RIP(%rsp)
1035 add $(ISF64_SIZE),%rsp /* remove entire intr stack frame */
1036 jmp L_unix_scall_continue /* continue system call entry */
1038 cmpl $(EXT(hi64_sysenter)), ISF64_RIP(%rsp)
1039 jne L_enter_lohandler_continue
1041 * Interrupt stack frame has been pushed on the temporary stack.
1042 * We have to switch to pcb stack and copy eflags.
1044 add $40,%rsp /* remove trapno/trapfn/err/rip/cs */
1045 push %rcx /* save %rcx - user stack pointer */
1046 mov 32(%rsp),%rcx /* top of intr stack -> pcb stack */
1047 xchg %rcx,%rsp /* switch to pcb stack */
1048 push $(USER_DS) /* ss */
1049 push (%rcx) /* saved %rcx into rsp slot */
1050 push 8(%rcx) /* rflags */
1051 mov (%rcx),%rcx /* restore %rcx */
1052 push $(SYSENTER_TF_CS) /* cs - not SYSENTER_CS for iret path */
1053 jmp L_sysenter_continue /* continue sysenter entry */
1056 Entry(hi64_double_fault)
1057 swapgs /* set %gs for cpu data */
1058 push $(LO_DOUBLE_FAULT)
1059 push $(T_DOUBLE_FAULT)
1061 cmpl $(KERNEL_UBER_BASE_HI32), ISF64_RIP+4(%rsp)
1062 jne L_enter_lohandler_continue /* trap not in uber-space */
1064 cmpl $(EXT(hi64_syscall)), ISF64_RIP(%rsp)
1065 jne L_enter_lohandler_continue
1067 mov ISF64_RSP(%rsp), %rsp
1068 jmp L_syscall_continue
1072 * General protection or segment-not-present fault.
1073 * Check for a GP/NP fault in the kernel_return
1074 * sequence; if there, report it as a GP/NP fault on the user's instruction.
1076 * rsp-> 0 ISF64_TRAPNO: trap code (NP or GP)
1077 * 8 ISF64_TRAPFN: trap function
1078 * 16 ISF64_ERR: segment number in error (error code)
1081 * 40 ISF64_RFLAGS: rflags
1084 * 64 old registers (trap is from kernel)
1086 Entry(hi64_gen_prot)
1088 push $(T_GENERAL_PROTECTION)
1089 jmp trap_check_kernel_exit /* check for kernel exit sequence */
1091 Entry(hi64_stack_fault)
1093 push $(T_STACK_FAULT)
1094 jmp trap_check_kernel_exit /* check for kernel exit sequence */
1098 push $(T_SEGMENT_NOT_PRESENT)
1099 /* indicate fault type */
1100 trap_check_kernel_exit:
1101 testb $3,ISF64_CS(%rsp)
1102 jnz L_enter_lohandler
1103 /* trap was from kernel mode, so */
1104 /* check for the kernel exit sequence */
1105 cmpl $(KERNEL_UBER_BASE_HI32), ISF64_RIP+4(%rsp)
1106 jne L_enter_lohandler_continue /* trap not in uber-space */
1108 cmpl $(EXT(ret32_iret)), ISF64_RIP(%rsp)
1110 cmpl $(EXT(ret32_set_ds)), ISF64_RIP(%rsp)
1111 je L_32bit_fault_set_seg
1112 cmpl $(EXT(ret32_set_es)), ISF64_RIP(%rsp)
1113 je L_32bit_fault_set_seg
1114 cmpl $(EXT(ret32_set_fs)), ISF64_RIP(%rsp)
1115 je L_32bit_fault_set_seg
1116 cmpl $(EXT(ret32_set_gs)), ISF64_RIP(%rsp)
1117 je L_32bit_fault_set_seg
1119 cmpl $(EXT(ret64_iret)), ISF64_RIP(%rsp)
1122 cmpl $(EXT(hi64_sysenter_user_arg_copy)), ISF64_RIP(%rsp)
1123 cmove ISF64_RSP(%rsp), %rsp
1128 * Here after taking an unexpected trap from kernel mode - perhaps
1129 * while running in the trampolines hereabouts.
1130 * Make sure we're not on the PCB stack, if so move to the kernel stack.
1131 * This is likely a fatal condition.
1132 * But first, try to be sure we have the kernel gs base active...
1134 cmpq $0, %gs:CPU_THIS /* test gs_base */
1135 js 1f /* -ve kernel addr, no swap */
1136 swapgs /* +ve user addr, swap */
1138 movq %rax, %gs:CPU_UBER_TMP /* save %rax */
1139 movq %gs:CPU_UBER_ISF, %rax /* PCB stack addr */
1141 cmpq $(PAGE_SIZE), %rax /* current stack in PCB? */
1142 movq %gs:CPU_UBER_TMP, %rax /* restore %rax */
1143 ja L_enter_lohandler_continue /* stack not in PCB */
1146 * Here if %rsp is in the PCB
1147 * Copy the interrupt stack frame from PCB stack to kernel stack
1149 movq %gs:CPU_KERNEL_STACK, %rax /* note: %rax restored below */
1151 pushq ISF64_SS(%rax)
1152 pushq ISF64_RSP(%rax)
1153 pushq ISF64_RFLAGS(%rax)
1154 pushq ISF64_CS(%rax)
1155 pushq ISF64_RIP(%rax)
1156 pushq ISF64_ERR(%rax)
1157 pushq ISF64_TRAPFN(%rax)
1158 pushq ISF64_TRAPNO(%rax)
1159 movq %gs:CPU_UBER_TMP, %rax /* restore %rax */
1160 jmp L_enter_lohandler_continue
1164 * GP/NP fault on IRET: CS or SS is in error.
1165 * All registers contain the user's values.
1168 * 0 ISF64_TRAPNO: trap code (NP or GP)
1169 * 8 ISF64_TRAPFN: trap function
1170 * 16 ISF64_ERR: segment number in error (error code)
1173 * 40 ISF64_RFLAGS: rflags
1175 * 56 ISF64_SS: ss --> new new trapno/trapfn
1176 * 64 pad --> new errcode
1181 * 104 user ss (16-byte aligned)
1184 mov %rax, ISF64_RIP(%rsp) /* save rax (we don`t need saved rip) */
1185 mov ISF64_TRAPNO(%rsp), %rax
1186 mov %rax, ISF64_SS(%rsp) /* put in user trap number */
1187 mov ISF64_ERR(%rsp), %rax
1188 mov %rax, 8+ISF64_SS(%rsp) /* put in user errcode */
1189 mov ISF64_RIP(%rsp), %rax /* restore rax */
1190 add $(ISF64_SS), %rsp /* reset to original frame */
1191 /* now treat as fault from user */
1196 mov %rax, ISF64_RIP(%rsp) /* save rax (we don`t need saved rip) */
1197 mov ISF64_TRAPNO(%rsp), %rax
1198 mov %rax, ISF64_SS(%rsp) /* put in user trap number */
1199 mov ISF64_ERR(%rsp), %rax
1200 mov %rax, 8+ISF64_SS(%rsp) /* put in user errcode */
1201 mov ISF64_RIP(%rsp), %rax /* restore rax */
1202 add $(ISF64_SS), %rsp /* reset to original frame */
1203 /* now treat as fault from user */
1208 * Fault restoring a segment register. All of the saved state is still
1209 * on the stack untouched since we didn't move the stack pointer.
1211 L_32bit_fault_set_seg:
1212 mov ISF64_TRAPNO(%rsp), %rax
1213 mov ISF64_ERR(%rsp), %rdx
1214 mov ISF64_RSP(%rsp), %rsp /* reload stack prior to fault */
1215 mov %rax,ISC32_TRAPNO(%rsp)
1216 mov %rdx,ISC32_ERR(%rsp)
1217 /* now treat as fault from user */
1218 /* except that all the state is */
1219 /* already saved - we just have to */
1220 /* move the trapno and error into */
1221 /* the compatibility frame */
1223 jmp L_32bit_enter_after_fault
1227 * Fatal exception handlers:
1229 Entry(db_task_dbl_fault64)
1230 push $(LO_DOUBLE_FAULT)
1231 push $(T_DOUBLE_FAULT)
1232 jmp L_enter_lohandler
1234 Entry(db_task_stk_fault64)
1235 push $(LO_DOUBLE_FAULT)
1236 push $(T_STACK_FAULT)
1237 jmp L_enter_lohandler
1240 push $(0) /* Error */
1241 push $(LO_MACHINE_CHECK)
1242 push $(T_MACHINE_CHECK)
1243 jmp L_enter_lohandler
1249 * All task 'exceptions' enter lo_alltraps:
1250 * esp -> x86_saved_state_t
1252 * The rest of the state is set up as:
1253 * cr3 -> kernel directory
1254 * esp -> low based stack
1257 * ss/ds/es -> KERNEL_DS
1259 * interrupts disabled
1260 * direction flag cleared
1263 movl R32_CS(%esp),%eax /* assume 32-bit state */
1264 cmpl $(SS_64),SS_FLAVOR(%esp)/* 64-bit? */
1266 movl R64_CS(%esp),%eax /* 64-bit user mode */
1270 /* user mode trap */
1273 movl %gs:CPU_ACTIVE_THREAD,%ecx
1274 movl TH_TASK(%ecx),%ebx
1276 /* Check for active vtimers in the current task */
1277 TASK_VTIMER_CHECK(%ebx, %ecx)
1279 movl %gs:CPU_KERNEL_STACK,%ebx
1280 xchgl %ebx,%esp /* switch to kernel stack */
1282 CCALL1(user_trap, %ebx) /* call user trap routine */
1283 /* user_trap() unmasks interrupts */
1284 cli /* hold off intrs - critical section */
1285 xorl %ecx,%ecx /* don't check if we're in the PFZ */
1288 * Return from trap or system call, checking for ASTs.
1289 * On lowbase PCB stack with intrs disabled
1291 Entry(return_from_trap)
1292 movl %gs:CPU_ACTIVE_THREAD, %esp
1293 movl TH_PCB_ISS(%esp),%esp /* switch back to PCB stack */
1294 movl %gs:CPU_PENDING_AST, %eax
1296 je return_to_user /* branch if no AST */
1297 LEXT(return_from_trap_with_ast)
1298 movl %gs:CPU_KERNEL_STACK, %ebx
1299 xchgl %ebx, %esp /* switch to kernel stack */
1301 testl %ecx, %ecx /* see if we need to check for an EIP in the PFZ */
1302 je 2f /* no, go handle the AST */
1303 cmpl $(SS_64), SS_FLAVOR(%ebx) /* are we a 64-bit task? */
1305 /* no... 32-bit user mode */
1306 movl R32_EIP(%ebx), %eax
1307 pushl %ebx /* save PCB stack */
1308 xorl %ebp, %ebp /* clear frame pointer */
1309 CCALL1(commpage_is_in_pfz32, %eax)
1310 popl %ebx /* retrieve pointer to PCB stack */
1312 je 2f /* not in the PFZ... go service AST */
1313 movl %eax, R32_EBX(%ebx) /* let the PFZ know we've pended an AST */
1314 xchgl %ebx, %esp /* switch back to PCB stack */
1316 1: /* 64-bit user mode */
1317 movl R64_RIP(%ebx), %ecx
1318 movl R64_RIP+4(%ebx), %eax
1319 pushl %ebx /* save PCB stack */
1320 xorl %ebp, %ebp /* clear frame pointer */
1321 CCALL2(commpage_is_in_pfz64, %ecx, %eax)
1322 popl %ebx /* retrieve pointer to PCB stack */
1324 je 2f /* not in the PFZ... go service AST */
1325 movl %eax, R64_RBX(%ebx) /* let the PFZ know we've pended an AST */
1326 xchgl %ebx, %esp /* switch back to PCB stack */
1329 sti /* interrupts always enabled on return to user mode */
1330 pushl %ebx /* save PCB stack */
1331 xorl %ebp, %ebp /* Clear framepointer */
1332 CCALL1(i386_astintr, $0) /* take the AST */
1335 popl %esp /* switch back to PCB stack (w/exc link) */
1337 xorl %ecx, %ecx /* don't check if we're in the PFZ */
1338 jmp EXT(return_from_trap) /* and check again (rare) */
1343 * Trap from kernel mode. No need to switch stacks.
1344 * Interrupts must be off here - we will set them to state at time of trap
1345 * as soon as it's safe for us to do so and not recurse doing preemption
1348 movl %esp, %eax /* saved state addr */
1349 pushl R32_EIP(%esp) /* Simulate a CALL from fault point */
1350 pushl %ebp /* Extend framepointer chain */
1352 CCALL1WITHSP(kernel_trap, %eax) /* Call kernel trap handler */
1357 movl %gs:CPU_PENDING_AST,%eax /* get pending asts */
1358 testl $ AST_URGENT,%eax /* any urgent preemption? */
1359 je ret_to_kernel /* no, nothing to do */
1360 cmpl $ T_PREEMPT,R32_TRAPNO(%esp)
1361 je ret_to_kernel /* T_PREEMPT handled in kernel_trap() */
1362 testl $ EFL_IF,R32_EFLAGS(%esp) /* interrupts disabled? */
1364 cmpl $0,%gs:CPU_PREEMPTION_LEVEL /* preemption disabled? */
1366 movl %gs:CPU_KERNEL_STACK,%eax
1369 and EXT(kernel_stack_mask),%ecx
1370 testl %ecx,%ecx /* are we on the kernel stack? */
1371 jne ret_to_kernel /* no, skip it */
1373 CCALL1(i386_astintr, $1) /* take the AST */
1377 * All interrupts on all tasks enter here with:
1378 * esp-> -> x86_saved_state_t
1380 * cr3 -> kernel directory
1381 * esp -> low based stack
1384 * ss/ds/es -> KERNEL_DS
1386 * interrupts disabled
1387 * direction flag cleared
1391 * test whether already on interrupt stack
1393 movl %gs:CPU_INT_STACK_TOP,%ecx
1396 leal -INTSTACK_SIZE(%ecx),%edx
1398 jb int_from_intstack
1400 xchgl %ecx,%esp /* switch to interrupt stack */
1402 movl %cr0,%eax /* get cr0 */
1403 orl $(CR0_TS),%eax /* or in TS bit */
1404 movl %eax,%cr0 /* set cr0 */
1406 subl $8, %esp /* for 16-byte stack alignment */
1407 pushl %ecx /* save pointer to old stack */
1408 movl %ecx,%gs:CPU_INT_STATE /* save intr state */
1410 TIME_INT_ENTRY /* do timing */
1412 movl %gs:CPU_ACTIVE_THREAD,%ecx
1413 movl TH_TASK(%ecx),%ebx
1415 /* Check for active vtimers in the current task */
1416 TASK_VTIMER_CHECK(%ebx, %ecx)
1418 incl %gs:CPU_PREEMPTION_LEVEL
1419 incl %gs:CPU_INTERRUPT_LEVEL
1421 movl %gs:CPU_INT_STATE, %eax
1422 CCALL1(interrupt, %eax) /* call generic interrupt routine */
1424 cli /* just in case we returned with intrs enabled */
1426 movl %eax,%gs:CPU_INT_STATE /* clear intr state pointer */
1428 decl %gs:CPU_INTERRUPT_LEVEL
1429 decl %gs:CPU_PREEMPTION_LEVEL
1431 TIME_INT_EXIT /* do timing */
1433 movl %gs:CPU_ACTIVE_THREAD,%eax
1434 movl TH_PCB_FPS(%eax),%eax /* get pcb's ifps */
1435 testl %eax, %eax /* Is there a context */
1436 je 1f /* Branch if not */
1437 cmpl $0, FP_VALID(%eax) /* Check fp_valid */
1438 jne 1f /* Branch if valid */
1442 movl %cr0,%eax /* get cr0 */
1443 orl $(CR0_TS),%eax /* or in TS bit */
1444 movl %eax,%cr0 /* set cr0 */
1446 popl %esp /* switch back to old stack */
1448 /* Load interrupted code segment into %eax */
1449 movl R32_CS(%esp),%eax /* assume 32-bit state */
1450 cmpl $(SS_64),SS_FLAVOR(%esp)/* 64-bit? */
1452 movl R64_CS(%esp),%eax /* 64-bit user mode */
1454 testb $3,%al /* user mode, */
1455 jnz ast_from_interrupt_user /* go handle potential ASTs */
1457 * we only want to handle preemption requests if
1458 * the interrupt fell in the kernel context
1459 * and preemption isn't disabled
1461 movl %gs:CPU_PENDING_AST,%eax
1462 testl $ AST_URGENT,%eax /* any urgent requests? */
1463 je ret_to_kernel /* no, nothing to do */
1465 cmpl $0,%gs:CPU_PREEMPTION_LEVEL /* preemption disabled? */
1466 jne ret_to_kernel /* yes, skip it */
1468 movl %gs:CPU_KERNEL_STACK,%eax
1471 and EXT(kernel_stack_mask),%ecx
1472 testl %ecx,%ecx /* are we on the kernel stack? */
1473 jne ret_to_kernel /* no, skip it */
1476 * Take an AST from kernel space. We don't need (and don't want)
1477 * to do as much as the case where the interrupt came from user
1480 CCALL1(i386_astintr, $1)
1486 * nested int - simple path, can't preempt etc on way out
1489 incl %gs:CPU_PREEMPTION_LEVEL
1490 incl %gs:CPU_INTERRUPT_LEVEL
1491 incl %gs:CPU_NESTED_ISTACK
1493 movl %esp, %edx /* x86_saved_state */
1494 CCALL1(interrupt, %edx)
1496 decl %gs:CPU_INTERRUPT_LEVEL
1497 decl %gs:CPU_PREEMPTION_LEVEL
1498 decl %gs:CPU_NESTED_ISTACK
1503 * Take an AST from an interrupted user
1505 ast_from_interrupt_user:
1506 movl %gs:CPU_PENDING_AST,%eax
1507 testl %eax,%eax /* pending ASTs? */
1508 je ret_to_user /* no, nothing to do */
1512 movl $1, %ecx /* check if we're in the PFZ */
1513 jmp EXT(return_from_trap_with_ast) /* return */
1518 * System call entries via INTR_GATE or sysenter:
1520 * esp -> x86_saved_state32_t
1521 * cr3 -> kernel directory
1522 * esp -> low based stack
1525 * ss/ds/es -> KERNEL_DS
1527 * interrupts disabled
1528 * direction flag cleared
1531 Entry(lo_unix_scall)
1534 movl %gs:CPU_KERNEL_STACK,%edi
1535 xchgl %edi,%esp /* switch to kernel stack */
1536 movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */
1537 movl TH_TASK(%ecx),%ebx /* point to current task */
1538 incl TH_SYSCALLS_UNIX(%ecx) /* increment call count */
1540 /* Check for active vtimers in the current task */
1541 TASK_VTIMER_CHECK(%ebx, %ecx)
1545 CCALL1(unix_syscall, %edi)
1547 * always returns through thread_exception_return
1551 Entry(lo_mach_scall)
1554 movl %gs:CPU_KERNEL_STACK,%edi
1555 xchgl %edi,%esp /* switch to kernel stack */
1556 movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */
1557 movl TH_TASK(%ecx),%ebx /* point to current task */
1558 incl TH_SYSCALLS_MACH(%ecx) /* increment call count */
1560 /* Check for active vtimers in the current task */
1561 TASK_VTIMER_CHECK(%ebx, %ecx)
1565 CCALL1(mach_call_munger, %edi)
1567 * always returns through thread_exception_return
1571 Entry(lo_mdep_scall)
1574 movl %gs:CPU_KERNEL_STACK,%edi
1575 xchgl %edi,%esp /* switch to kernel stack */
1576 movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */
1577 movl TH_TASK(%ecx),%ebx /* point to current task */
1579 /* Check for active vtimers in the current task */
1580 TASK_VTIMER_CHECK(%ebx, %ecx)
1584 CCALL1(machdep_syscall, %edi)
1586 * always returns through thread_exception_return
1596 * System call entries via syscall only:
1598 * esp -> x86_saved_state64_t
1599 * cr3 -> kernel directory
1600 * esp -> low based stack
1603 * ss/ds/es -> KERNEL_DS
1605 * interrupts disabled
1606 * direction flag cleared
1612 movl %gs:CPU_KERNEL_STACK,%edi
1613 xchgl %edi,%esp /* switch to kernel stack */
1615 movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */
1616 movl TH_TASK(%ecx),%ebx /* point to current task */
1618 /* Check for active vtimers in the current task */
1619 TASK_VTIMER_CHECK(%ebx, %ecx)
1622 * We can be here either for a mach, unix machdep or diag syscall,
1623 * as indicated by the syscall class:
1625 movl R64_RAX(%edi), %eax /* syscall number/class */
1627 andl $(SYSCALL_CLASS_MASK), %edx /* syscall class */
1628 cmpl $(SYSCALL_CLASS_MACH<<SYSCALL_CLASS_SHIFT), %edx
1629 je EXT(lo64_mach_scall)
1630 cmpl $(SYSCALL_CLASS_UNIX<<SYSCALL_CLASS_SHIFT), %edx
1631 je EXT(lo64_unix_scall)
1632 cmpl $(SYSCALL_CLASS_MDEP<<SYSCALL_CLASS_SHIFT), %edx
1633 je EXT(lo64_mdep_scall)
1634 cmpl $(SYSCALL_CLASS_DIAG<<SYSCALL_CLASS_SHIFT), %edx
1635 je EXT(lo64_diag_scall)
1639 /* Syscall class unknown */
1640 CCALL5(i386_exception, $(EXC_SYSCALL), %eax, $0, $1, $0)
1644 Entry(lo64_unix_scall)
1645 incl TH_SYSCALLS_UNIX(%ecx) /* increment call count */
1648 CCALL1(unix_syscall64, %edi)
1650 * always returns through thread_exception_return
1654 Entry(lo64_mach_scall)
1655 incl TH_SYSCALLS_MACH(%ecx) /* increment call count */
1658 CCALL1(mach_call_munger64, %edi)
1660 * always returns through thread_exception_return
1665 Entry(lo64_mdep_scall)
1668 CCALL1(machdep_syscall64, %edi)
1670 * always returns through thread_exception_return
1674 Entry(lo64_diag_scall)
1675 CCALL1(diagCall64, %edi) // Call diagnostics
1677 cli // Disable interruptions just in case
1678 cmpl $0,%eax // What kind of return is this?
1680 movl %edi, %esp // Get back the original stack
1681 jmp return_to_user // Normal return, do not check asts...
1683 CCALL5(i386_exception, $EXC_SYSCALL, $0x6000, $0, $1, $0)
1684 // pass what would be the diag syscall
1685 // error return - cause an exception
1691 * Compatibility mode's last gasp...
1695 CCALL1(panic_double_fault64, %eax)
1700 CCALL1(panic_machine_check64, %eax)
projects / apple / xnu.git / blob