]> git.saurik.com Git - apple/xnu.git/blob - osfmk/x86_64/idt64.s
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-1504.9.37.tar.gz
[apple/xnu.git] / osfmk / x86_64 / idt64.s
1 /*
2 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #include <i386/asm.h>
29 #include <assym.s>
30 #include <mach_kdb.h>
31 #include <i386/eflags.h>
32 #include <i386/rtclock.h>
33 #include <i386/trap.h>
34 #define _ARCH_I386_ASM_HELP_H_ /* Prevent inclusion of user header */
35 #include <mach/i386/syscall_sw.h>
36 #include <i386/postcode.h>
37 #include <i386/proc_reg.h>
38 #include <mach/exception_types.h>
39
40 #if DEBUG
41 #define DEBUG_IDT64 1
42 #endif
43
44 /*
45 * This is the low-level trap and interrupt handling code associated with
46 * the IDT. It also includes system call handlers for sysenter/syscall.
47 * The IDT itself is defined in mp_desc.c.
48 *
49 * Code here is structured as follows:
50 *
51 * stubs Code called directly from an IDT vector.
52 * All entry points have the "idt64_" prefix and they are built
53 * using macros expanded by the inclusion of idt_table.h.
54 * This code performs vector-dependent identification and jumps
55 * into the dispatch code.
56 *
57 * dispatch The dispatch code is responsible for saving the thread state
58 * (which is either 64-bit or 32-bit) and then jumping to the
59 * class handler identified by the stub.
60 *
61 * returns Code to restore state and return to the previous context.
62 *
63 * handlers There are several classes of handlers:
64 * interrupt - asynchronous events typically from external devices
65 * trap - synchronous events due to thread execution
66 * syscall - synchronous system call request
67 * fatal - fatal traps
68 */
69
70 /*
71 * Handlers:
72 */
73 #define HNDL_ALLINTRS EXT(hndl_allintrs)
74 #define HNDL_ALLTRAPS EXT(hndl_alltraps)
75 #define HNDL_SYSENTER EXT(hndl_sysenter)
76 #define HNDL_SYSCALL EXT(hndl_syscall)
77 #define HNDL_UNIX_SCALL EXT(hndl_unix_scall)
78 #define HNDL_MACH_SCALL EXT(hndl_mach_scall)
79 #define HNDL_MDEP_SCALL EXT(hndl_mdep_scall)
80 #define HNDL_DIAG_SCALL EXT(hndl_diag_scall)
81 #define HNDL_DOUBLE_FAULT EXT(hndl_double_fault)
82 #define HNDL_MACHINE_CHECK EXT(hndl_machine_check)
83
84 /*
85 * Nanosecond timing.
86 */
87
88 /*
89 * Nanotime returned in %rax.
90 * Computed from tsc based on the scale factor and an implicit 32 bit shift.
91 * This code must match what _rtc_nanotime_read does in
92 * machine_routines_asm.s. Failure to do so can
93 * result in "weird" timing results.
94 *
95 * Uses: %rsi, %rdi, %rdx, %rcx
96 */
97 #define NANOTIME \
98 movq %gs:CPU_NANOTIME,%rdi ; \
99 RTC_NANOTIME_READ_FAST()
100
101 /*
102 * Add 64-bit delta in register reg to timer pointed to by register treg.
103 */
104 #define TIMER_UPDATE(treg,reg,offset) \
105 addq reg,(offset)+TIMER_ALL(treg) /* add timer */
106
107 /*
108 * Add time delta to old timer and start new.
109 * Uses: %rsi, %rdi, %rdx, %rcx, %rax
110 */
111 #define TIMER_EVENT(old,new) \
112 NANOTIME /* %rax := nanosecs */ ; \
113 movq %rax,%rsi /* save timestamp */ ; \
114 movq %gs:CPU_ACTIVE_THREAD,%rcx /* get thread */ ; \
115 subq (old##_TIMER)+TIMER_TSTAMP(%rcx),%rax /* compute elapsed */ ; \
116 TIMER_UPDATE(%rcx,%rax,old##_TIMER) /* update timer */ ; \
117 leaq (new##_TIMER)(%rcx),%rcx /* point to new timer */; \
118 movq %rsi,TIMER_TSTAMP(%rcx) /* set timestamp */ ; \
119 movq %gs:CPU_PROCESSOR,%rdx /* get processor */ ; \
120 movq %rcx,THREAD_TIMER(%rdx) /* set current timer */ ; \
121 movq %rsi,%rax /* restore timestamp */ ; \
122 subq (old##_STATE)+TIMER_TSTAMP(%rdx),%rax /* compute elapsed */ ; \
123 TIMER_UPDATE(%rdx,%rax,old##_STATE) /* update timer */ ; \
124 leaq (new##_STATE)(%rdx),%rcx /* point to new state */; \
125 movq %rcx,CURRENT_STATE(%rdx) /* set current state */ ; \
126 movq %rsi,TIMER_TSTAMP(%rcx) /* set timestamp */
127
128 /*
129 * Update time on user trap entry.
130 * Uses: %rsi, %rdi, %rdx, %rcx, %rax
131 */
132 #define TIME_TRAP_UENTRY TIMER_EVENT(USER,SYSTEM)
133
134 /*
135 * update time on user trap exit.
136 * Uses: %rsi, %rdi, %rdx, %rcx, %rax
137 */
138 #define TIME_TRAP_UEXIT TIMER_EVENT(SYSTEM,USER)
139
140 /*
141 * update time on interrupt entry.
142 * Uses: %rsi, %rdi, %rdx, %rcx, %rax
143 * Saves processor state info on stack.
144 */
145 #define TIME_INT_ENTRY \
146 NANOTIME /* %rax := nanosecs */ ; \
147 movq %rax,%gs:CPU_INT_EVENT_TIME /* save in cpu data */ ; \
148 movq %rax,%rsi /* save timestamp */ ; \
149 movq %gs:CPU_PROCESSOR,%rdx /* get processor */ ; \
150 movq THREAD_TIMER(%rdx),%rcx /* get current timer */ ; \
151 subq TIMER_TSTAMP(%rcx),%rax /* compute elapsed */ ; \
152 TIMER_UPDATE(%rcx,%rax,0) /* update timer */ ; \
153 movq KERNEL_TIMER(%rdx),%rcx /* get kernel timer */ ; \
154 movq %rsi,TIMER_TSTAMP(%rcx) /* set timestamp */ ; \
155 movq %rsi,%rax /* restore timestamp */ ; \
156 movq CURRENT_STATE(%rdx),%rcx /* get current state */ ; \
157 pushq %rcx /* save state */ ; \
158 subq TIMER_TSTAMP(%rcx),%rax /* compute elapsed */ ; \
159 TIMER_UPDATE(%rcx,%rax,0) /* update timer */ ; \
160 leaq IDLE_STATE(%rdx),%rax /* get idle state */ ; \
161 cmpq %rax,%rcx /* compare current */ ; \
162 je 0f /* skip if equal */ ; \
163 leaq SYSTEM_STATE(%rdx),%rcx /* get system state */ ; \
164 movq %rcx,CURRENT_STATE(%rdx) /* set current state */ ; \
165 0: movq %rsi,TIMER_TSTAMP(%rcx) /* set timestamp */
166
167 /*
168 * update time on interrupt exit.
169 * Uses: %rsi, %rdi, %rdx, %rcx, %rax
170 * Restores processor state info from stack.
171 */
172 #define TIME_INT_EXIT \
173 NANOTIME /* %rax := nanosecs */ ; \
174 movq %rax,%gs:CPU_INT_EVENT_TIME /* save in cpu data */ ; \
175 movq %rax,%rsi /* save timestamp */ ; \
176 movq %gs:CPU_PROCESSOR,%rdx /* get processor */ ; \
177 movq KERNEL_TIMER(%rdx),%rcx /* get kernel timer */ ; \
178 subq TIMER_TSTAMP(%rcx),%rax /* compute elapsed */ ; \
179 TIMER_UPDATE(%rcx,%rax,0) /* update timer */ ; \
180 movq THREAD_TIMER(%rdx),%rcx /* interrupted timer */ ; \
181 movq %rsi,TIMER_TSTAMP(%rcx) /* set timestamp */ ; \
182 movq %rsi,%rax /* restore timestamp */ ; \
183 movq CURRENT_STATE(%rdx),%rcx /* get current state */ ; \
184 subq TIMER_TSTAMP(%rcx),%rax /* compute elapsed */ ; \
185 TIMER_UPDATE(%rcx,%rax,0) /* update timer */ ; \
186 popq %rcx /* restore state */ ; \
187 movq %rcx,CURRENT_STATE(%rdx) /* set current state */ ; \
188 movq %rsi,TIMER_TSTAMP(%rcx) /* set timestamp */
189
190 /*
191 * Check for vtimers for task.
192 * task_reg is register pointing to current task
193 * thread_reg is register pointing to current thread
194 */
195 #define TASK_VTIMER_CHECK(task_reg,thread_reg) \
196 cmpl $0,TASK_VTIMERS(task_reg) ; \
197 jz 1f ; \
198 orl $(AST_BSD),%gs:CPU_PENDING_AST /* Set pending AST */ ; \
199 lock ; \
200 orl $(AST_BSD),ACT_AST(thread_reg) /* Set thread AST */ ; \
201 1: ; \
202
203
204 /*
205 * Macros for calling into C functions.
206 * The stack is 16-byte aligned by masking.
207 */
208 #define CCALL(fn) \
209 mov %rsp, %r12 ;\
210 and $0xFFFFFFFFFFFFFFF0, %rsp ;\
211 call EXT(fn) ;\
212 mov %r12, %rsp
213
214 #define CCALL1(fn, arg1) \
215 mov arg1, %rdi ;\
216 CCALL(fn)
217
218 #define CCALL2(fn, arg1, arg2) \
219 mov arg1, %rdi ;\
220 CCALL(fn)
221
222 #define CCALL3(fn, arg1, arg2, arg3) \
223 mov arg1, %rdi ;\
224 mov arg2, %rsi ;\
225 mov arg3, %rdx ;\
226 CCALL(fn)
227
228 #if 1
229 #define PUSH_FUNCTION(func) \
230 sub $8, %rsp ;\
231 push %rax ;\
232 leaq func(%rip), %rax ;\
233 movq %rax, 8(%rsp) ;\
234 pop %rax
235 #else
236 #define PUSH_FUNCTION(func) pushq func
237 #endif
238
239 /* The wrapper for all non-special traps/interrupts */
240 /* Everything up to PUSH_FUNCTION is just to output
241 * the interrupt number out to the postcode display
242 */
243 #if DEBUG_IDT64
244 #define IDT_ENTRY_WRAPPER(n, f) \
245 push %rax ;\
246 POSTCODE2(0x6400+n) ;\
247 pop %rax ;\
248 PUSH_FUNCTION(f) ;\
249 pushq $(n) ;\
250 jmp L_dispatch
251 #else
252 #define IDT_ENTRY_WRAPPER(n, f) \
253 PUSH_FUNCTION(f) ;\
254 pushq $(n) ;\
255 jmp L_dispatch
256 #endif
257
258 /* A trap that comes with an error code already on the stack */
259 #define TRAP_ERR(n, f) \
260 Entry(f) ;\
261 IDT_ENTRY_WRAPPER(n, HNDL_ALLTRAPS)
262
263 /* A normal trap */
264 #define TRAP(n, f) \
265 Entry(f) ;\
266 pushq $0 ;\
267 IDT_ENTRY_WRAPPER(n, HNDL_ALLTRAPS)
268
269 #define USER_TRAP TRAP
270
271 /* An interrupt */
272 #define INTERRUPT(n) \
273 Entry(_intr_ ## n) ;\
274 pushq $0 ;\
275 IDT_ENTRY_WRAPPER(n, HNDL_ALLINTRS)
276
277 /* A trap with a special-case handler, hence we don't need to define anything */
278 #define TRAP_SPC(n, f)
279 #define TRAP_IST(n, f)
280 #define USER_TRAP_SPC(n, f)
281
282 /* Generate all the stubs */
283 #include "idt_table.h"
284
285 /*
286 * Common dispatch point.
287 * Determine what mode has been interrupted and save state accordingly.
288 */
289 L_dispatch:
290 cmpq $(KERNEL64_CS), ISF64_CS(%rsp)
291 je L_64bit_dispatch
292
293 swapgs
294
295 cmpl $(USER_CS), ISF64_CS(%rsp)
296 je L_32bit_dispatch /* 32-bit user task */
297 /* fall through to 64bit user dispatch */
298
299 /*
300 * Here for 64-bit user task or kernel
301 */
302 L_64bit_dispatch:
303 subq $(ISS64_OFFSET), %rsp
304 movl $(SS_64), SS_FLAVOR(%rsp)
305
306 /*
307 * Save segment regs - for completeness since theyre not used.
308 */
309 mov %fs, R64_FS(%rsp)
310 mov %gs, R64_GS(%rsp)
311
312 /* Save general-purpose registers */
313 mov %rax, R64_RAX(%rsp)
314 mov %rcx, R64_RCX(%rsp)
315 mov %rbx, R64_RBX(%rsp)
316 mov %rbp, R64_RBP(%rsp)
317 mov %r11, R64_R11(%rsp)
318 mov %r12, R64_R12(%rsp)
319 mov %r13, R64_R13(%rsp)
320 mov %r14, R64_R14(%rsp)
321 mov %r15, R64_R15(%rsp)
322
323 /* cr2 is significant only for page-faults */
324 mov %cr2, %rax
325 mov %rax, R64_CR2(%rsp)
326
327 /* Other registers (which may contain syscall args) */
328 mov %rdi, R64_RDI(%rsp) /* arg0 .. */
329 mov %rsi, R64_RSI(%rsp)
330 mov %rdx, R64_RDX(%rsp)
331 mov %r10, R64_R10(%rsp)
332 mov %r8, R64_R8(%rsp)
333 mov %r9, R64_R9(%rsp) /* .. arg5 */
334
335 mov R64_TRAPNO(%rsp), %ebx /* %ebx := trapno for later */
336 mov R64_TRAPFN(%rsp), %rdx /* %rdx := trapfn for later */
337 mov R64_CS(%rsp), %esi /* %esi := cs for later */
338
339 jmp L_common_dispatch
340
341 L_64bit_entry_reject:
342 /*
343 * Here for a 64-bit user attempting an invalid kernel entry.
344 */
345 pushq %rax
346 leaq HNDL_ALLTRAPS(%rip), %rax
347 movq %rax, ISF64_TRAPFN+8(%rsp)
348 popq %rax
349 movq $(T_INVALID_OPCODE), ISF64_TRAPNO(%rsp)
350 jmp L_64bit_dispatch
351
352 L_32bit_entry_check:
353 /*
354 * Check we're not a confused 64-bit user.
355 */
356 cmpl $(TASK_MAP_32BIT), %gs:CPU_TASK_MAP
357 jne L_64bit_entry_reject
358 /* fall through to 32-bit handler: */
359
360 L_32bit_dispatch: /* 32-bit user task */
361 subq $(ISC32_OFFSET), %rsp
362 movl $(SS_32), SS_FLAVOR(%rsp)
363
364 /*
365 * Save segment regs
366 */
367 mov %ds, R32_DS(%rsp)
368 mov %es, R32_ES(%rsp)
369 mov %fs, R32_FS(%rsp)
370 mov %gs, R32_GS(%rsp)
371
372 /*
373 * Save general 32-bit registers
374 */
375 mov %eax, R32_EAX(%rsp)
376 mov %ebx, R32_EBX(%rsp)
377 mov %ecx, R32_ECX(%rsp)
378 mov %edx, R32_EDX(%rsp)
379 mov %ebp, R32_EBP(%rsp)
380 mov %esi, R32_ESI(%rsp)
381 mov %edi, R32_EDI(%rsp)
382
383 /* Unconditionally save cr2; only meaningful on page faults */
384 mov %cr2, %rax
385 mov %eax, R32_CR2(%rsp)
386
387 /*
388 * Copy registers already saved in the machine state
389 * (in the interrupt stack frame) into the compat save area.
390 */
391 mov ISC32_RIP(%rsp), %eax
392 mov %eax, R32_EIP(%rsp)
393 mov ISC32_RFLAGS(%rsp), %eax
394 mov %eax, R32_EFLAGS(%rsp)
395 mov ISC32_CS(%rsp), %esi /* %esi := %cs for later */
396
397 mov %esi, R32_CS(%rsp)
398 mov ISC32_RSP(%rsp), %eax
399 mov %eax, R32_UESP(%rsp)
400 mov ISC32_SS(%rsp), %eax
401 mov %eax, R32_SS(%rsp)
402 L_32bit_dispatch_after_fault:
403 mov ISC32_TRAPNO(%rsp), %ebx /* %ebx := trapno for later */
404 mov %ebx, R32_TRAPNO(%rsp)
405 mov ISC32_ERR(%rsp), %eax
406 mov %eax, R32_ERR(%rsp)
407 mov ISC32_TRAPFN(%rsp), %rdx /* %rdx := trapfn for later */
408
409 L_common_dispatch:
410 /*
411 * On entering the kernel, we don't need to switch cr3
412 * because the kernel shares the user's address space.
413 * But we mark the kernel's cr3 as "active".
414 * If, however, the invalid cr3 flag is set, we have to flush tlbs
415 * since the kernel's mapping was changed while we were in userspace.
416 *
417 * But: if global no_shared_cr3 is TRUE we do switch to the kernel's cr3
418 * so that illicit accesses to userspace can be trapped.
419 */
420 mov %gs:CPU_KERNEL_CR3, %rcx
421 mov %rcx, %gs:CPU_ACTIVE_CR3
422 test $3, %esi /* user/kernel? */
423 jz 1f /* skip cr3 reload from kernel */
424 xor %rbp, %rbp
425 cmpl $0, EXT(no_shared_cr3)(%rip)
426 je 1f
427 mov %rcx, %cr3 /* load kernel cr3 */
428 jmp 2f /* and skip tlb flush test */
429 1:
430 cmpl $0, %gs:CPU_TLB_INVALID /* flush needed? */
431 je 2f /* - no */
432 movl $0, %gs:CPU_TLB_INVALID
433 mov %cr3, %rcx
434 mov %rcx, %cr3
435 2:
436 mov %gs:CPU_ACTIVE_THREAD, %rcx /* Get the active thread */
437 cmpq $0, ACT_PCB_IDS(%rcx) /* Is there a debug register state? */
438 je 3f
439 mov $0, %rcx /* If so, reset DR7 (the control) */
440 mov %rcx, %dr7
441 3:
442 addl $1,%gs:hwIntCnt(,%ebx,4) // Bump the trap/intr count
443 /* Dispatch the designated handler */
444 mov %rsp, %rdi /* rsp points to saved state */
445 jmp *%rdx
446
447 /*
448 * Control is passed here to return to user.
449 */
450 Entry(return_to_user)
451 TIME_TRAP_UEXIT
452
453 Entry(ret_to_user)
454 // XXX 'Be nice to tidy up this debug register restore sequence...
455 mov %gs:CPU_ACTIVE_THREAD, %rdx
456 movq ACT_PCB_IDS(%rdx),%rax /* Obtain this thread's debug state */
457
458 cmpq $0,%rax /* Is there a debug register context? */
459 je 2f /* branch if not */
460 cmpl $(TASK_MAP_32BIT), %gs:CPU_TASK_MAP /* Are we a 32-bit task? */
461 jne 1f
462 movl DS_DR0(%rax), %ecx /* If so, load the 32 bit DRs */
463 movq %rcx, %dr0
464 movl DS_DR1(%rax), %ecx
465 movq %rcx, %dr1
466 movl DS_DR2(%rax), %ecx
467 movq %rcx, %dr2
468 movl DS_DR3(%rax), %ecx
469 movq %rcx, %dr3
470 movl DS_DR7(%rax), %ecx
471 movq %rcx, %gs:CPU_DR7
472 jmp 2f
473 1:
474 mov DS64_DR0(%rax), %rcx /* Load the full width DRs*/
475 mov %rcx, %dr0
476 mov DS64_DR1(%rax), %rcx
477 mov %rcx, %dr1
478 mov DS64_DR2(%rax), %rcx
479 mov %rcx, %dr2
480 mov DS64_DR3(%rax), %rcx
481 mov %rcx, %dr3
482 mov DS64_DR7(%rax), %rcx
483 mov %rcx, %gs:CPU_DR7
484 2:
485 /*
486 * On exiting the kernel there's no need to switch cr3 since we're
487 * already running in the user's address space which includes the
488 * kernel. Nevertheless, we now mark the task's cr3 as active.
489 * However, there may be a defered tlb flush to deal with.
490 * This is a case where another cpu modified this task's address
491 * space while this thread was in the kernel.
492 * But, if no_shared_cr3 is set, we do need to switch cr3 at this point.
493 */
494 mov %gs:CPU_TASK_CR3, %rcx
495 mov %rcx, %gs:CPU_ACTIVE_CR3
496 movl %gs:CPU_TLB_INVALID, %eax
497 orl EXT(no_shared_cr3)(%rip), %eax
498 test %eax, %eax /* -no_shered_cr3 or flush required? */
499 jz 3f
500 movl $0, %gs:CPU_TLB_INVALID
501 mov %rcx, %cr3
502 3:
503
504 mov %gs:CPU_DR7, %rax /* Is there a debug control register?*/
505 cmp $0, %rax
506 je 4f
507 mov %rax, %dr7 /* Set DR7 */
508 movq $0, %gs:CPU_DR7
509 4:
510 cmpl $(SS_64), SS_FLAVOR(%rsp) /* 64-bit state? */
511 je L_64bit_return
512
513 L_32bit_return:
514 #if DEBUG_IDT64
515 cmpl $(SS_32), SS_FLAVOR(%rsp) /* 32-bit state? */
516 je 1f
517 cli
518 POSTCODE2(0x6432)
519 CCALL1(panic_idt64, %rsp)
520 1:
521 #endif /* DEBUG_IDT64 */
522
523 /*
524 * Restore registers into the machine state for iret.
525 */
526 movl R32_EIP(%rsp), %eax
527 movl %eax, ISC32_RIP(%rsp)
528 movl R32_EFLAGS(%rsp), %eax
529 movl %eax, ISC32_RFLAGS(%rsp)
530 movl R32_CS(%rsp), %eax
531 movl %eax, ISC32_CS(%rsp)
532 movl R32_UESP(%rsp), %eax
533 movl %eax, ISC32_RSP(%rsp)
534 movl R32_SS(%rsp), %eax
535 movl %eax, ISC32_SS(%rsp)
536
537 /*
538 * Restore general 32-bit registers
539 */
540 movl R32_EAX(%rsp), %eax
541 movl R32_EBX(%rsp), %ebx
542 movl R32_ECX(%rsp), %ecx
543 movl R32_EDX(%rsp), %edx
544 movl R32_EBP(%rsp), %ebp
545 movl R32_ESI(%rsp), %esi
546 movl R32_EDI(%rsp), %edi
547
548 /*
549 * Restore segment registers. We make take an exception here but
550 * we've got enough space left in the save frame area to absorb
551 * a hardware frame plus the trapfn and trapno
552 */
553 swapgs
554 EXT(ret32_set_ds):
555 movw R32_DS(%rsp), %ds
556 EXT(ret32_set_es):
557 movw R32_ES(%rsp), %es
558 EXT(ret32_set_fs):
559 movw R32_FS(%rsp), %fs
560 EXT(ret32_set_gs):
561 movw R32_GS(%rsp), %gs
562
563 /* pop compat frame + trapno, trapfn and error */
564 add $(ISC32_OFFSET)+8+8+8, %rsp
565 cmp $(SYSENTER_CS),ISF64_CS-8-8-8(%rsp)
566 /* test for fast entry/exit */
567 je L_fast_exit
568 EXT(ret32_iret):
569 iretq /* return from interrupt */
570
571 L_fast_exit:
572 pop %rdx /* user return eip */
573 pop %rcx /* pop and toss cs */
574 andl $(~EFL_IF), (%rsp) /* clear interrupts enable, sti below */
575 popf /* flags - carry denotes failure */
576 pop %rcx /* user return esp */
577 sti /* interrupts enabled after sysexit */
578 sysexit /* 32-bit sysexit */
579
580 ret_to_kernel:
581 #if DEBUG_IDT64
582 cmpl $(SS_64), SS_FLAVOR(%rsp) /* 64-bit state? */
583 je 1f
584 cli
585 POSTCODE2(0x6464)
586 CCALL1(panic_idt64, %rsp)
587 hlt
588 1:
589 cmpq $(KERNEL64_CS), R64_CS(%rsp)
590 je 2f
591 CCALL1(panic_idt64, %rsp)
592 hlt
593 2:
594 #endif
595
596 L_64bit_return:
597 testb $3, R64_CS(%rsp) /* returning to user-space? */
598 jz 1f
599 swapgs
600 1:
601
602 /*
603 * Restore general 64-bit registers
604 */
605 mov R64_R15(%rsp), %r15
606 mov R64_R14(%rsp), %r14
607 mov R64_R13(%rsp), %r13
608 mov R64_R12(%rsp), %r12
609 mov R64_R11(%rsp), %r11
610 mov R64_R10(%rsp), %r10
611 mov R64_R9(%rsp), %r9
612 mov R64_R8(%rsp), %r8
613 mov R64_RSI(%rsp), %rsi
614 mov R64_RDI(%rsp), %rdi
615 mov R64_RBP(%rsp), %rbp
616 mov R64_RDX(%rsp), %rdx
617 mov R64_RBX(%rsp), %rbx
618 mov R64_RCX(%rsp), %rcx
619 mov R64_RAX(%rsp), %rax
620
621 add $(ISS64_OFFSET)+24, %rsp /* pop saved state frame +
622 trapno + trapfn and error */
623 cmpl $(SYSCALL_CS),ISF64_CS-24(%rsp)
624 /* test for fast entry/exit */
625 je L_sysret
626 .globl _dump_iretq
627 EXT(ret64_iret):
628 iretq /* return from interrupt */
629
630 L_sysret:
631 /*
632 * Here to load rcx/r11/rsp and perform the sysret back to user-space.
633 * rcx user rip
634 * r1 user rflags
635 * rsp user stack pointer
636 */
637 mov ISF64_RIP-24(%rsp), %rcx
638 mov ISF64_RFLAGS-24(%rsp), %r11
639 mov ISF64_RSP-24(%rsp), %rsp
640 sysretq /* return from systen call */
641
642
643
644 /*
645 * System call handlers.
646 * These are entered via a syscall interrupt. The system call number in %rax
647 * is saved to the error code slot in the stack frame. We then branch to the
648 * common state saving code.
649 */
650
651 #ifndef UNIX_INT
652 #error NO UNIX INT!!!
653 #endif
654 Entry(idt64_unix_scall)
655 swapgs /* switch to kernel gs (cpu_data) */
656 L_unix_scall_continue:
657 pushq %rax /* save system call number */
658 PUSH_FUNCTION(HNDL_UNIX_SCALL)
659 pushq $(UNIX_INT)
660 jmp L_32bit_entry_check
661
662
663 Entry(idt64_mach_scall)
664 swapgs /* switch to kernel gs (cpu_data) */
665 L_mach_scall_continue:
666 pushq %rax /* save system call number */
667 PUSH_FUNCTION(HNDL_MACH_SCALL)
668 pushq $(MACH_INT)
669 jmp L_32bit_entry_check
670
671
672 Entry(idt64_mdep_scall)
673 swapgs /* switch to kernel gs (cpu_data) */
674 L_mdep_scall_continue:
675 pushq %rax /* save system call number */
676 PUSH_FUNCTION(HNDL_MDEP_SCALL)
677 pushq $(MACHDEP_INT)
678 jmp L_32bit_entry_check
679
680
681 Entry(idt64_diag_scall)
682 swapgs /* switch to kernel gs (cpu_data) */
683 L_diag_scall_continue:
684 push %rax /* save system call number */
685 PUSH_FUNCTION(HNDL_DIAG_SCALL)
686 pushq $(DIAG_INT)
687 jmp L_32bit_entry_check
688
689 Entry(hi64_syscall)
690 Entry(idt64_syscall)
691 swapgs /* Kapow! get per-cpu data area */
692 L_syscall_continue:
693 mov %rsp, %gs:CPU_UBER_TMP /* save user stack */
694 mov %gs:CPU_UBER_ISF, %rsp /* switch stack to pcb */
695
696 /*
697 * Save values in the ISF frame in the PCB
698 * to cons up the saved machine state.
699 */
700 movl $(USER_DS), ISF64_SS(%rsp)
701 movl $(SYSCALL_CS), ISF64_CS(%rsp) /* cs - a pseudo-segment */
702 mov %r11, ISF64_RFLAGS(%rsp) /* rflags */
703 mov %rcx, ISF64_RIP(%rsp) /* rip */
704 mov %gs:CPU_UBER_TMP, %rcx
705 mov %rcx, ISF64_RSP(%rsp) /* user stack */
706 mov %rax, ISF64_ERR(%rsp) /* err/rax - syscall code */
707 movq $(T_SYSCALL), ISF64_TRAPNO(%rsp) /* trapno */
708 leaq HNDL_SYSCALL(%rip), %r11;
709 movq %r11, ISF64_TRAPFN(%rsp)
710 jmp L_64bit_dispatch /* this can only be a 64-bit task */
711
712 /*
713 * sysenter entry point
714 * Requires user code to set up:
715 * edx: user instruction pointer (return address)
716 * ecx: user stack pointer
717 * on which is pushed stub ret addr and saved ebx
718 * Return to user-space is made using sysexit.
719 * Note: sysenter/sysexit cannot be used for calls returning a value in edx,
720 * or requiring ecx to be preserved.
721 */
722 Entry(hi64_sysenter)
723 Entry(idt64_sysenter)
724 movq (%rsp), %rsp
725 /*
726 * Push values on to the PCB stack
727 * to cons up the saved machine state.
728 */
729 push $(USER_DS) /* ss */
730 push %rcx /* uesp */
731 pushf /* flags */
732 push $(SYSENTER_CS) /* cs */
733 swapgs /* switch to kernel gs (cpu_data) */
734 L_sysenter_continue:
735 push %rdx /* eip */
736 push %rax /* err/eax - syscall code */
737 PUSH_FUNCTION(HNDL_SYSENTER)
738 pushq $(T_SYSENTER)
739 orl $(EFL_IF), ISF64_RFLAGS(%rsp)
740 jmp L_32bit_entry_check
741
742
743 Entry(idt64_page_fault)
744 PUSH_FUNCTION(HNDL_ALLTRAPS)
745 push %rax /* save %rax temporarily in trap slot */
746 leaq EXT(idt64_unix_scall_copy_args)(%rip), %rax
747 cmp %rax, ISF64_RIP(%rsp)
748 jne 1f
749 add $(ISF64_SIZE), %rsp /* remove entire intr stack frame */
750 jmp L_copy_args_continue /* continue system call entry */
751 1:
752 mov (%rsp), %rax /* restore %rax from trap slot */
753 movq $(T_PAGE_FAULT), (%rsp) /* set trap code */
754 jne L_dispatch
755
756
757 /*
758 * Debug trap. Check for single-stepping across system call into
759 * kernel. If this is the case, taking the debug trap has turned
760 * off single-stepping - save the flags register with the trace
761 * bit set.
762 */
763 Entry(idt64_debug)
764 push $0 /* error code */
765 PUSH_FUNCTION(HNDL_ALLTRAPS)
766 pushq $(T_DEBUG)
767
768 testb $3, ISF64_CS(%rsp)
769 jnz L_dispatch
770
771 /*
772 * trap came from kernel mode
773 */
774
775 push %rax /* save %rax temporarily */
776
777 leaq EXT(idt64_mach_scall)(%rip), %rax
778 cmp %rax, ISF64_RIP(%rsp)
779 jne 1f
780 pop %rax
781 add $(ISF64_SIZE),%rsp /* remove entire intr stack frame */
782 jmp L_mach_scall_continue /* continue system call entry */
783 1:
784 leaq EXT(idt64_mdep_scall)(%rip), %rax
785 cmp %rax, ISF64_RIP(%rsp)
786 jne 2f
787 pop %rax
788 add $(ISF64_SIZE),%rsp /* remove entire intr stack frame */
789 jmp L_mdep_scall_continue /* continue system call entry */
790 2:
791 leaq EXT(idt64_unix_scall)(%rip), %rax
792 cmp %rax, ISF64_RIP(%rsp)
793 jne 3f
794 pop %rax
795 add $(ISF64_SIZE),%rsp /* remove entire intr stack frame */
796 jmp L_unix_scall_continue /* continue system call entry */
797 3:
798 lea EXT(idt64_sysenter)(%rip), %rax
799 cmp %rax, ISF64_RIP(%rsp)
800 je 4f
801 pop %rax
802 jmp L_dispatch
803 4:
804 pop %rax
805 /*
806 * Interrupt stack frame has been pushed on the temporary stack.
807 * We have to switch to pcb stack and copy eflags.
808 */
809 add $40,%rsp /* remove trapno/trapfn/err/rip/cs */
810 push %rcx /* save %rcx - user stack pointer */
811 mov 40(%rsp),%rcx /* top of intr stack -> pcb stack */
812 xchg %rcx,%rsp /* switch to pcb stack */
813 push $(USER_DS) /* ss */
814 push (%rcx) /* saved %rcx into rsp slot */
815 push 8(%rcx) /* rflags */
816 mov (%rcx),%rcx /* restore %rcx */
817 push $(SYSENTER_TF_CS) /* cs - not SYSENTER_CS for iret path */
818 jmp L_sysenter_continue /* continue sysenter entry */
819
820
821
822 Entry(idt64_double_fault)
823 PUSH_FUNCTION(HNDL_DOUBLE_FAULT)
824 pushq $(T_DOUBLE_FAULT)
825
826 push %rax
827 leaq EXT(idt64_syscall)(%rip), %rax
828 cmp %rax, ISF64_RIP(%rsp)
829 pop %rax
830 jne L_dispatch
831
832 mov ISF64_RSP(%rsp), %rsp
833 jmp L_syscall_continue
834
835
836 /*
837 * General protection or segment-not-present fault.
838 * Check for a GP/NP fault in the kernel_return
839 * sequence; if there, report it as a GP/NP fault on the user's instruction.
840 *
841 * rsp-> 0: trap function
842 * 8: trap code (NP or GP)
843 * 16: segment number in error (error code)
844 * 24: rip
845 * 32: cs
846 * 40: rflags
847 * 48: rsp
848 * 56: ss
849 * 64: old registers (trap is from kernel)
850 */
851 Entry(idt64_gen_prot)
852 PUSH_FUNCTION(HNDL_ALLTRAPS)
853 pushq $(T_GENERAL_PROTECTION)
854 jmp trap_check_kernel_exit /* check for kernel exit sequence */
855
856 Entry(idt64_stack_fault)
857 PUSH_FUNCTION(HNDL_ALLTRAPS)
858 pushq $(T_STACK_FAULT)
859 jmp trap_check_kernel_exit /* check for kernel exit sequence */
860
861 Entry(idt64_segnp)
862 PUSH_FUNCTION(HNDL_ALLTRAPS)
863 pushq $(T_SEGMENT_NOT_PRESENT)
864 /* indicate fault type */
865 trap_check_kernel_exit:
866 testb $3,32(%rsp)
867 jnz L_dispatch
868 /*
869 * trap was from kernel mode,
870 * so check for the kernel exit sequence
871 */
872 push %rax
873
874 leaq EXT(ret32_iret)(%rip), %rax
875 cmp %rax, 24+8(%rsp)
876 je L_fault_iret
877 leaq EXT(ret64_iret)(%rip), %rax
878 cmp %rax, 24+8(%rsp)
879 je L_fault_iret
880 leaq EXT(ret32_set_ds)(%rip), %rax
881 cmp %rax, 24+8(%rsp)
882 je L_32bit_fault_set_seg
883 leaq EXT(ret32_set_es)(%rip), %rax
884 cmp %rax, 24+8(%rsp)
885 je L_32bit_fault_set_seg
886 leaq EXT(ret32_set_fs)(%rip), %rax
887 cmp %rax, 24+8(%rsp)
888 je L_32bit_fault_set_seg
889 leaq EXT(ret32_set_gs)(%rip), %rax
890 cmp %rax, 24+8(%rsp)
891 je L_32bit_fault_set_seg
892
893 leaq EXT(idt64_unix_scall_copy_args)(%rip), %rax
894 cmp %rax, 24+8(%rsp)
895 add $(ISF64_SIZE)+8, (%rsp)
896 je L_copy_args_continue
897
898 pop %rax
899 jmp L_dispatch
900
901
902 /*
903 * GP/NP fault on IRET: CS or SS is in error.
904 * Note that the user ss is originally 16-byte aligned, we'd popped the
905 * stack back to contain just the rip/cs/rflags/rsp/ss before issuing the iret.
906 * On taking the GP/NP fault on the iret instruction, the stack is 16-byte
907 * aligned before pushed the interrupt frame. Hence, an 8-byte padding exists.
908 *
909 * on SP is
910 * (- rax saved above, which is immediately popped)
911 * 0 function
912 * 8 trap number
913 * 16 errcode
914 * 24 rip
915 * 32 cs
916 * 40 rflags
917 * 48 rsp --> new trapfn
918 * 56 ss --> new trapno
919 * 64 pad --> new errcode
920 * 72 user rip
921 * 80 user cs
922 * 88 user rflags
923 * 96 user rsp
924 * 104 user ss (16-byte aligned)
925 */
926 L_fault_iret:
927 pop %rax /* recover saved %rax */
928 mov %rax, 24(%rsp) /* save rax (we don`t need saved rip) */
929 mov 0(%rsp), %rax /* get trap func */
930 mov %rax, 48(%rsp) /* put in user trap func */
931 mov 8(%rsp), %rax /* get trap number */
932 mov %rax, 56(%rsp) /* put in user trap number */
933 mov 16(%rsp), %rax /* get error code */
934 mov %rax, 64(%rsp) /* put in user errcode */
935 mov 24(%rsp), %rax /* restore rax */
936 add $48,%rsp /* reset to new trapfn */
937 /* now treat as fault from user */
938 jmp L_dispatch
939
940 /*
941 * Fault restoring a segment register. All of the saved state is still
942 * on the stack untouched since we haven't yet moved the stack pointer.
943 */
944 L_32bit_fault_set_seg:
945 pop %rax /* recover %rax from stack */
946 mov 0(%rsp), %rax /* get trap function */
947 mov 8(%rsp), %rcx /* get trap number */
948 mov 16(%rsp), %rdx /* get error code */
949 mov 48(%rsp), %rsp /* reset stack to saved state */
950 mov %rax,ISC32_TRAPFN(%rsp)
951 mov %rcx,ISC32_TRAPNO(%rsp)
952 mov %rdx,ISC32_ERR(%rsp)
953 /* now treat as fault from user */
954 /* except that all the state is */
955 /* already saved - we just have to */
956 /* move the trapno and error into */
957 /* the compatibility frame */
958 jmp L_32bit_dispatch_after_fault
959
960
961 /*
962 * Fatal exception handlers:
963 */
964 Entry(idt64_db_task_dbl_fault)
965 PUSH_FUNCTION(HNDL_DOUBLE_FAULT)
966 pushq $(T_DOUBLE_FAULT)
967 jmp L_dispatch
968
969 Entry(idt64_db_task_stk_fault)
970 PUSH_FUNCTION(HNDL_DOUBLE_FAULT)
971 pushq $(T_STACK_FAULT)
972 jmp L_dispatch
973
974 Entry(idt64_mc)
975 push $(0) /* Error */
976 PUSH_FUNCTION(HNDL_MACHINE_CHECK)
977 pushq $(T_MACHINE_CHECK)
978 jmp L_dispatch
979
980
981 /* All 'exceptions' enter hndl_alltraps:
982 * rsp -> x86_saved_state_t
983 * esi cs at trap
984 *
985 * The rest of the state is set up as:
986 * interrupts disabled
987 * direction flag cleared
988 */
989 Entry(hndl_alltraps)
990 mov %esi, %eax
991 testb $3, %al
992 jz trap_from_kernel
993
994 TIME_TRAP_UENTRY
995
996 movq %gs:CPU_ACTIVE_THREAD,%rdi
997 movq %rsp, ACT_PCB_ISS(%rdi) /* stash the PCB stack */
998 movq %rsp, %rdi /* also pass it as arg0 */
999 movq %gs:CPU_KERNEL_STACK,%rsp /* switch to kernel stack */
1000 sti
1001
1002 CCALL(user_trap) /* call user trap routine */
1003 cli /* hold off intrs - critical section */
1004 movq %gs:CPU_ACTIVE_THREAD,%rsp
1005 movq ACT_PCB_ISS(%rsp), %rsp /* switch back to PCB stack */
1006 xorl %ecx, %ecx /* don't check if we're in the PFZ */
1007
1008 #define CLI cli
1009 #define STI sti
1010
1011 Entry(return_from_trap)
1012 movl %gs:CPU_PENDING_AST,%eax
1013 testl %eax,%eax
1014 je EXT(return_to_user) /* branch if no AST */
1015
1016 L_return_from_trap_with_ast:
1017 movq %rsp, %r13
1018 movq %gs:CPU_KERNEL_STACK, %rsp
1019
1020 testl %ecx, %ecx /* see if we need to check for an EIP in the PFZ */
1021 je 2f /* no, go handle the AST */
1022 cmpl $(SS_64), SS_FLAVOR(%r13) /* are we a 64-bit task? */
1023 je 1f
1024 /* no... 32-bit user mode */
1025 movl R32_EIP(%r13), %edi
1026 CCALL(commpage_is_in_pfz32)
1027 testl %eax, %eax
1028 je 2f /* not in the PFZ... go service AST */
1029 movl %eax, R32_EBX(%r13) /* let the PFZ know we've pended an AST */
1030 movq %r13, %rsp /* switch back to PCB stack */
1031 jmp EXT(return_to_user)
1032 1:
1033 movq R64_RIP(%r13), %rdi
1034 CCALL(commpage_is_in_pfz64)
1035 testl %eax, %eax
1036 je 2f /* not in the PFZ... go service AST */
1037 movl %eax, R64_RBX(%r13) /* let the PFZ know we've pended an AST */
1038 movq %r13, %rsp /* switch back to PCB stack */
1039 jmp EXT(return_to_user)
1040 2:
1041 STI /* interrupts always enabled on return to user mode */
1042
1043 xor %edi, %edi /* zero %rdi */
1044 CCALL(i386_astintr) /* take the AST */
1045
1046 CLI
1047 movq %r13, %rsp /* switch back to PCB stack */
1048
1049 xorl %ecx, %ecx /* don't check if we're in the PFZ */
1050 jmp EXT(return_from_trap) /* and check again (rare) */
1051
1052 /*
1053 * Trap from kernel mode. No need to switch stacks.
1054 * Interrupts must be off here - we will set them to state at time of trap
1055 * as soon as it's safe for us to do so and not recurse doing preemption
1056 */
1057 hndl_kerntrap:
1058 trap_from_kernel:
1059
1060 movq %rsp, %rdi /* saved state addr */
1061 pushq R64_RIP(%rsp) /* Simulate a CALL from fault point */
1062 pushq %rbp /* Extend framepointer chain */
1063 movq %rsp, %rbp
1064 CCALL(kernel_trap) /* to kernel trap routine */
1065 popq %rbp
1066 addq $8, %rsp
1067 cli
1068
1069 movl %gs:CPU_PENDING_AST,%eax /* get pending asts */
1070 testl $(AST_URGENT),%eax /* any urgent preemption? */
1071 je ret_to_kernel /* no, nothing to do */
1072 cmpl $(T_PREEMPT),R64_TRAPNO(%rsp)
1073 je ret_to_kernel /* T_PREEMPT handled in kernel_trap() */
1074 testl $(EFL_IF),R64_RFLAGS(%rsp) /* interrupts disabled? */
1075 je ret_to_kernel
1076 cmpl $0,%gs:CPU_PREEMPTION_LEVEL /* preemption disabled? */
1077 jne ret_to_kernel
1078 movq %gs:CPU_KERNEL_STACK,%rax
1079 movq %rsp,%rcx
1080 xorq %rax,%rcx
1081 andq EXT(kernel_stack_mask)(%rip),%rcx
1082 testq %rcx,%rcx /* are we on the kernel stack? */
1083 jne ret_to_kernel /* no, skip it */
1084
1085 CCALL1(i386_astintr, $1) /* take the AST */
1086 jmp ret_to_kernel
1087
1088
1089 /*
1090 * All interrupts on all tasks enter here with:
1091 * rsp-> x86_saved_state_t
1092 * esi cs at trap
1093 *
1094 * interrupts disabled
1095 * direction flag cleared
1096 */
1097 Entry(hndl_allintrs)
1098 /*
1099 * test whether already on interrupt stack
1100 */
1101 movq %gs:CPU_INT_STACK_TOP,%rcx
1102 cmpq %rsp,%rcx
1103 jb 1f
1104 leaq -INTSTACK_SIZE(%rcx),%rdx
1105 cmpq %rsp,%rdx
1106 jb int_from_intstack
1107 1:
1108 xchgq %rcx,%rsp /* switch to interrupt stack */
1109
1110 mov %cr0,%rax /* get cr0 */
1111 orl $(CR0_TS),%eax /* or in TS bit */
1112 mov %rax,%cr0 /* set cr0 */
1113
1114 subq $8, %rsp /* for 16-byte stack alignment */
1115 pushq %rcx /* save pointer to old stack */
1116 movq %rcx,%gs:CPU_INT_STATE /* save intr state */
1117
1118 TIME_INT_ENTRY /* do timing */
1119
1120 incl %gs:CPU_PREEMPTION_LEVEL
1121 incl %gs:CPU_INTERRUPT_LEVEL
1122
1123 movq %gs:CPU_INT_STATE, %rdi
1124
1125 CCALL(interrupt) /* call generic interrupt routine */
1126
1127 cli /* just in case we returned with intrs enabled */
1128 xor %rax,%rax
1129 movq %rax,%gs:CPU_INT_STATE /* clear intr state pointer */
1130
1131 .globl EXT(return_to_iret)
1132 LEXT(return_to_iret) /* (label for kdb_kintr and hardclock) */
1133
1134 decl %gs:CPU_INTERRUPT_LEVEL
1135 decl %gs:CPU_PREEMPTION_LEVEL
1136
1137 TIME_INT_EXIT /* do timing */
1138
1139 movq %gs:CPU_ACTIVE_THREAD,%rax
1140 movq ACT_PCB(%rax),%rax /* get act`s PCB */
1141 movq PCB_FPS(%rax),%rax /* get pcb's ims.ifps */
1142 cmpq $0,%rax /* Is there a context */
1143 je 1f /* Branch if not */
1144 movl FP_VALID(%rax),%eax /* Load fp_valid */
1145 cmpl $0,%eax /* Check if valid */
1146 jne 1f /* Branch if valid */
1147 clts /* Clear TS */
1148 jmp 2f
1149 1:
1150 mov %cr0,%rax /* get cr0 */
1151 orl $(CR0_TS),%eax /* or in TS bit */
1152 mov %rax,%cr0 /* set cr0 */
1153 2:
1154 popq %rsp /* switch back to old stack */
1155
1156 /* Load interrupted code segment into %eax */
1157 movl R32_CS(%rsp),%eax /* assume 32-bit state */
1158 cmpl $(SS_64),SS_FLAVOR(%rsp)/* 64-bit? */
1159 #if DEBUG_IDT64
1160 jne 4f
1161 movl R64_CS(%rsp),%eax /* 64-bit user mode */
1162 jmp 3f
1163 4:
1164 cmpl $(SS_32),SS_FLAVOR(%rsp)
1165 je 3f
1166 POSTCODE2(0x6431)
1167 CCALL1(panic_idt64, %rsp)
1168 hlt
1169 #else
1170 jne 3f
1171 movl R64_CS(%rsp),%eax /* 64-bit user mode */
1172 #endif
1173 3:
1174 testb $3,%al /* user mode, */
1175 jnz ast_from_interrupt_user /* go handle potential ASTs */
1176 /*
1177 * we only want to handle preemption requests if
1178 * the interrupt fell in the kernel context
1179 * and preemption isn't disabled
1180 */
1181 movl %gs:CPU_PENDING_AST,%eax
1182 testl $(AST_URGENT),%eax /* any urgent requests? */
1183 je ret_to_kernel /* no, nothing to do */
1184
1185 cmpl $0,%gs:CPU_PREEMPTION_LEVEL /* preemption disabled? */
1186 jne ret_to_kernel /* yes, skip it */
1187
1188 movq %gs:CPU_KERNEL_STACK,%rax
1189 movq %rsp,%rcx
1190 xorq %rax,%rcx
1191 andq EXT(kernel_stack_mask)(%rip),%rcx
1192 testq %rcx,%rcx /* are we on the kernel stack? */
1193 jne ret_to_kernel /* no, skip it */
1194
1195 /*
1196 * Take an AST from kernel space. We don't need (and don't want)
1197 * to do as much as the case where the interrupt came from user
1198 * space.
1199 */
1200 CCALL1(i386_astintr, $1)
1201
1202 jmp ret_to_kernel
1203
1204
1205 /*
1206 * nested int - simple path, can't preempt etc on way out
1207 */
1208 int_from_intstack:
1209 incl %gs:CPU_PREEMPTION_LEVEL
1210 incl %gs:CPU_INTERRUPT_LEVEL
1211
1212 mov %rsp, %rdi /* x86_saved_state */
1213 CCALL(interrupt)
1214
1215 decl %gs:CPU_INTERRUPT_LEVEL
1216 decl %gs:CPU_PREEMPTION_LEVEL
1217
1218 #if DEBUG_IDT64
1219 CCALL1(panic_idt64, %rsp)
1220 POSTCODE2(0x6411)
1221 hlt
1222 #endif
1223 jmp ret_to_kernel
1224
1225 /*
1226 * Take an AST from an interrupted user
1227 */
1228 ast_from_interrupt_user:
1229 movl %gs:CPU_PENDING_AST,%eax
1230 testl %eax,%eax /* pending ASTs? */
1231 je EXT(ret_to_user) /* no, nothing to do */
1232
1233 TIME_TRAP_UENTRY
1234
1235 movl $1, %ecx /* check if we're in the PFZ */
1236 jmp L_return_from_trap_with_ast /* return */
1237
1238
1239 /* Syscall dispatch routines! */
1240
1241 /*
1242 *
1243 * 32bit Tasks
1244 * System call entries via INTR_GATE or sysenter:
1245 *
1246 * rsp -> x86_saved_state32_t
1247 * interrupts disabled
1248 * direction flag cleared
1249 */
1250
1251 Entry(hndl_sysenter)
1252 /*
1253 * We can be here either for a mach syscall or a unix syscall,
1254 * as indicated by the sign of the code:
1255 */
1256 movl R32_EAX(%rsp),%eax
1257 testl %eax,%eax
1258 js EXT(hndl_mach_scall) /* < 0 => mach */
1259 /* > 0 => unix */
1260
1261 Entry(hndl_unix_scall)
1262 /* If the caller (typically LibSystem) has recorded the cumulative size of
1263 * the arguments in EAX, copy them over from the user stack directly.
1264 * We recover from exceptions inline--if the copy loop doesn't complete
1265 * due to an exception, we fall back to copyin from compatibility mode.
1266 * We can potentially extend this mechanism to mach traps as well (DRK).
1267 */
1268 testl $(I386_SYSCALL_ARG_BYTES_MASK), %eax
1269 jz L_copy_args_continue
1270 movl %eax, %ecx
1271 mov %gs:CPU_UBER_ARG_STORE_VALID, %rbx
1272 shrl $(I386_SYSCALL_ARG_DWORDS_SHIFT), %ecx
1273 andl $(I386_SYSCALL_ARG_DWORDS_MASK), %ecx
1274 mov %gs:CPU_UBER_ARG_STORE, %rdi
1275 mov ISC32_RSP(%rsp), %rsi
1276 add $4, %rsi
1277 movl $0, (%rbx)
1278
1279 EXT(idt64_unix_scall_copy_args):
1280 rep movsl
1281 movl $1, (%rbx)
1282 L_copy_args_continue:
1283
1284 TIME_TRAP_UENTRY
1285
1286 movq %gs:CPU_KERNEL_STACK,%rdi
1287 xchgq %rdi,%rsp /* switch to kernel stack */
1288 movq %gs:CPU_ACTIVE_THREAD,%rcx /* get current thread */
1289 movq %rdi,ACT_PCB_ISS(%rcx)
1290 movq ACT_TASK(%rcx),%rbx /* point to current task */
1291 addl $1,TASK_SYSCALLS_UNIX(%rbx) /* increment call count */
1292
1293 /* Check for active vtimers in the current task */
1294 TASK_VTIMER_CHECK(%rbx,%rcx)
1295
1296 sti
1297
1298 CCALL(unix_syscall)
1299 /*
1300 * always returns through thread_exception_return
1301 */
1302
1303
1304 Entry(hndl_mach_scall)
1305 TIME_TRAP_UENTRY
1306
1307 movq %gs:CPU_KERNEL_STACK,%rdi
1308 xchgq %rdi,%rsp /* switch to kernel stack */
1309 movq %gs:CPU_ACTIVE_THREAD,%rcx /* get current thread */
1310 movq %rdi,ACT_PCB_ISS(%rcx)
1311 movq ACT_TASK(%rcx),%rbx /* point to current task */
1312 addl $1,TASK_SYSCALLS_MACH(%rbx) /* increment call count */
1313
1314 /* Check for active vtimers in the current task */
1315 TASK_VTIMER_CHECK(%rbx,%rcx)
1316
1317 sti
1318
1319 CCALL(mach_call_munger)
1320 /*
1321 * always returns through thread_exception_return
1322 */
1323
1324
1325 Entry(hndl_mdep_scall)
1326 TIME_TRAP_UENTRY
1327
1328 movq %gs:CPU_KERNEL_STACK,%rdi
1329 xchgq %rdi,%rsp /* switch to kernel stack */
1330
1331 /* Check for active vtimers in the current task */
1332 movq %gs:CPU_ACTIVE_THREAD,%rcx /* get current thread */
1333 movq ACT_TASK(%rcx),%rbx /* point to current task */
1334 TASK_VTIMER_CHECK(%rbx,%rcx)
1335
1336 sti
1337
1338 CCALL(machdep_syscall)
1339 /*
1340 * always returns through thread_exception_return
1341 */
1342
1343
1344 Entry(hndl_diag_scall)
1345 TIME_TRAP_UENTRY
1346
1347 movq %gs:CPU_KERNEL_STACK,%rdi
1348 xchgq %rdi,%rsp /* switch to kernel stack */
1349
1350 /* Check for active vtimers in the current task */
1351 movq %gs:CPU_ACTIVE_THREAD,%rcx /* get current thread */
1352 movq ACT_TASK(%rcx),%rbx /* point to current task */
1353 TASK_VTIMER_CHECK(%rbx,%rcx)
1354
1355 pushq %rdi /* push pcb stack so we can pop it later */
1356
1357 CCALL(diagCall) // Call diagnostics
1358 cli // Disable interruptions just in case they were enabled
1359 popq %rsp // Get back the original stack
1360
1361 cmpl $0,%eax // What kind of return is this?
1362 jne EXT(return_to_user) // Normal return, do not check asts...
1363
1364 CCALL3(i386_exception, $EXC_SYSCALL, $0x6000, $1)
1365 // pass what would be the diag syscall
1366 // error return - cause an exception
1367 /* no return */
1368
1369
1370
1371 /*
1372 * 64bit Tasks
1373 * System call entries via syscall only:
1374 *
1375 * rsp -> x86_saved_state64_t
1376 * interrupts disabled
1377 * direction flag cleared
1378 */
1379
1380 Entry(hndl_syscall)
1381 TIME_TRAP_UENTRY
1382
1383 movq %gs:CPU_KERNEL_STACK,%rdi
1384 xchgq %rdi,%rsp /* switch to kernel stack */
1385 movq %gs:CPU_ACTIVE_THREAD,%rcx /* get current thread */
1386 movq %rdi, ACT_PCB_ISS(%rcx)
1387 movq ACT_TASK(%rcx),%rbx /* point to current task */
1388
1389 /* Check for active vtimers in the current task */
1390 TASK_VTIMER_CHECK(%rbx,%rcx)
1391
1392 /*
1393 * We can be here either for a mach, unix machdep or diag syscall,
1394 * as indicated by the syscall class:
1395 */
1396 movl R64_RAX(%rdi), %eax /* syscall number/class */
1397 movl %eax, %edx
1398 andl $(SYSCALL_CLASS_MASK), %edx /* syscall class */
1399 cmpl $(SYSCALL_CLASS_MACH<<SYSCALL_CLASS_SHIFT), %edx
1400 je EXT(hndl_mach_scall64)
1401 cmpl $(SYSCALL_CLASS_UNIX<<SYSCALL_CLASS_SHIFT), %edx
1402 je EXT(hndl_unix_scall64)
1403 cmpl $(SYSCALL_CLASS_MDEP<<SYSCALL_CLASS_SHIFT), %edx
1404 je EXT(hndl_mdep_scall64)
1405 cmpl $(SYSCALL_CLASS_DIAG<<SYSCALL_CLASS_SHIFT), %edx
1406 je EXT(hndl_diag_scall64)
1407
1408 /* Syscall class unknown */
1409 CCALL3(i386_exception, $(EXC_SYSCALL), %rax, $1)
1410 /* no return */
1411
1412
1413 Entry(hndl_unix_scall64)
1414 addl $1,TASK_SYSCALLS_UNIX(%rbx) /* increment call count */
1415 sti
1416
1417 CCALL(unix_syscall64)
1418 /*
1419 * always returns through thread_exception_return
1420 */
1421
1422
1423 Entry(hndl_mach_scall64)
1424 addl $1,TASK_SYSCALLS_MACH(%rbx) /* increment call count */
1425 sti
1426
1427 CCALL(mach_call_munger64)
1428 /*
1429 * always returns through thread_exception_return
1430 */
1431
1432
1433
1434 Entry(hndl_mdep_scall64)
1435 sti
1436
1437 CCALL(machdep_syscall64)
1438 /*
1439 * always returns through thread_exception_return
1440 */
1441
1442
1443 Entry(hndl_diag_scall64)
1444 pushq %rdi // Push the previous stack
1445 CCALL(diagCall64) // Call diagnostics
1446 cli // Disable interruptions just in case
1447 popq %rsp // Get back the original stack
1448
1449 cmpl $0,%eax // What kind of return is this?
1450 jne EXT(return_to_user) // Normal return, do not check asts...
1451
1452 CCALL3(i386_exception, $EXC_SYSCALL, $0x6000, $1)
1453 /* no return */
1454
1455 Entry(hndl_machine_check)
1456 CCALL1(panic_machine_check64, %rsp)
1457 hlt
1458
1459 Entry(hndl_double_fault)
1460 CCALL1(panic_double_fault64, %rsp)
1461 hlt
mirror of XNU