]> git.saurik.com Git - apple/xnu.git/blob - osfmk/i386/locore.s
projects / apple / xnu.git / blob
? search:


ac1bb007a7a44a711303f18311f692432b69b032
[apple/xnu.git] / osfmk / i386 / locore.s
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57 #include <mach_rt.h>
58 #include <platforms.h>
59 #include <mach_kdb.h>
60 #include <mach_kgdb.h>
61 #include <mach_kdp.h>
62 #include <stat_time.h>
63 #include <mach_assert.h>
64
65 #include <sys/errno.h>
66 #include <i386/asm.h>
67 #include <i386/cpuid.h>
68 #include <i386/eflags.h>
69 #include <i386/proc_reg.h>
70 #include <i386/trap.h>
71 #include <assym.s>
72 #include <mach/exception_types.h>
73
74 #define _ARCH_I386_ASM_HELP_H_ /* Prevent inclusion of user header */
75 #include <mach/i386/syscall_sw.h>
76
77 #include <i386/mp.h>
78
79 /*
80 * PTmap is recursive pagemap at top of virtual address space.
81 * Within PTmap, the page directory can be found (third indirection).
82 */
83 .globl _PTmap,_PTD,_PTDpde
84 .set _PTmap,(PTDPTDI << PDESHIFT)
85 .set _PTD,_PTmap + (PTDPTDI * NBPG)
86 .set _PTDpde,_PTD + (PTDPTDI * PDESIZE)
87
88 /*
89 * APTmap, APTD is the alternate recursive pagemap.
90 * It's used when modifying another process's page tables.
91 */
92 .globl _APTmap,_APTD,_APTDpde
93 .set _APTmap,(APTDPTDI << PDESHIFT)
94 .set _APTD,_APTmap + (APTDPTDI * NBPG)
95 .set _APTDpde,_PTD + (APTDPTDI * PDESIZE)
96
97 #if __MACHO__
98 /* Under Mach-O, etext is a variable which contains
99 * the last text address
100 */
101 #define ETEXT_ADDR (EXT(etext))
102 #else
103 /* Under ELF and other non-Mach-O formats, the address of
104 * etext represents the last text address
105 */
106 #define ETEXT_ADDR $ EXT(etext)
107 #endif
108
109 #define CX(addr,reg) addr(,reg,4)
110
111 /*
112 * The following macros make calls into C code.
113 * They dynamically align the stack to 16 bytes.
114 * Arguments are moved (not pushed) onto the correctly aligned stack.
115 * NOTE: EDI is destroyed in the process, and hence cannot
116 * be directly used as a parameter. Users of this macro must
117 * independently preserve EDI (a non-volatile) if the routine is
118 * intended to be called from C, for instance.
119 */
120
121 #define CCALL(fn) \
122 movl %esp, %edi ;\
123 andl $0xFFFFFFF0, %esp ;\
124 call EXT(fn) ;\
125 movl %edi, %esp
126
127 #define CCALL1(fn, arg1) \
128 movl %esp, %edi ;\
129 subl $4, %esp ;\
130 andl $0xFFFFFFF0, %esp ;\
131 movl arg1, 0(%esp) ;\
132 call EXT(fn) ;\
133 movl %edi, %esp
134
135 #define CCALL2(fn, arg1, arg2) \
136 movl %esp, %edi ;\
137 subl $8, %esp ;\
138 andl $0xFFFFFFF0, %esp ;\
139 movl arg2, 4(%esp) ;\
140 movl arg1, 0(%esp) ;\
141 call EXT(fn) ;\
142 movl %edi, %esp
143
144 #define CCALL3(fn, arg1, arg2, arg3) \
145 movl %esp, %edi ;\
146 subl $12, %esp ;\
147 andl $0xFFFFFFF0, %esp ;\
148 movl arg3, 8(%esp) ;\
149 movl arg2, 4(%esp) ;\
150 movl arg1, 0(%esp) ;\
151 call EXT(fn) ;\
152 movl %edi, %esp
153
154 .text
155 locore_start:
156
157 /*
158 * Fault recovery.
159 */
160
161 #ifdef __MACHO__
162 #define RECOVERY_SECTION .section __VECTORS, __recover
163 #else
164 #define RECOVERY_SECTION .text
165 #define RECOVERY_SECTION .text
166 #endif
167
168 #define RECOVER_TABLE_START \
169 .align 2 ; \
170 .globl EXT(recover_table) ;\
171 LEXT(recover_table) ;\
172 .text
173
174 #define RECOVER(addr) \
175 .align 2; \
176 .long 9f ;\
177 .long addr ;\
178 .text ;\
179 9:
180
181 #define RECOVER_TABLE_END \
182 .align 2 ;\
183 .globl EXT(recover_table_end) ;\
184 LEXT(recover_table_end) ;\
185 .text
186
187 /*
188 * Allocate recovery and table.
189 */
190 RECOVERY_SECTION
191 RECOVER_TABLE_START
192
193 /*
194 * Timing routines.
195 */
196 Entry(timer_update)
197 movl 4(%esp),%ecx
198 movl 8(%esp),%eax
199 movl 12(%esp),%edx
200 movl %eax,TIMER_HIGHCHK(%ecx)
201 movl %edx,TIMER_LOW(%ecx)
202 movl %eax,TIMER_HIGH(%ecx)
203 ret
204
205 Entry(timer_grab)
206 movl 4(%esp),%ecx
207 0: movl TIMER_HIGH(%ecx),%edx
208 movl TIMER_LOW(%ecx),%eax
209 cmpl TIMER_HIGHCHK(%ecx),%edx
210 jne 0b
211 ret
212
213 #if STAT_TIME
214
215 #define TIME_TRAP_UENTRY
216 #define TIME_TRAP_UEXIT
217 #define TIME_INT_ENTRY
218 #define TIME_INT_EXIT
219
220 #else
221 /*
222 * Nanosecond timing.
223 */
224
225 /*
226 * Nanotime returned in %edx:%eax.
227 * Computed from tsc based on the scale factor
228 * and an implicit 32 bit shift.
229 * This code must match what _rtc_nanotime_read does in
230 * i386/machine_routines_asm.s. Failure to do so can
231 * result in "weird" timing results.
232 *
233 * Uses %eax, %ebx, %ecx, %edx, %esi, %edi.
234 */
235 #define RNT_INFO _rtc_nanotime_info
236 #define NANOTIME \
237 lea RNT_INFO,%edi ; \
238 0: ; \
239 movl RNT_GENERATION(%edi),%esi /* being updated? */ ; \
240 testl %esi,%esi ; \
241 jz 0b /* wait until done */ ; \
242 rdtsc ; \
243 subl RNT_TSC_BASE(%edi),%eax ; \
244 sbbl RNT_TSC_BASE+4(%edi),%edx /* tsc - tsc_base */ ; \
245 movl RNT_SCALE(%edi),%ecx /* * scale factor */ ; \
246 movl %edx,%ebx ; \
247 mull %ecx ; \
248 movl %ebx,%eax ; \
249 movl %edx,%ebx ; \
250 mull %ecx ; \
251 addl %ebx,%eax ; \
252 adcl $0,%edx ; \
253 addl RNT_NS_BASE(%edi),%eax /* + ns_base */ ; \
254 adcl RNT_NS_BASE+4(%edi),%edx ; \
255 cmpl RNT_GENERATION(%edi),%esi /* check for update */ ; \
256 jne 0b /* do it all again */
257
258
259 /*
260 * Add 64-bit delta in register dreg : areg to timer pointed to by register treg.
261 */
262 #define TIMER_UPDATE(treg,dreg,areg) \
263 addl TIMER_LOW(treg),areg /* add low bits */ ; \
264 adcl dreg,TIMER_HIGH(treg) /* add carry high bits */ ; \
265 movl areg,TIMER_LOW(treg) /* store updated low bit */ ; \
266 movl TIMER_HIGH(treg),dreg /* copy high bits */ ; \
267 movl dreg,TIMER_HIGHCHK(treg) /* to high check */
268
269 /*
270 * Add time delta to old timer and start new.
271 */
272 #define TIMER_EVENT(old,new) \
273 NANOTIME /* edx:eax nanosecs */ ; \
274 movl %eax,%esi /* save timestamp */ ; \
275 movl %edx,%edi /* save timestamp */ ; \
276 movl %gs:CPU_PROCESSOR,%ebx /* get current processor */ ; \
277 movl THREAD_TIMER(%ebx),%ecx /* get current timer */ ; \
278 subl TIMER_TSTAMP(%ecx),%eax /* compute elapsed time */ ; \
279 sbbl TIMER_TSTAMP+4(%ecx),%edx /* compute elapsed time */ ; \
280 TIMER_UPDATE(%ecx,%edx,%eax) /* update timer */ ; \
281 addl $(new##_TIMER-old##_TIMER),%ecx /* point to new timer */ ; \
282 movl %esi,TIMER_TSTAMP(%ecx) /* set timestamp */ ; \
283 movl %edi,TIMER_TSTAMP+4(%ecx) /* set timestamp */ ; \
284 movl %ecx,THREAD_TIMER(%ebx) /* set current timer */ ; \
285 movl %esi,%eax /* restore timestamp */ ; \
286 movl %edi,%edx /* restore timestamp */ ; \
287 movl CURRENT_STATE(%ebx),%ecx /* current state */ ; \
288 subl TIMER_TSTAMP(%ecx),%eax /* compute elapsed time */ ; \
289 sbbl TIMER_TSTAMP+4(%ecx),%edx /* compute elapsed time */ ; \
290 TIMER_UPDATE(%ecx,%edx,%eax) /* update timer */ ; \
291 addl $(new##_STATE-old##_STATE),%ecx /* point to new state */ ; \
292 movl %ecx,CURRENT_STATE(%ebx) /* set current state */ ; \
293 movl %esi,TIMER_TSTAMP(%ecx) /* set timestamp */ ; \
294 movl %edi,TIMER_TSTAMP+4(%ecx) /* set timestamp */
295
296 /*
297 * Update time on user trap entry.
298 * Uses %eax,%ebx,%ecx,%edx,%esi,%edi.
299 */
300 #define TIME_TRAP_UENTRY TIMER_EVENT(USER,SYSTEM)
301
302 /*
303 * update time on user trap exit.
304 * Uses %eax,%ebx,%ecx,%edx,%esi,%edi.
305 */
306 #define TIME_TRAP_UEXIT TIMER_EVENT(SYSTEM,USER)
307
308 /*
309 * update time on interrupt entry.
310 * Uses %eax,%ebx,%ecx,%edx,%esi,%edi.
311 * Saves processor state info on stack.
312 */
313 #define TIME_INT_ENTRY \
314 NANOTIME /* edx:eax nanosecs */ ; \
315 movl %eax,%gs:CPU_INT_EVENT_TIME /* save in cpu data */ ; \
316 movl %edx,%gs:CPU_INT_EVENT_TIME+4 /* save in cpu data */ ; \
317 movl %eax,%esi /* save timestamp */ ; \
318 movl %edx,%edi /* save timestamp */ ; \
319 movl %gs:CPU_PROCESSOR,%ebx /* get current processor */ ; \
320 movl THREAD_TIMER(%ebx),%ecx /* get current timer */ ; \
321 subl TIMER_TSTAMP(%ecx),%eax /* compute elapsed time */ ; \
322 sbbl TIMER_TSTAMP+4(%ecx),%edx /* compute elapsed time */ ; \
323 TIMER_UPDATE(%ecx,%edx,%eax) /* update timer */ ; \
324 movl KERNEL_TIMER(%ebx),%ecx /* point to kernel timer */ ; \
325 movl %esi,TIMER_TSTAMP(%ecx) /* set timestamp */ ; \
326 movl %edi,TIMER_TSTAMP+4(%ecx) /* set timestamp */ ; \
327 movl %esi,%eax /* restore timestamp */ ; \
328 movl %edi,%edx /* restore timestamp */ ; \
329 movl CURRENT_STATE(%ebx),%ecx /* get current state */ ; \
330 pushl %ecx /* save state */ ; \
331 subl TIMER_TSTAMP(%ecx),%eax /* compute elapsed time */ ; \
332 sbbl TIMER_TSTAMP+4(%ecx),%edx /* compute elapsed time */ ; \
333 TIMER_UPDATE(%ecx,%edx,%eax) /* update timer */ ; \
334 leal IDLE_STATE(%ebx),%eax /* get idle state */ ; \
335 cmpl %eax,%ecx /* compare current state */ ; \
336 je 0f /* skip if equal */ ; \
337 leal SYSTEM_STATE(%ebx),%ecx /* get system state */ ; \
338 movl %ecx,CURRENT_STATE(%ebx) /* set current state */ ; \
339 0: movl %esi,TIMER_TSTAMP(%ecx) /* set timestamp */ ; \
340 movl %edi,TIMER_TSTAMP+4(%ecx) /* set timestamp */
341
342 /*
343 * update time on interrupt exit.
344 * Uses %eax,%ebx,%ecx,%edx,%esi,%edi.
345 * Restores processor state info from stack.
346 */
347 #define TIME_INT_EXIT \
348 NANOTIME /* edx:eax nanosecs */ ; \
349 movl %eax,%gs:CPU_INT_EVENT_TIME /* save in cpu data */ ; \
350 movl %edx,%gs:CPU_INT_EVENT_TIME+4 /* save in cpu data */ ; \
351 movl %eax,%esi /* save timestamp */ ; \
352 movl %edx,%edi /* save timestamp */ ; \
353 movl %gs:CPU_PROCESSOR,%ebx /* get current processor */ ; \
354 movl KERNEL_TIMER(%ebx),%ecx /* point to kernel timer */ ; \
355 subl TIMER_TSTAMP(%ecx),%eax /* compute elapsed time */ ; \
356 sbbl TIMER_TSTAMP+4(%ecx),%edx /* compute elapsed time */ ; \
357 TIMER_UPDATE(%ecx,%edx,%eax) /* update timer */ ; \
358 movl THREAD_TIMER(%ebx),%ecx /* interrupted timer */ ; \
359 movl %esi,TIMER_TSTAMP(%ecx) /* set timestamp */ ; \
360 movl %edi,TIMER_TSTAMP+4(%ecx) /* set timestamp */ ; \
361 movl %esi,%eax /* restore timestamp */ ; \
362 movl %edi,%edx /* restore timestamp */ ; \
363 movl CURRENT_STATE(%ebx),%ecx /* get current state */ ; \
364 subl TIMER_TSTAMP(%ecx),%eax /* compute elapsed time */ ; \
365 sbbl TIMER_TSTAMP+4(%ecx),%edx /* compute elapsed time */ ; \
366 TIMER_UPDATE(%ecx,%edx,%eax) /* update timer */ ; \
367 popl %ecx /* restore state */ ; \
368 movl %ecx,CURRENT_STATE(%ebx) /* set current state */ ; \
369 movl %esi,TIMER_TSTAMP(%ecx) /* set timestamp */ ; \
370 movl %edi,TIMER_TSTAMP+4(%ecx) /* set timestamp */
371
372 #endif /* STAT_TIME */
373
374 #undef PDEBUG
375
376 #ifdef PDEBUG
377
378 /*
379 * Traditional, not ANSI.
380 */
381 #define CAH(label) \
382 .data ;\
383 .globl label/**/count ;\
384 label/**/count: ;\
385 .long 0 ;\
386 .globl label/**/limit ;\
387 label/**/limit: ;\
388 .long 0 ;\
389 .text ;\
390 addl $1,%ss:label/**/count ;\
391 cmpl $0,label/**/limit ;\
392 jz label/**/exit ;\
393 pushl %eax ;\
394 label/**/loop: ;\
395 movl %ss:label/**/count,%eax ;\
396 cmpl %eax,%ss:label/**/limit ;\
397 je label/**/loop ;\
398 popl %eax ;\
399 label/**/exit:
400
401 #else /* PDEBUG */
402
403 #define CAH(label)
404
405 #endif /* PDEBUG */
406
407 #if MACH_KDB
408 /*
409 * Last-ditch debug code to handle faults that might result
410 * from entering kernel (from collocated server) on an invalid
411 * stack. On collocated entry, there's no hardware-initiated
412 * stack switch, so a valid stack must be in place when an
413 * exception occurs, or we may double-fault.
414 *
415 * In case of a double-fault, our only recourse is to switch
416 * hardware "tasks", so that we avoid using the current stack.
417 *
418 * The idea here is just to get the processor into the debugger,
419 * post-haste. No attempt is made to fix up whatever error got
420 * us here, so presumably continuing from the debugger will
421 * simply land us here again -- at best.
422 */
423 #if 0
424 /*
425 * Note that the per-fault entry points are not currently
426 * functional. The only way to make them work would be to
427 * set up separate TSS's for each fault type, which doesn't
428 * currently seem worthwhile. (The offset part of a task
429 * gate is always ignored.) So all faults that task switch
430 * currently resume at db_task_start.
431 */
432 /*
433 * Double fault (Murphy's point) - error code (0) on stack
434 */
435 Entry(db_task_dbl_fault)
436 popl %eax
437 movl $(T_DOUBLE_FAULT),%ebx
438 jmp db_task_start
439 /*
440 * Segment not present - error code on stack
441 */
442 Entry(db_task_seg_np)
443 popl %eax
444 movl $(T_SEGMENT_NOT_PRESENT),%ebx
445 jmp db_task_start
446 /*
447 * Stack fault - error code on (current) stack
448 */
449 Entry(db_task_stk_fault)
450 popl %eax
451 movl $(T_STACK_FAULT),%ebx
452 jmp db_task_start
453 /*
454 * General protection fault - error code on stack
455 */
456 Entry(db_task_gen_prot)
457 popl %eax
458 movl $(T_GENERAL_PROTECTION),%ebx
459 jmp db_task_start
460 #endif /* 0 */
461 /*
462 * The entry point where execution resumes after last-ditch debugger task
463 * switch.
464 */
465 Entry(db_task_start)
466 movl %esp,%edx
467 subl $(ISS32_SIZE),%edx
468 movl %edx,%esp /* allocate x86_saved_state on stack */
469 movl %eax,R_ERR(%esp)
470 movl %ebx,R_TRAPNO(%esp)
471 pushl %edx
472 CPU_NUMBER(%edx)
473 movl CX(EXT(master_dbtss),%edx),%edx
474 movl TSS_LINK(%edx),%eax
475 pushl %eax /* pass along selector of previous TSS */
476 call EXT(db_tss_to_frame)
477 popl %eax /* get rid of TSS selector */
478 call EXT(db_trap_from_asm)
479 addl $0x4,%esp
480 /*
481 * And now...?
482 */
483 iret /* ha, ha, ha... */
484 #endif /* MACH_KDB */
485
486 /*
487 * Called as a function, makes the current thread
488 * return from the kernel as if from an exception.
489 */
490
491 .globl EXT(thread_exception_return)
492 .globl EXT(thread_bootstrap_return)
493 LEXT(thread_exception_return)
494 LEXT(thread_bootstrap_return)
495 cli
496 movl %gs:CPU_KERNEL_STACK,%ecx
497 movl (%ecx),%esp /* switch back to PCB stack */
498 jmp EXT(return_from_trap)
499
500 Entry(call_continuation)
501 movl S_ARG0,%eax /* get continuation */
502 movl S_ARG1,%edx /* continuation param */
503 movl S_ARG2,%ecx /* wait result */
504 movl %gs:CPU_KERNEL_STACK,%esp /* pop the stack */
505 xorl %ebp,%ebp /* zero frame pointer */
506 subl $8,%esp /* align the stack */
507 pushl %ecx
508 pushl %edx
509 call *%eax /* call continuation */
510 addl $16,%esp
511 movl %gs:CPU_ACTIVE_THREAD,%eax
512 pushl %eax
513 call EXT(thread_terminate)
514
515
516
517 /*******************************************************************************************************
518 *
519 * All 64 bit task 'exceptions' enter lo_alltraps:
520 * esp -> x86_saved_state_t
521 *
522 * The rest of the state is set up as:
523 * cr3 -> kernel directory
524 * esp -> low based stack
525 * gs -> CPU_DATA_GS
526 * cs -> KERNEL_CS
527 * ss/ds/es -> KERNEL_DS
528 *
529 * interrupts disabled
530 * direction flag cleared
531 */
532 Entry(lo_alltraps)
533 movl R_CS(%esp),%eax /* assume 32-bit state */
534 cmpl $(SS_64),SS_FLAVOR(%esp)/* 64-bit? */
535 jne 1f
536 movl R64_CS(%esp),%eax /* 64-bit user mode */
537 1:
538 testb $3,%al
539 jz trap_from_kernel
540 /* user mode trap */
541 TIME_TRAP_UENTRY
542
543 movl %gs:CPU_ACTIVE_THREAD,%ecx
544 movl ACT_TASK(%ecx),%ebx
545
546 /* Check for active vtimers in the current task */
547 cmpl $0,TASK_VTIMERS(%ebx)
548 jz 1f
549
550 /* Set a pending AST */
551 orl $(AST_BSD),%gs:CPU_PENDING_AST
552
553 /* Set a thread AST (atomic) */
554 lock
555 orl $(AST_BSD),ACT_AST(%ecx)
556
557 1:
558 movl %gs:CPU_KERNEL_STACK,%ebx
559 xchgl %ebx,%esp /* switch to kernel stack */
560 sti
561
562 CCALL1(user_trap, %ebx) /* call user trap routine */
563 cli /* hold off intrs - critical section */
564 popl %esp /* switch back to PCB stack */
565
566 /*
567 * Return from trap or system call, checking for ASTs.
568 * On lowbase PCB stack with intrs disabled
569 */
570 LEXT(return_from_trap)
571 movl %gs:CPU_PENDING_AST,%eax
572 testl %eax,%eax
573 je EXT(return_to_user) /* branch if no AST */
574
575 movl %gs:CPU_KERNEL_STACK,%ebx
576 xchgl %ebx,%esp /* switch to kernel stack */
577 sti /* interrupts always enabled on return to user mode */
578
579 pushl %ebx /* save PCB stack */
580 xorl %ebp,%ebp /* Clear framepointer */
581 CCALL1(i386_astintr, $0) /* take the AST */
582 cli
583 popl %esp /* switch back to PCB stack (w/exc link) */
584 jmp EXT(return_from_trap) /* and check again (rare) */
585
586 LEXT(return_to_user)
587 TIME_TRAP_UEXIT
588
589 LEXT(ret_to_user)
590 cmpl $0, %gs:CPU_IS64BIT
591 je EXT(lo_ret_to_user)
592 jmp EXT(lo64_ret_to_user)
593
594
595
596 /*
597 * Trap from kernel mode. No need to switch stacks.
598 * Interrupts must be off here - we will set them to state at time of trap
599 * as soon as it's safe for us to do so and not recurse doing preemption
600 */
601 trap_from_kernel:
602 movl %esp, %eax /* saved state addr */
603 pushl R_EIP(%esp) /* Simulate a CALL from fault point */
604 pushl %ebp /* Extend framepointer chain */
605 movl %esp, %ebp
606 CCALL1(kernel_trap, %eax) /* Call kernel trap handler */
607 popl %ebp
608 addl $4, %esp
609 cli
610
611 movl %gs:CPU_PENDING_AST,%eax /* get pending asts */
612 testl $ AST_URGENT,%eax /* any urgent preemption? */
613 je ret_to_kernel /* no, nothing to do */
614 cmpl $ T_PREEMPT,R_TRAPNO(%esp)
615 je ret_to_kernel /* T_PREEMPT handled in kernel_trap() */
616 testl $ EFL_IF,R_EFLAGS(%esp) /* interrupts disabled? */
617 je ret_to_kernel
618 cmpl $0,%gs:CPU_PREEMPTION_LEVEL /* preemption disabled? */
619 jne ret_to_kernel
620 movl %gs:CPU_KERNEL_STACK,%eax
621 movl %esp,%ecx
622 xorl %eax,%ecx
623 andl $(-KERNEL_STACK_SIZE),%ecx
624 testl %ecx,%ecx /* are we on the kernel stack? */
625 jne ret_to_kernel /* no, skip it */
626
627 CCALL1(i386_astintr, $1) /* take the AST */
628
629 ret_to_kernel:
630 cmpl $0, %gs:CPU_IS64BIT
631 je EXT(lo_ret_to_kernel)
632 jmp EXT(lo64_ret_to_kernel)
633
634
635
636 /*******************************************************************************************************
637 *
638 * All interrupts on all tasks enter here with:
639 * esp-> -> x86_saved_state_t
640 *
641 * cr3 -> kernel directory
642 * esp -> low based stack
643 * gs -> CPU_DATA_GS
644 * cs -> KERNEL_CS
645 * ss/ds/es -> KERNEL_DS
646 *
647 * interrupts disabled
648 * direction flag cleared
649 */
650 Entry(lo_allintrs)
651 /*
652 * test whether already on interrupt stack
653 */
654 movl %gs:CPU_INT_STACK_TOP,%ecx
655 cmpl %esp,%ecx
656 jb 1f
657 leal -INTSTACK_SIZE(%ecx),%edx
658 cmpl %esp,%edx
659 jb int_from_intstack
660 1:
661 xchgl %ecx,%esp /* switch to interrupt stack */
662
663 movl %cr0,%eax /* get cr0 */
664 orl $(CR0_TS),%eax /* or in TS bit */
665 movl %eax,%cr0 /* set cr0 */
666
667 subl $8, %esp /* for 16-byte stack alignment */
668 pushl %ecx /* save pointer to old stack */
669 movl %ecx,%gs:CPU_INT_STATE /* save intr state */
670
671 TIME_INT_ENTRY /* do timing */
672
673 movl %gs:CPU_ACTIVE_THREAD,%ecx
674 movl ACT_TASK(%ecx),%ebx
675
676 /* Check for active vtimers in the current task */
677 cmpl $0,TASK_VTIMERS(%ebx)
678 jz 1f
679
680 /* Set a pending AST */
681 orl $(AST_BSD),%gs:CPU_PENDING_AST
682
683 /* Set a thread AST (atomic) */
684 lock
685 orl $(AST_BSD),ACT_AST(%ecx)
686
687 1:
688 incl %gs:CPU_PREEMPTION_LEVEL
689 incl %gs:CPU_INTERRUPT_LEVEL
690
691 movl %gs:CPU_INT_STATE, %eax
692 CCALL1(PE_incoming_interrupt, %eax) /* call generic interrupt routine */
693
694 cli /* just in case we returned with intrs enabled */
695 xorl %eax,%eax
696 movl %eax,%gs:CPU_INT_STATE /* clear intr state pointer */
697
698 decl %gs:CPU_INTERRUPT_LEVEL
699 decl %gs:CPU_PREEMPTION_LEVEL
700
701 TIME_INT_EXIT /* do timing */
702
703 movl %gs:CPU_ACTIVE_THREAD,%eax
704 movl ACT_PCB(%eax),%eax /* get act`s PCB */
705 movl PCB_FPS(%eax),%eax /* get pcb's ims.ifps */
706 cmpl $0,%eax /* Is there a context */
707 je 1f /* Branch if not */
708 movl FP_VALID(%eax),%eax /* Load fp_valid */
709 cmpl $0,%eax /* Check if valid */
710 jne 1f /* Branch if valid */
711 clts /* Clear TS */
712 jmp 2f
713 1:
714 movl %cr0,%eax /* get cr0 */
715 orl $(CR0_TS),%eax /* or in TS bit */
716 movl %eax,%cr0 /* set cr0 */
717 2:
718 popl %esp /* switch back to old stack */
719
720 /* Load interrupted code segment into %eax */
721 movl R_CS(%esp),%eax /* assume 32-bit state */
722 cmpl $(SS_64),SS_FLAVOR(%esp)/* 64-bit? */
723 jne 3f
724 movl R64_CS(%esp),%eax /* 64-bit user mode */
725 3:
726 testb $3,%al /* user mode, */
727 jnz ast_from_interrupt_user /* go handle potential ASTs */
728 /*
729 * we only want to handle preemption requests if
730 * the interrupt fell in the kernel context
731 * and preemption isn't disabled
732 */
733 movl %gs:CPU_PENDING_AST,%eax
734 testl $ AST_URGENT,%eax /* any urgent requests? */
735 je ret_to_kernel /* no, nothing to do */
736
737 cmpl $0,%gs:CPU_PREEMPTION_LEVEL /* preemption disabled? */
738 jne ret_to_kernel /* yes, skip it */
739
740 movl %gs:CPU_KERNEL_STACK,%eax
741 movl %esp,%ecx
742 xorl %eax,%ecx
743 andl $(-KERNEL_STACK_SIZE),%ecx
744 testl %ecx,%ecx /* are we on the kernel stack? */
745 jne ret_to_kernel /* no, skip it */
746
747 /*
748 * Take an AST from kernel space. We don't need (and don't want)
749 * to do as much as the case where the interrupt came from user
750 * space.
751 */
752 CCALL1(i386_astintr, $1)
753
754 jmp ret_to_kernel
755
756
757 /*
758 * nested int - simple path, can't preempt etc on way out
759 */
760 int_from_intstack:
761 incl %gs:CPU_PREEMPTION_LEVEL
762 incl %gs:CPU_INTERRUPT_LEVEL
763
764 movl %esp, %edx /* x86_saved_state */
765 CCALL1(PE_incoming_interrupt, %edx)
766
767 decl %gs:CPU_INTERRUPT_LEVEL
768 decl %gs:CPU_PREEMPTION_LEVEL
769
770 jmp ret_to_kernel
771
772 /*
773 * Take an AST from an interrupted user
774 */
775 ast_from_interrupt_user:
776 movl %gs:CPU_PENDING_AST,%eax
777 testl %eax,%eax /* pending ASTs? */
778 je EXT(ret_to_user) /* no, nothing to do */
779
780 TIME_TRAP_UENTRY
781
782 jmp EXT(return_from_trap) /* return */
783
784
785 /*******************************************************************************************************
786 *
787 * 32bit Tasks
788 * System call entries via INTR_GATE or sysenter:
789 *
790 * esp -> x86_saved_state32_t
791 * cr3 -> kernel directory
792 * esp -> low based stack
793 * gs -> CPU_DATA_GS
794 * cs -> KERNEL_CS
795 * ss/ds/es -> KERNEL_DS
796 *
797 * interrupts disabled
798 * direction flag cleared
799 */
800
801 Entry(lo_sysenter)
802 /*
803 * We can be here either for a mach syscall or a unix syscall,
804 * as indicated by the sign of the code:
805 */
806 movl R_EAX(%esp),%eax
807 testl %eax,%eax
808 js EXT(lo_mach_scall) /* < 0 => mach */
809 /* > 0 => unix */
810
811 Entry(lo_unix_scall)
812 TIME_TRAP_UENTRY
813
814 movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */
815 movl ACT_TASK(%ecx),%ebx /* point to current task */
816 addl $1,TASK_SYSCALLS_UNIX(%ebx) /* increment call count */
817
818 /* Check for active vtimers in the current task */
819 cmpl $0,TASK_VTIMERS(%ebx)
820 jz 1f
821
822 /* Set a pending AST */
823 orl $(AST_BSD),%gs:CPU_PENDING_AST
824
825 /* Set a thread AST (atomic) */
826 lock
827 orl $(AST_BSD),ACT_AST(%ecx)
828
829 1:
830 movl %gs:CPU_KERNEL_STACK,%ebx
831 xchgl %ebx,%esp /* switch to kernel stack */
832
833 sti
834
835 CCALL1(unix_syscall, %ebx)
836 /*
837 * always returns through thread_exception_return
838 */
839
840
841 Entry(lo_mach_scall)
842 TIME_TRAP_UENTRY
843
844 movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */
845 movl ACT_TASK(%ecx),%ebx /* point to current task */
846 addl $1,TASK_SYSCALLS_MACH(%ebx) /* increment call count */
847
848 /* Check for active vtimers in the current task */
849 cmpl $0,TASK_VTIMERS(%ebx)
850 jz 1f
851
852 /* Set a pending AST */
853 orl $(AST_BSD),%gs:CPU_PENDING_AST
854
855 /* Set a thread AST (atomic) */
856 lock
857 orl $(AST_BSD),ACT_AST(%ecx)
858
859 1:
860 movl %gs:CPU_KERNEL_STACK,%ebx
861 xchgl %ebx,%esp /* switch to kernel stack */
862
863 sti
864
865 CCALL1(mach_call_munger, %ebx)
866 /*
867 * always returns through thread_exception_return
868 */
869
870
871 Entry(lo_mdep_scall)
872 TIME_TRAP_UENTRY
873
874 movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */
875 movl ACT_TASK(%ecx),%ebx /* point to current task */
876
877 /* Check for active vtimers in the current task */
878 cmpl $0,TASK_VTIMERS(%ebx)
879 jz 1f
880
881 /* Set a pending AST */
882 orl $(AST_BSD),%gs:CPU_PENDING_AST
883
884 /* Set a thread AST (atomic) */
885 lock
886 orl $(AST_BSD),ACT_AST(%ecx)
887
888 1:
889 movl %gs:CPU_KERNEL_STACK,%ebx
890 xchgl %ebx,%esp /* switch to kernel stack */
891
892 sti
893
894 CCALL1(machdep_syscall, %ebx)
895 /*
896 * always returns through thread_exception_return
897 */
898
899
900 Entry(lo_diag_scall)
901 TIME_TRAP_UENTRY
902
903 movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */
904 movl ACT_TASK(%ecx),%ebx /* point to current task */
905
906 /* Check for active vtimers in the current task */
907 cmpl $0,TASK_VTIMERS(%ebx)
908 jz 1f
909
910 /* Set a pending AST */
911 orl $(AST_BSD),%gs:CPU_PENDING_AST
912
913 /* Set a thread AST (atomic) */
914 lock
915 orl $(AST_BSD),ACT_AST(%ecx)
916
917 1:
918 movl %gs:CPU_KERNEL_STACK,%ebx // Get the address of the kernel stack
919 xchgl %ebx,%esp // Switch to it, saving the previous
920
921 CCALL1(diagCall, %ebx) // Call diagnostics
922
923 cmpl $0,%eax // What kind of return is this?
924 je 2f
925 cli // Disable interruptions just in case they were enabled
926 popl %esp // Get back the original stack
927 jmp EXT(return_to_user) // Normal return, do not check asts...
928 2:
929 CCALL3(i386_exception, $EXC_SYSCALL, $0x6000, $1)
930 // pass what would be the diag syscall
931 // error return - cause an exception
932 /* no return */
933
934
935
936 /*******************************************************************************************************
937 *
938 * 64bit Tasks
939 * System call entries via syscall only:
940 *
941 * esp -> x86_saved_state64_t
942 * cr3 -> kernel directory
943 * esp -> low based stack
944 * gs -> CPU_DATA_GS
945 * cs -> KERNEL_CS
946 * ss/ds/es -> KERNEL_DS
947 *
948 * interrupts disabled
949 * direction flag cleared
950 */
951
952 Entry(lo_syscall)
953 /*
954 * We can be here either for a mach, unix machdep or diag syscall,
955 * as indicated by the syscall class:
956 */
957 movl R64_RAX(%esp), %eax /* syscall number/class */
958 movl %eax, %ebx
959 andl $(SYSCALL_CLASS_MASK), %ebx /* syscall class */
960 cmpl $(SYSCALL_CLASS_MACH<<SYSCALL_CLASS_SHIFT), %ebx
961 je EXT(lo64_mach_scall)
962 cmpl $(SYSCALL_CLASS_UNIX<<SYSCALL_CLASS_SHIFT), %ebx
963 je EXT(lo64_unix_scall)
964 cmpl $(SYSCALL_CLASS_MDEP<<SYSCALL_CLASS_SHIFT), %ebx
965 je EXT(lo64_mdep_scall)
966 cmpl $(SYSCALL_CLASS_DIAG<<SYSCALL_CLASS_SHIFT), %ebx
967 je EXT(lo64_diag_scall)
968
969 movl %gs:CPU_KERNEL_STACK,%ebx
970 xchgl %ebx,%esp /* switch to kernel stack */
971
972 sti
973
974 /* Syscall class unknown */
975 CCALL3(i386_exception, $(EXC_SYSCALL), %eax, $1)
976 /* no return */
977
978
979 Entry(lo64_unix_scall)
980 TIME_TRAP_UENTRY
981
982 movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */
983 movl ACT_TASK(%ecx),%ebx /* point to current task */
984 addl $1,TASK_SYSCALLS_UNIX(%ebx) /* increment call count */
985
986 /* Check for active vtimers in the current task */
987 cmpl $0,TASK_VTIMERS(%ebx)
988 jz 1f
989
990 /* Set a pending AST */
991 orl $(AST_BSD),%gs:CPU_PENDING_AST
992
993 /* Set a thread AST (atomic) */
994 lock
995 orl $(AST_BSD),ACT_AST(%ecx)
996
997 1:
998 movl %gs:CPU_KERNEL_STACK,%ebx
999 xchgl %ebx,%esp /* switch to kernel stack */
1000
1001 sti
1002
1003 CCALL1(unix_syscall64, %ebx)
1004 /*
1005 * always returns through thread_exception_return
1006 */
1007
1008
1009 Entry(lo64_mach_scall)
1010 TIME_TRAP_UENTRY
1011
1012 movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */
1013 movl ACT_TASK(%ecx),%ebx /* point to current task */
1014 addl $1,TASK_SYSCALLS_MACH(%ebx) /* increment call count */
1015
1016 /* Check for active vtimers in the current task */
1017 cmpl $0,TASK_VTIMERS(%ebx)
1018 jz 1f
1019
1020 /* Set a pending AST */
1021 orl $(AST_BSD),%gs:CPU_PENDING_AST
1022
1023 lock
1024 orl $(AST_BSD),ACT_AST(%ecx)
1025
1026 1:
1027 movl %gs:CPU_KERNEL_STACK,%ebx
1028 xchgl %ebx,%esp /* switch to kernel stack */
1029
1030 sti
1031
1032 CCALL1(mach_call_munger64, %ebx)
1033 /*
1034 * always returns through thread_exception_return
1035 */
1036
1037
1038
1039 Entry(lo64_mdep_scall)
1040 TIME_TRAP_UENTRY
1041
1042 movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */
1043 movl ACT_TASK(%ecx),%ebx /* point to current task */
1044
1045 /* Check for active vtimers in the current task */
1046 cmpl $0,TASK_VTIMERS(%ebx)
1047 jz 1f
1048
1049 /* Set a pending AST */
1050 orl $(AST_BSD),%gs:CPU_PENDING_AST
1051
1052 /* Set a thread AST (atomic) */
1053 lock
1054 orl $(AST_BSD),ACT_AST(%ecx)
1055
1056 1:
1057 movl %gs:CPU_KERNEL_STACK,%ebx
1058 xchgl %ebx,%esp /* switch to kernel stack */
1059
1060 sti
1061
1062 CCALL1(machdep_syscall64, %ebx)
1063 /*
1064 * always returns through thread_exception_return
1065 */
1066
1067
1068 Entry(lo64_diag_scall)
1069 TIME_TRAP_UENTRY
1070
1071 movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */
1072 movl ACT_TASK(%ecx),%ebx /* point to current task */
1073
1074 /* Check for active vtimers in the current task */
1075 cmpl $0,TASK_VTIMERS(%ebx)
1076 jz 1f
1077
1078 /* Set a pending AST */
1079 orl $(AST_BSD),%gs:CPU_PENDING_AST
1080
1081 /* Set a thread AST (atomic) */
1082 lock
1083 orl $(AST_BSD),ACT_AST(%ecx)
1084
1085 1:
1086 movl %gs:CPU_KERNEL_STACK,%ebx // Get the address of the kernel stack
1087 xchgl %ebx,%esp // Switch to it, saving the previous
1088
1089 CCALL1(diagCall64, %ebx) // Call diagnostics
1090
1091 cmpl $0,%eax // What kind of return is this?
1092 je 2f
1093 cli // Disable interruptions just in case they were enabled
1094 popl %esp // Get back the original stack
1095 jmp EXT(return_to_user) // Normal return, do not check asts...
1096 2:
1097 CCALL3(i386_exception, $EXC_SYSCALL, $0x6000, $1)
1098 // pass what would be the diag syscall
1099 // error return - cause an exception
1100 /* no return */
1101
1102 /*\f*/
1103 /*
1104 * Utility routines.
1105 */
1106
1107
1108 /*
1109 * Copy from user/kernel address space.
1110 * arg0: window offset or kernel address
1111 * arg1: kernel address
1112 * arg2: byte count
1113 */
1114 Entry(copyinphys_user)
1115 movl $(USER_WINDOW_SEL),%ecx /* user data segment access through kernel window */
1116 mov %cx,%ds
1117
1118 Entry(copyinphys_kern)
1119 movl $(PHYS_WINDOW_SEL),%ecx /* physical access through kernel window */
1120 mov %cx,%es
1121 jmp copyin_common
1122
1123 Entry(copyin_user)
1124 movl $(USER_WINDOW_SEL),%ecx /* user data segment access through kernel window */
1125 mov %cx,%ds
1126
1127 Entry(copyin_kern)
1128
1129 copyin_common:
1130 pushl %esi
1131 pushl %edi /* save registers */
1132
1133 movl 8+S_ARG0,%esi /* get source - window offset or kernel address */
1134 movl 8+S_ARG1,%edi /* get destination - kernel address */
1135 movl 8+S_ARG2,%edx /* get count */
1136
1137 cld /* count up */
1138 movl %edx,%ecx /* move by longwords first */
1139 shrl $2,%ecx
1140 RECOVERY_SECTION
1141 RECOVER(copyin_fail)
1142 rep
1143 movsl /* move longwords */
1144 movl %edx,%ecx /* now move remaining bytes */
1145 andl $3,%ecx
1146 RECOVERY_SECTION
1147 RECOVER(copyin_fail)
1148 rep
1149 movsb
1150 xorl %eax,%eax /* return 0 for success */
1151 copyin_ret:
1152 mov %ss,%cx /* restore kernel data and extended segments */
1153 mov %cx,%ds
1154 mov %cx,%es
1155
1156 popl %edi /* restore registers */
1157 popl %esi
1158 ret /* and return */
1159
1160 copyin_fail:
1161 movl $(EFAULT),%eax /* return error for failure */
1162 jmp copyin_ret /* pop frame and return */
1163
1164
1165
1166 /*
1167 * Copy string from user/kern address space.
1168 * arg0: window offset or kernel address
1169 * arg1: kernel address
1170 * arg2: max byte count
1171 * arg3: actual byte count (OUT)
1172 */
1173 Entry(copyinstr_kern)
1174 mov %ds,%cx
1175 jmp copyinstr_common
1176
1177 Entry(copyinstr_user)
1178 movl $(USER_WINDOW_SEL),%ecx /* user data segment access through kernel window */
1179
1180 copyinstr_common:
1181 mov %cx,%fs
1182
1183 pushl %esi
1184 pushl %edi /* save registers */
1185
1186 movl 8+S_ARG0,%esi /* get source - window offset or kernel address */
1187 movl 8+S_ARG1,%edi /* get destination - kernel address */
1188 movl 8+S_ARG2,%edx /* get count */
1189
1190 xorl %eax,%eax /* set to 0 here so that the high 24 bits */
1191 /* are 0 for the cmpl against 0 */
1192 2:
1193 RECOVERY_SECTION
1194 RECOVER(copystr_fail) /* copy bytes... */
1195 movb %fs:(%esi),%al
1196 incl %esi
1197 testl %edi,%edi /* if kernel address is ... */
1198 jz 3f /* not NULL */
1199 movb %al,(%edi) /* copy the byte */
1200 incl %edi
1201 3:
1202 testl %eax,%eax /* did we just stuff the 0-byte? */
1203 jz 4f /* yes, return 0 status already in %eax */
1204 decl %edx /* decrement #bytes left in buffer */
1205 jnz 2b /* buffer not full so copy in another byte */
1206 movl $(ENAMETOOLONG),%eax /* buffer full but no 0-byte: ENAMETOOLONG */
1207 4:
1208 movl 8+S_ARG3,%edi /* get OUT len ptr */
1209 cmpl $0,%edi
1210 jz copystr_ret /* if null, just return */
1211 subl 8+S_ARG0,%esi
1212 movl %esi,(%edi) /* else set OUT arg to xfer len */
1213 copystr_ret:
1214 popl %edi /* restore registers */
1215 popl %esi
1216 ret /* and return */
1217
1218 copystr_fail:
1219 movl $(EFAULT),%eax /* return error for failure */
1220 jmp copystr_ret /* pop frame and return */
1221
1222
1223 /*
1224 * Copy to user/kern address space.
1225 * arg0: kernel address
1226 * arg1: window offset or kernel address
1227 * arg2: byte count
1228 */
1229 ENTRY(copyoutphys_user)
1230 movl $(USER_WINDOW_SEL),%ecx /* user data segment access through kernel window */
1231 mov %cx,%es
1232
1233 ENTRY(copyoutphys_kern)
1234 movl $(PHYS_WINDOW_SEL),%ecx /* physical access through kernel window */
1235 mov %cx,%ds
1236 jmp copyout_common
1237
1238 ENTRY(copyout_user)
1239 movl $(USER_WINDOW_SEL),%ecx /* user data segment access through kernel window */
1240 mov %cx,%es
1241
1242 ENTRY(copyout_kern)
1243
1244 copyout_common:
1245 pushl %esi
1246 pushl %edi /* save registers */
1247
1248 movl 8+S_ARG0,%esi /* get source - kernel address */
1249 movl 8+S_ARG1,%edi /* get destination - window offset or kernel address */
1250 movl 8+S_ARG2,%edx /* get count */
1251
1252 cld /* count up */
1253 movl %edx,%ecx /* move by longwords first */
1254 shrl $2,%ecx
1255 RECOVERY_SECTION
1256 RECOVER(copyout_fail)
1257 rep
1258 movsl
1259 movl %edx,%ecx /* now move remaining bytes */
1260 andl $3,%ecx
1261 RECOVERY_SECTION
1262 RECOVER(copyout_fail)
1263 rep
1264 movsb /* move */
1265 xorl %eax,%eax /* return 0 for success */
1266 copyout_ret:
1267 mov %ss,%cx /* restore kernel segment */
1268 mov %cx,%es
1269 mov %cx,%ds
1270
1271 popl %edi /* restore registers */
1272 popl %esi
1273 ret /* and return */
1274
1275 copyout_fail:
1276 movl $(EFAULT),%eax /* return error for failure */
1277 jmp copyout_ret /* pop frame and return */
1278
1279 /*
1280 * io register must not be used on slaves (no AT bus)
1281 */
1282 #define ILL_ON_SLAVE
1283
1284
1285 #if MACH_ASSERT
1286
1287 #define ARG0 B_ARG0
1288 #define ARG1 B_ARG1
1289 #define ARG2 B_ARG2
1290 #define PUSH_FRAME FRAME
1291 #define POP_FRAME EMARF
1292
1293 #else /* MACH_ASSERT */
1294
1295 #define ARG0 S_ARG0
1296 #define ARG1 S_ARG1
1297 #define ARG2 S_ARG2
1298 #define PUSH_FRAME
1299 #define POP_FRAME
1300
1301 #endif /* MACH_ASSERT */
1302
1303
1304 #if MACH_KDB || MACH_ASSERT
1305
1306 /*
1307 * Following routines are also defined as macros in i386/pio.h
1308 * Compile then when MACH_KDB is configured so that they
1309 * can be invoked from the debugger.
1310 */
1311
1312 /*
1313 * void outb(unsigned char *io_port,
1314 * unsigned char byte)
1315 *
1316 * Output a byte to an IO port.
1317 */
1318 ENTRY(outb)
1319 PUSH_FRAME
1320 ILL_ON_SLAVE
1321 movl ARG0,%edx /* IO port address */
1322 movl ARG1,%eax /* data to output */
1323 outb %al,%dx /* send it out */
1324 POP_FRAME
1325 ret
1326
1327 /*
1328 * unsigned char inb(unsigned char *io_port)
1329 *
1330 * Input a byte from an IO port.
1331 */
1332 ENTRY(inb)
1333 PUSH_FRAME
1334 ILL_ON_SLAVE
1335 movl ARG0,%edx /* IO port address */
1336 xor %eax,%eax /* clear high bits of register */
1337 inb %dx,%al /* get the byte */
1338 POP_FRAME
1339 ret
1340
1341 /*
1342 * void outw(unsigned short *io_port,
1343 * unsigned short word)
1344 *
1345 * Output a word to an IO port.
1346 */
1347 ENTRY(outw)
1348 PUSH_FRAME
1349 ILL_ON_SLAVE
1350 movl ARG0,%edx /* IO port address */
1351 movl ARG1,%eax /* data to output */
1352 outw %ax,%dx /* send it out */
1353 POP_FRAME
1354 ret
1355
1356 /*
1357 * unsigned short inw(unsigned short *io_port)
1358 *
1359 * Input a word from an IO port.
1360 */
1361 ENTRY(inw)
1362 PUSH_FRAME
1363 ILL_ON_SLAVE
1364 movl ARG0,%edx /* IO port address */
1365 xor %eax,%eax /* clear high bits of register */
1366 inw %dx,%ax /* get the word */
1367 POP_FRAME
1368 ret
1369
1370 /*
1371 * void outl(unsigned int *io_port,
1372 * unsigned int byte)
1373 *
1374 * Output an int to an IO port.
1375 */
1376 ENTRY(outl)
1377 PUSH_FRAME
1378 ILL_ON_SLAVE
1379 movl ARG0,%edx /* IO port address*/
1380 movl ARG1,%eax /* data to output */
1381 outl %eax,%dx /* send it out */
1382 POP_FRAME
1383 ret
1384
1385 /*
1386 * unsigned int inl(unsigned int *io_port)
1387 *
1388 * Input an int from an IO port.
1389 */
1390 ENTRY(inl)
1391 PUSH_FRAME
1392 ILL_ON_SLAVE
1393 movl ARG0,%edx /* IO port address */
1394 inl %dx,%eax /* get the int */
1395 POP_FRAME
1396 ret
1397
1398 #endif /* MACH_KDB || MACH_ASSERT*/
1399
1400 /*
1401 * void loutb(unsigned byte *io_port,
1402 * unsigned byte *data,
1403 * unsigned int count)
1404 *
1405 * Output an array of bytes to an IO port.
1406 */
1407 ENTRY(loutb)
1408 ENTRY(outsb)
1409 PUSH_FRAME
1410 ILL_ON_SLAVE
1411 movl %esi,%eax /* save register */
1412 movl ARG0,%edx /* get io port number */
1413 movl ARG1,%esi /* get data address */
1414 movl ARG2,%ecx /* get count */
1415 cld /* count up */
1416 rep
1417 outsb /* output */
1418 movl %eax,%esi /* restore register */
1419 POP_FRAME
1420 ret
1421
1422
1423 /*
1424 * void loutw(unsigned short *io_port,
1425 * unsigned short *data,
1426 * unsigned int count)
1427 *
1428 * Output an array of shorts to an IO port.
1429 */
1430 ENTRY(loutw)
1431 ENTRY(outsw)
1432 PUSH_FRAME
1433 ILL_ON_SLAVE
1434 movl %esi,%eax /* save register */
1435 movl ARG0,%edx /* get io port number */
1436 movl ARG1,%esi /* get data address */
1437 movl ARG2,%ecx /* get count */
1438 cld /* count up */
1439 rep
1440 outsw /* output */
1441 movl %eax,%esi /* restore register */
1442 POP_FRAME
1443 ret
1444
1445 /*
1446 * void loutw(unsigned short io_port,
1447 * unsigned int *data,
1448 * unsigned int count)
1449 *
1450 * Output an array of longs to an IO port.
1451 */
1452 ENTRY(loutl)
1453 ENTRY(outsl)
1454 PUSH_FRAME
1455 ILL_ON_SLAVE
1456 movl %esi,%eax /* save register */
1457 movl ARG0,%edx /* get io port number */
1458 movl ARG1,%esi /* get data address */
1459 movl ARG2,%ecx /* get count */
1460 cld /* count up */
1461 rep
1462 outsl /* output */
1463 movl %eax,%esi /* restore register */
1464 POP_FRAME
1465 ret
1466
1467
1468 /*
1469 * void linb(unsigned char *io_port,
1470 * unsigned char *data,
1471 * unsigned int count)
1472 *
1473 * Input an array of bytes from an IO port.
1474 */
1475 ENTRY(linb)
1476 ENTRY(insb)
1477 PUSH_FRAME
1478 ILL_ON_SLAVE
1479 movl %edi,%eax /* save register */
1480 movl ARG0,%edx /* get io port number */
1481 movl ARG1,%edi /* get data address */
1482 movl ARG2,%ecx /* get count */
1483 cld /* count up */
1484 rep
1485 insb /* input */
1486 movl %eax,%edi /* restore register */
1487 POP_FRAME
1488 ret
1489
1490
1491 /*
1492 * void linw(unsigned short *io_port,
1493 * unsigned short *data,
1494 * unsigned int count)
1495 *
1496 * Input an array of shorts from an IO port.
1497 */
1498 ENTRY(linw)
1499 ENTRY(insw)
1500 PUSH_FRAME
1501 ILL_ON_SLAVE
1502 movl %edi,%eax /* save register */
1503 movl ARG0,%edx /* get io port number */
1504 movl ARG1,%edi /* get data address */
1505 movl ARG2,%ecx /* get count */
1506 cld /* count up */
1507 rep
1508 insw /* input */
1509 movl %eax,%edi /* restore register */
1510 POP_FRAME
1511 ret
1512
1513
1514 /*
1515 * void linl(unsigned short io_port,
1516 * unsigned int *data,
1517 * unsigned int count)
1518 *
1519 * Input an array of longs from an IO port.
1520 */
1521 ENTRY(linl)
1522 ENTRY(insl)
1523 PUSH_FRAME
1524 ILL_ON_SLAVE
1525 movl %edi,%eax /* save register */
1526 movl ARG0,%edx /* get io port number */
1527 movl ARG1,%edi /* get data address */
1528 movl ARG2,%ecx /* get count */
1529 cld /* count up */
1530 rep
1531 insl /* input */
1532 movl %eax,%edi /* restore register */
1533 POP_FRAME
1534 ret
1535
1536 /*
1537 * int rdmsr_carefully(uint32_t msr, uint32_t *lo, uint32_t *hi)
1538 */
1539 ENTRY(rdmsr_carefully)
1540 movl S_ARG0, %ecx
1541 RECOVERY_SECTION
1542 RECOVER(rdmsr_fail)
1543 rdmsr
1544 movl S_ARG1, %ecx
1545 movl %eax, (%ecx)
1546 movl S_ARG2, %ecx
1547 movl %edx, (%ecx)
1548 movl $0, %eax
1549 ret
1550
1551 rdmsr_fail:
1552 movl $1, %eax
1553 ret
1554
1555 /*
1556 * Done with recovery table.
1557 */
1558 RECOVERY_SECTION
1559 RECOVER_TABLE_END
1560
1561 .data
1562 dr_msk:
1563 .long ~0x000f0003
1564 .long ~0x00f0000c
1565 .long ~0x0f000030
1566 .long ~0xf00000c0
1567 ENTRY(dr_addr)
1568 .long 0,0,0,0
1569 .long 0,0,0,0
1570
1571 .text
1572
1573 #ifndef SYMMETRY
1574
1575 /*
1576 * ffs(mask)
1577 */
1578 ENTRY(ffs)
1579 bsfl S_ARG0, %eax
1580 jz 0f
1581 incl %eax
1582 ret
1583 0: xorl %eax, %eax
1584 ret
1585
1586 /*
1587 * cpu_shutdown()
1588 * Force reboot
1589 */
1590
1591 null_idtr:
1592 .word 0
1593 .long 0
1594
1595 Entry(cpu_shutdown)
1596 lidt null_idtr /* disable the interrupt handler */
1597 xor %ecx,%ecx /* generate a divide by zero */
1598 div %ecx,%eax /* reboot now */
1599 ret /* this will "never" be executed */
1600
1601 #endif /* SYMMETRY */
1602
1603
1604 /*
1605 * setbit(int bitno, int *s) - set bit in bit string
1606 */
1607 ENTRY(setbit)
1608 movl S_ARG0, %ecx /* bit number */
1609 movl S_ARG1, %eax /* address */
1610 btsl %ecx, (%eax) /* set bit */
1611 ret
1612
1613 /*
1614 * clrbit(int bitno, int *s) - clear bit in bit string
1615 */
1616 ENTRY(clrbit)
1617 movl S_ARG0, %ecx /* bit number */
1618 movl S_ARG1, %eax /* address */
1619 btrl %ecx, (%eax) /* clear bit */
1620 ret
1621
1622 /*
1623 * ffsbit(int *s) - find first set bit in bit string
1624 */
1625 ENTRY(ffsbit)
1626 movl S_ARG0, %ecx /* address */
1627 movl $0, %edx /* base offset */
1628 0:
1629 bsfl (%ecx), %eax /* check argument bits */
1630 jnz 1f /* found bit, return */
1631 addl $4, %ecx /* increment address */
1632 addl $32, %edx /* increment offset */
1633 jmp 0b /* try again */
1634 1:
1635 addl %edx, %eax /* return offset */
1636 ret
1637
1638 /*
1639 * testbit(int nr, volatile void *array)
1640 *
1641 * Test to see if the bit is set within the bit string
1642 */
1643
1644 ENTRY(testbit)
1645 movl S_ARG0,%eax /* Get the bit to test */
1646 movl S_ARG1,%ecx /* get the array string */
1647 btl %eax,(%ecx)
1648 sbbl %eax,%eax
1649 ret
1650
1651 ENTRY(get_pc)
1652 movl 4(%ebp),%eax
1653 ret
1654
1655 ENTRY(minsecurity)
1656 pushl %ebp
1657 movl %esp,%ebp
1658 /*
1659 * jail: set the EIP to "jail" to block a kernel thread.
1660 * Useful to debug synchronization problems on MPs.
1661 */
1662 ENTRY(jail)
1663 jmp EXT(jail)
1664
1665 /*
1666 * unsigned int
1667 * div_scale(unsigned int dividend,
1668 * unsigned int divisor,
1669 * unsigned int *scale)
1670 *
1671 * This function returns (dividend << *scale) //divisor where *scale
1672 * is the largest possible value before overflow. This is used in
1673 * computation where precision must be achieved in order to avoid
1674 * floating point usage.
1675 *
1676 * Algorithm:
1677 * *scale = 0;
1678 * while (((dividend >> *scale) >= divisor))
1679 * (*scale)++;
1680 * *scale = 32 - *scale;
1681 * return ((dividend << *scale) / divisor);
1682 */
1683 ENTRY(div_scale)
1684 PUSH_FRAME
1685 xorl %ecx, %ecx /* *scale = 0 */
1686 xorl %eax, %eax
1687 movl ARG0, %edx /* get dividend */
1688 0:
1689 cmpl ARG1, %edx /* if (divisor > dividend) */
1690 jle 1f /* goto 1f */
1691 addl $1, %ecx /* (*scale)++ */
1692 shrdl $1, %edx, %eax /* dividend >> 1 */
1693 shrl $1, %edx /* dividend >> 1 */
1694 jmp 0b /* goto 0b */
1695 1:
1696 divl ARG1 /* (dividend << (32 - *scale)) / divisor */
1697 movl ARG2, %edx /* get scale */
1698 movl $32, (%edx) /* *scale = 32 */
1699 subl %ecx, (%edx) /* *scale -= %ecx */
1700 POP_FRAME
1701 ret
1702
1703 /*
1704 * unsigned int
1705 * mul_scale(unsigned int multiplicand,
1706 * unsigned int multiplier,
1707 * unsigned int *scale)
1708 *
1709 * This function returns ((multiplicand * multiplier) >> *scale) where
1710 * scale is the largest possible value before overflow. This is used in
1711 * computation where precision must be achieved in order to avoid
1712 * floating point usage.
1713 *
1714 * Algorithm:
1715 * *scale = 0;
1716 * while (overflow((multiplicand * multiplier) >> *scale))
1717 * (*scale)++;
1718 * return ((multiplicand * multiplier) >> *scale);
1719 */
1720 ENTRY(mul_scale)
1721 PUSH_FRAME
1722 xorl %ecx, %ecx /* *scale = 0 */
1723 movl ARG0, %eax /* get multiplicand */
1724 mull ARG1 /* multiplicand * multiplier */
1725 0:
1726 cmpl $0, %edx /* if (!overflow()) */
1727 je 1f /* goto 1 */
1728 addl $1, %ecx /* (*scale)++ */
1729 shrdl $1, %edx, %eax /* (multiplicand * multiplier) >> 1 */
1730 shrl $1, %edx /* (multiplicand * multiplier) >> 1 */
1731 jmp 0b
1732 1:
1733 movl ARG2, %edx /* get scale */
1734 movl %ecx, (%edx) /* set *scale */
1735 POP_FRAME
1736 ret
1737
1738
1739
1740 /*
1741 * Double-fault exception handler task. The last gasp...
1742 */
1743 Entry(df_task_start)
1744 CCALL1(panic_double_fault, $(T_DOUBLE_FAULT))
1745 hlt
1746
1747
1748 /*
1749 * machine-check handler task. The last gasp...
1750 */
1751 Entry(mc_task_start)
1752 CCALL1(panic_machine_check, $(T_MACHINE_CHECK))
1753 hlt
1754
1755 /*
1756 * Compatibility mode's last gasp...
1757 */
1758 Entry(lo_df64)
1759 movl %esp, %eax
1760 CCALL1(panic_double_fault64, %eax)
1761 hlt
1762
1763 Entry(lo_mc64)
1764 movl %esp, %eax
1765 CCALL1(panic_machine_check64, %eax)
1766 hlt
1767
mirror of XNU