X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/378393581903b274cb7a4d18e0d978071a6b592d..7e4a7d3939db04e70062ae6c7bf24b8c8b2f5a7c:/osfmk/i386/locore.s diff --git a/osfmk/i386/locore.s b/osfmk/i386/locore.s index a4ba462cc..b58b7ece7 100644 --- a/osfmk/i386/locore.s +++ b/osfmk/i386/locore.s @@ -1,23 +1,29 @@ /* - * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * The contents of this file constitute Original Code as defined in and - * are subject to the Apple Public Source License Version 1.1 (the - * "License"). You may not use this file except in compliance with the - * License. Please obtain a copy of the License at - * http://www.apple.com/publicsource and read it before using this file. + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. * - * This Original Code and all software distributed under the License are - * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the - * License for the specific language governing rights and limitations - * under the License. + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ @@ -60,16 +66,23 @@ #include #include #include +#include +#include #include #include #include #include +#include -#include +#define _ARCH_I386_ASM_HELP_H_ /* Prevent inclusion of user header */ +#include -#define PREEMPT_DEBUG_LOG 0 +#include +#define CLI cli +#define STI sti + /* * PTmap is recursive pagemap at top of virtual address space. * Within PTmap, the page directory can be found (third indirection). @@ -102,6 +115,55 @@ #define CX(addr,reg) addr(,reg,4) +/* + * The following macros make calls into C code. + * They dynamically align the stack to 16 bytes. + * Arguments are moved (not pushed) onto the correctly aligned stack. + * NOTE: EDI is destroyed in the process, and hence cannot + * be directly used as a parameter. Users of this macro must + * independently preserve EDI (a non-volatile) if the routine is + * intended to be called from C, for instance. + */ + +#define CCALL(fn) \ + movl %esp, %edi ;\ + andl $0xFFFFFFF0, %esp ;\ + call EXT(fn) ;\ + movl %edi, %esp + +#define CCALL1(fn, arg1) \ + movl %esp, %edi ;\ + subl $4, %esp ;\ + andl $0xFFFFFFF0, %esp ;\ + movl arg1, 0(%esp) ;\ + call EXT(fn) ;\ + movl %edi, %esp + +#define CCALL2(fn, arg1, arg2) \ + movl %esp, %edi ;\ + subl $8, %esp ;\ + andl $0xFFFFFFF0, %esp ;\ + movl arg2, 4(%esp) ;\ + movl arg1, 0(%esp) ;\ + call EXT(fn) ;\ + movl %edi, %esp + +/* + * CCALL5 is used for callee functions with 3 arguments but + * where arg2 (a3:a2) and arg3 (a5:a4) are 64-bit values. + */ +#define CCALL5(fn, a1, a2, a3, a4, a5) \ + movl %esp, %edi ;\ + subl $20, %esp ;\ + andl $0xFFFFFFF0, %esp ;\ + movl a5, 16(%esp) ;\ + movl a4, 12(%esp) ;\ + movl a3, 8(%esp) ;\ + movl a2, 4(%esp) ;\ + movl a1, 0(%esp) ;\ + call EXT(fn) ;\ + movl %edi, %esp + .text locore_start: @@ -111,7 +173,6 @@ locore_start: #ifdef __MACHO__ #define RECOVERY_SECTION .section __VECTORS, __recover -#define RETRY_SECTION .section __VECTORS, __retries #else #define RECOVERY_SECTION .text #define RECOVERY_SECTION .text @@ -134,37 +195,22 @@ LEXT(recover_table) ;\ .align 2 ;\ .globl EXT(recover_table_end) ;\ LEXT(recover_table_end) ;\ - .text + .long 0 /* workaround see comment below */ ;\ + .text ; -/* - * Retry table for certain successful faults. +/* TODO FIXME + * the .long 0 is to work around a linker bug (insert radar# here) + * basically recover_table_end has zero size and bumps up right against saved_esp in acpi_wakeup.s + * recover_table_end is in __RECOVER,__vectors and saved_esp is in __SLEEP,__data, but they're right next to each + * other and so the linker combines them and incorrectly relocates everything referencing recover_table_end to point + * into the SLEEP section */ -#define RETRY_TABLE_START \ - .align 3; \ - .globl EXT(retry_table) ;\ -LEXT(retry_table) ;\ - .text - -#define RETRY(addr) \ - .align 3 ;\ - .long 9f ;\ - .long addr ;\ - .text ;\ -9: - -#define RETRY_TABLE_END \ - .align 3; \ - .globl EXT(retry_table_end) ;\ -LEXT(retry_table_end) ;\ - .text /* - * Allocate recovery and retry tables. + * Allocate recovery and table. */ RECOVERY_SECTION RECOVER_TABLE_START - RETRY_SECTION - RETRY_TABLE_START /* * Timing routines. @@ -199,166 +245,131 @@ Entry(timer_grab) */ /* - * Low 32-bits of nanotime returned in %eax. - * Computed from tsc using conversion scale/shift from per-cpu data. - * Uses %ecx and %edx. + * Nanotime returned in %edx:%eax. + * Computed from tsc based on the scale factor + * and an implicit 32 bit shift. + * + * Uses %eax, %ebx, %ecx, %edx, %esi, %edi. */ -#define NANOTIME32 \ - pushl %esi /* save %esi */ ;\ - movl %gs:CPU_THIS,%esi /* per-cpu data ptr */ ;\ - addl $(CPU_RTC_NANOTIME),%esi /* esi -> per-cpu nanotime*/ ;\ - rdtsc /* edx:eax = tsc */ ;\ - subl RTN_TSC(%esi),%eax /* eax = (tsc - base_tsc) */ ;\ - mull RTN_SCALE(%esi) /* eax *= scale */ ;\ - movl RTN_SHIFT(%esi),%ecx /* ecx = shift */ ;\ - shrdl %cl,%edx,%eax /* edx:eax >> shift */ ;\ - andb $32,%cl /* shift == 32? */ ;\ - cmovnel %edx,%eax /* %eax = %edx if so */ ;\ - addl RTN_NANOS(%esi),%eax /* add base ns */ ;\ - popl %esi +#define NANOTIME \ + mov %gs:CPU_NANOTIME,%edi ; \ + RTC_NANOTIME_READ_FAST() + /* - * Add 32-bit ns delta in register dreg to timer pointed to by register treg. + * Add 64-bit delta in register dreg : areg to timer pointed to by register treg. */ -#define TIMER_UPDATE(treg,dreg) \ - addl TIMER_LOW(treg),dreg /* add delta low bits */ ;\ - adcl $0,TIMER_HIGHCHK(treg) /* add carry check bits */ ;\ - movl dreg,TIMER_LOW(treg) /* store updated low bit */ ;\ - movl TIMER_HIGHCHK(treg),dreg /* copy high check bits */ ;\ - movl dreg,TIMER_HIGH(treg) /* to high bita */ +#define TIMER_UPDATE(treg,dreg,areg,offset) \ + addl (TIMER_LOW+(offset))(treg),areg /* add low bits */ ;\ + adcl dreg,(TIMER_HIGH+(offset))(treg) /* add carry high bits */ ;\ + movl areg,(TIMER_LOW+(offset))(treg) /* store updated low bit */ ;\ + movl (TIMER_HIGH+(offset))(treg),dreg /* copy high bits */ ;\ + movl dreg,(TIMER_HIGHCHK+(offset))(treg) /* to high check */ /* * Add time delta to old timer and start new. */ -#define TIMER_EVENT(old,new) \ - pushl %eax /* must be invariant */ ;\ - cli /* block interrupts */ ;\ - NANOTIME32 /* eax low bits nanosecs */ ;\ - movl %gs:CPU_PROCESSOR,%ecx /* get current processor */ ;\ - movl CURRENT_TIMER(%ecx),%ecx /* get current timer */ ;\ - movl %eax,%edx /* save timestamp in %edx */ ;\ - subl TIMER_TSTAMP(%ecx),%eax /* compute elapsed time */ ;\ - TIMER_UPDATE(%ecx,%eax) /* update timer struct */ ;\ - addl $(new##_TIMER-old##_TIMER),%ecx /* point to new timer */ ;\ - movl %edx,TIMER_TSTAMP(%ecx) /* set timestamp */ ;\ - movl %gs:CPU_PROCESSOR,%edx /* get current processor */ ;\ - movl %ecx,CURRENT_TIMER(%edx) /* set current timer */ ;\ - sti /* interrupts on */ ;\ - popl %eax /* must be invariant */ +#define TIMER_EVENT(old,new) \ + NANOTIME /* edx:eax nanosecs */ ; \ + movl %eax,%esi /* save timestamp */ ; \ + movl %edx,%edi /* save timestamp */ ; \ + movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */ ; \ + subl (old##_TIMER)+TIMER_TSTAMP(%ecx),%eax /* compute elapsed time */ ; \ + sbbl (old##_TIMER)+TIMER_TSTAMP+4(%ecx),%edx /* compute elapsed time */ ; \ + TIMER_UPDATE(%ecx,%edx,%eax,old##_TIMER) /* update timer */ ; \ + movl %esi,(new##_TIMER)+TIMER_TSTAMP(%ecx) /* set timestamp */ ; \ + movl %edi,(new##_TIMER)+TIMER_TSTAMP+4(%ecx) /* set timestamp */ ; \ + leal (new##_TIMER)(%ecx), %ecx /* compute new timer pointer */ ; \ + movl %gs:CPU_PROCESSOR,%ebx /* get current processor */ ; \ + movl %ecx,THREAD_TIMER(%ebx) /* set current timer */ ; \ + movl %esi,%eax /* restore timestamp */ ; \ + movl %edi,%edx /* restore timestamp */ ; \ + subl (old##_STATE)+TIMER_TSTAMP(%ebx),%eax /* compute elapsed time */ ; \ + sbbl (old##_STATE)+TIMER_TSTAMP+4(%ebx),%edx /* compute elapsed time */ ; \ + TIMER_UPDATE(%ebx,%edx,%eax,old##_STATE) /* update timer */ ; \ + leal (new##_STATE)(%ebx),%ecx /* compute new state pointer */ ; \ + movl %ecx,CURRENT_STATE(%ebx) /* set current state */ ; \ + movl %esi,TIMER_TSTAMP(%ecx) /* set timestamp */ ; \ + movl %edi,TIMER_TSTAMP+4(%ecx) /* set timestamp */ /* * Update time on user trap entry. - * Uses %ecx,%edx. + * Uses %eax,%ebx,%ecx,%edx,%esi,%edi. */ -#define TIME_TRAP_UENTRY TIMER_EVENT(USER,SYSTEM) +#define TIME_TRAP_UENTRY TIMER_EVENT(USER,SYSTEM) /* * update time on user trap exit. - * Uses %ecx,%edx. + * Uses %eax,%ebx,%ecx,%edx,%esi,%edi. */ -#define TIME_TRAP_UEXIT TIMER_EVENT(SYSTEM,USER) +#define TIME_TRAP_UEXIT TIMER_EVENT(SYSTEM,USER) /* * update time on interrupt entry. - * Uses %eax,%ecx,%edx. - */ -#define TIME_INT_ENTRY \ - NANOTIME32 /* eax low bits nanosecs */ ;\ - movl %gs:CPU_PROCESSOR,%ecx /* get current processor */ ;\ - movl CURRENT_TIMER(%ecx),%ecx /* get current timer */ ;\ - movl %eax,%edx /* save timestamp in %edx */ ;\ - subl TIMER_TSTAMP(%ecx),%eax /* compute elapsed time */ ;\ - TIMER_UPDATE(%ecx,%eax) /* update timer struct */ ;\ - movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */ ;\ - addl $(SYSTEM_TIMER),%ecx /* point to sys timer */ ;\ - movl %edx,TIMER_TSTAMP(%ecx) /* set timestamp */ + * Uses %eax,%ebx,%ecx,%edx,%esi,%edi. + * Saves processor state info on stack. + */ +#define TIME_INT_ENTRY \ + NANOTIME /* edx:eax nanosecs */ ; \ + movl %eax,%gs:CPU_INT_EVENT_TIME /* save in cpu data */ ; \ + movl %edx,%gs:CPU_INT_EVENT_TIME+4 /* save in cpu data */ ; \ + movl %eax,%esi /* save timestamp */ ; \ + movl %edx,%edi /* save timestamp */ ; \ + movl %gs:CPU_PROCESSOR,%ebx /* get current processor */ ; \ + movl THREAD_TIMER(%ebx),%ecx /* get current timer */ ; \ + subl TIMER_TSTAMP(%ecx),%eax /* compute elapsed time */ ; \ + sbbl TIMER_TSTAMP+4(%ecx),%edx /* compute elapsed time */ ; \ + TIMER_UPDATE(%ecx,%edx,%eax,0) /* update timer */ ; \ + movl KERNEL_TIMER(%ebx),%ecx /* point to kernel timer */ ; \ + movl %esi,TIMER_TSTAMP(%ecx) /* set timestamp */ ; \ + movl %edi,TIMER_TSTAMP+4(%ecx) /* set timestamp */ ; \ + movl %esi,%eax /* restore timestamp */ ; \ + movl %edi,%edx /* restore timestamp */ ; \ + movl CURRENT_STATE(%ebx),%ecx /* get current state */ ; \ + pushl %ecx /* save state */ ; \ + subl TIMER_TSTAMP(%ecx),%eax /* compute elapsed time */ ; \ + sbbl TIMER_TSTAMP+4(%ecx),%edx /* compute elapsed time */ ; \ + TIMER_UPDATE(%ecx,%edx,%eax,0) /* update timer */ ; \ + leal IDLE_STATE(%ebx),%eax /* get idle state */ ; \ + cmpl %eax,%ecx /* compare current state */ ; \ + je 0f /* skip if equal */ ; \ + leal SYSTEM_STATE(%ebx),%ecx /* get system state */ ; \ + movl %ecx,CURRENT_STATE(%ebx) /* set current state */ ; \ +0: movl %esi,TIMER_TSTAMP(%ecx) /* set timestamp */ ; \ + movl %edi,TIMER_TSTAMP+4(%ecx) /* set timestamp */ /* * update time on interrupt exit. - * Uses %eax, %ecx, %edx. - */ -#define TIME_INT_EXIT \ - NANOTIME32 /* eax low bits nanosecs */ ;\ - movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */ ;\ - addl $(SYSTEM_TIMER),%ecx /* point to sys timer */ ;\ - movl %eax,%edx /* save timestamp in %edx */ ;\ - subl TIMER_TSTAMP(%ecx),%eax /* compute elapsed time */ ;\ - TIMER_UPDATE(%ecx,%eax) /* update timer struct */ ;\ - movl %gs:CPU_PROCESSOR,%ecx /* get current processor */ ;\ - movl CURRENT_TIMER(%ecx),%ecx /* interrupted timer */ ;\ - movl %edx,TIMER_TSTAMP(%ecx) /* set timestamp */ + * Uses %eax,%ebx,%ecx,%edx,%esi,%edi. + * Restores processor state info from stack. + */ +#define TIME_INT_EXIT \ + NANOTIME /* edx:eax nanosecs */ ; \ + movl %eax,%gs:CPU_INT_EVENT_TIME /* save in cpu data */ ; \ + movl %edx,%gs:CPU_INT_EVENT_TIME+4 /* save in cpu data */ ; \ + movl %eax,%esi /* save timestamp */ ; \ + movl %edx,%edi /* save timestamp */ ; \ + movl %gs:CPU_PROCESSOR,%ebx /* get current processor */ ; \ + movl KERNEL_TIMER(%ebx),%ecx /* point to kernel timer */ ; \ + subl TIMER_TSTAMP(%ecx),%eax /* compute elapsed time */ ; \ + sbbl TIMER_TSTAMP+4(%ecx),%edx /* compute elapsed time */ ; \ + TIMER_UPDATE(%ecx,%edx,%eax,0) /* update timer */ ; \ + movl THREAD_TIMER(%ebx),%ecx /* interrupted timer */ ; \ + movl %esi,TIMER_TSTAMP(%ecx) /* set timestamp */ ; \ + movl %edi,TIMER_TSTAMP+4(%ecx) /* set timestamp */ ; \ + movl %esi,%eax /* restore timestamp */ ; \ + movl %edi,%edx /* restore timestamp */ ; \ + movl CURRENT_STATE(%ebx),%ecx /* get current state */ ; \ + subl TIMER_TSTAMP(%ecx),%eax /* compute elapsed time */ ; \ + sbbl TIMER_TSTAMP+4(%ecx),%edx /* compute elapsed time */ ; \ + TIMER_UPDATE(%ecx,%edx,%eax,0) /* update timer */ ; \ + popl %ecx /* restore state */ ; \ + movl %ecx,CURRENT_STATE(%ebx) /* set current state */ ; \ + movl %esi,TIMER_TSTAMP(%ecx) /* set timestamp */ ; \ + movl %edi,TIMER_TSTAMP+4(%ecx) /* set timestamp */ #endif /* STAT_TIME */ -/* - * Encapsulate the transfer of exception stack frames between a PCB - * and a thread stack. Since the whole point of these is to emulate - * a call or exception that changes privilege level, both macros - * assume that there is no user esp or ss stored in the source - * frame (because there was no change of privilege to generate them). - */ - -/* - * Transfer a stack frame from a thread's user stack to its PCB. - * We assume the thread and stack addresses have been loaded into - * registers (our arguments). - * - * The macro overwrites edi, esi, ecx and whatever registers hold the - * thread and stack addresses (which can't be one of the above three). - * The thread address is overwritten with the address of its saved state - * (where the frame winds up). - * - * Must be called on kernel stack. - */ -#define FRAME_STACK_TO_PCB(thread, stkp) ;\ - movl ACT_PCB(thread),thread /* get act`s PCB */ ;\ - leal PCB_ISS(thread),%edi /* point to PCB`s saved state */;\ - movl %edi,thread /* save for later */ ;\ - movl stkp,%esi /* point to start of frame */ ;\ - movl $ R_UESP,%ecx ;\ - sarl $2,%ecx /* word count for transfer */ ;\ - cld /* we`re incrementing */ ;\ - rep ;\ - movsl /* transfer the frame */ ;\ - addl $ R_UESP,stkp /* derive true "user" esp */ ;\ - movl stkp,R_UESP(thread) /* store in PCB */ ;\ - movl $0,%ecx ;\ - mov %ss,%cx /* get current ss */ ;\ - movl %ecx,R_SS(thread) /* store in PCB */ - -/* - * Transfer a stack frame from a thread's PCB to the stack pointed - * to by the PCB. We assume the thread address has been loaded into - * a register (our argument). - * - * The macro overwrites edi, esi, ecx and whatever register holds the - * thread address (which can't be one of the above three). The - * thread address is overwritten with the address of its saved state - * (where the frame winds up). - * - * Must be called on kernel stack. - */ -#define FRAME_PCB_TO_STACK(thread) ;\ - movl ACT_PCB(thread),%esi /* get act`s PCB */ ;\ - leal PCB_ISS(%esi),%esi /* point to PCB`s saved state */;\ - movl R_UESP(%esi),%edi /* point to end of dest frame */;\ - movl ACT_MAP(thread),%ecx /* get act's map */ ;\ - movl MAP_PMAP(%ecx),%ecx /* get map's pmap */ ;\ - cmpl EXT(kernel_pmap), %ecx /* If kernel loaded task */ ;\ - jz 1f /* use kernel data segment */ ;\ - movl $ USER_DS,%cx /* else use user data segment */;\ - mov %cx,%es ;\ -1: ;\ - movl $ R_UESP,%ecx ;\ - subl %ecx,%edi /* derive start of frame */ ;\ - movl %edi,thread /* save for later */ ;\ - sarl $2,%ecx /* word count for transfer */ ;\ - cld /* we`re incrementing */ ;\ - rep ;\ - movsl /* transfer the frame */ ;\ - mov %ss,%cx /* restore kernel segments */ ;\ - mov %cx,%es - #undef PDEBUG #ifdef PDEBUG @@ -391,7 +402,7 @@ label/**/exit: #define CAH(label) #endif /* PDEBUG */ - + #if MACH_KDB /* * Last-ditch debug code to handle faults that might result @@ -452,13 +463,13 @@ Entry(db_task_gen_prot) */ Entry(db_task_start) movl %esp,%edx - subl $ISS_SIZE,%edx - movl %edx,%esp /* allocate i386_saved_state on stack */ - movl %eax,R_ERR(%esp) - movl %ebx,R_TRAPNO(%esp) + subl $(ISS32_SIZE),%edx + movl %edx,%esp /* allocate x86_saved_state on stack */ + movl %eax,R32_ERR(%esp) + movl %ebx,R32_TRAPNO(%esp) pushl %edx CPU_NUMBER(%edx) - movl CX(EXT(mp_dbtss),%edx),%edx + movl CX(EXT(master_dbtss),%edx),%edx movl TSS_LINK(%edx),%eax pushl %eax /* pass along selector of previous TSS */ call EXT(db_tss_to_frame) @@ -472,572 +483,211 @@ Entry(db_task_start) #endif /* MACH_KDB */ /* - * Trap/interrupt entry points. - * - * All traps must create the following save area on the PCB "stack": - * - * gs - * fs - * es - * ds - * edi - * esi - * ebp - * cr2 if page fault - otherwise unused - * ebx - * edx - * ecx - * eax - * trap number - * error code - * eip - * cs - * eflags - * user esp - if from user - * user ss - if from user - * es - if from V86 thread - * ds - if from V86 thread - * fs - if from V86 thread - * gs - if from V86 thread - * - */ - -/* - * General protection or segment-not-present fault. - * Check for a GP/NP fault in the kernel_return - * sequence; if there, report it as a GP/NP fault on the user's instruction. - * - * esp-> 0: trap code (NP or GP) - * 4: segment number in error - * 8 eip - * 12 cs - * 16 eflags - * 20 old registers (trap is from kernel) - */ -Entry(t_gen_prot) - pushl $(T_GENERAL_PROTECTION) /* indicate fault type */ - jmp trap_check_kernel_exit /* check for kernel exit sequence */ - -Entry(t_segnp) - pushl $(T_SEGMENT_NOT_PRESENT) - /* indicate fault type */ - -trap_check_kernel_exit: - testl $(EFL_VM),16(%esp) /* is trap from V86 mode? */ - jnz EXT(alltraps) /* isn`t kernel trap if so */ - testl $3,12(%esp) /* is trap from kernel mode? */ - jne EXT(alltraps) /* if so: */ - /* check for the kernel exit sequence */ - cmpl $ EXT(kret_iret),8(%esp) /* on IRET? */ - je fault_iret - cmpl $ EXT(kret_popl_ds),8(%esp) /* popping DS? */ - je fault_popl_ds - cmpl $ EXT(kret_popl_es),8(%esp) /* popping ES? */ - je fault_popl_es - cmpl $ EXT(kret_popl_fs),8(%esp) /* popping FS? */ - je fault_popl_fs - cmpl $ EXT(kret_popl_gs),8(%esp) /* popping GS? */ - je fault_popl_gs -take_fault: /* if none of the above: */ - jmp EXT(alltraps) /* treat as normal trap. */ - -/* - * GP/NP fault on IRET: CS or SS is in error. - * All registers contain the user's values. - * - * on SP is - * 0 trap number - * 4 errcode - * 8 eip - * 12 cs --> trapno - * 16 efl --> errcode - * 20 user eip - * 24 user cs - * 28 user eflags - * 32 user esp - * 36 user ss + * Called as a function, makes the current thread + * return from the kernel as if from an exception. + * We will consult with DTrace if this is a + * newly created thread and we need to fire a probe. */ -fault_iret: - movl %eax,8(%esp) /* save eax (we don`t need saved eip) */ - popl %eax /* get trap number */ - movl %eax,12-4(%esp) /* put in user trap number */ - popl %eax /* get error code */ - movl %eax,16-8(%esp) /* put in user errcode */ - popl %eax /* restore eax */ - CAH(fltir) - jmp EXT(alltraps) /* take fault */ -/* - * Fault restoring a segment register. The user's registers are still - * saved on the stack. The offending segment register has not been - * popped. - */ -fault_popl_ds: - popl %eax /* get trap number */ - popl %edx /* get error code */ - addl $12,%esp /* pop stack to user regs */ - jmp push_es /* (DS on top of stack) */ -fault_popl_es: - popl %eax /* get trap number */ - popl %edx /* get error code */ - addl $12,%esp /* pop stack to user regs */ - jmp push_fs /* (ES on top of stack) */ -fault_popl_fs: - popl %eax /* get trap number */ - popl %edx /* get error code */ - addl $12,%esp /* pop stack to user regs */ - jmp push_gs /* (FS on top of stack) */ -fault_popl_gs: - popl %eax /* get trap number */ - popl %edx /* get error code */ - addl $12,%esp /* pop stack to user regs */ - jmp push_segregs /* (GS on top of stack) */ - -push_es: - pushl %es /* restore es, */ -push_fs: - pushl %fs /* restore fs, */ -push_gs: - pushl %gs /* restore gs. */ -push_segregs: - movl %eax,R_TRAPNO(%esp) /* set trap number */ - movl %edx,R_ERR(%esp) /* set error code */ - CAH(fltpp) - jmp trap_set_segs /* take trap */ + .globl EXT(thread_exception_return) + .globl EXT(thread_bootstrap_return) +LEXT(thread_bootstrap_return) +#if CONFIG_DTRACE + call EXT(dtrace_thread_bootstrap) +#endif -/* - * Debug trap. Check for single-stepping across system call into - * kernel. If this is the case, taking the debug trap has turned - * off single-stepping - save the flags register with the trace - * bit set. - */ -Entry(t_debug) - testl $(EFL_VM),8(%esp) /* is trap from V86 mode? */ - jnz 0f /* isn`t kernel trap if so */ - testl $3,4(%esp) /* is trap from kernel mode? */ - jnz 0f /* if so: */ - cmpl $syscall_entry,(%esp) /* system call entry? */ - jne 1f /* if so: */ - /* flags are sitting where syscall */ - /* wants them */ - addl $8,%esp /* remove eip/cs */ - jmp syscall_entry_2 /* continue system call entry */ - -1: cmpl $trap_unix_addr,(%esp) - jne 0f - addl $8,%esp - jmp trap_unix_2 - -0: pushl $0 /* otherwise: */ - pushl $(T_DEBUG) /* handle as normal */ - jmp EXT(alltraps) /* debug fault */ +LEXT(thread_exception_return) + CLI + movl %gs:CPU_KERNEL_STACK,%ecx -/* - * Page fault traps save cr2. - */ -Entry(t_page_fault) - pushl $(T_PAGE_FAULT) /* mark a page fault trap */ - pusha /* save the general registers */ - movl %cr2,%eax /* get the faulting address */ - movl %eax,12(%esp) /* save in esp save slot */ - jmp trap_push_segs /* continue fault */ + movl (%ecx),%esp /* switch back to PCB stack */ + xorl %ecx,%ecx /* don't check if we're in the PFZ */ + jmp EXT(return_from_trap) -/* - * All 'exceptions' enter here with: - * esp-> trap number - * error code - * old eip - * old cs - * old eflags - * old esp if trapped from user - * old ss if trapped from user +Entry(call_continuation) + movl S_ARG0,%eax /* get continuation */ + movl S_ARG1,%edx /* continuation param */ + movl S_ARG2,%ecx /* wait result */ + movl %gs:CPU_KERNEL_STACK,%esp /* pop the stack */ + xorl %ebp,%ebp /* zero frame pointer */ + subl $8,%esp /* align the stack */ + pushl %ecx + pushl %edx + call *%eax /* call continuation */ + addl $16,%esp + movl %gs:CPU_ACTIVE_THREAD,%eax + pushl %eax + call EXT(thread_terminate) + + + +/******************************************************************************************************* * - * NB: below use of CPU_NUMBER assumes that macro will use correct - * segment register for any kernel data accesses. - */ -Entry(alltraps) - pusha /* save the general registers */ -trap_push_segs: - pushl %ds /* save the segment registers */ - pushl %es - pushl %fs - pushl %gs - -trap_set_segs: - movl %ss,%ax - movl %ax,%ds - movl %ax,%es /* switch to kernel data seg */ - cld /* clear direction flag */ - testl $(EFL_VM),R_EFLAGS(%esp) /* in V86 mode? */ - jnz trap_from_user /* user mode trap if so */ - testb $3,R_CS(%esp) /* user mode trap? */ - jnz trap_from_user - cmpl $0,%gs:CPU_ACTIVE_KLOADED - je trap_from_kernel /* if clear, truly in kernel */ -#ifdef FIXME - cmpl ETEXT_ADDR,R_EIP(%esp) /* pc within kernel? */ - jb trap_from_kernel -#endif -trap_from_kloaded: - /* - * We didn't enter here "through" PCB (i.e., using ring 0 stack), - * so transfer the stack frame into the PCB explicitly, then - * start running on resulting "PCB stack". We have to set - * up a simulated "uesp" manually, since there's none in the - * frame. - */ - mov $ CPU_DATA_GS,%dx - mov %dx,%gs - CAH(atstart) - movl %gs:CPU_ACTIVE_KLOADED,%ebx - movl %gs:CPU_KERNEL_STACK,%eax - xchgl %esp,%eax - FRAME_STACK_TO_PCB(%ebx,%eax) - CAH(atend) - jmp EXT(take_trap) + * All 64 bit task 'exceptions' enter lo_alltraps: + * esp -> x86_saved_state_t + * + * The rest of the state is set up as: + * cr3 -> kernel directory + * esp -> low based stack + * gs -> CPU_DATA_GS + * cs -> KERNEL32_CS + * ss/ds/es -> KERNEL_DS + * + * interrupts disabled + * direction flag cleared + */ +Entry(lo_alltraps) + movl R32_CS(%esp),%eax /* assume 32-bit state */ + cmpl $(SS_64),SS_FLAVOR(%esp)/* 64-bit? */ + jne 1f + movl R64_CS(%esp),%eax /* 64-bit user mode */ +1: + testb $3,%al + jz trap_from_kernel + /* user mode trap */ + TIME_TRAP_UENTRY -trap_from_user: - mov $ CPU_DATA_GS,%ax - mov %ax,%gs + movl %gs:CPU_ACTIVE_THREAD,%ecx + movl ACT_TASK(%ecx),%ebx - TIME_TRAP_UENTRY + /* Check for active vtimers in the current task */ + cmpl $0,TASK_VTIMERS(%ebx) + jz 1f + + /* Set a pending AST */ + orl $(AST_BSD),%gs:CPU_PENDING_AST + /* Set a thread AST (atomic) */ + lock + orl $(AST_BSD),ACT_AST(%ecx) + +1: movl %gs:CPU_KERNEL_STACK,%ebx - xchgl %ebx,%esp /* switch to kernel stack */ - /* user regs pointer already set */ -LEXT(take_trap) - pushl %ebx /* record register save area */ - pushl %ebx /* pass register save area to trap */ - call EXT(user_trap) /* call user trap routine */ - movl 4(%esp),%esp /* switch back to PCB stack */ + xchgl %ebx,%esp /* switch to kernel stack */ + sti + CCALL1(user_trap, %ebx) /* call user trap routine */ + cli /* hold off intrs - critical section */ + popl %esp /* switch back to PCB stack */ + xorl %ecx,%ecx /* don't check if we're in the PFZ */ + /* * Return from trap or system call, checking for ASTs. - * On PCB stack. - */ - + * On lowbase PCB stack with intrs disabled + */ LEXT(return_from_trap) - movl %gs:CPU_PENDING_AST,%edx - cmpl $0,%edx - je EXT(return_to_user) /* if we need an AST: */ - - movl %gs:CPU_KERNEL_STACK,%esp - /* switch to kernel stack */ - pushl $0 /* push preemption flag */ - call EXT(i386_astintr) /* take the AST */ - addl $4,%esp /* pop preemption flag */ + movl %gs:CPU_PENDING_AST, %eax + testl %eax, %eax + je EXT(return_to_user) /* branch if no AST */ + +LEXT(return_from_trap_with_ast) + movl %gs:CPU_KERNEL_STACK, %ebx + xchgl %ebx, %esp /* switch to kernel stack */ + + testl %ecx, %ecx /* see if we need to check for an EIP in the PFZ */ + je 2f /* no, go handle the AST */ + cmpl $(SS_64), SS_FLAVOR(%ebx) /* are we a 64-bit task? */ + je 1f + /* no... 32-bit user mode */ + movl R32_EIP(%ebx), %eax + pushl %ebx /* save PCB stack */ + xorl %ebp, %ebp /* clear frame pointer */ + CCALL1(commpage_is_in_pfz32, %eax) + popl %ebx /* retrieve pointer to PCB stack */ + testl %eax, %eax + je 2f /* not in the PFZ... go service AST */ + movl %eax, R32_EBX(%ebx) /* let the PFZ know we've pended an AST */ + xchgl %ebx, %esp /* switch back to PCB stack */ + jmp EXT(return_to_user) +1: /* 64-bit user mode */ + movl R64_RIP(%ebx), %ecx + movl R64_RIP+4(%ebx), %eax + pushl %ebx /* save PCB stack */ + xorl %ebp, %ebp /* clear frame pointer */ + CCALL2(commpage_is_in_pfz64, %ecx, %eax) + popl %ebx /* retrieve pointer to PCB stack */ + testl %eax, %eax + je 2f /* not in the PFZ... go service AST */ + movl %eax, R64_RBX(%ebx) /* let the PFZ know we've pended an AST */ + xchgl %ebx, %esp /* switch back to PCB stack */ + jmp EXT(return_to_user) +2: + STI /* interrupts always enabled on return to user mode */ + pushl %ebx /* save PCB stack */ + xorl %ebp, %ebp /* Clear framepointer */ + CCALL1(i386_astintr, $0) /* take the AST */ + CLI + popl %esp /* switch back to PCB stack (w/exc link) */ + + xorl %ecx, %ecx /* don't check if we're in the PFZ */ jmp EXT(return_from_trap) /* and check again (rare) */ - /* ASTs after this point will */ - /* have to wait */ -/* - * Arrange the checks needed for kernel-loaded (or kernel-loading) - * threads so that branch is taken in kernel-loaded case. - */ LEXT(return_to_user) TIME_TRAP_UEXIT - cmpl $0,%gs:CPU_ACTIVE_KLOADED - jnz EXT(return_xfer_stack) - movl %gs:CPU_ACTIVE_THREAD, %ebx /* get active thread */ - -#if MACH_RT -#if MACH_ASSERT - cmpl $0,%gs:CPU_PREEMPTION_LEVEL - je EXT(return_from_kernel) - int $3 -#endif /* MACH_ASSERT */ -#endif /* MACH_RT */ - -/* - * Return from kernel mode to interrupted thread. - */ - -LEXT(return_from_kernel) -LEXT(kret_popl_gs) - popl %gs /* restore segment registers */ -LEXT(kret_popl_fs) - popl %fs -LEXT(kret_popl_es) - popl %es -LEXT(kret_popl_ds) - popl %ds - popa /* restore general registers */ - addl $8,%esp /* discard trap number and error code */ - -LEXT(kret_iret) - iret /* return from interrupt */ - + +LEXT(ret_to_user) + cmpl $0, %gs:CPU_IS64BIT + je EXT(lo_ret_to_user) + jmp EXT(lo64_ret_to_user) -LEXT(return_xfer_stack) - /* - * If we're on PCB stack in a kernel-loaded task, we have - * to transfer saved state back to thread stack and swap - * stack pointers here, because the hardware's not going - * to do so for us. - */ - CAH(rxsstart) - movl %gs:CPU_KERNEL_STACK,%esp - movl %gs:CPU_ACTIVE_KLOADED,%eax - FRAME_PCB_TO_STACK(%eax) - movl %eax,%esp - CAH(rxsend) - jmp EXT(return_from_kernel) -/* - * Hate to put this here, but setting up a separate swap_func for - * kernel-loaded threads no longer works, since thread executes - * "for a while" (i.e., until it reaches glue code) when first - * created, even if it's nominally suspended. Hence we can't - * transfer the PCB when the thread first resumes, because we - * haven't initialized it yet. - */ -/* - * Have to force transfer to new stack "manually". Use a string - * move to transfer all of our saved state to the stack pointed - * to by iss.uesp, then install a pointer to it as our current - * stack pointer. - */ -LEXT(return_kernel_loading) - movl %gs:CPU_KERNEL_STACK,%esp - movl %gs:CPU_ACTIVE_THREAD, %ebx /* get active thread */ - movl %ebx,%edx /* save for later */ - FRAME_PCB_TO_STACK(%ebx) - movl %ebx,%esp /* start running on new stack */ - movl $0,%gs:CPU_ACTIVE_KLOADED /* set cached indicator */ - jmp EXT(return_from_kernel) /* - * Trap from kernel mode. No need to switch stacks or load segment registers. + * Trap from kernel mode. No need to switch stacks. + * Interrupts must be off here - we will set them to state at time of trap + * as soon as it's safe for us to do so and not recurse doing preemption */ trap_from_kernel: -#if MACH_KDB || MACH_KGDB - mov $ CPU_DATA_GS,%ax - mov %ax,%gs - movl %esp,%ebx /* save current stack */ - - cmpl EXT(int_stack_high),%esp /* on an interrupt stack? */ - jb 6f /* OK if so */ - -#if MACH_KGDB - cmpl $0,EXT(kgdb_active) /* Unexpected trap in kgdb */ - je 0f /* no */ - - pushl %esp /* Already on kgdb stack */ - cli - call EXT(kgdb_trap) - addl $4,%esp - jmp EXT(return_from_kernel) -0: /* should kgdb handle this exception? */ - cmpl $(T_NO_FPU),R_TRAPNO(%esp) /* FPU disabled? */ - je 2f /* yes */ - cmpl $(T_PAGE_FAULT),R_TRAPNO(%esp) /* page fault? */ - je 2f /* yes */ -1: - cli /* disable interrupts */ - CPU_NUMBER(%edx) /* get CPU number */ - movl CX(EXT(kgdb_stacks),%edx),%ebx - xchgl %ebx,%esp /* switch to kgdb stack */ - pushl %ebx /* pass old sp as an arg */ - call EXT(kgdb_from_kernel) - popl %esp /* switch back to kernel stack */ - jmp EXT(return_from_kernel) -2: -#endif /* MACH_KGDB */ - -#if MACH_KDB - cmpl $0,EXT(db_active) /* could trap be from ddb? */ - je 3f /* no */ - CPU_NUMBER(%edx) /* see if this CPU is in ddb */ - cmpl $0,CX(EXT(kdb_active),%edx) - je 3f /* no */ - pushl %esp - call EXT(db_trap_from_asm) - addl $0x4,%esp - jmp EXT(return_from_kernel) - -3: - /* - * Dilemma: don't want to switch to kernel_stack if trap - * "belongs" to ddb; don't want to switch to db_stack if - * trap "belongs" to kernel. So have to duplicate here the - * set of trap types that kernel_trap() handles. Note that - * "unexpected" page faults will not be handled by kernel_trap(). - * In this panic-worthy case, we fall into the debugger with - * kernel_stack containing the call chain that led to the - * bogus fault. - */ - movl R_TRAPNO(%esp),%edx - cmpl $(T_PAGE_FAULT),%edx - je 4f - cmpl $(T_NO_FPU),%edx - je 4f - cmpl $(T_FPU_FAULT),%edx - je 4f - cmpl $(T_FLOATING_POINT_ERROR),%edx - je 4f - cmpl $(T_PREEMPT),%edx - jne 7f -4: -#endif /* MACH_KDB */ + movl %esp, %eax /* saved state addr */ + pushl R32_EIP(%esp) /* Simulate a CALL from fault point */ + pushl %ebp /* Extend framepointer chain */ + movl %esp, %ebp + CCALL1(kernel_trap, %eax) /* Call kernel trap handler */ + popl %ebp + addl $4, %esp + cli - cmpl %gs:CPU_KERNEL_STACK,%esp - /* if not already on kernel stack, */ - ja 5f /* check some more */ - cmpl %gs:CPU_ACTIVE_STACK,%esp - ja 6f /* on kernel stack: no switch */ -5: - movl %gs:CPU_KERNEL_STACK,%esp -6: - pushl %ebx /* save old stack */ - pushl %ebx /* pass as parameter */ - call EXT(kernel_trap) /* to kernel trap routine */ - addl $4,%esp /* pop parameter */ - testl %eax,%eax - jne 8f - /* - * If kernel_trap returns false, trap wasn't handled. - */ -7: -#if MACH_KDB - CPU_NUMBER(%edx) - movl CX(EXT(db_stacks),%edx),%esp - pushl %ebx /* pass old stack as parameter */ - call EXT(db_trap_from_asm) -#endif /* MACH_KDB */ -#if MACH_KGDB - cli /* disable interrupts */ - CPU_NUMBER(%edx) /* get CPU number */ - movl CX(EXT(kgdb_stacks),%edx),%esp - pushl %ebx /* pass old stack as parameter */ - call EXT(kgdb_from_kernel) -#endif /* MACH_KGDB */ - addl $4,%esp /* pop parameter */ - testl %eax,%eax - jne 8f - /* - * Likewise, if kdb_trap/kgdb_from_kernel returns false, trap - * wasn't handled. - */ - pushl %ebx /* pass old stack as parameter */ - call EXT(panic_trap) - addl $4,%esp /* pop parameter */ -8: - movl %ebx,%esp /* get old stack (from callee-saves reg) */ -#else /* MACH_KDB || MACH_KGDB */ - pushl %esp /* pass parameter */ - call EXT(kernel_trap) /* to kernel trap routine */ - addl $4,%esp /* pop parameter */ -#endif /* MACH_KDB || MACH_KGDB */ - -#if MACH_RT movl %gs:CPU_PENDING_AST,%eax /* get pending asts */ testl $ AST_URGENT,%eax /* any urgent preemption? */ - je EXT(return_from_kernel) /* no, nothing to do */ - cmpl $ T_PREEMPT,48(%esp) /* preempt request? */ - jne EXT(return_from_kernel) /* no, nothing to do */ + je ret_to_kernel /* no, nothing to do */ + cmpl $ T_PREEMPT,R32_TRAPNO(%esp) + je ret_to_kernel /* T_PREEMPT handled in kernel_trap() */ + testl $ EFL_IF,R32_EFLAGS(%esp) /* interrupts disabled? */ + je ret_to_kernel + cmpl $0,%gs:CPU_PREEMPTION_LEVEL /* preemption disabled? */ + jne ret_to_kernel movl %gs:CPU_KERNEL_STACK,%eax movl %esp,%ecx xorl %eax,%ecx - andl $(-KERNEL_STACK_SIZE),%ecx + and EXT(kernel_stack_mask),%ecx testl %ecx,%ecx /* are we on the kernel stack? */ - jne EXT(return_from_kernel) /* no, skip it */ - -#if PREEMPT_DEBUG_LOG - pushl 28(%esp) /* stack pointer */ - pushl 24+4(%esp) /* frame pointer */ - pushl 56+8(%esp) /* stack pointer */ - pushl $0f - call EXT(log_thread_action) - addl $16, %esp - .data -0: String "trap preempt eip" - .text -#endif /* PREEMPT_DEBUG_LOG */ - - pushl $1 /* push preemption flag */ - call EXT(i386_astintr) /* take the AST */ - addl $4,%esp /* pop preemption flag */ -#endif /* MACH_RT */ - - jmp EXT(return_from_kernel) + jne ret_to_kernel /* no, skip it */ -/* - * Called as a function, makes the current thread - * return from the kernel as if from an exception. - */ + CCALL1(i386_astintr, $1) /* take the AST */ - .globl EXT(thread_exception_return) - .globl EXT(thread_bootstrap_return) -LEXT(thread_exception_return) -LEXT(thread_bootstrap_return) - movl %esp,%ecx /* get kernel stack */ - or $(KERNEL_STACK_SIZE-1),%ecx - movl -3-IKS_SIZE(%ecx),%esp /* switch back to PCB stack */ - jmp EXT(return_from_trap) +ret_to_kernel: + cmpl $0, %gs:CPU_IS64BIT + je EXT(lo_ret_to_kernel) + jmp EXT(lo64_ret_to_kernel) -Entry(call_continuation) - movl S_ARG0,%eax /* get continuation */ - movl S_ARG1,%edx /* continuation param */ - movl S_ARG2,%ecx /* wait result */ - movl %esp,%ebp /* get kernel stack */ - or $(KERNEL_STACK_SIZE-1),%ebp - addl $(-3-IKS_SIZE),%ebp - movl %ebp,%esp /* pop the stack */ - xorl %ebp,%ebp /* zero frame pointer */ - pushl %ecx - pushl %edx - call *%eax /* call continuation */ - addl $8,%esp - movl %gs:CPU_ACTIVE_THREAD,%eax - pushl %eax - call EXT(thread_terminate) -#if 0 -#define LOG_INTERRUPT(info,msg) \ - pushal ; \ - pushl msg ; \ - pushl info ; \ - call EXT(log_thread_action) ; \ - add $8,%esp ; \ - popal -#define CHECK_INTERRUPT_TIME(n) \ - pushal ; \ - pushl $n ; \ - call EXT(check_thread_time) ; \ - add $4,%esp ; \ - popal -#else -#define LOG_INTERRUPT(info,msg) -#define CHECK_INTERRUPT_TIME(n) -#endif - -.data -imsg_start: - String "interrupt start" -imsg_end: - String "interrupt end" - -.text -/* - * All interrupts enter here. - * old %eax on stack; interrupt number in %eax. - */ -Entry(all_intrs) - pushl %ecx /* save registers */ - pushl %edx - cld /* clear direction flag */ - - pushl %ds /* save segment registers */ - pushl %es - pushl %fs - pushl %gs - mov %ss,%dx /* switch to kernel segments */ - mov %dx,%ds - mov %dx,%es - mov $ CPU_DATA_GS,%dx - mov %dx,%gs +/******************************************************************************************************* + * + * All interrupts on all tasks enter here with: + * esp-> -> x86_saved_state_t + * + * cr3 -> kernel directory + * esp -> low based stack + * gs -> CPU_DATA_GS + * cs -> KERNEL32_CS + * ss/ds/es -> KERNEL_DS + * + * interrupts disabled + * direction flag cleared + */ +Entry(lo_allintrs) /* * test whether already on interrupt stack */ @@ -1047,709 +697,442 @@ Entry(all_intrs) leal -INTSTACK_SIZE(%ecx),%edx cmpl %esp,%edx jb int_from_intstack -1: - movl %esp,%edx /* & i386_interrupt_state */ +1: xchgl %ecx,%esp /* switch to interrupt stack */ + movl %cr0,%eax /* get cr0 */ + orl $(CR0_TS),%eax /* or in TS bit */ + movl %eax,%cr0 /* set cr0 */ + + subl $8, %esp /* for 16-byte stack alignment */ pushl %ecx /* save pointer to old stack */ - pushl %edx /* pass &i386_interrupt_state to pe_incoming_interrupt */ - pushl %eax /* push trap number */ + movl %ecx,%gs:CPU_INT_STATE /* save intr state */ TIME_INT_ENTRY /* do timing */ -#if MACH_RT + movl %gs:CPU_ACTIVE_THREAD,%ecx + movl ACT_TASK(%ecx),%ebx + + /* Check for active vtimers in the current task */ + cmpl $0,TASK_VTIMERS(%ebx) + jz 1f + + /* Set a pending AST */ + orl $(AST_BSD),%gs:CPU_PENDING_AST + + /* Set a thread AST (atomic) */ + lock + orl $(AST_BSD),ACT_AST(%ecx) + +1: incl %gs:CPU_PREEMPTION_LEVEL -#endif /* MACH_RT */ incl %gs:CPU_INTERRUPT_LEVEL - call EXT(PE_incoming_interrupt) /* call generic interrupt routine */ - addl $8,%esp /* Pop trap number and eip */ + movl %gs:CPU_INT_STATE, %eax + CCALL1(interrupt, %eax) /* call generic interrupt routine */ - .globl EXT(return_to_iret) -LEXT(return_to_iret) /* (label for kdb_kintr and hardclock) */ + cli /* just in case we returned with intrs enabled */ + xorl %eax,%eax + movl %eax,%gs:CPU_INT_STATE /* clear intr state pointer */ decl %gs:CPU_INTERRUPT_LEVEL - -#if MACH_RT decl %gs:CPU_PREEMPTION_LEVEL -#endif /* MACH_RT */ TIME_INT_EXIT /* do timing */ + movl %gs:CPU_ACTIVE_THREAD,%eax + movl ACT_PCB(%eax),%eax /* get act`s PCB */ + movl PCB_FPS(%eax),%eax /* get pcb's ims.ifps */ + cmpl $0,%eax /* Is there a context */ + je 1f /* Branch if not */ + movl FP_VALID(%eax),%eax /* Load fp_valid */ + cmpl $0,%eax /* Check if valid */ + jne 1f /* Branch if valid */ + clts /* Clear TS */ + jmp 2f +1: + movl %cr0,%eax /* get cr0 */ + orl $(CR0_TS),%eax /* or in TS bit */ + movl %eax,%cr0 /* set cr0 */ +2: popl %esp /* switch back to old stack */ - movl %gs:CPU_PENDING_AST,%eax - testl %eax,%eax /* any pending asts? */ - je 1f /* no, nothing to do */ - testl $(EFL_VM),I_EFL(%esp) /* if in V86 */ - jnz ast_from_interrupt /* take it */ - testb $3,I_CS(%esp) /* user mode, */ - jnz ast_from_interrupt /* take it */ -#ifdef FIXME - cmpl ETEXT_ADDR,I_EIP(%esp) /* if within kernel-loaded task, */ - jnb ast_from_interrupt /* take it */ -#endif + /* Load interrupted code segment into %eax */ + movl R32_CS(%esp),%eax /* assume 32-bit state */ + cmpl $(SS_64),SS_FLAVOR(%esp)/* 64-bit? */ + jne 3f + movl R64_CS(%esp),%eax /* 64-bit user mode */ +3: + testb $3,%al /* user mode, */ + jnz ast_from_interrupt_user /* go handle potential ASTs */ + /* + * we only want to handle preemption requests if + * the interrupt fell in the kernel context + * and preemption isn't disabled + */ + movl %gs:CPU_PENDING_AST,%eax + testl $ AST_URGENT,%eax /* any urgent requests? */ + je ret_to_kernel /* no, nothing to do */ + + cmpl $0,%gs:CPU_PREEMPTION_LEVEL /* preemption disabled? */ + jne ret_to_kernel /* yes, skip it */ -#if MACH_RT - cmpl $0,%gs:CPU_PREEMPTION_LEVEL /* preemption masked? */ - jne 1f /* yes, skip it */ - testl $ AST_URGENT,%eax /* any urgent requests? */ - je 1f /* no, skip it */ - cmpl $ EXT(locore_end),I_EIP(%esp) /* are we in locore code? */ - jb 1f /* yes, skip it */ movl %gs:CPU_KERNEL_STACK,%eax movl %esp,%ecx xorl %eax,%ecx - andl $(-KERNEL_STACK_SIZE),%ecx - testl %ecx,%ecx /* are we on the kernel stack? */ - jne 1f /* no, skip it */ + and EXT(kernel_stack_mask),%ecx + testl %ecx,%ecx /* are we on the kernel stack? */ + jne ret_to_kernel /* no, skip it */ -/* - * Take an AST from kernel space. We don't need (and don't want) - * to do as much as the case where the interrupt came from user - * space. - */ -#if PREEMPT_DEBUG_LOG - pushl $0 - pushl $0 - pushl I_EIP+8(%esp) - pushl $0f - call EXT(log_thread_action) - addl $16, %esp - .data -0: String "intr preempt eip" - .text -#endif /* PREEMPT_DEBUG_LOG */ + /* + * Take an AST from kernel space. We don't need (and don't want) + * to do as much as the case where the interrupt came from user + * space. + */ + CCALL1(i386_astintr, $1) - sti - pushl $1 /* push preemption flag */ - call EXT(i386_astintr) /* take the AST */ - addl $4,%esp /* pop preemption flag */ -#endif /* MACH_RT */ + jmp ret_to_kernel -1: - pop %gs - pop %fs - pop %es /* restore segment regs */ - pop %ds - pop %edx - pop %ecx - pop %eax - iret /* return to caller */ +/* + * nested int - simple path, can't preempt etc on way out + */ int_from_intstack: -#if MACH_RT incl %gs:CPU_PREEMPTION_LEVEL -#endif /* MACH_RT */ - incl %gs:CPU_INTERRUPT_LEVEL - movl %esp, %edx /* i386_interrupt_state */ - pushl %edx /* pass &i386_interrupt_state to PE_incoming_interrupt /* - - pushl %eax /* Push trap number */ - - call EXT(PE_incoming_interrupt) - addl $20,%esp /* pop i386_interrupt_state, gs,fs,es,ds */ - -LEXT(return_to_iret_i) /* ( label for kdb_kintr) */ - - addl $4,%esp /* pop trap number */ + movl %esp, %edx /* x86_saved_state */ + CCALL1(interrupt, %edx) decl %gs:CPU_INTERRUPT_LEVEL - -#if MACH_RT decl %gs:CPU_PREEMPTION_LEVEL -#endif /* MACH_RT */ - pop %edx /* must have been on kernel segs */ - pop %ecx - pop %eax /* no ASTs */ - iret + jmp ret_to_kernel /* - * Take an AST from an interrupt. - * On PCB stack. - * sp-> es -> edx - * ds -> ecx - * edx -> eax - * ecx -> trapno - * eax -> code - * eip - * cs - * efl - * esp - * ss + * Take an AST from an interrupted user */ -ast_from_interrupt: - pop %gs - pop %fs - pop %es /* restore all registers ... */ - pop %ds - popl %edx - popl %ecx - popl %eax - sti /* Reenable interrupts */ - pushl $0 /* zero code */ - pushl $0 /* zero trap number */ - pusha /* save general registers */ - push %ds /* save segment registers */ - push %es - push %fs - push %gs - mov %ss,%dx /* switch to kernel segments */ - mov %dx,%ds - mov %dx,%es - mov $ CPU_DATA_GS,%dx - mov %dx,%gs - - /* - * See if we interrupted a kernel-loaded thread executing - * in its own task. - */ - CPU_NUMBER(%edx) - testl $(EFL_VM),R_EFLAGS(%esp) /* in V86 mode? */ - jnz 0f /* user mode trap if so */ - testb $3,R_CS(%esp) - jnz 0f /* user mode, back to normal */ -#ifdef FIXME - cmpl ETEXT_ADDR,R_EIP(%esp) - jb 0f /* not kernel-loaded, back to normal */ -#endif +ast_from_interrupt_user: + movl %gs:CPU_PENDING_AST,%eax + testl %eax,%eax /* pending ASTs? */ + je EXT(ret_to_user) /* no, nothing to do */ - /* - * Transfer the current stack frame by hand into the PCB. - */ - CAH(afistart) - movl %gs:CPU_ACTIVE_KLOADED,%eax - movl %gs:CPU_KERNEL_STACK,%ebx - xchgl %ebx,%esp - FRAME_STACK_TO_PCB(%eax,%ebx) - CAH(afiend) - TIME_TRAP_UENTRY - jmp 3f -0: TIME_TRAP_UENTRY - movl %gs:CPU_KERNEL_STACK,%eax - /* switch to kernel stack */ - xchgl %eax,%esp -3: - pushl %eax - pushl $0 /* push preemption flag */ - call EXT(i386_astintr) /* take the AST */ - addl $4,%esp /* pop preemption flag */ - popl %esp /* back to PCB stack */ - jmp EXT(return_from_trap) /* return */ + movl $1, %ecx /* check if we're in the PFZ */ + jmp EXT(return_from_trap_with_ast) /* return */ -#if MACH_KDB || MACH_KGDB -/* - * kdb_kintr: enter kdb from keyboard interrupt. - * Chase down the stack frames until we find one whose return - * address is the interrupt handler. At that point, we have: - * - * frame-> saved %ebp - * return address in interrupt handler - * ivect - * saved SPL - * return address == return_to_iret_i - * saved %edx - * saved %ecx - * saved %eax - * saved %eip - * saved %cs - * saved %efl + +/******************************************************************************************************* * - * OR: - * frame-> saved %ebp - * return address in interrupt handler - * ivect - * saved SPL - * return address == return_to_iret - * pointer to save area on old stack - * [ saved %ebx, if accurate timing ] + * 32bit Tasks + * System call entries via INTR_GATE or sysenter: * - * old stack: saved %es - * saved %ds - * saved %edx - * saved %ecx - * saved %eax - * saved %eip - * saved %cs - * saved %efl + * esp -> x86_saved_state32_t + * cr3 -> kernel directory + * esp -> low based stack + * gs -> CPU_DATA_GS + * cs -> KERNEL32_CS + * ss/ds/es -> KERNEL_DS * - * Call kdb, passing it that register save area. + * interrupts disabled + * direction flag cleared */ -#if MACH_KGDB -Entry(kgdb_kintr) -#endif /* MACH_KGDB */ -#if MACH_KDB -Entry(kdb_kintr) -#endif /* MACH_KDB */ - movl %ebp,%eax /* save caller`s frame pointer */ - movl $ EXT(return_to_iret),%ecx /* interrupt return address 1 */ - movl $ EXT(return_to_iret_i),%edx /* interrupt return address 2 */ - -0: cmpl 16(%eax),%ecx /* does this frame return to */ - /* interrupt handler (1)? */ - je 1f - cmpl $kdb_from_iret,16(%eax) - je 1f - cmpl 16(%eax),%edx /* interrupt handler (2)? */ - je 2f /* if not: */ - cmpl $kdb_from_iret_i,16(%eax) - je 2f - movl (%eax),%eax /* try next frame */ - jmp 0b +Entry(lo_sysenter) + /* + * We can be here either for a mach syscall or a unix syscall, + * as indicated by the sign of the code: + */ + movl R32_EAX(%esp),%eax + testl %eax,%eax + js EXT(lo_mach_scall) /* < 0 => mach */ + /* > 0 => unix */ + +Entry(lo_unix_scall) + TIME_TRAP_UENTRY -1: movl $kdb_from_iret,16(%eax) /* returns to kernel/user stack */ - ret + movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */ + movl ACT_TASK(%ecx),%ebx /* point to current task */ + addl $1,TASK_SYSCALLS_UNIX(%ebx) /* increment call count */ -2: movl $kdb_from_iret_i,16(%eax) - /* returns to interrupt stack */ - ret + /* Check for active vtimers in the current task */ + cmpl $0,TASK_VTIMERS(%ebx) + jz 1f -/* - * On return from keyboard interrupt, we will execute - * kdb_from_iret_i - * if returning to an interrupt on the interrupt stack - * kdb_from_iret - * if returning to an interrupt on the user or kernel stack - */ -kdb_from_iret: - /* save regs in known locations */ - pushl %ebx /* caller`s %ebx is in reg */ - pushl %ebp - pushl %esi - pushl %edi - push %fs - push %gs -#if MACH_KGDB - cli - pushl %esp /* pass regs */ - call EXT(kgdb_kentry) /* to kgdb */ - addl $4,%esp /* pop parameters */ -#endif /* MACH_KGDB */ -#if MACH_KDB - pushl %esp /* pass regs */ - call EXT(kdb_kentry) /* to kdb */ - addl $4,%esp /* pop parameters */ -#endif /* MACH_KDB */ - pop %gs /* restore registers */ - pop %fs - popl %edi - popl %esi - popl %ebp - popl %ebx - jmp EXT(return_to_iret) /* normal interrupt return */ - -kdb_from_iret_i: /* on interrupt stack */ - pop %edx /* restore saved registers */ - pop %ecx - pop %eax - pushl $0 /* zero error code */ - pushl $0 /* zero trap number */ - pusha /* save general registers */ - push %ds /* save segment registers */ - push %es - push %fs - push %gs -#if MACH_KGDB - cli /* disable interrupts */ - CPU_NUMBER(%edx) /* get CPU number */ - movl CX(EXT(kgdb_stacks),%edx),%ebx - xchgl %ebx,%esp /* switch to kgdb stack */ - pushl %ebx /* pass old sp as an arg */ - call EXT(kgdb_from_kernel) - popl %esp /* switch back to interrupt stack */ -#endif /* MACH_KGDB */ -#if MACH_KDB - pushl %esp /* pass regs, */ - pushl $0 /* code, */ - pushl $-1 /* type to kdb */ - call EXT(kdb_trap) - addl $12,%esp -#endif /* MACH_KDB */ - pop %gs /* restore segment registers */ - pop %fs - pop %es - pop %ds - popa /* restore general registers */ - addl $8,%esp - iret - -#endif /* MACH_KDB || MACH_KGDB */ + /* Set a pending AST */ + orl $(AST_BSD),%gs:CPU_PENDING_AST + /* Set a thread AST (atomic) */ + lock + orl $(AST_BSD),ACT_AST(%ecx) + +1: + movl %gs:CPU_KERNEL_STACK,%ebx + xchgl %ebx,%esp /* switch to kernel stack */ -/* - * Mach RPC enters through a call gate, like a system call. - */ + sti -Entry(mach_rpc) - pushf /* save flags as soon as possible */ - pushl %eax /* save system call number */ - pushl $0 /* clear trap number slot */ + CCALL1(unix_syscall, %ebx) + /* + * always returns through thread_exception_return + */ - pusha /* save the general registers */ - pushl %ds /* and the segment registers */ - pushl %es - pushl %fs - pushl %gs - mov %ss,%dx /* switch to kernel data segment */ - mov %dx,%ds - mov %dx,%es - mov $ CPU_DATA_GS,%dx - mov %dx,%gs +Entry(lo_mach_scall) + TIME_TRAP_UENTRY -/* - * Shuffle eflags,eip,cs into proper places - */ + movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */ + movl ACT_TASK(%ecx),%ebx /* point to current task */ + addl $1,TASK_SYSCALLS_MACH(%ebx) /* increment call count */ - movl R_EIP(%esp),%ebx /* eflags are in EIP slot */ - movl R_CS(%esp),%ecx /* eip is in CS slot */ - movl R_EFLAGS(%esp),%edx /* cs is in EFLAGS slot */ - movl %ecx,R_EIP(%esp) /* fix eip */ - movl %edx,R_CS(%esp) /* fix cs */ - movl %ebx,R_EFLAGS(%esp) /* fix eflags */ + /* Check for active vtimers in the current task */ + cmpl $0,TASK_VTIMERS(%ebx) + jz 1f - TIME_TRAP_UENTRY + /* Set a pending AST */ + orl $(AST_BSD),%gs:CPU_PENDING_AST - negl %eax /* get system call number */ - shll $4,%eax /* manual indexing */ + /* Set a thread AST (atomic) */ + lock + orl $(AST_BSD),ACT_AST(%ecx) + +1: + movl %gs:CPU_KERNEL_STACK,%ebx + xchgl %ebx,%esp /* switch to kernel stack */ -/* - * Check here for mach_rpc from kernel-loaded task -- - * - Note that kernel-loaded task returns via real return. - * We didn't enter here "through" PCB (i.e., using ring 0 stack), - * so transfer the stack frame into the PCB explicitly, then - * start running on resulting "PCB stack". We have to set - * up a simulated "uesp" manually, since there's none in the - * frame. - */ - cmpl $0,%gs:CPU_ACTIVE_KLOADED - jz 2f - CAH(mrstart) - movl %gs:CPU_ACTIVE_KLOADED,%ebx - movl %gs:CPU_KERNEL_STACK,%edx - xchgl %edx,%esp + sti - FRAME_STACK_TO_PCB(%ebx,%edx) - CAH(mrend) + CCALL1(mach_call_munger, %ebx) + /* + * always returns through thread_exception_return + */ - jmp 3f -2: - movl %gs:CPU_KERNEL_STACK,%ebx - /* get current kernel stack */ - xchgl %ebx,%esp /* switch stacks - %ebx points to */ - /* user registers. */ +Entry(lo_mdep_scall) + TIME_TRAP_UENTRY -3: + movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */ + movl ACT_TASK(%ecx),%ebx /* point to current task */ -/* - * Register use on entry: - * eax contains syscall number - * ebx contains user regs pointer - */ -#undef RPC_TRAP_REGISTERS -#ifdef RPC_TRAP_REGISTERS - pushl R_ESI(%ebx) - pushl R_EDI(%ebx) - pushl R_ECX(%ebx) - pushl R_EDX(%ebx) -#else - movl EXT(mach_trap_table)(%eax),%ecx - /* get number of arguments */ - jecxz 2f /* skip argument copy if none */ - movl R_UESP(%ebx),%esi /* get user stack pointer */ - lea 4(%esi,%ecx,4),%esi /* skip user return address, */ - /* and point past last argument */ - movl %gs:CPU_ACTIVE_KLOADED,%edx - /* point to current thread */ - orl %edx,%edx /* if ! kernel-loaded, check addr */ - jz 4f /* else */ - mov %ds,%dx /* kernel data segment access */ - jmp 5f -4: - cmpl $(VM_MAX_ADDRESS),%esi /* in user space? */ - ja mach_call_addr /* address error if not */ - movl $ USER_DS,%edx /* user data segment access */ -5: - mov %dx,%fs - movl %esp,%edx /* save kernel ESP for error recovery */ -1: - subl $4,%esi - RECOVERY_SECTION - RECOVER(mach_call_addr_push) - pushl %fs:(%esi) /* push argument on stack */ - loop 1b /* loop for all arguments */ -#endif + /* Check for active vtimers in the current task */ + cmpl $0,TASK_VTIMERS(%ebx) + jz 1f -/* - * Register use on entry: - * eax contains syscall number << 4 - * mach_call_munger is declared regparm(1), so the first arg is %eax - */ -2: + /* Set a pending AST */ + orl $(AST_BSD),%gs:CPU_PENDING_AST - call EXT(mach_call_munger) + /* Set a thread AST (atomic) */ + lock + orl $(AST_BSD),ACT_AST(%ecx) - movl %esp,%ecx /* get kernel stack */ - or $(KERNEL_STACK_SIZE-1),%ecx - movl -3-IKS_SIZE(%ecx),%esp /* switch back to PCB stack */ - movl %eax,R_EAX(%esp) /* save return value */ - jmp EXT(return_from_trap) /* return to user */ +1: + movl %gs:CPU_KERNEL_STACK,%ebx + xchgl %ebx,%esp /* switch to kernel stack */ + sti -/* - * Special system call entry for "int 0x80", which has the "eflags" - * register saved at the right place already. - * Fall back to the common syscall path after saving the registers. - * - * esp -> old eip - * old cs - * old eflags - * old esp if trapped from user - * old ss if trapped from user - * - * XXX: for the moment, we don't check for int 0x80 from kernel mode. - */ -Entry(syscall_int80) - pushl %eax /* save system call number */ - pushl $0 /* clear trap number slot */ + CCALL1(machdep_syscall, %ebx) + /* + * always returns through thread_exception_return + */ - pusha /* save the general registers */ - pushl %ds /* and the segment registers */ - pushl %es - pushl %fs - pushl %gs - mov %ss,%dx /* switch to kernel data segment */ - mov %dx,%ds - mov %dx,%es - mov $ CPU_DATA_GS,%dx - mov %dx,%gs +Entry(lo_diag_scall) + TIME_TRAP_UENTRY - jmp syscall_entry_3 + movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */ + movl ACT_TASK(%ecx),%ebx /* point to current task */ -/* - * System call enters through a call gate. Flags are not saved - - * we must shuffle stack to look like trap save area. + /* Check for active vtimers in the current task */ + cmpl $0,TASK_VTIMERS(%ebx) + jz 1f + + /* Set a pending AST */ + orl $(AST_BSD),%gs:CPU_PENDING_AST + + /* Set a thread AST (atomic) */ + lock + orl $(AST_BSD),ACT_AST(%ecx) + +1: + movl %gs:CPU_KERNEL_STACK,%ebx // Get the address of the kernel stack + xchgl %ebx,%esp // Switch to it, saving the previous + + CCALL1(diagCall, %ebx) // Call diagnostics + + cmpl $0,%eax // What kind of return is this? + je 2f + cli // Disable interruptions just in case they were enabled + popl %esp // Get back the original stack + jmp EXT(return_to_user) // Normal return, do not check asts... +2: + CCALL5(i386_exception, $EXC_SYSCALL, $0x6000, $0, $1, $0) + // pass what would be the diag syscall + // error return - cause an exception + /* no return */ + + + +/******************************************************************************************************* * - * esp-> old eip - * old cs - * old esp - * old ss + * 64bit Tasks + * System call entries via syscall only: * - * eax contains system call number. + * esp -> x86_saved_state64_t + * cr3 -> kernel directory + * esp -> low based stack + * gs -> CPU_DATA_GS + * cs -> KERNEL32_CS + * ss/ds/es -> KERNEL_DS * - * NB: below use of CPU_NUMBER assumes that macro will use correct - * correct segment register for any kernel data accesses. - */ -Entry(syscall) -syscall_entry: - pushf /* save flags as soon as possible */ -syscall_entry_2: - pushl %eax /* save system call number */ - pushl $0 /* clear trap number slot */ - - pusha /* save the general registers */ - pushl %ds /* and the segment registers */ - pushl %es - pushl %fs - pushl %gs - - mov %ss,%dx /* switch to kernel data segment */ - mov %dx,%ds - mov %dx,%es - mov $ CPU_DATA_GS,%dx - mov %dx,%gs - -/* - * Shuffle eflags,eip,cs into proper places + * interrupts disabled + * direction flag cleared */ - movl R_EIP(%esp),%ebx /* eflags are in EIP slot */ - movl R_CS(%esp),%ecx /* eip is in CS slot */ - movl R_EFLAGS(%esp),%edx /* cs is in EFLAGS slot */ - movl %ecx,R_EIP(%esp) /* fix eip */ - movl %edx,R_CS(%esp) /* fix cs */ - movl %ebx,R_EFLAGS(%esp) /* fix eflags */ - -syscall_entry_3: -/* - * Check here for syscall from kernel-loaded task -- - * We didn't enter here "through" PCB (i.e., using ring 0 stack), - * so transfer the stack frame into the PCB explicitly, then - * start running on resulting "PCB stack". We have to set - * up a simulated "uesp" manually, since there's none in the - * frame. - */ - cmpl $0,%gs:CPU_ACTIVE_KLOADED - jz 0f - CAH(scstart) - movl %gs:CPU_ACTIVE_KLOADED,%ebx - movl %gs:CPU_KERNEL_STACK,%edx - xchgl %edx,%esp - FRAME_STACK_TO_PCB(%ebx,%edx) - CAH(scend) +Entry(lo_syscall) TIME_TRAP_UENTRY - jmp 1f -0: - TIME_TRAP_UENTRY + /* + * We can be here either for a mach, unix machdep or diag syscall, + * as indicated by the syscall class: + */ + movl R64_RAX(%esp), %eax /* syscall number/class */ + movl %eax, %ebx + andl $(SYSCALL_CLASS_MASK), %ebx /* syscall class */ + cmpl $(SYSCALL_CLASS_MACH<map */ - movl MAP_PMAP(%ecx),%ecx /* get map->pmap */ - cmpl EXT(kernel_pmap), %ecx - jz 1f - movl $ USER_DS,%cx /* user data segment access */ - mov %cx,%ds -1: - cmpl %esi,%eax - jb copyin_fail /* fail if wrap-around */ cld /* count up */ movl %edx,%ecx /* move by longwords first */ shrl $2,%ecx @@ -1798,66 +1183,62 @@ ENTRY(copyin) rep movsb xorl %eax,%eax /* return 0 for success */ -copy_ret: - mov %ss,%di /* restore kernel data segment */ - mov %di,%ds +copyin_ret: + mov %ss,%cx /* restore kernel data and extended segments */ + mov %cx,%ds + mov %cx,%es popl %edi /* restore registers */ popl %esi ret /* and return */ copyin_fail: - movl $ EFAULT,%eax /* return error for failure */ - jmp copy_ret /* pop frame and return */ + movl $(EFAULT),%eax /* return error for failure */ + jmp copyin_ret /* pop frame and return */ + + /* - * Copy string from user address space. - * arg0: user address + * Copy string from user/kern address space. + * arg0: window offset or kernel address * arg1: kernel address * arg2: max byte count * arg3: actual byte count (OUT) */ -Entry(copyinstr) +Entry(copyinstr_kern) + mov %ds,%cx + jmp copyinstr_common + +Entry(copyinstr_user) + movl $(USER_WINDOW_SEL),%ecx /* user data segment access through kernel window */ + +copyinstr_common: + mov %cx,%fs + pushl %esi pushl %edi /* save registers */ - movl 8+S_ARG0,%esi /* get user start address */ - movl 8+S_ARG1,%edi /* get kernel destination address */ + movl 8+S_ARG0,%esi /* get source - window offset or kernel address */ + movl 8+S_ARG1,%edi /* get destination - kernel address */ movl 8+S_ARG2,%edx /* get count */ - lea 0(%esi,%edx),%eax /* get user end address + 1 */ - - movl %gs:CPU_ACTIVE_THREAD,%ecx /* get active thread */ - movl ACT_MAP(%ecx),%ecx /* get act->map */ - movl MAP_PMAP(%ecx),%ecx /* get map->pmap */ - cmpl EXT(kernel_pmap), %ecx - jne 0f - mov %ds,%cx /* kernel data segment access */ - jmp 1f -0: - movl $ USER_DS,%cx /* user data segment access */ -1: - mov %cx,%fs - xorl %eax,%eax - cmpl $0,%edx - je 4f + xorl %eax,%eax /* set to 0 here so that the high 24 bits */ + /* are 0 for the cmpl against 0 */ 2: RECOVERY_SECTION RECOVER(copystr_fail) /* copy bytes... */ - movb %fs:(%esi),%eax + movb %fs:(%esi),%al incl %esi testl %edi,%edi /* if kernel address is ... */ jz 3f /* not NULL */ - movb %eax,(%edi) /* copy the byte */ + movb %al,(%edi) /* copy the byte */ incl %edi 3: - decl %edx - je 5f /* Zero count.. error out */ - cmpl $0,%eax - jne 2b /* .. a NUL found? */ - jmp 4f /* return zero (%eax) */ -5: - movl $ ENAMETOOLONG,%eax /* String is too long.. */ + testl %eax,%eax /* did we just stuff the 0-byte? */ + jz 4f /* yes, return 0 status already in %eax */ + decl %edx /* decrement #bytes left in buffer */ + jnz 2b /* buffer not full so copy in another byte */ + movl $(ENAMETOOLONG),%eax /* buffer full but no 0-byte: ENAMETOOLONG */ 4: movl 8+S_ARG3,%edi /* get OUT len ptr */ cmpl $0,%edi @@ -1870,272 +1251,66 @@ copystr_ret: ret /* and return */ copystr_fail: - movl $ EFAULT,%eax /* return error for failure */ - jmp copy_ret /* pop frame and return */ + movl $(EFAULT),%eax /* return error for failure */ + jmp copystr_ret /* pop frame and return */ + /* - * Copy to user address space. + * Copy to user/kern address space. * arg0: kernel address - * arg1: user address + * arg1: window offset or kernel address * arg2: byte count */ -Entry(copyoutmsg) -ENTRY(copyout) - pushl %esi - pushl %edi /* save registers */ - pushl %ebx - - movl 12+S_ARG0,%esi /* get kernel start address */ - movl 12+S_ARG1,%edi /* get user start address */ - movl 12+S_ARG2,%edx /* get count */ +ENTRY(copyoutphys_user) + movl $(USER_WINDOW_SEL),%ecx /* user data segment access through kernel window */ + mov %cx,%es - leal 0(%edi,%edx),%eax /* get user end address + 1 */ +ENTRY(copyoutphys_kern) + movl $(PHYS_WINDOW_SEL),%ecx /* physical access through kernel window */ + mov %cx,%ds + jmp copyout_common - movl %gs:CPU_ACTIVE_THREAD,%ecx /* get active thread */ - movl ACT_MAP(%ecx),%ecx /* get act->map */ - movl MAP_PMAP(%ecx),%ecx /* get map->pmap */ - cmpl EXT(kernel_pmap), %ecx - jne 0f - mov %ds,%cx /* else kernel data segment access */ - jmp 1f -0: - movl $ USER_DS,%cx -1: +ENTRY(copyout_user) + movl $(USER_WINDOW_SEL),%ecx /* user data segment access through kernel window */ mov %cx,%es -/* - * Check whether user address space is writable - * before writing to it - hardware is broken. - * - * Skip check if "user" address is really in - * kernel space (i.e., if it's in a kernel-loaded - * task). - * - * Register usage: - * esi/edi source/dest pointers for rep/mov - * ecx counter for rep/mov - * edx counts down from 3rd arg - * eax count of bytes for each (partial) page copy - * ebx shadows edi, used to adjust edx - */ - movl %edi,%ebx /* copy edi for syncing up */ -copyout_retry: - /* if restarting after a partial copy, put edx back in sync, */ - addl %ebx,%edx /* edx -= (edi - ebx); */ - subl %edi,%edx / - movl %edi,%ebx /* ebx = edi; */ +ENTRY(copyout_kern) + +copyout_common: + pushl %esi + pushl %edi /* save registers */ + + movl 8+S_ARG0,%esi /* get source - kernel address */ + movl 8+S_ARG1,%edi /* get destination - window offset or kernel address */ + movl 8+S_ARG2,%edx /* get count */ -/* - * Copy only what fits on the current destination page. - * Check for write-fault again on the next page. - */ - leal NBPG(%edi),%eax /* point to */ - andl $(-NBPG),%eax /* start of next page */ - subl %edi,%eax /* get number of bytes to that point */ - cmpl %edx,%eax /* bigger than count? */ - jle 1f /* if so, */ - movl %edx,%eax /* use count */ -1: cld /* count up */ - movl %eax,%ecx /* move by longwords first */ + movl %edx,%ecx /* move by longwords first */ shrl $2,%ecx RECOVERY_SECTION RECOVER(copyout_fail) - RETRY_SECTION - RETRY(copyout_retry) rep movsl - movl %eax,%ecx /* now move remaining bytes */ + movl %edx,%ecx /* now move remaining bytes */ andl $3,%ecx RECOVERY_SECTION RECOVER(copyout_fail) - RETRY_SECTION - RETRY(copyout_retry) rep movsb /* move */ - movl %edi,%ebx /* copy edi for syncing up */ - subl %eax,%edx /* and decrement count */ - jg copyout_retry /* restart on next page if not done */ xorl %eax,%eax /* return 0 for success */ copyout_ret: - mov %ss,%di /* restore kernel segment */ - mov %di,%es + mov %ss,%cx /* restore kernel segment */ + mov %cx,%es + mov %cx,%ds - popl %ebx popl %edi /* restore registers */ popl %esi ret /* and return */ copyout_fail: - movl $ EFAULT,%eax /* return error for failure */ + movl $(EFAULT),%eax /* return error for failure */ jmp copyout_ret /* pop frame and return */ -/* - * FPU routines. - */ - -/* - * Initialize FPU. - */ -ENTRY(_fninit) - fninit - ret - -/* - * Read control word - */ -ENTRY(_fstcw) - pushl %eax /* get stack space */ - fstcw (%esp) - popl %eax - ret - -/* - * Set control word - */ -ENTRY(_fldcw) - fldcw 4(%esp) - ret - -/* - * Read status word - */ -ENTRY(_fnstsw) - xor %eax,%eax /* clear high 16 bits of eax */ - fnstsw %ax /* read FP status */ - ret - -/* - * Clear FPU exceptions - */ -ENTRY(_fnclex) - fnclex - ret - -/* - * Clear task-switched flag. - */ -ENTRY(_clts) - clts - ret - -/* - * Save complete FPU state. Save error for later. - */ -ENTRY(_fpsave) - movl 4(%esp),%eax /* get save area pointer */ - fnsave (%eax) /* save complete state, including */ - /* errors */ - ret - -/* - * Restore FPU state. - */ -ENTRY(_fprestore) - movl 4(%esp),%eax /* get save area pointer */ - frstor (%eax) /* restore complete state */ - ret - -/* - * Set cr3 - */ -ENTRY(set_cr3) - CPU_NUMBER(%eax) - orl 4(%esp), %eax - /* - * Don't set PDBR to a new value (hence invalidating the - * "paging cache") if the new value matches the current one. - */ - movl %cr3,%edx /* get current cr3 value */ - cmpl %eax,%edx - je 0f /* if two are equal, don't set */ - movl %eax,%cr3 /* load it (and flush cache) */ -0: - ret - -/* - * Read cr3 - */ -ENTRY(get_cr3) - movl %cr3,%eax - andl $(~0x7), %eax /* remove cpu number */ - ret - -/* - * Flush TLB - */ -ENTRY(flush_tlb) - movl %cr3,%eax /* flush tlb by reloading CR3 */ - movl %eax,%cr3 /* with itself */ - ret - -/* - * Read cr2 - */ -ENTRY(get_cr2) - movl %cr2,%eax - ret - -/* - * Read cr4 - */ -ENTRY(get_cr4) - .byte 0x0f,0x20,0xe0 /* movl %cr4, %eax */ - ret - -/* - * Write cr4 - */ -ENTRY(set_cr4) - movl 4(%esp), %eax - .byte 0x0f,0x22,0xe0 /* movl %eax, %cr4 */ - ret - -/* - * Read ldtr - */ -Entry(get_ldt) - xorl %eax,%eax - sldt %ax - ret - -/* - * Set ldtr - */ -Entry(set_ldt) - lldt 4(%esp) - ret - -/* - * Read task register. - */ -ENTRY(get_tr) - xorl %eax,%eax - str %ax - ret - -/* - * Set task register. Also clears busy bit of task descriptor. - */ -ENTRY(set_tr) - movl S_ARG0,%eax /* get task segment number */ - subl $8,%esp /* push space for SGDT */ - sgdt 2(%esp) /* store GDT limit and base (linear) */ - movl 4(%esp),%edx /* address GDT */ - movb $(K_TSS),5(%edx,%eax) /* fix access byte in task descriptor */ - ltr %ax /* load task register */ - addl $8,%esp /* clear stack */ - ret /* and return */ - -/* - * Set task-switched flag. - */ -ENTRY(_setts) - movl %cr0,%eax /* get cr0 */ - orl $(CR0_TS),%eax /* or in TS bit */ - movl %eax,%cr0 /* set cr0 */ - ret - /* * io register must not be used on slaves (no AT bus) */ @@ -2160,103 +1335,6 @@ ENTRY(_setts) #endif /* MACH_ASSERT */ - -#if MACH_KDB || MACH_ASSERT - -/* - * Following routines are also defined as macros in i386/pio.h - * Compile then when MACH_KDB is configured so that they - * can be invoked from the debugger. - */ - -/* - * void outb(unsigned char *io_port, - * unsigned char byte) - * - * Output a byte to an IO port. - */ -ENTRY(outb) - PUSH_FRAME - ILL_ON_SLAVE - movl ARG0,%edx /* IO port address */ - movl ARG1,%eax /* data to output */ - outb %al,%dx /* send it out */ - POP_FRAME - ret - -/* - * unsigned char inb(unsigned char *io_port) - * - * Input a byte from an IO port. - */ -ENTRY(inb) - PUSH_FRAME - ILL_ON_SLAVE - movl ARG0,%edx /* IO port address */ - xor %eax,%eax /* clear high bits of register */ - inb %dx,%al /* get the byte */ - POP_FRAME - ret - -/* - * void outw(unsigned short *io_port, - * unsigned short word) - * - * Output a word to an IO port. - */ -ENTRY(outw) - PUSH_FRAME - ILL_ON_SLAVE - movl ARG0,%edx /* IO port address */ - movl ARG1,%eax /* data to output */ - outw %ax,%dx /* send it out */ - POP_FRAME - ret - -/* - * unsigned short inw(unsigned short *io_port) - * - * Input a word from an IO port. - */ -ENTRY(inw) - PUSH_FRAME - ILL_ON_SLAVE - movl ARG0,%edx /* IO port address */ - xor %eax,%eax /* clear high bits of register */ - inw %dx,%ax /* get the word */ - POP_FRAME - ret - -/* - * void outl(unsigned int *io_port, - * unsigned int byte) - * - * Output an int to an IO port. - */ -ENTRY(outl) - PUSH_FRAME - ILL_ON_SLAVE - movl ARG0,%edx /* IO port address*/ - movl ARG1,%eax /* data to output */ - outl %eax,%dx /* send it out */ - POP_FRAME - ret - -/* - * unsigned int inl(unsigned int *io_port) - * - * Input an int from an IO port. - */ -ENTRY(inl) - PUSH_FRAME - ILL_ON_SLAVE - movl ARG0,%edx /* IO port address */ - inl %dx,%eax /* get the int */ - POP_FRAME - ret - -#endif /* MACH_KDB || MACH_ASSERT*/ - /* * void loutb(unsigned byte *io_port, * unsigned byte *data, @@ -2393,79 +1471,6 @@ ENTRY(insl) POP_FRAME ret - -/* - * int inst_fetch(int eip, int cs); - * - * Fetch instruction byte. Return -1 if invalid address. - */ - .globl EXT(inst_fetch) -LEXT(inst_fetch) - movl S_ARG1, %eax /* get segment */ - movw %ax,%fs /* into FS */ - movl S_ARG0, %eax /* get offset */ - RETRY_SECTION - RETRY(EXT(inst_fetch)) /* re-load FS on retry */ - RECOVERY_SECTION - RECOVER(EXT(inst_fetch_fault)) - movzbl %fs:(%eax),%eax /* load instruction byte */ - ret - -LEXT(inst_fetch_fault) - movl $-1,%eax /* return -1 if error */ - ret - - -#if MACH_KDP -/* - * kdp_copy_kmem(char *src, char *dst, int count) - * - * Similar to copyin except that both addresses are kernel addresses. - */ - -ENTRY(kdp_copy_kmem) - pushl %esi - pushl %edi /* save registers */ - - movl 8+S_ARG0,%esi /* get kernel start address */ - movl 8+S_ARG1,%edi /* get kernel destination address */ - - movl 8+S_ARG2,%edx /* get count */ - - lea 0(%esi,%edx),%eax /* get kernel end address + 1 */ - - cmpl %esi,%eax - jb kdp_vm_read_fail /* fail if wrap-around */ - cld /* count up */ - movl %edx,%ecx /* move by longwords first */ - shrl $2,%ecx - RECOVERY_SECTION - RECOVER(kdp_vm_read_fail) - rep - movsl /* move longwords */ - movl %edx,%ecx /* now move remaining bytes */ - andl $3,%ecx - RECOVERY_SECTION - RECOVER(kdp_vm_read_fail) - rep - movsb -kdp_vm_read_done: - movl 8+S_ARG2,%edx /* get count */ - subl %ecx,%edx /* Return number of bytes transfered */ - movl %edx,%eax - - popl %edi /* restore registers */ - popl %esi - ret /* and return */ - -kdp_vm_read_fail: - xorl %eax,%eax /* didn't copy a thing. */ - - popl %edi - popl %esi - ret -#endif - /* * int rdmsr_carefully(uint32_t msr, uint32_t *lo, uint32_t *hi) */ @@ -2486,76 +1491,10 @@ rdmsr_fail: ret /* - * Done with recovery and retry tables. + * Done with recovery table. */ RECOVERY_SECTION RECOVER_TABLE_END - RETRY_SECTION - RETRY_TABLE_END - - - -ENTRY(dr6) - movl %db6, %eax - ret - -/* dr(address, type, len, persistence) - */ -ENTRY(dr0) - movl S_ARG0, %eax - movl %eax,EXT(dr_addr) - movl %eax, %db0 - movl $0, %ecx - jmp 0f -ENTRY(dr1) - movl S_ARG0, %eax - movl %eax,EXT(dr_addr)+1*4 - movl %eax, %db1 - movl $2, %ecx - jmp 0f -ENTRY(dr2) - movl S_ARG0, %eax - movl %eax,EXT(dr_addr)+2*4 - movl %eax, %db2 - movl $4, %ecx - jmp 0f - -ENTRY(dr3) - movl S_ARG0, %eax - movl %eax,EXT(dr_addr)+3*4 - movl %eax, %db3 - movl $6, %ecx - -0: - pushl %ebp - movl %esp, %ebp - - movl %db7, %edx - movl %edx,EXT(dr_addr)+4*4 - andl dr_msk(,%ecx,2),%edx /* clear out new entry */ - movl %edx,EXT(dr_addr)+5*4 - movzbl B_ARG3, %eax - andb $3, %al - shll %cl, %eax - orl %eax, %edx - - movzbl B_ARG1, %eax - andb $3, %al - addb $0x10, %ecx - shll %cl, %eax - orl %eax, %edx - - movzbl B_ARG2, %eax - andb $3, %al - addb $0x2, %ecx - shll %cl, %eax - orl %eax, %edx - - movl %edx, %db7 - movl %edx,EXT(dr_addr)+7*4 - movl %edx, %eax - leave - ret .data dr_msk: @@ -2566,16 +1505,8 @@ dr_msk: ENTRY(dr_addr) .long 0,0,0,0 .long 0,0,0,0 - .text - -ENTRY(get_cr0) - movl %cr0, %eax - ret -ENTRY(set_cr0) - movl 4(%esp), %eax - movl %eax, %cr0 - ret + .text #ifndef SYMMETRY @@ -2659,34 +1590,6 @@ ENTRY(get_pc) movl 4(%ebp),%eax ret -#if ETAP - -ENTRY(etap_get_pc) - movl 4(%ebp), %eax /* fetch pc of caller */ - ret - -ENTRY(tvals_to_etap) - movl S_ARG0, %eax - movl $1000000000, %ecx - mull %ecx - addl S_ARG1, %eax - adc $0, %edx - ret - -/* etap_time_t - * etap_time_sub(etap_time_t stop, etap_time_t start) - * - * 64bit subtract, returns stop - start - */ -ENTRY(etap_time_sub) - movl S_ARG0, %eax /* stop.low */ - movl S_ARG1, %edx /* stop.hi */ - subl S_ARG2, %eax /* stop.lo - start.lo */ - sbbl S_ARG3, %edx /* stop.hi - start.hi */ - ret - -#endif /* ETAP */ - ENTRY(minsecurity) pushl %ebp movl %esp,%ebp @@ -2770,174 +1673,33 @@ ENTRY(mul_scale) POP_FRAME ret -#ifdef MACH_BSD -/* - * BSD System call entry point.. - */ - -Entry(trap_unix_syscall) -trap_unix_addr: - pushf /* save flags as soon as possible */ -trap_unix_2: - pushl %eax /* save system call number */ - pushl $0 /* clear trap number slot */ - - pusha /* save the general registers */ - pushl %ds /* and the segment registers */ - pushl %es - pushl %fs - pushl %gs - - mov %ss,%dx /* switch to kernel data segment */ - mov %dx,%ds - mov %dx,%es - mov $ CPU_DATA_GS,%dx - mov %dx,%gs - -/* - * Shuffle eflags,eip,cs into proper places - */ - - movl R_EIP(%esp),%ebx /* eflags are in EIP slot */ - movl R_CS(%esp),%ecx /* eip is in CS slot */ - movl R_EFLAGS(%esp),%edx /* cs is in EFLAGS slot */ - movl %ecx,R_EIP(%esp) /* fix eip */ - movl %edx,R_CS(%esp) /* fix cs */ - movl %ebx,R_EFLAGS(%esp) /* fix eflags */ - - TIME_TRAP_UENTRY - - negl %eax /* get system call number */ - shll $4,%eax /* manual indexing */ - - movl %gs:CPU_KERNEL_STACK,%ebx - /* get current kernel stack */ - xchgl %ebx,%esp /* switch stacks - %ebx points to */ - /* user registers. */ - -/* - * Register use on entry: - * eax contains syscall number - * ebx contains user regs pointer - */ - CAH(call_call) - pushl %ebx /* Push the regs set onto stack */ - call EXT(unix_syscall) - popl %ebx - movl %esp,%ecx /* get kernel stack */ - or $(KERNEL_STACK_SIZE-1),%ecx - movl -3-IKS_SIZE(%ecx),%esp /* switch back to PCB stack */ - movl %eax,R_EAX(%esp) /* save return value */ - jmp EXT(return_from_trap) /* return to user */ - -/* - * Entry point for machdep system calls.. - */ - -Entry(trap_machdep_syscall) - pushf /* save flags as soon as possible */ - pushl %eax /* save system call number */ - pushl $0 /* clear trap number slot */ - - pusha /* save the general registers */ - pushl %ds /* and the segment registers */ - pushl %es - pushl %fs - pushl %gs - - mov %ss,%dx /* switch to kernel data segment */ - mov %dx,%ds - mov %dx,%es - mov $ CPU_DATA_GS,%dx - mov %dx,%gs + /* - * Shuffle eflags,eip,cs into proper places + * Double-fault exception handler task. The last gasp... */ +Entry(df_task_start) + CCALL1(panic_double_fault32, $(T_DOUBLE_FAULT)) + hlt - movl R_EIP(%esp),%ebx /* eflags are in EIP slot */ - movl R_CS(%esp),%ecx /* eip is in CS slot */ - movl R_EFLAGS(%esp),%edx /* cs is in EFLAGS slot */ - movl %ecx,R_EIP(%esp) /* fix eip */ - movl %edx,R_CS(%esp) /* fix cs */ - movl %ebx,R_EFLAGS(%esp) /* fix eflags */ - - TIME_TRAP_UENTRY - - negl %eax /* get system call number */ - shll $4,%eax /* manual indexing */ - - movl %gs:CPU_KERNEL_STACK,%ebx - /* get current kernel stack */ - xchgl %ebx,%esp /* switch stacks - %ebx points to */ - /* user registers. */ /* - * Register use on entry: - * eax contains syscall number - * ebx contains user regs pointer + * machine-check handler task. The last gasp... */ - CAH(call_call) - pushl %ebx - call EXT(machdep_syscall) - popl %ebx - movl %esp,%ecx /* get kernel stack */ - or $(KERNEL_STACK_SIZE-1),%ecx - movl -3-IKS_SIZE(%ecx),%esp /* switch back to PCB stack */ - movl %eax,R_EAX(%esp) /* save return value */ - jmp EXT(return_from_trap) /* return to user */ - -Entry(trap_mach25_syscall) - pushf /* save flags as soon as possible */ - pushl %eax /* save system call number */ - pushl $0 /* clear trap number slot */ - - pusha /* save the general registers */ - pushl %ds /* and the segment registers */ - pushl %es - pushl %fs - pushl %gs - - mov %ss,%dx /* switch to kernel data segment */ - mov %dx,%ds - mov %dx,%es - mov $ CPU_DATA_GS,%dx - mov %dx,%gs +Entry(mc_task_start) + CCALL1(panic_machine_check32, $(T_MACHINE_CHECK)) + hlt /* - * Shuffle eflags,eip,cs into proper places + * Compatibility mode's last gasp... */ +Entry(lo_df64) + movl %esp, %eax + CCALL1(panic_double_fault64, %eax) + hlt - movl R_EIP(%esp),%ebx /* eflags are in EIP slot */ - movl R_CS(%esp),%ecx /* eip is in CS slot */ - movl R_EFLAGS(%esp),%edx /* cs is in EFLAGS slot */ - movl %ecx,R_EIP(%esp) /* fix eip */ - movl %edx,R_CS(%esp) /* fix cs */ - movl %ebx,R_EFLAGS(%esp) /* fix eflags */ +Entry(lo_mc64) + movl %esp, %eax + CCALL1(panic_machine_check64, %eax) + hlt - TIME_TRAP_UENTRY - - negl %eax /* get system call number */ - shll $4,%eax /* manual indexing */ - - movl %gs:CPU_KERNEL_STACK,%ebx - /* get current kernel stack */ - xchgl %ebx,%esp /* switch stacks - %ebx points to */ - /* user registers. */ - -/* - * Register use on entry: - * eax contains syscall number - * ebx contains user regs pointer - */ - CAH(call_call) - pushl %ebx - call EXT(mach25_syscall) - popl %ebx - movl %esp,%ecx /* get kernel stack */ - or $(KERNEL_STACK_SIZE-1),%ecx - movl -3-IKS_SIZE(%ecx),%esp /* switch back to PCB stack */ - movl %eax,R_EAX(%esp) /* save return value */ - jmp EXT(return_from_trap) /* return to user */ - -#endif