X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/3e170ce000f1506b7b5d2c5c7faec85ceabb573d..527f99514973766e9c0382a4d8550dfb00f54939:/osfmk/i386/pcb.c diff --git a/osfmk/i386/pcb.c b/osfmk/i386/pcb.c index 84456b323..5fb9112ea 100644 --- a/osfmk/i386/pcb.c +++ b/osfmk/i386/pcb.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2012 Apple Inc. All rights reserved. + * Copyright (c) 2000-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -54,7 +54,6 @@ * the rights to redistribute these changes. */ -#include #include #include @@ -77,6 +76,7 @@ #include #include #include +#include #include #include #include @@ -94,11 +94,6 @@ #include #include /* LAPIC_PMC_SWI_VECTOR */ -#if KPERF -#include -#include -#endif - #if HYPERVISOR #include #endif @@ -122,6 +117,11 @@ unsigned int _MachineStateCount[] = { [x86_AVX_STATE32] = x86_AVX_STATE32_COUNT, [x86_AVX_STATE64] = x86_AVX_STATE64_COUNT, [x86_AVX_STATE] = x86_AVX_STATE_COUNT, +#if !defined(RC_HIDE_XNU_J137) + [x86_AVX512_STATE32] = x86_AVX512_STATE32_COUNT, + [x86_AVX512_STATE64] = x86_AVX512_STATE64_COUNT, + [x86_AVX512_STATE] = x86_AVX512_STATE_COUNT, +#endif /* not RC_HIDE_XNU_J137 */ }; zone_t iss_zone; /* zone for saved_state area */ @@ -131,7 +131,7 @@ zone_t ids_zone; /* zone for debug_state area */ extern void Thread_continue(void); extern void Load_context( - thread_t thread); + thread_t thread) __attribute__((noreturn)); static void get_exception_state32(thread_t thread, x86_exception_state32_t *es); @@ -394,6 +394,15 @@ machine_load_context( Load_context(new); } +static inline void pmap_switch_context(thread_t ot, thread_t nt, int cnum) { + pmap_assert(ml_get_interrupts_enabled() == FALSE); + vm_map_t nmap = nt->map, omap = ot->map; + if ((omap != nmap) || (nmap->pmap->pagezero_accessible)) { + PMAP_DEACTIVATE_MAP(omap, ot, cnum); + PMAP_ACTIVATE_MAP(nmap, nt, cnum); + } +} + /* * Switch to a new thread. * Save the old thread`s kernel state or continuation, @@ -405,22 +414,22 @@ machine_switch_context( thread_continue_t continuation, thread_t new) { -#if MACH_RT - assert(current_cpu_datap()->cpu_active_stack == old->kernel_stack); -#endif -#if KPERF - kperf_kpc_cswitch(old, new); -#endif + assert(current_cpu_datap()->cpu_active_stack == old->kernel_stack); + +#if KPC + kpc_off_cpu(old); +#endif /* KPC */ + /* * Save FP registers if in use. */ - fpu_save_context(old); + fpu_switch_context(old, new); old->machine.specFlags &= ~OnProc; new->machine.specFlags |= OnProc; /* - * Monitor the stack depth and report new max, + * Monitor the stack depth and report new max, * not worrying about races. */ vm_offset_t depth = current_stack_depth(); @@ -435,7 +444,7 @@ machine_switch_context( * Switch address maps if need be, even if not switching tasks. * (A server activation may be "borrowing" a client map.) */ - PMAP_SWITCH_CONTEXT(old, new, cpu_number()); + pmap_switch_context(old, new, cpu_number()); /* * Load the rest of the user state for the new thread @@ -458,8 +467,8 @@ machine_processor_shutdown( #if CONFIG_VMX vmx_suspend(); #endif - fpu_save_context(thread); - PMAP_SWITCH_CONTEXT(thread, processor->idle_thread, cpu_number()); + fpu_switch_context(thread, NULL); + pmap_switch_context(thread, processor->idle_thread, cpu_number()); return(Shutdown_context(thread, doshutdown, processor)); } @@ -864,8 +873,12 @@ machine_thread_set_state( } case x86_FLOAT_STATE32: + case x86_AVX_STATE32: +#if !defined(RC_HIDE_XNU_J137) + case x86_AVX512_STATE32: +#endif /* not RC_HIDE_XNU_J137 */ { - if (count != x86_FLOAT_STATE32_COUNT) + if (count != _MachineStateCount[flavor]) return(KERN_INVALID_ARGUMENT); if (thread_is_64bit(thr_act)) @@ -875,11 +888,15 @@ machine_thread_set_state( } case x86_FLOAT_STATE64: + case x86_AVX_STATE64: +#if !defined(RC_HIDE_XNU_J137) + case x86_AVX512_STATE64: +#endif /* not RC_HIDE_XNU_J137 */ { - if (count != x86_FLOAT_STATE64_COUNT) + if (count != _MachineStateCount[flavor]) return(KERN_INVALID_ARGUMENT); - if ( !thread_is_64bit(thr_act)) + if (!thread_is_64bit(thr_act)) return(KERN_INVALID_ARGUMENT); return fpu_set_fxstate(thr_act, tstate, flavor); @@ -904,49 +921,33 @@ machine_thread_set_state( return(KERN_INVALID_ARGUMENT); } - case x86_AVX_STATE32: - { - if (count != x86_AVX_STATE32_COUNT) - return(KERN_INVALID_ARGUMENT); - - if (thread_is_64bit(thr_act)) - return(KERN_INVALID_ARGUMENT); - - return fpu_set_fxstate(thr_act, tstate, flavor); - } - - case x86_AVX_STATE64: - { - if (count != x86_AVX_STATE64_COUNT) - return(KERN_INVALID_ARGUMENT); - - if (!thread_is_64bit(thr_act)) - return(KERN_INVALID_ARGUMENT); - - return fpu_set_fxstate(thr_act, tstate, flavor); - } - case x86_AVX_STATE: +#if !defined(RC_HIDE_XNU_J137) + case x86_AVX512_STATE: +#endif { x86_avx_state_t *state; - if (count != x86_AVX_STATE_COUNT) + if (count != _MachineStateCount[flavor]) return(KERN_INVALID_ARGUMENT); state = (x86_avx_state_t *)tstate; - if (state->ash.flavor == x86_AVX_STATE64 && - state->ash.count == x86_FLOAT_STATE64_COUNT && + /* Flavors are defined to have sequential values: 32-bit, 64-bit, non-specific */ + /* 64-bit flavor? */ + if (state->ash.flavor == (flavor - 1) && + state->ash.count == _MachineStateCount[flavor - 1] && thread_is_64bit(thr_act)) { return fpu_set_fxstate(thr_act, (thread_state_t)&state->ufs.as64, - x86_FLOAT_STATE64); + flavor - 1); } - if (state->ash.flavor == x86_FLOAT_STATE32 && - state->ash.count == x86_FLOAT_STATE32_COUNT && + /* 32-bit flavor? */ + if (state->ash.flavor == (flavor - 2) && + state->ash.count == _MachineStateCount[flavor - 2] && !thread_is_64bit(thr_act)) { return fpu_set_fxstate(thr_act, (thread_state_t)&state->ufs.as32, - x86_FLOAT_STATE32); + flavor - 2); } return(KERN_INVALID_ARGUMENT); } @@ -992,8 +993,6 @@ machine_thread_set_state( return set_thread_state32(thr_act, &state->uts.ts32); } else return(KERN_INVALID_ARGUMENT); - - break; } case x86_DEBUG_STATE32: { @@ -1112,6 +1111,24 @@ machine_thread_get_state( break; } +#if !defined(RC_HIDE_XNU_J137) + case THREAD_STATE_FLAVOR_LIST_10_13: + { + if (*count < 6) + return (KERN_INVALID_ARGUMENT); + + tstate[0] = x86_THREAD_STATE; + tstate[1] = x86_FLOAT_STATE; + tstate[2] = x86_EXCEPTION_STATE; + tstate[3] = x86_DEBUG_STATE; + tstate[4] = x86_AVX_STATE; + tstate[5] = x86_AVX512_STATE; + + *count = 6; + break; + } + +#endif case x86_SAVED_STATE32: { x86_saved_state32_t *state; @@ -1221,58 +1238,64 @@ machine_thread_get_state( } case x86_AVX_STATE32: +#if !defined(RC_HIDE_XNU_J137) + case x86_AVX512_STATE32: +#endif { - if (*count != x86_AVX_STATE32_COUNT) + if (*count != _MachineStateCount[flavor]) return(KERN_INVALID_ARGUMENT); if (thread_is_64bit(thr_act)) return(KERN_INVALID_ARGUMENT); - *count = x86_AVX_STATE32_COUNT; + *count = _MachineStateCount[flavor]; return fpu_get_fxstate(thr_act, tstate, flavor); } case x86_AVX_STATE64: +#if !defined(RC_HIDE_XNU_J137) + case x86_AVX512_STATE64: +#endif { - if (*count != x86_AVX_STATE64_COUNT) + if (*count != _MachineStateCount[flavor]) return(KERN_INVALID_ARGUMENT); if ( !thread_is_64bit(thr_act)) return(KERN_INVALID_ARGUMENT); - *count = x86_AVX_STATE64_COUNT; + *count = _MachineStateCount[flavor]; return fpu_get_fxstate(thr_act, tstate, flavor); } case x86_AVX_STATE: +#if !defined(RC_HIDE_XNU_J137) + case x86_AVX512_STATE: +#endif { x86_avx_state_t *state; - kern_return_t kret; + thread_state_t fstate; - if (*count < x86_AVX_STATE_COUNT) + if (*count < _MachineStateCount[flavor]) return(KERN_INVALID_ARGUMENT); + *count = _MachineStateCount[flavor]; state = (x86_avx_state_t *)tstate; - bzero((char *)state, sizeof(x86_avx_state_t)); + bzero((char *)state, *count * sizeof(int)); + if (thread_is_64bit(thr_act)) { - state->ash.flavor = x86_AVX_STATE64; - state->ash.count = x86_AVX_STATE64_COUNT; - kret = fpu_get_fxstate(thr_act, - (thread_state_t)&state->ufs.as64, - x86_AVX_STATE64); + flavor -= 1; /* 64-bit flavor */ + fstate = (thread_state_t) &state->ufs.as64; } else { - state->ash.flavor = x86_AVX_STATE32; - state->ash.count = x86_AVX_STATE32_COUNT; - kret = fpu_get_fxstate(thr_act, - (thread_state_t)&state->ufs.as32, - x86_AVX_STATE32); + flavor -= 2; /* 32-bit flavor */ + fstate = (thread_state_t) &state->ufs.as32; } - *count = x86_AVX_STATE_COUNT; + state->ash.flavor = flavor; + state->ash.count = _MachineStateCount[flavor]; - return(kret); + return fpu_get_fxstate(thr_act, fstate, flavor); } case x86_THREAD_STATE32: @@ -1639,6 +1662,9 @@ machine_thread_switch_addrmode(thread_t thread) */ machine_thread_create(thread, thread->task); + /* Adjust FPU state */ + fpu_switch_addrmode(thread, task_has_64BitAddr(thread->task)); + /* If we're switching ourselves, reset the pcb addresses etc. */ if (thread == current_thread()) { boolean_t istate = ml_set_interrupts_enabled(FALSE); @@ -1739,16 +1765,17 @@ machine_stack_attach( assert(stack); thread->kernel_stack = stack; + thread_initialize_kernel_state(thread); statep = STACK_IKS(stack); #if defined(__x86_64__) statep->k_rip = (unsigned long) Thread_continue; statep->k_rbx = (unsigned long) thread_continue; - statep->k_rsp = (unsigned long) (STACK_IKS(stack) - 1); + statep->k_rsp = (unsigned long) STACK_IKS(stack); #else statep->k_eip = (unsigned long) Thread_continue; statep->k_ebx = (unsigned long) thread_continue; - statep->k_esp = (unsigned long) (STACK_IKS(stack) - 1); + statep->k_esp = (unsigned long) STACK_IKS(stack); #endif return; @@ -1767,9 +1794,7 @@ machine_stack_handoff(thread_t old, assert(new); assert(old); -#if KPERF - kperf_kpc_cswitch(old, new); -#endif + kpc_off_cpu(old); stack = old->kernel_stack; if (stack == old->reserved_stack) { @@ -1784,12 +1809,12 @@ machine_stack_handoff(thread_t old, */ new->kernel_stack = stack; - fpu_save_context(old); + fpu_switch_context(old, new); old->machine.specFlags &= ~OnProc; new->machine.specFlags |= OnProc; - PMAP_SWITCH_CONTEXT(old, new, cpu_number()); + pmap_switch_context(old, new, cpu_number()); act_machine_switch_pcb(old, new); #if HYPERVISOR @@ -1797,6 +1822,7 @@ machine_stack_handoff(thread_t old, #endif machine_set_current_thread(new); + thread_initialize_kernel_state(new); return; }