X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/e2fac8b15b12a7979f72090454d850e612fc5b13..4d15aeb193b2c68f1d38666c317f8d3734f5f083:/osfmk/i386/acpi.c diff --git a/osfmk/i386/acpi.c b/osfmk/i386/acpi.c index 17143604c..955301c6d 100644 --- a/osfmk/i386/acpi.c +++ b/osfmk/i386/acpi.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2009 Apple Inc. All rights reserved. + * Copyright (c) 2000-2012 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -26,51 +26,76 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ +#include +#include +#include #include +#include #include -#include -#include +#if CONFIG_MTRR #include +#endif +#if HYPERVISOR +#include +#endif +#if CONFIG_VMX #include +#endif +#include #include #include #include #include #include #include +#if CONFIG_MCA #include +#endif #include +#include + #include +#include +#include #include +#include +#include #if HIBERNATION #include #endif #include +#include +#if CONFIG_SLEEP extern void acpi_sleep_cpu(acpi_sleep_callback, void * refcon); -extern char acpi_wake_start[]; -extern char acpi_wake_end[]; - -extern void set_kbd_leds(int leds); +extern void acpi_wake_prot(void); +#endif +extern kern_return_t IOCPURunPlatformQuiesceActions(void); +extern kern_return_t IOCPURunPlatformActiveActions(void); +extern kern_return_t IOCPURunPlatformHaltRestartActions(uint32_t message); extern void fpinit(void); vm_offset_t acpi_install_wake_handler(void) { - /* copy wake code to ACPI_WAKE_ADDR in low memory */ - bcopy_phys(kvtophys((vm_offset_t)acpi_wake_start), - (addr64_t) ACPI_WAKE_ADDR, - acpi_wake_end - acpi_wake_start); +#if CONFIG_SLEEP + install_real_mode_bootstrap(acpi_wake_prot); + return REAL_MODE_BOOTSTRAP_OFFSET; +#else + return 0; +#endif +} - /* flush cache */ - wbinvd(); +#if CONFIG_SLEEP - /* return physical address of the wakeup code */ - return ACPI_WAKE_ADDR; -} +unsigned int save_kdebug_enable = 0; +static uint64_t acpi_sleep_abstime; +static uint64_t acpi_idle_abstime; +static uint64_t acpi_wake_abstime, acpi_wake_postrebase_abstime; +boolean_t deep_idle_rebase = TRUE; #if HIBERNATION struct acpi_hibernate_callback_data { @@ -89,20 +114,20 @@ acpi_hibernate(void *refcon) if (current_cpu_datap()->cpu_hibernate) { - cpu_IA32e_enable(current_cpu_datap()); - mode = hibernate_write_image(); if( mode == kIOHibernatePostWriteHalt ) { // off HIBLOG("power off\n"); + IOCPURunPlatformHaltRestartActions(kPEHaltCPU); if (PE_halt_restart) (*PE_halt_restart)(kPEHaltCPU); } else if( mode == kIOHibernatePostWriteRestart ) { // restart HIBLOG("restart\n"); + IOCPURunPlatformHaltRestartActions(kPERestartCPU); if (PE_halt_restart) (*PE_halt_restart)(kPERestartCPU); } else @@ -114,47 +139,58 @@ acpi_hibernate(void *refcon) cpu_datap(0)->cpu_hibernate = 0; } - /* - * If we're in 64-bit mode, drop back into legacy mode during sleep. - */ - cpu_IA32e_disable(current_cpu_datap()); - } +#if CONFIG_VMX + vmx_suspend(); +#endif + kdebug_enable = 0; + + IOCPURunPlatformQuiesceActions(); + + acpi_sleep_abstime = mach_absolute_time(); + (data->func)(data->refcon); /* should never get here! */ } -#endif +#endif /* HIBERNATION */ +#endif /* CONFIG_SLEEP */ -static uint64_t acpi_sleep_abstime; +extern void slave_pstart(void); +extern void hibernate_rebuild_vm_structs(void); + +extern unsigned int wake_nkdbufs; void acpi_sleep_kernel(acpi_sleep_callback func, void *refcon) { #if HIBERNATION acpi_hibernate_callback_data_t data; - boolean_t did_hibernate; #endif + boolean_t did_hibernate; unsigned int cpu; kern_return_t rc; unsigned int my_cpu; + uint64_t start; + uint64_t elapsed = 0; + uint64_t elapsed_trace_start = 0; - kprintf("acpi_sleep_kernel hib=%d\n", - current_cpu_datap()->cpu_hibernate); + kprintf("acpi_sleep_kernel hib=%d, cpu=%d\n", + current_cpu_datap()->cpu_hibernate, cpu_number()); - /* Geta ll CPUs to be in the "off" state */ - my_cpu = cpu_number(); + /* Get all CPUs to be in the "off" state */ + my_cpu = cpu_number(); for (cpu = 0; cpu < real_ncpus; cpu += 1) { if (cpu == my_cpu) continue; rc = pmCPUExitHaltToOff(cpu); if (rc != KERN_SUCCESS) - panic("Error %d trying to transition CPU %d to OFF", - rc, cpu); + panic("Error %d trying to transition CPU %d to OFF", + rc, cpu); } - /* shutdown local APIC before passing control to BIOS */ + /* shutdown local APIC before passing control to firmware */ lapic_shutdown(); #if HIBERNATION @@ -165,43 +201,61 @@ acpi_sleep_kernel(acpi_sleep_callback func, void *refcon) /* Save power management timer state */ pmTimerSave(); - /* - * Turn off VT, otherwise switching to legacy mode will fail - */ - vmx_suspend(); +#if HYPERVISOR + /* Notify hypervisor that we are about to sleep */ + hv_suspend(); +#endif /* - * If we're in 64-bit mode, drop back into legacy mode during sleep. + * Enable FPU/SIMD unit for potential hibernate acceleration */ - cpu_IA32e_disable(current_cpu_datap()); + clear_ts(); + + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 0) | DBG_FUNC_START, 0, 0, 0, 0, 0); + + save_kdebug_enable = kdebug_enable; + kdebug_enable = 0; acpi_sleep_abstime = mach_absolute_time(); +#if CONFIG_SLEEP /* * Save master CPU state and sleep platform. * Will not return until platform is woken up, * or if sleep failed. */ + uint64_t old_cr3 = x86_64_pre_sleep(); #if HIBERNATION acpi_sleep_cpu(acpi_hibernate, &data); #else +#if CONFIG_VMX + vmx_suspend(); +#endif acpi_sleep_cpu(func, refcon); #endif + acpi_wake_abstime = mach_absolute_time(); + /* Rebase TSC->absolute time conversion, using timestamp + * recorded before sleep. + */ + rtc_nanotime_init(acpi_sleep_abstime); + acpi_wake_postrebase_abstime = start = mach_absolute_time(); + assert(start >= acpi_sleep_abstime); + + x86_64_post_sleep(old_cr3); + +#endif /* CONFIG_SLEEP */ + /* Reset UART if kprintf is enabled. * However kprintf should not be used before rtc_sleep_wakeup() * for compatibility with firewire kprintf. */ if (FALSE == disable_serial_output) - serial_init(); + pal_serial_init(); #if HIBERNATION if (current_cpu_datap()->cpu_hibernate) { - int i; - for (i = 0; i < PMAP_NWINDOWS; i++) - *current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CMAP = 0; - current_cpu_datap()->cpu_hibernate = 0; did_hibernate = TRUE; } else @@ -213,19 +267,30 @@ acpi_sleep_kernel(acpi_sleep_callback func, void *refcon) /* Re-enable mode (including 64-bit if applicable) */ cpu_mode_init(current_cpu_datap()); +#if CONFIG_MCA /* Re-enable machine check handling */ mca_cpu_init(); +#endif +#if CONFIG_MTRR /* restore MTRR settings */ mtrr_update_cpu(); +#endif - /* - * Restore VT mode - */ - vmx_resume(); + /* update CPU microcode */ + ucode_update_wake(); +#if CONFIG_MTRR /* set up PAT following boot processor power up */ pat_init(); +#endif + +#if CONFIG_VMX + /* + * Restore VT mode + */ + vmx_resume(did_hibernate); +#endif /* * Go through all of the CPUs and mark them as requiring @@ -233,32 +298,191 @@ acpi_sleep_kernel(acpi_sleep_callback func, void *refcon) */ pmMarkAllCPUsOff(); - /* let the realtime clock reset */ - rtc_sleep_wakeup(acpi_sleep_abstime); - if (did_hibernate) - hibernate_machine_init(); - - /* re-enable and re-init local apic */ + /* re-enable and re-init local apic (prior to starting timers) */ if (lapic_probe()) lapic_configure(); +#if HIBERNATION + hibernate_rebuild_vm_structs(); +#endif + + elapsed += mach_absolute_time() - start; + + rtc_decrementer_configure(); + kdebug_enable = save_kdebug_enable; + + if (kdebug_enable == 0) { + if (wake_nkdbufs) { + start = mach_absolute_time(); + kdebug_trace_start(wake_nkdbufs, NULL, TRUE); + elapsed_trace_start += mach_absolute_time() - start; + } + } + start = mach_absolute_time(); + + /* Reconfigure FP/SIMD unit */ + init_fpu(); + clear_ts(); + + IOCPURunPlatformActiveActions(); + +#if HIBERNATION + if (did_hibernate) { + elapsed += mach_absolute_time() - start; + + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_START, elapsed, elapsed_trace_start, 0, 0, 0); + hibernate_machine_init(); + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_END, 0, 0, 0, 0, 0); + + current_cpu_datap()->cpu_hibernate = 0; + + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 0) | DBG_FUNC_END, 0, 0, 0, 0, 0); + } else +#endif /* HIBERNATION */ + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 0) | DBG_FUNC_END, 0, 0, 0, 0, 0); + /* Restore power management register state */ pmCPUMarkRunning(current_cpu_datap()); /* Restore power management timer state */ pmTimerRestore(); - /* Restart tick interrupts from the LAPIC timer */ - rtc_lapic_start_ticking(); - - fpinit(); - clear_fpu(); + /* Restart timer interrupts */ + rtc_timer_start(); #if HIBERNATION - if (did_hibernate) - enable_preemption(); kprintf("ret from acpi_sleep_cpu hib=%d\n", did_hibernate); #endif + +#if CONFIG_SLEEP + /* Becase we don't save the bootstrap page, and we share it + * between sleep and mp slave init, we need to recreate it + * after coming back from sleep or hibernate */ + install_real_mode_bootstrap(slave_pstart); +#endif +} + +/* + * acpi_idle_kernel is called by the ACPI Platform kext to request the kernel + * to idle the boot processor in the deepest C-state for S0 sleep. All slave + * processors are expected already to have been offlined in the deepest C-state. + * + * The contract with ACPI is that although the kernel is called with interrupts + * disabled, interrupts may need to be re-enabled to dismiss any pending timer + * interrupt. However, the callback function will be called once this has + * occurred and interrupts are guaranteed to be disabled at that time, + * and to remain disabled during C-state entry, exit (wake) and return + * from acpi_idle_kernel. + */ +void +acpi_idle_kernel(acpi_sleep_callback func, void *refcon) +{ + boolean_t istate = ml_get_interrupts_enabled(); + + kprintf("acpi_idle_kernel, cpu=%d, interrupts %s\n", + cpu_number(), istate ? "enabled" : "disabled"); + + assert(cpu_number() == master_cpu); + + /* Cancel any pending deadline */ + setPop(0); + while (lapic_is_interrupting(LAPIC_TIMER_VECTOR)) { + (void) ml_set_interrupts_enabled(TRUE); + setPop(0); + ml_set_interrupts_enabled(FALSE); + } + + /* + * Call back to caller to indicate that interrupts will remain + * disabled while we deep idle, wake and return. + */ + func(refcon); + + acpi_idle_abstime = mach_absolute_time(); + + KERNEL_DEBUG_CONSTANT( + MACHDBG_CODE(DBG_MACH_SCHED, MACH_DEEP_IDLE) | DBG_FUNC_START, + acpi_idle_abstime, deep_idle_rebase, 0, 0, 0); + + /* + * Disable tracing during S0-sleep + * unless overridden by sysctl -w tsc.deep_idle_rebase=0 + */ + if (deep_idle_rebase) { + save_kdebug_enable = kdebug_enable; + kdebug_enable = 0; + } + + /* + * Call into power-management to enter the lowest C-state. + * Note when called on the boot processor this routine will + * return directly when awoken. + */ + pmCPUHalt(PM_HALT_SLEEP); + + /* + * Get wakeup time relative to the TSC which has progressed. + * Then rebase nanotime to reflect time not progressing over sleep + * - unless overriden so that tracing can occur during deep_idle. + */ + acpi_wake_abstime = mach_absolute_time(); + if (deep_idle_rebase) { + rtc_sleep_wakeup(acpi_idle_abstime); + kdebug_enable = save_kdebug_enable; + } + acpi_wake_postrebase_abstime = mach_absolute_time(); + assert(mach_absolute_time() >= acpi_idle_abstime); + + KERNEL_DEBUG_CONSTANT( + MACHDBG_CODE(DBG_MACH_SCHED, MACH_DEEP_IDLE) | DBG_FUNC_END, + acpi_wake_abstime, acpi_wake_abstime - acpi_idle_abstime, 0, 0, 0); + + /* Like S3 sleep, turn on tracing if trace_wake boot-arg is present */ + if (kdebug_enable == 0) { + if (wake_nkdbufs) + kdebug_trace_start(wake_nkdbufs, NULL, TRUE); + } + + IOCPURunPlatformActiveActions(); + + /* Restart timer interrupts */ + rtc_timer_start(); +} + +extern char real_mode_bootstrap_end[]; +extern char real_mode_bootstrap_base[]; + +void +install_real_mode_bootstrap(void *prot_entry) +{ + /* + * Copy the boot entry code to the real-mode vector area REAL_MODE_BOOTSTRAP_OFFSET. + * This is in page 1 which has been reserved for this purpose by + * machine_startup() from the boot processor. + * The slave boot code is responsible for switching to protected + * mode and then jumping to the common startup, _start(). + */ + bcopy_phys(kvtophys((vm_offset_t) real_mode_bootstrap_base), + (addr64_t) REAL_MODE_BOOTSTRAP_OFFSET, + real_mode_bootstrap_end-real_mode_bootstrap_base); + + /* + * Set the location at the base of the stack to point to the + * common startup entry. + */ + ml_phys_write_word( + PROT_MODE_START+REAL_MODE_BOOTSTRAP_OFFSET, + (unsigned int)kvtophys((vm_offset_t)prot_entry)); + + /* Flush caches */ + __asm__("wbinvd"); +} + +boolean_t +ml_recent_wake(void) { + uint64_t ctime = mach_absolute_time(); + assert(ctime > acpi_wake_postrebase_abstime); + return ((ctime - acpi_wake_postrebase_abstime) < 5 * NSEC_PER_SEC); }