X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/316670eb35587141e969394ae8537d66b9211e80..refs/heads/master:/osfmk/i386/lapic_native.c diff --git a/osfmk/i386/lapic_native.c b/osfmk/i386/lapic_native.c index 3e6991974..82acb30f6 100644 --- a/osfmk/i386/lapic_native.c +++ b/osfmk/i386/lapic_native.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2008-2011 Apple Inc. All rights reserved. + * Copyright (c) 2008-2020 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* @@ -56,26 +56,22 @@ #include #endif -#if CONFIG_COUNTERS -#include -#endif - #include -#if MP_DEBUG -#define PAUSE delay(1000000) -#define DBG(x...) kprintf(x) +#if MP_DEBUG +#define PAUSE delay(1000000) +#define DBG(x...) kprintf(x) #else #define DBG(x...) #define PAUSE -#endif /* MP_DEBUG */ +#endif /* MP_DEBUG */ -lapic_ops_table_t *lapic_ops; /* Lapic operations switch */ +lapic_ops_table_t *lapic_ops; /* Lapic operations switch */ -static vm_map_offset_t lapic_pbase; /* Physical base memory-mapped regs */ -static vm_offset_t lapic_vbase; /* Virtual base memory-mapped regs */ +static vm_map_offset_t lapic_pbase; /* Physical base memory-mapped regs */ +static vm_offset_t lapic_vbase; /* Virtual base memory-mapped regs */ -static i386_intr_func_t lapic_intr_func[LAPIC_FUNC_TABLE_SIZE]; +static i386_intr_func_t lapic_intr_func[LAPIC_FUNC_TABLE_SIZE]; /* TRUE if local APIC was enabled by the OS not by the BIOS */ static boolean_t lapic_os_enabled = FALSE; @@ -86,70 +82,111 @@ static uint64_t lapic_error_time_threshold = 0; static unsigned lapic_master_error_count = 0; static unsigned lapic_error_count_threshold = 5; static boolean_t lapic_dont_panic = FALSE; +int lapic_max_interrupt_cpunum = 0; + +typedef enum { + APIC_MODE_UNKNOWN = 0, + APIC_MODE_XAPIC = 1, + APIC_MODE_X2APIC = 2 +} apic_mode_t; + +static apic_mode_t apic_mode_before_sleep = APIC_MODE_UNKNOWN; #ifdef MP_DEBUG void lapic_cpu_map_dump(void) { - int i; + int i; for (i = 0; i < MAX_CPUS; i++) { - if (cpu_to_lapic[i] == -1) + if (cpu_to_lapic[i] == -1) { continue; + } kprintf("cpu_to_lapic[%d]: %d\n", - i, cpu_to_lapic[i]); + i, cpu_to_lapic[i]); } for (i = 0; i < MAX_LAPICIDS; i++) { - if (lapic_to_cpu[i] == -1) + if (lapic_to_cpu[i] == -1) { continue; + } kprintf("lapic_to_cpu[%d]: %d\n", - i, lapic_to_cpu[i]); + i, lapic_to_cpu[i]); } } #endif /* MP_DEBUG */ static void -legacy_init(void) +map_local_apic(void) { - int result; - vm_map_entry_t entry; vm_map_offset_t lapic_vbase64; - /* Establish a map to the local apic */ - - lapic_vbase64 = (vm_offset_t)vm_map_min(kernel_map); - result = vm_map_find_space(kernel_map, - &lapic_vbase64, - round_page(LAPIC_SIZE), 0, - VM_MAKE_TAG(VM_MEMORY_IOKIT), &entry); - /* Convert 64-bit vm_map_offset_t to "pointer sized" vm_offset_t - */ - lapic_vbase = (vm_offset_t) lapic_vbase64; - if (result != KERN_SUCCESS) { - panic("legacy_init: vm_map_find_entry FAILED (err=%d)", result); + int result; + kern_return_t kr; + vm_map_entry_t entry; + + if (lapic_vbase == 0) { + lapic_vbase64 = (vm_offset_t)vm_map_min(kernel_map); + result = vm_map_find_space(kernel_map, + &lapic_vbase64, + round_page(LAPIC_SIZE), 0, + 0, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_IOKIT, + &entry); + /* Convert 64-bit vm_map_offset_t to "pointer sized" vm_offset_t + */ + lapic_vbase = (vm_offset_t) lapic_vbase64; + if (result != KERN_SUCCESS) { + panic("legacy_init: vm_map_find_entry FAILED (err=%d)", result); + } + vm_map_unlock(kernel_map); + + /* + * Map in the local APIC non-cacheable, as recommended by Intel + * in section 8.4.1 of the "System Programming Guide". + * In fact, this is redundant because EFI will have assigned an + * MTRR physical range containing the local APIC's MMIO space as + * UC and this will override the default PAT setting. + */ + kr = pmap_enter(pmap_kernel(), + lapic_vbase, + (ppnum_t) i386_btop(lapic_pbase), + VM_PROT_READ | VM_PROT_WRITE, + VM_PROT_NONE, + VM_WIMG_IO, + TRUE); + + assert(kr == KERN_SUCCESS); } - vm_map_unlock(kernel_map); +} + +static void +legacy_init(void) +{ + uint32_t lo, hi; + rdmsr(MSR_IA32_APIC_BASE, lo, hi); + if ((lo & MSR_IA32_APIC_BASE_EXTENDED) != 0) { + /* + * If we're already in x2APIC mode, we MUST disable the local APIC + * before transitioning back to legacy APIC mode. + */ + lo &= ~(MSR_IA32_APIC_BASE_ENABLE | MSR_IA32_APIC_BASE_EXTENDED); + wrmsr64(MSR_IA32_APIC_BASE, ((uint64_t)hi) << 32 | lo); + wrmsr64(MSR_IA32_APIC_BASE, ((uint64_t)hi) << 32 | lo | MSR_IA32_APIC_BASE_ENABLE); + } /* - * Map in the local APIC non-cacheable, as recommended by Intel - * in section 8.4.1 of the "System Programming Guide". - * In fact, this is redundant because EFI will have assigned an - * MTRR physical range containing the local APIC's MMIO space as - * UC and this will override the default PAT setting. + * Set flat delivery model, logical processor id + * This should already be the default set. */ - pmap_enter(pmap_kernel(), - lapic_vbase, - (ppnum_t) i386_btop(lapic_pbase), - VM_PROT_READ|VM_PROT_WRITE, - VM_PROT_NONE, - VM_WIMG_IO, - TRUE); + LAPIC_WRITE(DFR, LAPIC_DFR_FLAT); + LAPIC_WRITE(LDR, (get_cpu_number()) << LAPIC_LDR_SHIFT); } static uint32_t legacy_read(lapic_register_t reg) { - return *LAPIC_MMIO(reg); + return *LAPIC_MMIO(reg); } static void @@ -158,22 +195,48 @@ legacy_write(lapic_register_t reg, uint32_t value) *LAPIC_MMIO(reg) = value; } +static uint64_t +legacy_read_icr(void) +{ + return (((uint64_t)*LAPIC_MMIO(ICRD)) << 32) | ((uint64_t)*LAPIC_MMIO(ICR)); +} + +static void +legacy_write_icr(uint32_t dst, uint32_t cmd) +{ + *LAPIC_MMIO(ICRD) = dst << LAPIC_ICRD_DEST_SHIFT; + *LAPIC_MMIO(ICR) = cmd; +} + static lapic_ops_table_t legacy_ops = { legacy_init, legacy_read, - legacy_write + legacy_write, + legacy_read_icr, + legacy_write_icr }; +boolean_t is_x2apic = FALSE; + static void x2apic_init(void) { + uint32_t lo; + uint32_t hi; + + rdmsr(MSR_IA32_APIC_BASE, lo, hi); + if ((lo & MSR_IA32_APIC_BASE_EXTENDED) == 0) { + lo |= MSR_IA32_APIC_BASE_EXTENDED; + wrmsr(MSR_IA32_APIC_BASE, lo, hi); + kprintf("x2APIC mode enabled\n"); + } } static uint32_t x2apic_read(lapic_register_t reg) { - uint32_t lo; - uint32_t hi; + uint32_t lo; + uint32_t hi; rdmsr(LAPIC_MSR(reg), lo, hi); return lo; @@ -185,47 +248,182 @@ x2apic_write(lapic_register_t reg, uint32_t value) wrmsr(LAPIC_MSR(reg), value, 0); } +static uint64_t +x2apic_read_icr(void) +{ + return rdmsr64(LAPIC_MSR(ICR));; +} + +static void +x2apic_write_icr(uint32_t dst, uint32_t cmd) +{ + wrmsr(LAPIC_MSR(ICR), cmd, dst); +} + static lapic_ops_table_t x2apic_ops = { x2apic_init, x2apic_read, - x2apic_write + x2apic_write, + x2apic_read_icr, + x2apic_write_icr }; +/* + * Used by APs to determine their APIC IDs; assumes master CPU has initialized + * the local APIC interfaces. + */ +uint32_t +lapic_safe_apicid(void) +{ + uint32_t lo; + uint32_t hi; + boolean_t is_lapic_enabled, is_local_x2apic; + + rdmsr(MSR_IA32_APIC_BASE, lo, hi); + is_lapic_enabled = (lo & MSR_IA32_APIC_BASE_ENABLE) != 0; + is_local_x2apic = (lo & MSR_IA32_APIC_BASE_EXTENDED) != 0; + + if (is_lapic_enabled && is_local_x2apic) { + return x2apic_read(ID); + } else if (is_lapic_enabled) { + return (*LAPIC_MMIO(ID) >> LAPIC_ID_SHIFT) & LAPIC_ID_MASK; + } else { + panic("Unknown Local APIC state!"); + /*NORETURN*/ + } +} + +static void +lapic_reinit(bool for_wake) +{ + uint32_t lo; + uint32_t hi; + boolean_t is_boot_processor; + boolean_t is_lapic_enabled; + boolean_t is_local_x2apic; + + rdmsr(MSR_IA32_APIC_BASE, lo, hi); + is_boot_processor = (lo & MSR_IA32_APIC_BASE_BSP) != 0; + is_lapic_enabled = (lo & MSR_IA32_APIC_BASE_ENABLE) != 0; + is_local_x2apic = (lo & MSR_IA32_APIC_BASE_EXTENDED) != 0; + + /* + * If we're configured for x2apic mode and we're being asked to transition + * to legacy APIC mode, OR if we're in legacy APIC mode and we're being + * asked to transition to x2apic mode, call LAPIC_INIT(). + */ + if ((!is_local_x2apic && is_x2apic) || (is_local_x2apic && !is_x2apic)) { + LAPIC_INIT(); + /* Now re-read after LAPIC_INIT() */ + rdmsr(MSR_IA32_APIC_BASE, lo, hi); + is_lapic_enabled = (lo & MSR_IA32_APIC_BASE_ENABLE) != 0; + is_local_x2apic = (lo & MSR_IA32_APIC_BASE_EXTENDED) != 0; + } + + if ((!is_lapic_enabled && !is_local_x2apic)) { + panic("Unexpected local APIC state\n"); + } + + /* + * If we did not select the same APIC mode as we had before sleep, flag + * that as an error (and panic on debug/development kernels). Note that + * we might get here with for_wake == true for the first boot case. In + * that case, apic_mode_before_sleep will be UNKNOWN (since we haven't + * slept yet), so we do not need to do any APIC checks. + */ + if (for_wake && + ((apic_mode_before_sleep == APIC_MODE_XAPIC && !is_lapic_enabled) || + (apic_mode_before_sleep == APIC_MODE_X2APIC && !is_local_x2apic))) { + kprintf("Inconsistent APIC state after wake (was %d before sleep, " + "now is %d)", apic_mode_before_sleep, + is_lapic_enabled ? APIC_MODE_XAPIC : APIC_MODE_X2APIC); +#if DEBUG || DEVELOPMENT + kprintf("HALTING.\n"); + /* + * Unfortunately, we cannot safely panic here because the + * executing CPU might not be fully initialized. The best + * we can do is just print a message to the console and + * halt. + */ + asm volatile ("cli; hlt;" ::: "memory"); +#endif + } +} + +void +lapic_init_slave(void) +{ + lapic_reinit(false); +#if DEBUG || DEVELOPMENT + if (rdmsr64(MSR_IA32_APIC_BASE) & MSR_IA32_APIC_BASE_BSP) { + panic("Calling lapic_init_slave() on the boot processor\n"); + } +#endif +} void lapic_init(void) { - uint32_t lo; - uint32_t hi; - boolean_t is_boot_processor; - boolean_t is_lapic_enabled; - boolean_t is_x2apic; + uint32_t lo; + uint32_t hi; + boolean_t is_boot_processor; + boolean_t is_lapic_enabled; /* Examine the local APIC state */ rdmsr(MSR_IA32_APIC_BASE, lo, hi); is_boot_processor = (lo & MSR_IA32_APIC_BASE_BSP) != 0; is_lapic_enabled = (lo & MSR_IA32_APIC_BASE_ENABLE) != 0; is_x2apic = (lo & MSR_IA32_APIC_BASE_EXTENDED) != 0; - lapic_pbase = (lo & MSR_IA32_APIC_BASE_BASE); + lapic_pbase = (lo & MSR_IA32_APIC_BASE_BASE); kprintf("MSR_IA32_APIC_BASE 0x%llx %s %s mode %s\n", lapic_pbase, - is_lapic_enabled ? "enabled" : "disabled", - is_x2apic ? "extended" : "legacy", - is_boot_processor ? "BSP" : "AP"); - if (!is_boot_processor || !is_lapic_enabled) + is_lapic_enabled ? "enabled" : "disabled", + is_x2apic ? "extended" : "legacy", + is_boot_processor ? "BSP" : "AP"); + if (!is_boot_processor || !is_lapic_enabled) { panic("Unexpected local APIC state\n"); + } + + /* + * If x2APIC is available and not already enabled, enable it. + * Unless overriden by boot-arg. + */ + if (!is_x2apic && (cpuid_features() & CPUID_FEATURE_x2APIC)) { + /* + * If no x2apic boot-arg was set and if we're running under a VMM, + * autoenable x2APIC mode. + */ + if (PE_parse_boot_argn("x2apic", &is_x2apic, sizeof(is_x2apic)) == FALSE && + cpuid_vmm_info()->cpuid_vmm_family != CPUID_VMM_FAMILY_NONE) { + is_x2apic = TRUE; + } + kprintf("x2APIC supported %s be enabled\n", + is_x2apic ? "and will" : "but will not"); + } lapic_ops = is_x2apic ? &x2apic_ops : &legacy_ops; - lapic_ops->init(); + if (lapic_pbase != 0) { + /* + * APs might need to consult the local APIC via the MMIO interface + * to get their APIC IDs. + */ + map_local_apic(); + } else if (!is_x2apic) { + panic("Local APIC physical address was not set."); + } + + LAPIC_INIT(); - if ((LAPIC_READ(VERSION)&LAPIC_VERSION_MASK) < 0x14) { + kprintf("ID: 0x%x LDR: 0x%x\n", LAPIC_READ(ID), LAPIC_READ(LDR)); + if ((LAPIC_READ(VERSION) & LAPIC_VERSION_MASK) < 0x14) { panic("Local APIC version 0x%x, 0x14 or more expected\n", - (LAPIC_READ(VERSION)&LAPIC_VERSION_MASK)); + (LAPIC_READ(VERSION) & LAPIC_VERSION_MASK)); } /* Set up the lapic_id <-> cpu_number map and add this boot processor */ lapic_cpu_map_init(); - lapic_cpu_map((LAPIC_READ(ID)>>LAPIC_ID_SHIFT)&LAPIC_ID_MASK, 0); + lapic_cpu_map(lapic_safe_apicid(), 0); + current_cpu_datap()->cpu_phys_number = cpu_to_lapic[0]; kprintf("Boot cpu local APIC id 0x%x\n", cpu_to_lapic[0]); } @@ -238,7 +436,7 @@ lapic_esr_read(void) return LAPIC_READ(ERROR_STATUS); } -static void +static void lapic_esr_clear(void) { LAPIC_WRITE(ERROR_STATUS, 0); @@ -253,7 +451,8 @@ static const char *DM_str[8] = { "NMI", "Reset", "Invalid", - "ExtINT"}; + "ExtINT" +}; static const char *TMR_str[] = { "OneShot", @@ -265,12 +464,12 @@ static const char *TMR_str[] = { void lapic_dump(void) { - int i; + int i; #define BOOL(a) ((a)?' ':'!') #define VEC(lvt) \ LAPIC_READ(lvt)&LAPIC_LVT_VECTOR_MASK -#define DS(lvt) \ +#define DS(lvt) \ (LAPIC_READ(lvt)&LAPIC_LVT_DS_PENDING)?" SendPending" : "Idle" #define DM(lvt) \ DM_str[(LAPIC_READ(lvt)>>LAPIC_LVT_DM_SHIFT)&LAPIC_LVT_DM_MASK] @@ -281,93 +480,99 @@ lapic_dump(void) #define IP(lvt) \ (LAPIC_READ(lvt)&LAPIC_LVT_IP_PLRITY_LOW)? "Low " : "High" - kprintf("LAPIC %d at %p version 0x%x\n", - (LAPIC_READ(ID)>>LAPIC_ID_SHIFT)&LAPIC_ID_MASK, - (void *) lapic_vbase, - LAPIC_READ(VERSION)&LAPIC_VERSION_MASK); + kprintf("LAPIC %d at %p version 0x%x\n", + lapic_safe_apicid(), + (void *) lapic_vbase, + LAPIC_READ(VERSION) & LAPIC_VERSION_MASK); kprintf("Priorities: Task 0x%x Arbitration 0x%x Processor 0x%x\n", - LAPIC_READ(TPR)&LAPIC_TPR_MASK, - LAPIC_READ(APR)&LAPIC_APR_MASK, - LAPIC_READ(PPR)&LAPIC_PPR_MASK); + LAPIC_READ(TPR) & LAPIC_TPR_MASK, + LAPIC_READ(APR) & LAPIC_APR_MASK, + LAPIC_READ(PPR) & LAPIC_PPR_MASK); kprintf("Destination Format 0x%x Logical Destination 0x%x\n", - LAPIC_READ(DFR)>>LAPIC_DFR_SHIFT, - LAPIC_READ(LDR)>>LAPIC_LDR_SHIFT); + is_x2apic ? 0 : LAPIC_READ(DFR) >> LAPIC_DFR_SHIFT, + LAPIC_READ(LDR) >> LAPIC_LDR_SHIFT); kprintf("%cEnabled %cFocusChecking SV 0x%x\n", - BOOL(LAPIC_READ(SVR)&LAPIC_SVR_ENABLE), - BOOL(!(LAPIC_READ(SVR)&LAPIC_SVR_FOCUS_OFF)), - LAPIC_READ(SVR) & LAPIC_SVR_MASK); + BOOL(LAPIC_READ(SVR) & LAPIC_SVR_ENABLE), + BOOL(!(LAPIC_READ(SVR) & LAPIC_SVR_FOCUS_OFF)), + LAPIC_READ(SVR) & LAPIC_SVR_MASK); #if CONFIG_MCA - if (mca_is_cmci_present()) + if (mca_is_cmci_present()) { kprintf("LVT_CMCI: Vector 0x%02x [%s] %s %cmasked\n", - VEC(LVT_CMCI), - DM(LVT_CMCI), - DS(LVT_CMCI), - MASK(LVT_CMCI)); + VEC(LVT_CMCI), + DM(LVT_CMCI), + DS(LVT_CMCI), + MASK(LVT_CMCI)); + } #endif kprintf("LVT_TIMER: Vector 0x%02x %s %cmasked %s\n", - VEC(LVT_TIMER), - DS(LVT_TIMER), - MASK(LVT_TIMER), - TMR_str[(LAPIC_READ(LVT_TIMER) >> LAPIC_LVT_TMR_SHIFT) - & LAPIC_LVT_TMR_MASK]); + VEC(LVT_TIMER), + DS(LVT_TIMER), + MASK(LVT_TIMER), + TMR_str[(LAPIC_READ(LVT_TIMER) >> LAPIC_LVT_TMR_SHIFT) + & LAPIC_LVT_TMR_MASK]); kprintf(" Initial Count: 0x%08x \n", LAPIC_READ(TIMER_INITIAL_COUNT)); kprintf(" Current Count: 0x%08x \n", LAPIC_READ(TIMER_CURRENT_COUNT)); kprintf(" Divide Config: 0x%08x \n", LAPIC_READ(TIMER_DIVIDE_CONFIG)); kprintf("LVT_PERFCNT: Vector 0x%02x [%s] %s %cmasked\n", - VEC(LVT_PERFCNT), - DM(LVT_PERFCNT), - DS(LVT_PERFCNT), - MASK(LVT_PERFCNT)); + VEC(LVT_PERFCNT), + DM(LVT_PERFCNT), + DS(LVT_PERFCNT), + MASK(LVT_PERFCNT)); kprintf("LVT_THERMAL: Vector 0x%02x [%s] %s %cmasked\n", - VEC(LVT_THERMAL), - DM(LVT_THERMAL), - DS(LVT_THERMAL), - MASK(LVT_THERMAL)); + VEC(LVT_THERMAL), + DM(LVT_THERMAL), + DS(LVT_THERMAL), + MASK(LVT_THERMAL)); kprintf("LVT_LINT0: Vector 0x%02x [%s][%s][%s] %s %cmasked\n", - VEC(LVT_LINT0), - DM(LVT_LINT0), - TM(LVT_LINT0), - IP(LVT_LINT0), - DS(LVT_LINT0), - MASK(LVT_LINT0)); + VEC(LVT_LINT0), + DM(LVT_LINT0), + TM(LVT_LINT0), + IP(LVT_LINT0), + DS(LVT_LINT0), + MASK(LVT_LINT0)); kprintf("LVT_LINT1: Vector 0x%02x [%s][%s][%s] %s %cmasked\n", - VEC(LVT_LINT1), - DM(LVT_LINT1), - TM(LVT_LINT1), - IP(LVT_LINT1), - DS(LVT_LINT1), - MASK(LVT_LINT1)); + VEC(LVT_LINT1), + DM(LVT_LINT1), + TM(LVT_LINT1), + IP(LVT_LINT1), + DS(LVT_LINT1), + MASK(LVT_LINT1)); kprintf("LVT_ERROR: Vector 0x%02x %s %cmasked\n", - VEC(LVT_ERROR), - DS(LVT_ERROR), - MASK(LVT_ERROR)); + VEC(LVT_ERROR), + DS(LVT_ERROR), + MASK(LVT_ERROR)); kprintf("ESR: %08x \n", lapic_esr_read()); kprintf(" "); - for(i=0xf; i>=0; i--) - kprintf("%x%x%x%x",i,i,i,i); + for (i = 0xf; i >= 0; i--) { + kprintf("%x%x%x%x", i, i, i, i); + } kprintf("\n"); kprintf("TMR: 0x"); - for(i=7; i>=0; i--) - kprintf("%08x",LAPIC_READ_OFFSET(TMR_BASE, i)); + for (i = 7; i >= 0; i--) { + kprintf("%08x", LAPIC_READ_OFFSET(TMR_BASE, i)); + } kprintf("\n"); kprintf("IRR: 0x"); - for(i=7; i>=0; i--) - kprintf("%08x",LAPIC_READ_OFFSET(IRR_BASE, i)); + for (i = 7; i >= 0; i--) { + kprintf("%08x", LAPIC_READ_OFFSET(IRR_BASE, i)); + } kprintf("\n"); kprintf("ISR: 0x"); - for(i=7; i >= 0; i--) - kprintf("%08x",LAPIC_READ_OFFSET(ISR_BASE, i)); + for (i = 7; i >= 0; i--) { + kprintf("%08x", LAPIC_READ_OFFSET(ISR_BASE, i)); + } kprintf("\n"); } boolean_t lapic_probe(void) { - uint32_t lo; - uint32_t hi; + uint32_t lo; + uint32_t hi; - if (cpuid_features() & CPUID_FEATURE_APIC) + if (cpuid_features() & CPUID_FEATURE_APIC) { return TRUE; + } if (cpuid_family() == 6 || cpuid_family() == 15) { /* @@ -385,6 +590,10 @@ lapic_probe(void) * Re-initialize cpu features info and re-check. */ cpuid_set_info(); + /* We expect this codepath will never be traversed + * due to EFI enabling the APIC. Reducing the APIC + * interrupt base dynamically is not supported. + */ if (cpuid_features() & CPUID_FEATURE_APIC) { printf("Local APIC discovered and enabled\n"); lapic_os_enabled = TRUE; @@ -397,20 +606,25 @@ lapic_probe(void) } void -lapic_shutdown(void) +lapic_shutdown(bool for_sleep) { uint32_t lo; uint32_t hi; uint32_t value; + if (for_sleep == true) { + apic_mode_before_sleep = (is_x2apic ? APIC_MODE_X2APIC : APIC_MODE_XAPIC); + } + /* Shutdown if local APIC was enabled by OS */ - if (lapic_os_enabled == FALSE) + if (lapic_os_enabled == FALSE) { return; + } mp_disable_preemption(); /* ExtINT: masked */ - if (get_cpu_number() == master_cpu) { + if (get_cpu_number() <= lapic_max_interrupt_cpunum) { value = LAPIC_READ(LVT_LINT0); value |= LAPIC_LVT_MASKED; LAPIC_WRITE(LVT_LINT0, value); @@ -437,10 +651,16 @@ lapic_shutdown(void) mp_enable_preemption(); } +boolean_t +cpu_can_exit(int cpu) +{ + return cpu > lapic_max_interrupt_cpunum; +} + void -lapic_configure(void) +lapic_configure(bool for_wake) { - int value; + int value; if (lapic_error_time_threshold == 0 && cpu_number() == 0) { nanoseconds_to_absolutetime(NSEC_PER_SEC >> 2, &lapic_error_time_threshold); @@ -449,9 +669,17 @@ lapic_configure(void) } } - /* Set flat delivery model, logical processor id */ - LAPIC_WRITE(DFR, LAPIC_DFR_FLAT); - LAPIC_WRITE(LDR, (get_cpu_number()) << LAPIC_LDR_SHIFT); + if (cpu_number() == 0) { + if (!PE_parse_boot_argn("intcpumax", &lapic_max_interrupt_cpunum, sizeof(lapic_max_interrupt_cpunum))) { + lapic_max_interrupt_cpunum = ((cpuid_features() & CPUID_FEATURE_HTT) ? 1 : 0); + } + } + + /* + * Reinitialize the APIC (handles the case where we're configured to use the X2APIC + * but firmware configured the Legacy APIC): + */ + lapic_reinit(for_wake); /* Accept all */ LAPIC_WRITE(TPR, 0); @@ -459,7 +687,7 @@ lapic_configure(void) LAPIC_WRITE(SVR, LAPIC_VECTOR(SPURIOUS) | LAPIC_SVR_ENABLE); /* ExtINT */ - if (get_cpu_number() == master_cpu) { + if (get_cpu_number() <= lapic_max_interrupt_cpunum) { value = LAPIC_READ(LVT_LINT0); value &= ~LAPIC_LVT_MASKED; value |= LAPIC_LVT_DM_EXTINT; @@ -477,12 +705,13 @@ lapic_configure(void) #if CONFIG_MCA /* CMCI, if available */ - if (mca_is_cmci_present()) + if (mca_is_cmci_present()) { LAPIC_WRITE(LVT_CMCI, LAPIC_VECTOR(CMCI)); + } #endif if (((cpu_number() == master_cpu) && lapic_errors_masked == FALSE) || - (cpu_number() != master_cpu)) { + (cpu_number() != master_cpu)) { lapic_esr_clear(); LAPIC_WRITE(LVT_ERROR, LAPIC_VECTOR(ERROR)); } @@ -490,16 +719,16 @@ lapic_configure(void) void lapic_set_timer( - boolean_t interrupt_unmasked, - lapic_timer_mode_t mode, - lapic_timer_divide_t divisor, - lapic_timer_count_t initial_count) + boolean_t interrupt_unmasked, + lapic_timer_mode_t mode, + lapic_timer_divide_t divisor, + lapic_timer_count_t initial_count) { - uint32_t timer_vector; + uint32_t timer_vector; mp_disable_preemption(); timer_vector = LAPIC_READ(LVT_TIMER); - timer_vector &= ~(LAPIC_LVT_MASKED|LAPIC_LVT_PERIODIC);; + timer_vector &= ~(LAPIC_LVT_MASKED | LAPIC_LVT_PERIODIC);; timer_vector |= interrupt_unmasked ? 0 : LAPIC_LVT_MASKED; timer_vector |= (mode == periodic) ? LAPIC_LVT_PERIODIC : 0; LAPIC_WRITE(LVT_TIMER, timer_vector); @@ -510,17 +739,17 @@ lapic_set_timer( void lapic_config_timer( - boolean_t interrupt_unmasked, - lapic_timer_mode_t mode, - lapic_timer_divide_t divisor) + boolean_t interrupt_unmasked, + lapic_timer_mode_t mode, + lapic_timer_divide_t divisor) { - uint32_t timer_vector; + uint32_t timer_vector; mp_disable_preemption(); timer_vector = LAPIC_READ(LVT_TIMER); timer_vector &= ~(LAPIC_LVT_MASKED | - LAPIC_LVT_PERIODIC | - LAPIC_LVT_TSC_DEADLINE); + LAPIC_LVT_PERIODIC | + LAPIC_LVT_TSC_DEADLINE); timer_vector |= interrupt_unmasked ? 0 : LAPIC_LVT_MASKED; timer_vector |= (mode == periodic) ? LAPIC_LVT_PERIODIC : 0; LAPIC_WRITE(LVT_TIMER, timer_vector); @@ -534,19 +763,19 @@ lapic_config_timer( void lapic_config_tsc_deadline_timer(void) { - uint32_t timer_vector; + uint32_t timer_vector; DBG("lapic_config_tsc_deadline_timer()\n"); mp_disable_preemption(); timer_vector = LAPIC_READ(LVT_TIMER); timer_vector &= ~(LAPIC_LVT_MASKED | - LAPIC_LVT_PERIODIC); + LAPIC_LVT_PERIODIC); timer_vector |= LAPIC_LVT_TSC_DEADLINE; LAPIC_WRITE(LVT_TIMER, timer_vector); /* Serialize writes per Intel OSWG */ do { - lapic_set_tsc_deadline_timer(rdtsc64() + (1ULL<<32)); + lapic_set_tsc_deadline_timer(rdtsc64() + (1ULL << 32)); } while (lapic_get_tsc_deadline_timer() == 0); lapic_set_tsc_deadline_timer(0); @@ -556,7 +785,7 @@ lapic_config_tsc_deadline_timer(void) void lapic_set_timer_fast( - lapic_timer_count_t initial_count) + lapic_timer_count_t initial_count) { LAPIC_WRITE(LVT_TIMER, LAPIC_READ(LVT_TIMER) & ~LAPIC_LVT_MASKED); LAPIC_WRITE(TIMER_INITIAL_COUNT, initial_count); @@ -577,23 +806,27 @@ lapic_get_tsc_deadline_timer(void) void lapic_get_timer( - lapic_timer_mode_t *mode, - lapic_timer_divide_t *divisor, - lapic_timer_count_t *initial_count, - lapic_timer_count_t *current_count) + lapic_timer_mode_t *mode, + lapic_timer_divide_t *divisor, + lapic_timer_count_t *initial_count, + lapic_timer_count_t *current_count) { mp_disable_preemption(); - if (mode) + if (mode) { *mode = (LAPIC_READ(LVT_TIMER) & LAPIC_LVT_PERIODIC) ? - periodic : one_shot; - if (divisor) + periodic : one_shot; + } + if (divisor) { *divisor = LAPIC_READ(TIMER_DIVIDE_CONFIG) & LAPIC_TIMER_DIVIDE_MASK; - if (initial_count) + } + if (initial_count) { *initial_count = LAPIC_READ(TIMER_INITIAL_COUNT); - if (current_count) + } + if (current_count) { *current_count = LAPIC_READ(TIMER_CURRENT_COUNT); + } mp_enable_preemption(); -} +} static inline void _lapic_end_of_interrupt(void) @@ -607,11 +840,15 @@ lapic_end_of_interrupt(void) _lapic_end_of_interrupt(); } -void lapic_unmask_perfcnt_interrupt(void) { +void +lapic_unmask_perfcnt_interrupt(void) +{ LAPIC_WRITE(LVT_PERFCNT, LAPIC_VECTOR(PERFCNT)); } -void lapic_set_perfcnt_interrupt_mask(boolean_t mask) { +void +lapic_set_perfcnt_interrupt_mask(boolean_t mask) +{ uint32_t m = (mask ? LAPIC_LVT_MASKED : 0); LAPIC_WRITE(LVT_PERFCNT, LAPIC_VECTOR(PERFCNT) | m); } @@ -619,8 +856,9 @@ void lapic_set_perfcnt_interrupt_mask(boolean_t mask) { void lapic_set_intr_func(int vector, i386_intr_func_t func) { - if (vector > lapic_interrupt_base) + if (vector > lapic_interrupt_base) { vector -= lapic_interrupt_base; + } switch (vector) { case LAPIC_NMI_INTERRUPT: @@ -634,19 +872,21 @@ lapic_set_intr_func(int vector, i386_intr_func_t func) break; default: panic("lapic_set_intr_func(%d,%p) invalid vector\n", - vector, func); + vector, func); } } -void lapic_set_pmi_func(i386_intr_func_t func) { +void +lapic_set_pmi_func(i386_intr_func_t func) +{ lapic_set_intr_func(LAPIC_VECTOR(PERFCNT), func); } int lapic_interrupt(int interrupt_num, x86_saved_state_t *state) { - int retval = 0; - int esr = -1; + int retval = 0; + int esr = -1; interrupt_num -= lapic_interrupt_base; if (interrupt_num < 0) { @@ -654,18 +894,19 @@ lapic_interrupt(int interrupt_num, x86_saved_state_t *state) lapic_intr_func[LAPIC_NMI_INTERRUPT] != NULL) { retval = (*lapic_intr_func[LAPIC_NMI_INTERRUPT])(state); return retval; - } - else + } else { return 0; + } } - switch(interrupt_num) { + switch (interrupt_num) { case LAPIC_TIMER_INTERRUPT: case LAPIC_THERMAL_INTERRUPT: case LAPIC_INTERPROCESSOR_INTERRUPT: case LAPIC_PM_INTERRUPT: - if (lapic_intr_func[interrupt_num] != NULL) + if (lapic_intr_func[interrupt_num] != NULL) { (void) (*lapic_intr_func[interrupt_num])(state); + } _lapic_end_of_interrupt(); retval = 1; break; @@ -684,8 +925,9 @@ lapic_interrupt(int interrupt_num, x86_saved_state_t *state) } break; case LAPIC_CMCI_INTERRUPT: - if (lapic_intr_func[interrupt_num] != NULL) + if (lapic_intr_func[interrupt_num] != NULL) { (void) (*lapic_intr_func[interrupt_num])(state); + } /* return 0 for plaform expert to handle */ break; case LAPIC_ERROR_INTERRUPT: @@ -711,7 +953,7 @@ lapic_interrupt(int interrupt_num, x86_saved_state_t *state) lapic_dump(); if ((debug_boot_arg && (lapic_dont_panic == FALSE)) || - cpu_number() != master_cpu) { + cpu_number() != master_cpu) { panic("Local APIC error, ESR: %d\n", esr); } @@ -723,8 +965,7 @@ lapic_interrupt(int interrupt_num, x86_saved_state_t *state) LAPIC_WRITE(LVT_ERROR, LAPIC_READ(LVT_ERROR) | LAPIC_LVT_MASKED); printf("Local APIC: errors masked\n"); } - } - else { + } else { lapic_last_master_error = abstime; lapic_master_error_count = 0; } @@ -739,18 +980,13 @@ lapic_interrupt(int interrupt_num, x86_saved_state_t *state) /* No EOI required here */ retval = 1; break; - case LAPIC_PMC_SW_INTERRUPT: - { -#if CONFIG_COUNTERS - thread_t old, new; - ml_get_csw_threads(&old, &new); - - if (pmc_context_switch(old, new) == TRUE) { - retval = 1; - /* No EOI required for SWI */ - } -#endif /* CONFIG_COUNTERS */ - } + case LAPIC_PMC_SW_INTERRUPT: + { + } + break; + case LAPIC_KICK_INTERRUPT: + _lapic_end_of_interrupt(); + retval = 1; break; } @@ -762,18 +998,19 @@ lapic_smm_restore(void) { boolean_t state; - if (lapic_os_enabled == FALSE) + if (lapic_os_enabled == FALSE) { return; + } state = ml_set_interrupts_enabled(FALSE); - if (LAPIC_ISR_IS_SET(LAPIC_REDUCED_INTERRUPT_BASE, TIMER)) { + if (LAPIC_ISR_IS_SET(LAPIC_REDUCED_INTERRUPT_BASE, TIMER)) { /* * Bogus SMI handler enables interrupts but does not know about * local APIC interrupt sources. When APIC timer counts down to * zero while in SMM, local APIC will end up waiting for an EOI * but no interrupt was delivered to the OS. - */ + */ _lapic_end_of_interrupt(); /* @@ -793,20 +1030,23 @@ lapic_smm_restore(void) void lapic_send_ipi(int cpu, int vector) { - boolean_t state; + boolean_t state; - if (vector < lapic_interrupt_base) + if (vector < lapic_interrupt_base) { vector += lapic_interrupt_base; + } state = ml_set_interrupts_enabled(FALSE); - /* Wait for pending outgoing send to complete */ - while (LAPIC_READ(ICR) & LAPIC_ICR_DS_PENDING) { - cpu_pause(); + /* X2APIC's ICR doesn't have a pending bit. */ + if (!is_x2apic) { + /* Wait for pending outgoing send to complete */ + while (LAPIC_READ_ICR() & LAPIC_ICR_DS_PENDING) { + cpu_pause(); + } } - LAPIC_WRITE(ICRD, cpu_to_lapic[cpu] << LAPIC_ICRD_DEST_SHIFT); - LAPIC_WRITE(ICR, vector | LAPIC_ICR_DM_FIXED); + LAPIC_WRITE_ICR(cpu_to_lapic[cpu], vector | LAPIC_ICR_DM_FIXED); (void) ml_set_interrupts_enabled(state); } @@ -818,24 +1058,25 @@ lapic_send_ipi(int cpu, int vector) boolean_t lapic_is_interrupt_pending(void) { - int i; + int i; for (i = 0; i < 8; i += 1) { if ((LAPIC_READ_OFFSET(IRR_BASE, i) != 0) || - (LAPIC_READ_OFFSET(ISR_BASE, i) != 0)) - return (TRUE); + (LAPIC_READ_OFFSET(ISR_BASE, i) != 0)) { + return TRUE; + } } - return (FALSE); + return FALSE; } boolean_t lapic_is_interrupting(uint8_t vector) { - int i; - int bit; - uint32_t irr; - uint32_t isr; + int i; + int bit; + uint32_t irr; + uint32_t isr; i = vector / 32; bit = 1 << (vector % 32); @@ -843,35 +1084,39 @@ lapic_is_interrupting(uint8_t vector) irr = LAPIC_READ_OFFSET(IRR_BASE, i); isr = LAPIC_READ_OFFSET(ISR_BASE, i); - if ((irr | isr) & bit) - return (TRUE); + if ((irr | isr) & bit) { + return TRUE; + } - return (FALSE); + return FALSE; } void lapic_interrupt_counts(uint64_t intrs[256]) { - int i; - int j; - int bit; - uint32_t irr; - uint32_t isr; + int i; + int j; + int bit; + uint32_t irr; + uint32_t isr; - if (intrs == NULL) + if (intrs == NULL) { return; + } for (i = 0; i < 8; i += 1) { irr = LAPIC_READ_OFFSET(IRR_BASE, i); isr = LAPIC_READ_OFFSET(ISR_BASE, i); - if ((isr | irr) == 0) + if ((isr | irr) == 0) { continue; + } for (j = (i == 0) ? 16 : 0; j < 32; j += 1) { bit = (32 * i) + j; - if ((isr | irr) & (1 << j)) + if ((isr | irr) & (1 << j)) { intrs[bit] += 1; + } } } } @@ -879,13 +1124,13 @@ lapic_interrupt_counts(uint64_t intrs[256]) void lapic_disable_timer(void) { - uint32_t lvt_timer; + uint32_t lvt_timer; /* - * If we're in deadline timer mode, + * If we're in deadline timer mode, * simply clear the deadline timer, otherwise * mask the timer interrupt and clear the countdown. - */ + */ lvt_timer = LAPIC_READ(LVT_TIMER); if (lvt_timer & LAPIC_LVT_TSC_DEADLINE) { wrmsr64(MSR_IA32_TSC_DEADLINE, 0); @@ -896,3 +1141,27 @@ lapic_disable_timer(void) } } +/* SPI returning the CMCI vector */ +uint8_t +lapic_get_cmci_vector(void) +{ + uint8_t cmci_vector = 0; +#if CONFIG_MCA + /* CMCI, if available */ + if (mca_is_cmci_present()) { + cmci_vector = LAPIC_VECTOR(CMCI); + } +#endif + return cmci_vector; +} + +#if DEVELOPMENT || DEBUG +extern void lapic_trigger_MC(void); +void +lapic_trigger_MC(void) +{ + /* A 64-bit access to any register will do it. */ + volatile uint64_t dummy = *(volatile uint64_t *) (volatile void *) LAPIC_MMIO(ID); + dummy++; +} +#endif