osfmk/i386/lapic.c

   1 /*
   2  * Copyright (c) 2008-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31
  32 #include <mach/mach_types.h>
  33 #include <mach/kern_return.h>
  34
  35 #include <kern/kern_types.h>
  36 #include <kern/cpu_number.h>
  37 #include <kern/cpu_data.h>
  38 #include <kern/assert.h>
  39 #include <kern/machine.h>
  40 #include <kern/debug.h>
  41
  42 #include <vm/vm_map.h>
  43 #include <vm/vm_kern.h>
  44
  45 #include <i386/lapic.h>
  46 #include <i386/cpuid.h>
  47 #include <i386/proc_reg.h>
  48 #include <i386/machine_cpu.h>
  49 #include <i386/misc_protos.h>
  50 #include <i386/mp.h>
  51 #include <i386/mtrr.h>
  52 #include <i386/postcode.h>
  53 #include <i386/cpu_threads.h>
  54 #include <i386/machine_routines.h>
  55 #if CONFIG_MCA
  56 #include <i386/machine_check.h>
  57 #endif
  58
  59 #if CONFIG_COUNTERS
  60 #include <pmc/pmc.h>
  61 #endif
  62
  63 #if MACH_KDB
  64 #include <machine/db_machdep.h>
  65 #endif
  66
  67 #include <sys/kdebug.h>
  68
  69 #if     MP_DEBUG
  70 #define PAUSE           delay(1000000)
  71 #define DBG(x...)       kprintf(x)
  72 #else
  73 #define DBG(x...)
  74 #define PAUSE
  75 #endif  /* MP_DEBUG */
  76
  77 /* Initialize lapic_id so cpu_number() works on non SMP systems */
  78 unsigned long   lapic_id_initdata = 0;
  79 unsigned long   lapic_id = (unsigned long)&lapic_id_initdata;
  80 vm_offset_t     lapic_start;
  81
  82 static i386_intr_func_t lapic_intr_func[LAPIC_FUNC_TABLE_SIZE];
  83
  84 /* TRUE if local APIC was enabled by the OS not by the BIOS */
  85 static boolean_t lapic_os_enabled = FALSE;
  86
  87 static boolean_t lapic_errors_masked = FALSE;
  88 static uint64_t lapic_last_master_error = 0;
  89 static uint64_t lapic_error_time_threshold = 0;
  90 static unsigned lapic_master_error_count = 0;
  91 static unsigned lapic_error_count_threshold = 5;
  92 static boolean_t lapic_dont_panic = FALSE;
  93
  94 /* Base vector for local APIC interrupt sources */
  95 int lapic_interrupt_base = LAPIC_DEFAULT_INTERRUPT_BASE;
  96
  97 #define         MAX_LAPICIDS    (LAPIC_ID_MAX+1)
  98 int             lapic_to_cpu[MAX_LAPICIDS];
  99 int             cpu_to_lapic[MAX_CPUS];
 100
 101 static void
 102 lapic_cpu_map_init(void)
 103 {
 104         int     i;
 105
 106         for (i = 0; i < MAX_CPUS; i++)
 107                 cpu_to_lapic[i] = -1;
 108         for (i = 0; i < MAX_LAPICIDS; i++)
 109                 lapic_to_cpu[i] = -1;
 110 }
 111
 112 void
 113 lapic_cpu_map(int apic_id, int cpu)
 114 {
 115         assert(apic_id < MAX_LAPICIDS);
 116         assert(cpu < MAX_CPUS);
 117         cpu_to_lapic[cpu] = apic_id;
 118         lapic_to_cpu[apic_id] = cpu;
 119 }
 120
 121 /*
 122  * Retrieve the local apic ID a cpu.
 123  *
 124  * Returns the local apic ID for the given processor.
 125  * If the processor does not exist or apic not configured, returns -1.
 126  */
 127
 128 uint32_t
 129 ml_get_apicid(uint32_t cpu)
 130 {
 131         if(cpu >= (uint32_t)MAX_CPUS)
 132                 return 0xFFFFFFFF;      /* Return -1 if cpu too big */
 133
 134         /* Return the apic ID (or -1 if not configured) */
 135         return (uint32_t)cpu_to_lapic[cpu];
 136
 137 }
 138
 139 uint32_t
 140 ml_get_cpuid(uint32_t lapic_index)
 141 {
 142         if(lapic_index >= (uint32_t)MAX_LAPICIDS)
 143                 return 0xFFFFFFFF;      /* Return -1 if cpu too big */
 144
 145         /* Return the cpu ID (or -1 if not configured) */
 146         return (uint32_t)lapic_to_cpu[lapic_index];
 147
 148 }
 149
 150
 151 #ifdef MP_DEBUG
 152 static void
 153 lapic_cpu_map_dump(void)
 154 {
 155         int     i;
 156
 157         for (i = 0; i < MAX_CPUS; i++) {
 158                 if (cpu_to_lapic[i] == -1)
 159                         continue;
 160                 kprintf("cpu_to_lapic[%d]: %d\n",
 161                         i, cpu_to_lapic[i]);
 162         }
 163         for (i = 0; i < MAX_LAPICIDS; i++) {
 164                 if (lapic_to_cpu[i] == -1)
 165                         continue;
 166                 kprintf("lapic_to_cpu[%d]: %d\n",
 167                         i, lapic_to_cpu[i]);
 168         }
 169 }
 170 #endif /* MP_DEBUG */
 171
 172 void
 173 lapic_init(void)
 174 {
 175         int             result;
 176         vm_map_entry_t  entry;
 177         uint32_t        lo;
 178         uint32_t        hi;
 179         boolean_t       is_boot_processor;
 180         boolean_t       is_lapic_enabled;
 181         vm_offset_t     lapic_base;
 182
 183         /* Examine the local APIC state */
 184         rdmsr(MSR_IA32_APIC_BASE, lo, hi);
 185         is_boot_processor = (lo & MSR_IA32_APIC_BASE_BSP) != 0;
 186         is_lapic_enabled  = (lo & MSR_IA32_APIC_BASE_ENABLE) != 0;
 187         lapic_base = (lo &  MSR_IA32_APIC_BASE_BASE);
 188         kprintf("MSR_IA32_APIC_BASE %p %s %s\n", (void *) lapic_base,
 189                 is_lapic_enabled ? "enabled" : "disabled",
 190                 is_boot_processor ? "BSP" : "AP");
 191         if (!is_boot_processor || !is_lapic_enabled)
 192                 panic("Unexpected local APIC state\n");
 193
 194         /* Establish a map to the local apic */
 195         lapic_start = (vm_offset_t)vm_map_min(kernel_map);
 196         result = vm_map_find_space(kernel_map,
 197                                    (vm_map_address_t *) &lapic_start,
 198                                    round_page(LAPIC_SIZE), 0,
 199                                    VM_MAKE_TAG(VM_MEMORY_IOKIT), &entry);
 200         if (result != KERN_SUCCESS) {
 201                 panic("smp_init: vm_map_find_entry FAILED (err=%d)", result);
 202         }
 203         vm_map_unlock(kernel_map);
 204 /* Map in the local APIC non-cacheable, as recommended by Intel
 205  * in section 8.4.1 of the "System Programming Guide".
 206  */
 207         pmap_enter(pmap_kernel(),
 208                         lapic_start,
 209                         (ppnum_t) i386_btop(lapic_base),
 210                         VM_PROT_READ|VM_PROT_WRITE,
 211                         VM_WIMG_IO,
 212                         TRUE);
 213         lapic_id = (unsigned long)(lapic_start + LAPIC_ID);
 214
 215         if ((LAPIC_READ(VERSION)&LAPIC_VERSION_MASK) < 0x14) {
 216                 panic("Local APIC version 0x%x, 0x14 or more expected\n",
 217                         (LAPIC_READ(VERSION)&LAPIC_VERSION_MASK));
 218         }
 219
 220         /* Set up the lapic_id <-> cpu_number map and add this boot processor */
 221         lapic_cpu_map_init();
 222         lapic_cpu_map((LAPIC_READ(ID)>>LAPIC_ID_SHIFT)&LAPIC_ID_MASK, 0);
 223         kprintf("Boot cpu local APIC id 0x%x\n", cpu_to_lapic[0]);
 224 }
 225
 226
 227 static int
 228 lapic_esr_read(void)
 229 {
 230         /* write-read register */
 231         LAPIC_WRITE(ERROR_STATUS, 0);
 232         return LAPIC_READ(ERROR_STATUS);
 233 }
 234
 235 static void
 236 lapic_esr_clear(void)
 237 {
 238         LAPIC_WRITE(ERROR_STATUS, 0);
 239         LAPIC_WRITE(ERROR_STATUS, 0);
 240 }
 241
 242 static const char *DM_str[8] = {
 243         "Fixed",
 244         "Lowest Priority",
 245         "Invalid",
 246         "Invalid",
 247         "NMI",
 248         "Reset",
 249         "Invalid",
 250         "ExtINT"};
 251
 252 void
 253 lapic_dump(void)
 254 {
 255         int     i;
 256
 257 #define BOOL(a) ((a)?' ':'!')
 258 #define VEC(lvt) \
 259         LAPIC_READ(lvt)&LAPIC_LVT_VECTOR_MASK
 260 #define DS(lvt) \
 261         (LAPIC_READ(lvt)&LAPIC_LVT_DS_PENDING)?" SendPending" : "Idle"
 262 #define DM(lvt) \
 263         DM_str[(LAPIC_READ(lvt)>>LAPIC_LVT_DM_SHIFT)&LAPIC_LVT_DM_MASK]
 264 #define MASK(lvt) \
 265         BOOL(LAPIC_READ(lvt)&LAPIC_LVT_MASKED)
 266 #define TM(lvt) \
 267         (LAPIC_READ(lvt)&LAPIC_LVT_TM_LEVEL)? "Level" : "Edge"
 268 #define IP(lvt) \
 269         (LAPIC_READ(lvt)&LAPIC_LVT_IP_PLRITY_LOW)? "Low " : "High"
 270
 271         kprintf("LAPIC %d at %p version 0x%x\n",
 272                 (LAPIC_READ(ID)>>LAPIC_ID_SHIFT)&LAPIC_ID_MASK,
 273                 (void *) lapic_start,
 274                 LAPIC_READ(VERSION)&LAPIC_VERSION_MASK);
 275         kprintf("Priorities: Task 0x%x  Arbitration 0x%x  Processor 0x%x\n",
 276                 LAPIC_READ(TPR)&LAPIC_TPR_MASK,
 277                 LAPIC_READ(APR)&LAPIC_APR_MASK,
 278                 LAPIC_READ(PPR)&LAPIC_PPR_MASK);
 279         kprintf("Destination Format 0x%x Logical Destination 0x%x\n",
 280                 LAPIC_READ(DFR)>>LAPIC_DFR_SHIFT,
 281                 LAPIC_READ(LDR)>>LAPIC_LDR_SHIFT);
 282         kprintf("%cEnabled %cFocusChecking SV 0x%x\n",
 283                 BOOL(LAPIC_READ(SVR)&LAPIC_SVR_ENABLE),
 284                 BOOL(!(LAPIC_READ(SVR)&LAPIC_SVR_FOCUS_OFF)),
 285                 LAPIC_READ(SVR) & LAPIC_SVR_MASK);
 286 #if CONFIG_MCA
 287         if (mca_is_cmci_present())
 288                 kprintf("LVT_CMCI:    Vector 0x%02x [%s] %s %cmasked\n",
 289                         VEC(LVT_CMCI),
 290                         DM(LVT_CMCI),
 291                         DS(LVT_CMCI),
 292                         MASK(LVT_CMCI));
 293 #endif
 294         kprintf("LVT_TIMER:   Vector 0x%02x %s %cmasked %s\n",
 295                 VEC(LVT_TIMER),
 296                 DS(LVT_TIMER),
 297                 MASK(LVT_TIMER),
 298                 (LAPIC_READ(LVT_TIMER)&LAPIC_LVT_PERIODIC)?"Periodic":"OneShot");
 299         kprintf("  Initial Count: 0x%08x \n", LAPIC_READ(TIMER_INITIAL_COUNT));
 300         kprintf("  Current Count: 0x%08x \n", LAPIC_READ(TIMER_CURRENT_COUNT));
 301         kprintf("  Divide Config: 0x%08x \n", LAPIC_READ(TIMER_DIVIDE_CONFIG));
 302         kprintf("LVT_PERFCNT: Vector 0x%02x [%s] %s %cmasked\n",
 303                 VEC(LVT_PERFCNT),
 304                 DM(LVT_PERFCNT),
 305                 DS(LVT_PERFCNT),
 306                 MASK(LVT_PERFCNT));
 307         kprintf("LVT_THERMAL: Vector 0x%02x [%s] %s %cmasked\n",
 308                 VEC(LVT_THERMAL),
 309                 DM(LVT_THERMAL),
 310                 DS(LVT_THERMAL),
 311                 MASK(LVT_THERMAL));
 312         kprintf("LVT_LINT0:   Vector 0x%02x [%s][%s][%s] %s %cmasked\n",
 313                 VEC(LVT_LINT0),
 314                 DM(LVT_LINT0),
 315                 TM(LVT_LINT0),
 316                 IP(LVT_LINT0),
 317                 DS(LVT_LINT0),
 318                 MASK(LVT_LINT0));
 319         kprintf("LVT_LINT1:   Vector 0x%02x [%s][%s][%s] %s %cmasked\n",
 320                 VEC(LVT_LINT1),
 321                 DM(LVT_LINT1),
 322                 TM(LVT_LINT1),
 323                 IP(LVT_LINT1),
 324                 DS(LVT_LINT1),
 325                 MASK(LVT_LINT1));
 326         kprintf("LVT_ERROR:   Vector 0x%02x %s %cmasked\n",
 327                 VEC(LVT_ERROR),
 328                 DS(LVT_ERROR),
 329                 MASK(LVT_ERROR));
 330         kprintf("ESR: %08x \n", lapic_esr_read());
 331         kprintf("       ");
 332         for(i=0xf; i>=0; i--)
 333                 kprintf("%x%x%x%x",i,i,i,i);
 334         kprintf("\n");
 335         kprintf("TMR: 0x");
 336         for(i=7; i>=0; i--)
 337                 kprintf("%08x",LAPIC_READ_OFFSET(TMR_BASE, i*0x10));
 338         kprintf("\n");
 339         kprintf("IRR: 0x");
 340         for(i=7; i>=0; i--)
 341                 kprintf("%08x",LAPIC_READ_OFFSET(IRR_BASE, i*0x10));
 342         kprintf("\n");
 343         kprintf("ISR: 0x");
 344         for(i=7; i >= 0; i--)
 345                 kprintf("%08x",LAPIC_READ_OFFSET(ISR_BASE, i*0x10));
 346         kprintf("\n");
 347 }
 348
 349 #if MACH_KDB
 350 /*
 351  *      Displays apic junk
 352  *
 353  *      da
 354  */
 355 void
 356 db_apic(__unused db_expr_t addr,
 357         __unused int have_addr,
 358         __unused db_expr_t count,
 359         __unused char *modif)
 360 {
 361
 362         lapic_dump();
 363
 364         return;
 365 }
 366
 367 #endif
 368
 369 boolean_t
 370 lapic_probe(void)
 371 {
 372         uint32_t        lo;
 373         uint32_t        hi;
 374
 375         if (cpuid_features() & CPUID_FEATURE_APIC)
 376                 return TRUE;
 377
 378         if (cpuid_family() == 6 || cpuid_family() == 15) {
 379                 /*
 380                  * Mobile Pentiums:
 381                  * There may be a local APIC which wasn't enabled by BIOS.
 382                  * So we try to enable it explicitly.
 383                  */
 384                 rdmsr(MSR_IA32_APIC_BASE, lo, hi);
 385                 lo &= ~MSR_IA32_APIC_BASE_BASE;
 386                 lo |= MSR_IA32_APIC_BASE_ENABLE | LAPIC_START;
 387                 lo |= MSR_IA32_APIC_BASE_ENABLE;
 388                 wrmsr(MSR_IA32_APIC_BASE, lo, hi);
 389
 390                 /*
 391                  * Re-initialize cpu features info and re-check.
 392                  */
 393                 cpuid_set_info();
 394                 if (cpuid_features() & CPUID_FEATURE_APIC) {
 395                         printf("Local APIC discovered and enabled\n");
 396                         lapic_os_enabled = TRUE;
 397                         lapic_interrupt_base = LAPIC_REDUCED_INTERRUPT_BASE;
 398                         return TRUE;
 399                 }
 400         }
 401
 402         return FALSE;
 403 }
 404
 405 void
 406 lapic_shutdown(void)
 407 {
 408         uint32_t lo;
 409         uint32_t hi;
 410         uint32_t value;
 411
 412         /* Shutdown if local APIC was enabled by OS */
 413         if (lapic_os_enabled == FALSE)
 414                 return;
 415
 416         mp_disable_preemption();
 417
 418         /* ExtINT: masked */
 419         if (get_cpu_number() == master_cpu) {
 420                 value = LAPIC_READ(LVT_LINT0);
 421                 value |= LAPIC_LVT_MASKED;
 422                 LAPIC_WRITE(LVT_LINT0, value);
 423         }
 424
 425         /* Error: masked */
 426         LAPIC_WRITE(LVT_ERROR, LAPIC_READ(LVT_ERROR) | LAPIC_LVT_MASKED);
 427
 428         /* Timer: masked */
 429         LAPIC_WRITE(LVT_TIMER, LAPIC_READ(LVT_TIMER) | LAPIC_LVT_MASKED);
 430
 431         /* Perfmon: masked */
 432         LAPIC_WRITE(LVT_PERFCNT, LAPIC_READ(LVT_PERFCNT) | LAPIC_LVT_MASKED);
 433
 434         /* APIC software disabled */
 435         LAPIC_WRITE(SVR, LAPIC_READ(SVR) & ~LAPIC_SVR_ENABLE);
 436
 437         /* Bypass the APIC completely and update cpu features */
 438         rdmsr(MSR_IA32_APIC_BASE, lo, hi);
 439         lo &= ~MSR_IA32_APIC_BASE_ENABLE;
 440         wrmsr(MSR_IA32_APIC_BASE, lo, hi);
 441         cpuid_set_info();
 442
 443         mp_enable_preemption();
 444 }
 445
 446 void
 447 lapic_configure(void)
 448 {
 449         int     value;
 450
 451         if (lapic_error_time_threshold == 0 && cpu_number() == 0) {
 452                 nanoseconds_to_absolutetime(NSEC_PER_SEC >> 2, &lapic_error_time_threshold);
 453                 if (!PE_parse_boot_argn("lapic_dont_panic", &lapic_dont_panic, sizeof(lapic_dont_panic))) {
 454                         lapic_dont_panic = FALSE;
 455                 }
 456         }
 457
 458         /* Set flat delivery model, logical processor id */
 459         LAPIC_WRITE(DFR, LAPIC_DFR_FLAT);
 460         LAPIC_WRITE(LDR, (get_cpu_number()) << LAPIC_LDR_SHIFT);
 461
 462         /* Accept all */
 463         LAPIC_WRITE(TPR, 0);
 464
 465         LAPIC_WRITE(SVR, LAPIC_VECTOR(SPURIOUS) | LAPIC_SVR_ENABLE);
 466
 467         /* ExtINT */
 468         if (get_cpu_number() == master_cpu) {
 469                 value = LAPIC_READ(LVT_LINT0);
 470                 value &= ~LAPIC_LVT_MASKED;
 471                 value |= LAPIC_LVT_DM_EXTINT;
 472                 LAPIC_WRITE(LVT_LINT0, value);
 473         }
 474
 475         /* Timer: unmasked, one-shot */
 476         LAPIC_WRITE(LVT_TIMER, LAPIC_VECTOR(TIMER));
 477
 478         /* Perfmon: unmasked */
 479         LAPIC_WRITE(LVT_PERFCNT, LAPIC_VECTOR(PERFCNT));
 480
 481         /* Thermal: unmasked */
 482         LAPIC_WRITE(LVT_THERMAL, LAPIC_VECTOR(THERMAL));
 483
 484 #if CONFIG_MCA
 485         /* CMCI, if available */
 486         if (mca_is_cmci_present())
 487                 LAPIC_WRITE(LVT_CMCI, LAPIC_VECTOR(CMCI));
 488 #endif
 489
 490         if (((cpu_number() == master_cpu) && lapic_errors_masked == FALSE) ||
 491                 (cpu_number() != master_cpu)) {
 492                 lapic_esr_clear();
 493                 LAPIC_WRITE(LVT_ERROR, LAPIC_VECTOR(ERROR));
 494         }
 495 }
 496
 497 void
 498 lapic_set_timer(
 499         boolean_t               interrupt_unmasked,
 500         lapic_timer_mode_t      mode,
 501         lapic_timer_divide_t    divisor,
 502         lapic_timer_count_t     initial_count)
 503 {
 504         boolean_t       state;
 505         uint32_t        timer_vector;
 506
 507         state = ml_set_interrupts_enabled(FALSE);
 508         timer_vector = LAPIC_READ(LVT_TIMER);
 509         timer_vector &= ~(LAPIC_LVT_MASKED|LAPIC_LVT_PERIODIC);;
 510         timer_vector |= interrupt_unmasked ? 0 : LAPIC_LVT_MASKED;
 511         timer_vector |= (mode == periodic) ? LAPIC_LVT_PERIODIC : 0;
 512         LAPIC_WRITE(LVT_TIMER, timer_vector);
 513         LAPIC_WRITE(TIMER_DIVIDE_CONFIG, divisor);
 514         LAPIC_WRITE(TIMER_INITIAL_COUNT, initial_count);
 515         ml_set_interrupts_enabled(state);
 516 }
 517
 518 void
 519 lapic_get_timer(
 520         lapic_timer_mode_t      *mode,
 521         lapic_timer_divide_t    *divisor,
 522         lapic_timer_count_t     *initial_count,
 523         lapic_timer_count_t     *current_count)
 524 {
 525         boolean_t       state;
 526
 527         state = ml_set_interrupts_enabled(FALSE);
 528         if (mode)
 529                 *mode = (LAPIC_READ(LVT_TIMER) & LAPIC_LVT_PERIODIC) ?
 530                                 periodic : one_shot;
 531         if (divisor)
 532                 *divisor = LAPIC_READ(TIMER_DIVIDE_CONFIG) & LAPIC_TIMER_DIVIDE_MASK;
 533         if (initial_count)
 534                 *initial_count = LAPIC_READ(TIMER_INITIAL_COUNT);
 535         if (current_count)
 536                 *current_count = LAPIC_READ(TIMER_CURRENT_COUNT);
 537         ml_set_interrupts_enabled(state);
 538 }
 539
 540 static inline void
 541 _lapic_end_of_interrupt(void)
 542 {
 543         LAPIC_WRITE(EOI, 0);
 544 }
 545
 546 void
 547 lapic_end_of_interrupt(void)
 548 {
 549         _lapic_end_of_interrupt();
 550 }
 551
 552 void lapic_unmask_perfcnt_interrupt(void) {
 553         LAPIC_WRITE(LVT_PERFCNT, LAPIC_VECTOR(PERFCNT));
 554 }
 555
 556 void
 557 lapic_set_intr_func(int vector, i386_intr_func_t func)
 558 {
 559         if (vector > lapic_interrupt_base)
 560                 vector -= lapic_interrupt_base;
 561
 562         switch (vector) {
 563         case LAPIC_NMI_INTERRUPT:
 564         case LAPIC_INTERPROCESSOR_INTERRUPT:
 565         case LAPIC_TIMER_INTERRUPT:
 566         case LAPIC_THERMAL_INTERRUPT:
 567         case LAPIC_PERFCNT_INTERRUPT:
 568         case LAPIC_CMCI_INTERRUPT:
 569         case LAPIC_PM_INTERRUPT:
 570                 lapic_intr_func[vector] = func;
 571                 break;
 572         default:
 573                 panic("lapic_set_intr_func(%d,%p) invalid vector\n",
 574                         vector, func);
 575         }
 576 }
 577
 578 int
 579 lapic_interrupt(int interrupt_num, x86_saved_state_t *state)
 580 {
 581         int     retval = 0;
 582         int     esr = -1;
 583
 584         interrupt_num -= lapic_interrupt_base;
 585         if (interrupt_num < 0) {
 586                 if (interrupt_num == (LAPIC_NMI_INTERRUPT - lapic_interrupt_base) &&
 587                     lapic_intr_func[LAPIC_NMI_INTERRUPT] != NULL) {
 588                         retval = (*lapic_intr_func[LAPIC_NMI_INTERRUPT])(state);
 589                         _lapic_end_of_interrupt();
 590                         return retval;
 591                 }
 592                 else
 593                         return 0;
 594         }
 595
 596         switch(interrupt_num) {
 597         case LAPIC_TIMER_INTERRUPT:
 598         case LAPIC_THERMAL_INTERRUPT:
 599         case LAPIC_INTERPROCESSOR_INTERRUPT:
 600         case LAPIC_PM_INTERRUPT:
 601                 if (lapic_intr_func[interrupt_num] != NULL)
 602                         (void) (*lapic_intr_func[interrupt_num])(state);
 603                 _lapic_end_of_interrupt();
 604                 retval = 1;
 605                 break;
 606         case LAPIC_PERFCNT_INTERRUPT:
 607                 /* If a function has been registered, invoke it.  Otherwise,
 608                  * pass up to IOKit.
 609                  */
 610                 if (lapic_intr_func[interrupt_num] != NULL) {
 611                         (void) (*lapic_intr_func[interrupt_num])(state);
 612                         /* Unmask the interrupt since we don't expect legacy users
 613                          * to be responsible for it.
 614                          */
 615                         lapic_unmask_perfcnt_interrupt();
 616                         _lapic_end_of_interrupt();
 617                         retval = 1;
 618                 }
 619                 break;
 620         case LAPIC_CMCI_INTERRUPT:
 621                 if (lapic_intr_func[interrupt_num] != NULL)
 622                         (void) (*lapic_intr_func[interrupt_num])(state);
 623                 /* return 0 for plaform expert to handle */
 624                 break;
 625         case LAPIC_ERROR_INTERRUPT:
 626                 /* We treat error interrupts on APs as fatal.
 627                  * The current interrupt steering scheme directs most
 628                  * external interrupts to the BSP (HPET interrupts being
 629                  * a notable exception); hence, such an error
 630                  * on an AP may signify LVT corruption (with "may" being
 631                  * the operative word). On the BSP, we adopt a more
 632                  * lenient approach, in the interests of enhancing
 633                  * debuggability and reducing fragility.
 634                  * If "lapic_error_count_threshold" error interrupts
 635                  * occur within "lapic_error_time_threshold" absolute
 636                  * time units, we mask the error vector and log. The
 637                  * error interrupts themselves are likely
 638                  * side effects of issues which are beyond the purview of
 639                  * the local APIC interrupt handler, however. The Error
 640                  * Status Register value (the illegal destination
 641                  * vector code is one observed in practice) indicates
 642                  * the immediate cause of the error.
 643                  */
 644                 esr = lapic_esr_read();
 645                 lapic_dump();
 646
 647                 if ((debug_boot_arg && (lapic_dont_panic == FALSE)) ||
 648                         cpu_number() != master_cpu) {
 649                         panic("Local APIC error, ESR: %d\n", esr);
 650                 }
 651
 652                 if (cpu_number() == master_cpu) {
 653                         uint64_t abstime = mach_absolute_time();
 654                         if ((abstime - lapic_last_master_error) < lapic_error_time_threshold) {
 655                                 if (lapic_master_error_count++ > lapic_error_count_threshold) {
 656                                         lapic_errors_masked = TRUE;
 657                                         LAPIC_WRITE(LVT_ERROR, LAPIC_READ(LVT_ERROR) | LAPIC_LVT_MASKED);
 658                                         printf("Local APIC: errors masked\n");
 659                                 }
 660                         }
 661                         else {
 662                                 lapic_last_master_error = abstime;
 663                                 lapic_master_error_count = 0;
 664                         }
 665                         printf("Local APIC error on master CPU, ESR: %d, error count this run: %d\n", esr, lapic_master_error_count);
 666                 }
 667
 668                 _lapic_end_of_interrupt();
 669                 retval = 1;
 670                 break;
 671         case LAPIC_SPURIOUS_INTERRUPT:
 672                 kprintf("SPIV\n");
 673                 /* No EOI required here */
 674                 retval = 1;
 675                 break;
 676         case LAPIC_PMC_SW_INTERRUPT:
 677                 {
 678 #if CONFIG_COUNTERS
 679                         thread_t old, new;
 680                         ml_get_csw_threads(&old, &new);
 681
 682                         if (pmc_context_switch(old, new) == TRUE) {
 683                                 retval = 1;
 684                                 /* No EOI required for SWI */
 685                         }
 686 #endif /* CONFIG_COUNTERS */
 687                 }
 688                 break;
 689         }
 690
 691         return retval;
 692 }
 693
 694 void
 695 lapic_smm_restore(void)
 696 {
 697         boolean_t state;
 698
 699         if (lapic_os_enabled == FALSE)
 700                 return;
 701
 702         state = ml_set_interrupts_enabled(FALSE);
 703
 704         if (LAPIC_ISR_IS_SET(LAPIC_REDUCED_INTERRUPT_BASE, TIMER)) {
 705                 /*
 706                  * Bogus SMI handler enables interrupts but does not know about
 707                  * local APIC interrupt sources. When APIC timer counts down to
 708                  * zero while in SMM, local APIC will end up waiting for an EOI
 709                  * but no interrupt was delivered to the OS.
 710                  */
 711                 _lapic_end_of_interrupt();
 712
 713                 /*
 714                  * timer is one-shot, trigger another quick countdown to trigger
 715                  * another timer interrupt.
 716                  */
 717                 if (LAPIC_READ(TIMER_CURRENT_COUNT) == 0) {
 718                         LAPIC_WRITE(TIMER_INITIAL_COUNT, 1);
 719                 }
 720
 721                 kprintf("lapic_smm_restore\n");
 722         }
 723
 724         ml_set_interrupts_enabled(state);
 725 }
 726
 727 void
 728 lapic_send_ipi(int cpu, int vector)
 729 {
 730         boolean_t       state;
 731
 732         if (vector < lapic_interrupt_base)
 733                 vector += lapic_interrupt_base;
 734
 735         state = ml_set_interrupts_enabled(FALSE);
 736
 737         /* Wait for pending outgoing send to complete */
 738         while (LAPIC_READ(ICR) & LAPIC_ICR_DS_PENDING) {
 739                 cpu_pause();
 740         }
 741
 742         LAPIC_WRITE(ICRD, cpu_to_lapic[cpu] << LAPIC_ICRD_DEST_SHIFT);
 743         LAPIC_WRITE(ICR, vector | LAPIC_ICR_DM_FIXED);
 744
 745         (void) ml_set_interrupts_enabled(state);
 746 }