2 * Copyright (c) 2008 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
32 #include <mach/mach_types.h>
33 #include <mach/kern_return.h>
35 #include <kern/kern_types.h>
36 #include <kern/cpu_number.h>
37 #include <kern/cpu_data.h>
38 #include <kern/assert.h>
39 #include <kern/machine.h>
41 #include <vm/vm_map.h>
42 #include <vm/vm_kern.h>
44 #include <i386/lapic.h>
45 #include <i386/cpuid.h>
46 #include <i386/proc_reg.h>
47 #include <i386/machine_cpu.h>
48 #include <i386/misc_protos.h>
50 #include <i386/mtrr.h>
51 #include <i386/postcode.h>
52 #include <i386/cpu_threads.h>
53 #include <i386/trap.h>
54 #include <i386/machine_routines.h>
55 #include <i386/machine_check.h>
58 #include <machine/db_machdep.h>
61 #include <sys/kdebug.h>
64 #define PAUSE delay(1000000)
65 #define DBG(x...) kprintf(x)
71 /* Initialize lapic_id so cpu_number() works on non SMP systems */
72 unsigned long lapic_id_initdata
= 0;
73 unsigned long lapic_id
= (unsigned long)&lapic_id_initdata
;
74 vm_offset_t lapic_start
;
76 static i386_intr_func_t lapic_intr_func
[LAPIC_FUNC_TABLE_SIZE
];
78 /* TRUE if local APIC was enabled by the OS not by the BIOS */
79 static boolean_t lapic_os_enabled
= FALSE
;
81 /* Base vector for local APIC interrupt sources */
82 int lapic_interrupt_base
= LAPIC_DEFAULT_INTERRUPT_BASE
;
84 int lapic_to_cpu
[MAX_CPUS
];
85 int cpu_to_lapic
[MAX_CPUS
];
88 lapic_cpu_map_init(void)
92 for (i
= 0; i
< MAX_CPUS
; i
++) {
99 lapic_cpu_map(int apic_id
, int cpu
)
101 cpu_to_lapic
[cpu
] = apic_id
;
102 lapic_to_cpu
[apic_id
] = cpu
;
106 * Retrieve the local apic ID a cpu.
108 * Returns the local apic ID for the given processor.
109 * If the processor does not exist or apic not configured, returns -1.
113 ml_get_apicid(uint32_t cpu
)
115 if(cpu
>= (uint32_t)MAX_CPUS
)
116 return 0xFFFFFFFF; /* Return -1 if cpu too big */
118 /* Return the apic ID (or -1 if not configured) */
119 return (uint32_t)cpu_to_lapic
[cpu
];
125 lapic_cpu_map_dump(void)
129 for (i
= 0; i
< MAX_CPUS
; i
++) {
130 if (cpu_to_lapic
[i
] == -1)
132 kprintf("cpu_to_lapic[%d]: %d\n",
135 for (i
= 0; i
< MAX_CPUS
; i
++) {
136 if (lapic_to_cpu
[i
] == -1)
138 kprintf("lapic_to_cpu[%d]: %d\n",
142 #endif /* MP_DEBUG */
148 vm_map_entry_t entry
;
151 boolean_t is_boot_processor
;
152 boolean_t is_lapic_enabled
;
153 vm_offset_t lapic_base
;
155 /* Examine the local APIC state */
156 rdmsr(MSR_IA32_APIC_BASE
, lo
, hi
);
157 is_boot_processor
= (lo
& MSR_IA32_APIC_BASE_BSP
) != 0;
158 is_lapic_enabled
= (lo
& MSR_IA32_APIC_BASE_ENABLE
) != 0;
159 lapic_base
= (lo
& MSR_IA32_APIC_BASE_BASE
);
160 kprintf("MSR_IA32_APIC_BASE 0x%x %s %s\n", lapic_base
,
161 is_lapic_enabled
? "enabled" : "disabled",
162 is_boot_processor
? "BSP" : "AP");
163 if (!is_boot_processor
|| !is_lapic_enabled
)
164 panic("Unexpected local APIC state\n");
166 /* Establish a map to the local apic */
167 lapic_start
= vm_map_min(kernel_map
);
168 result
= vm_map_find_space(kernel_map
,
169 (vm_map_address_t
*) &lapic_start
,
170 round_page(LAPIC_SIZE
), 0,
171 VM_MAKE_TAG(VM_MEMORY_IOKIT
), &entry
);
172 if (result
!= KERN_SUCCESS
) {
173 panic("smp_init: vm_map_find_entry FAILED (err=%d)", result
);
175 vm_map_unlock(kernel_map
);
176 /* Map in the local APIC non-cacheable, as recommended by Intel
177 * in section 8.4.1 of the "System Programming Guide".
179 pmap_enter(pmap_kernel(),
181 (ppnum_t
) i386_btop(lapic_base
),
182 VM_PROT_READ
|VM_PROT_WRITE
,
185 lapic_id
= (unsigned long)(lapic_start
+ LAPIC_ID
);
187 if ((LAPIC_READ(VERSION
)&LAPIC_VERSION_MASK
) < 0x14) {
188 printf("Local APIC version 0x%x, 0x14 or greater expected\n",
189 (LAPIC_READ(VERSION
)&LAPIC_VERSION_MASK
));
192 /* Set up the lapic_id <-> cpu_number map and add this boot processor */
193 lapic_cpu_map_init();
194 lapic_cpu_map((LAPIC_READ(ID
)>>LAPIC_ID_SHIFT
)&LAPIC_ID_MASK
, 0);
195 kprintf("Boot cpu local APIC id 0x%x\n", cpu_to_lapic
[0]);
202 /* write-read register */
203 LAPIC_WRITE(ERROR_STATUS
, 0);
204 return LAPIC_READ(ERROR_STATUS
);
208 lapic_esr_clear(void)
210 LAPIC_WRITE(ERROR_STATUS
, 0);
211 LAPIC_WRITE(ERROR_STATUS
, 0);
214 static const char *DM_str
[8] = {
229 #define BOOL(a) ((a)?' ':'!')
231 LAPIC_READ(lvt)&LAPIC_LVT_VECTOR_MASK
233 (LAPIC_READ(lvt)&LAPIC_LVT_DS_PENDING)?" SendPending" : "Idle"
235 DM_str[(LAPIC_READ(lvt)>>LAPIC_LVT_DM_SHIFT)&LAPIC_LVT_DM_MASK]
237 BOOL(LAPIC_READ(lvt)&LAPIC_LVT_MASKED)
239 (LAPIC_READ(lvt)&LAPIC_LVT_TM_LEVEL)? "Level" : "Edge"
241 (LAPIC_READ(lvt)&LAPIC_LVT_IP_PLRITY_LOW)? "Low " : "High"
243 kprintf("LAPIC %d at 0x%x version 0x%x\n",
244 (LAPIC_READ(ID
)>>LAPIC_ID_SHIFT
)&LAPIC_ID_MASK
,
246 LAPIC_READ(VERSION
)&LAPIC_VERSION_MASK
);
247 kprintf("Priorities: Task 0x%x Arbitration 0x%x Processor 0x%x\n",
248 LAPIC_READ(TPR
)&LAPIC_TPR_MASK
,
249 LAPIC_READ(APR
)&LAPIC_APR_MASK
,
250 LAPIC_READ(PPR
)&LAPIC_PPR_MASK
);
251 kprintf("Destination Format 0x%x Logical Destination 0x%x\n",
252 LAPIC_READ(DFR
)>>LAPIC_DFR_SHIFT
,
253 LAPIC_READ(LDR
)>>LAPIC_LDR_SHIFT
);
254 kprintf("%cEnabled %cFocusChecking SV 0x%x\n",
255 BOOL(LAPIC_READ(SVR
)&LAPIC_SVR_ENABLE
),
256 BOOL(!(LAPIC_READ(SVR
)&LAPIC_SVR_FOCUS_OFF
)),
257 LAPIC_READ(SVR
) & LAPIC_SVR_MASK
);
258 kprintf("LVT_TIMER: Vector 0x%02x %s %cmasked %s\n",
262 (LAPIC_READ(LVT_TIMER
)&LAPIC_LVT_PERIODIC
)?"Periodic":"OneShot");
263 kprintf(" Initial Count: 0x%08x \n", LAPIC_READ(TIMER_INITIAL_COUNT
));
264 kprintf(" Current Count: 0x%08x \n", LAPIC_READ(TIMER_CURRENT_COUNT
));
265 kprintf(" Divide Config: 0x%08x \n", LAPIC_READ(TIMER_DIVIDE_CONFIG
));
266 kprintf("LVT_PERFCNT: Vector 0x%02x [%s] %s %cmasked\n",
271 kprintf("LVT_THERMAL: Vector 0x%02x [%s] %s %cmasked\n",
276 kprintf("LVT_LINT0: Vector 0x%02x [%s][%s][%s] %s %cmasked\n",
283 kprintf("LVT_LINT1: Vector 0x%02x [%s][%s][%s] %s %cmasked\n",
290 kprintf("LVT_ERROR: Vector 0x%02x %s %cmasked\n",
294 kprintf("ESR: %08x \n", lapic_esr_read());
296 for(i
=0xf; i
>=0; i
--)
297 kprintf("%x%x%x%x",i
,i
,i
,i
);
301 kprintf("%08x",LAPIC_READ_OFFSET(TMR_BASE
, i
*0x10));
305 kprintf("%08x",LAPIC_READ_OFFSET(IRR_BASE
, i
*0x10));
308 for(i
=7; i
>= 0; i
--)
309 kprintf("%08x",LAPIC_READ_OFFSET(ISR_BASE
, i
*0x10));
320 db_apic(__unused db_expr_t addr
,
321 __unused
int have_addr
,
322 __unused db_expr_t count
,
323 __unused
char *modif
)
339 if (cpuid_features() & CPUID_FEATURE_APIC
)
342 if (cpuid_family() == 6 || cpuid_family() == 15) {
345 * There may be a local APIC which wasn't enabled by BIOS.
346 * So we try to enable it explicitly.
348 rdmsr(MSR_IA32_APIC_BASE
, lo
, hi
);
349 lo
&= ~MSR_IA32_APIC_BASE_BASE
;
350 lo
|= MSR_IA32_APIC_BASE_ENABLE
| LAPIC_START
;
351 lo
|= MSR_IA32_APIC_BASE_ENABLE
;
352 wrmsr(MSR_IA32_APIC_BASE
, lo
, hi
);
355 * Re-initialize cpu features info and re-check.
358 if (cpuid_features() & CPUID_FEATURE_APIC
) {
359 printf("Local APIC discovered and enabled\n");
360 lapic_os_enabled
= TRUE
;
361 lapic_interrupt_base
= LAPIC_REDUCED_INTERRUPT_BASE
;
376 /* Shutdown if local APIC was enabled by OS */
377 if (lapic_os_enabled
== FALSE
)
380 mp_disable_preemption();
383 if (get_cpu_number() == master_cpu
) {
384 value
= LAPIC_READ(LVT_LINT0
);
385 value
|= LAPIC_LVT_MASKED
;
386 LAPIC_WRITE(LVT_LINT0
, value
);
390 LAPIC_WRITE(LVT_TIMER
, LAPIC_READ(LVT_TIMER
) | LAPIC_LVT_MASKED
);
392 /* Perfmon: masked */
393 LAPIC_WRITE(LVT_PERFCNT
, LAPIC_READ(LVT_PERFCNT
) | LAPIC_LVT_MASKED
);
396 LAPIC_WRITE(LVT_ERROR
, LAPIC_READ(LVT_ERROR
) | LAPIC_LVT_MASKED
);
398 /* APIC software disabled */
399 LAPIC_WRITE(SVR
, LAPIC_READ(SVR
) & ~LAPIC_SVR_ENABLE
);
401 /* Bypass the APIC completely and update cpu features */
402 rdmsr(MSR_IA32_APIC_BASE
, lo
, hi
);
403 lo
&= ~MSR_IA32_APIC_BASE_ENABLE
;
404 wrmsr(MSR_IA32_APIC_BASE
, lo
, hi
);
407 mp_enable_preemption();
411 lapic_configure(void)
415 /* Set flat delivery model, logical processor id */
416 LAPIC_WRITE(DFR
, LAPIC_DFR_FLAT
);
417 LAPIC_WRITE(LDR
, (get_cpu_number()) << LAPIC_LDR_SHIFT
);
422 LAPIC_WRITE(SVR
, LAPIC_VECTOR(SPURIOUS
) | LAPIC_SVR_ENABLE
);
425 if (get_cpu_number() == master_cpu
) {
426 value
= LAPIC_READ(LVT_LINT0
);
427 value
&= ~LAPIC_LVT_MASKED
;
428 value
|= LAPIC_LVT_DM_EXTINT
;
429 LAPIC_WRITE(LVT_LINT0
, value
);
432 /* Timer: unmasked, one-shot */
433 LAPIC_WRITE(LVT_TIMER
, LAPIC_VECTOR(TIMER
));
435 /* Perfmon: unmasked */
436 LAPIC_WRITE(LVT_PERFCNT
, LAPIC_VECTOR(PERFCNT
));
438 /* Thermal: unmasked */
439 LAPIC_WRITE(LVT_THERMAL
, LAPIC_VECTOR(THERMAL
));
443 LAPIC_WRITE(LVT_ERROR
, LAPIC_VECTOR(ERROR
));
449 lapic_timer_mode_t mode
,
450 lapic_timer_divide_t divisor
,
451 lapic_timer_count_t initial_count
)
454 uint32_t timer_vector
;
456 state
= ml_set_interrupts_enabled(FALSE
);
457 timer_vector
= LAPIC_READ(LVT_TIMER
);
458 timer_vector
&= ~(LAPIC_LVT_MASKED
|LAPIC_LVT_PERIODIC
);;
459 timer_vector
|= interrupt
? 0 : LAPIC_LVT_MASKED
;
460 timer_vector
|= (mode
== periodic
) ? LAPIC_LVT_PERIODIC
: 0;
461 LAPIC_WRITE(LVT_TIMER
, timer_vector
);
462 LAPIC_WRITE(TIMER_DIVIDE_CONFIG
, divisor
);
463 LAPIC_WRITE(TIMER_INITIAL_COUNT
, initial_count
);
464 ml_set_interrupts_enabled(state
);
469 lapic_timer_mode_t
*mode
,
470 lapic_timer_divide_t
*divisor
,
471 lapic_timer_count_t
*initial_count
,
472 lapic_timer_count_t
*current_count
)
476 state
= ml_set_interrupts_enabled(FALSE
);
478 *mode
= (LAPIC_READ(LVT_TIMER
) & LAPIC_LVT_PERIODIC
) ?
481 *divisor
= LAPIC_READ(TIMER_DIVIDE_CONFIG
) & LAPIC_TIMER_DIVIDE_MASK
;
483 *initial_count
= LAPIC_READ(TIMER_INITIAL_COUNT
);
485 *current_count
= LAPIC_READ(TIMER_CURRENT_COUNT
);
486 ml_set_interrupts_enabled(state
);
490 _lapic_end_of_interrupt(void)
496 lapic_end_of_interrupt(void)
498 _lapic_end_of_interrupt();
502 lapic_set_intr_func(int vector
, i386_intr_func_t func
)
504 if (vector
> lapic_interrupt_base
)
505 vector
-= lapic_interrupt_base
;
508 case LAPIC_NMI_INTERRUPT
:
509 case LAPIC_INTERPROCESSOR_INTERRUPT
:
510 case LAPIC_TIMER_INTERRUPT
:
511 case LAPIC_THERMAL_INTERRUPT
:
512 case LAPIC_PERFCNT_INTERRUPT
:
513 lapic_intr_func
[vector
] = func
;
516 panic("lapic_set_intr_func(%d,%p) invalid vector\n",
522 lapic_interrupt(int interrupt
, x86_saved_state_t
*state
)
526 interrupt
-= lapic_interrupt_base
;
528 if (interrupt
== (LAPIC_NMI_INTERRUPT
- lapic_interrupt_base
) &&
529 lapic_intr_func
[LAPIC_NMI_INTERRUPT
] != NULL
) {
530 retval
= (*lapic_intr_func
[LAPIC_NMI_INTERRUPT
])(state
);
531 _lapic_end_of_interrupt();
539 case LAPIC_TIMER_INTERRUPT
:
540 case LAPIC_THERMAL_INTERRUPT
:
541 case LAPIC_INTERPROCESSOR_INTERRUPT
:
542 if (lapic_intr_func
[interrupt
] != NULL
)
543 (void) (*lapic_intr_func
[interrupt
])(state
);
544 if (interrupt
== LAPIC_PERFCNT_INTERRUPT
)
545 LAPIC_WRITE(LVT_PERFCNT
, LAPIC_VECTOR(PERFCNT
));
546 _lapic_end_of_interrupt();
549 case LAPIC_ERROR_INTERRUPT
:
551 panic("Local APIC error\n");
552 _lapic_end_of_interrupt();
555 case LAPIC_SPURIOUS_INTERRUPT
:
557 /* No EOI required here */
566 lapic_smm_restore(void)
570 if (lapic_os_enabled
== FALSE
)
573 state
= ml_set_interrupts_enabled(FALSE
);
575 if (LAPIC_ISR_IS_SET(LAPIC_REDUCED_INTERRUPT_BASE
, TIMER
)) {
577 * Bogus SMI handler enables interrupts but does not know about
578 * local APIC interrupt sources. When APIC timer counts down to
579 * zero while in SMM, local APIC will end up waiting for an EOI
580 * but no interrupt was delivered to the OS.
582 _lapic_end_of_interrupt();
585 * timer is one-shot, trigger another quick countdown to trigger
586 * another timer interrupt.
588 if (LAPIC_READ(TIMER_CURRENT_COUNT
) == 0) {
589 LAPIC_WRITE(TIMER_INITIAL_COUNT
, 1);
592 kprintf("lapic_smm_restore\n");
595 ml_set_interrupts_enabled(state
);