2 * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
28 #include <i386/machine_routines.h>
29 #include <i386/io_map_entries.h>
30 #include <i386/cpuid.h>
32 #include <kern/processor.h>
33 #include <kern/machine.h>
34 #include <kern/cpu_data.h>
35 #include <kern/cpu_number.h>
36 #include <kern/thread.h>
37 #include <i386/cpu_data.h>
38 #include <i386/machine_cpu.h>
40 #include <i386/mp_events.h>
41 #include <i386/cpu_threads.h>
42 #include <i386/pmap.h>
43 #include <i386/misc_protos.h>
44 #include <i386/pmCPU.h>
45 #include <i386/proc_reg.h>
46 #include <mach/vm_param.h>
48 #include <i386/db_machdep.h>
49 #include <ddb/db_aout.h>
50 #include <ddb/db_access.h>
51 #include <ddb/db_sym.h>
52 #include <ddb/db_variables.h>
53 #include <ddb/db_command.h>
54 #include <ddb/db_output.h>
55 #include <ddb/db_expr.h>
58 #define MIN(a,b) ((a)<(b)? (a) : (b))
61 #define DBG(x...) kprintf("DBG: " x)
66 extern void initialize_screen(Boot_Video
*, unsigned int);
67 extern thread_t
Shutdown_context(thread_t thread
, void (*doshutdown
)(processor_t
),processor_t processor
);
68 extern void wakeup(void *);
69 extern unsigned KernelRelocOffset
;
71 static int max_cpus_initialized
= 0;
73 unsigned int LockTimeOut
= 12500000;
74 unsigned int MutexSpin
= 0;
76 #define MAX_CPUS_SET 0x1
77 #define MAX_CPUS_WAIT 0x2
79 /* IO memory map services */
81 /* Map memory map IO space */
82 vm_offset_t
ml_io_map(
83 vm_offset_t phys_addr
,
86 return(io_map(phys_addr
,size
,VM_WIMG_IO
));
89 /* boot memory allocation */
90 vm_offset_t
ml_static_malloc(
91 __unused vm_size_t size
)
93 return((vm_offset_t
)NULL
);
97 void ml_get_bouncepool_info(vm_offset_t
*phys_addr
, vm_size_t
*size
)
99 *phys_addr
= bounce_pool_base
;
100 *size
= bounce_pool_size
;
108 return (vm_offset_t
)((paddr
-KernelRelocOffset
) | LINEAR_KERNEL_ADDRESS
);
115 return (vm_offset_t
)((unsigned) paddr
| LINEAR_KERNEL_ADDRESS
);
120 * Routine: ml_static_mfree
128 vm_offset_t vaddr_cur
;
131 // if (vaddr < VM_MIN_KERNEL_ADDRESS) return;
133 assert((vaddr
& (PAGE_SIZE
-1)) == 0); /* must be page aligned */
135 for (vaddr_cur
= vaddr
;
136 vaddr_cur
< round_page_32(vaddr
+size
);
137 vaddr_cur
+= PAGE_SIZE
) {
138 ppn
= pmap_find_phys(kernel_pmap
, (addr64_t
)vaddr_cur
);
139 if (ppn
!= (vm_offset_t
)NULL
) {
140 pmap_remove(kernel_pmap
, (addr64_t
)vaddr_cur
, (addr64_t
)(vaddr_cur
+PAGE_SIZE
));
141 vm_page_create(ppn
,(ppn
+1));
142 vm_page_wire_count
--;
148 /* virtual to physical on wired pages */
149 vm_offset_t
ml_vtophys(
152 return kvtophys(vaddr
);
155 /* Interrupt handling */
157 /* Initialize Interrupts */
158 void ml_init_interrupt(void)
160 (void) ml_set_interrupts_enabled(TRUE
);
163 /* Get Interrupts Enabled */
164 boolean_t
ml_get_interrupts_enabled(void)
168 __asm__
volatile("pushf; popl %0" : "=r" (flags
));
169 return (flags
& EFL_IF
) != 0;
172 /* Set Interrupts Enabled */
173 boolean_t
ml_set_interrupts_enabled(boolean_t enable
)
177 __asm__
volatile("pushf; popl %0" : "=r" (flags
));
182 myast
= ast_pending();
184 if ( (get_preemption_level() == 0) && (*myast
& AST_URGENT
) ) {
185 __asm__
volatile("sti");
186 __asm__
volatile ("int $0xff");
188 __asm__
volatile ("sti");
192 __asm__
volatile("cli");
195 return (flags
& EFL_IF
) != 0;
198 /* Check if running at interrupt context */
199 boolean_t
ml_at_interrupt_context(void)
201 return get_interrupt_level() != 0;
204 /* Generate a fake interrupt */
205 void ml_cause_interrupt(void)
207 panic("ml_cause_interrupt not defined yet on Intel");
210 void ml_thread_policy(
213 unsigned policy_info
)
215 if (policy_id
== MACHINE_GROUP
)
216 thread_bind(thread
, master_processor
);
218 if (policy_info
& MACHINE_NETWORK_WORKLOOP
) {
219 spl_t s
= splsched();
223 set_priority(thread
, thread
->priority
+ 1);
225 thread_unlock(thread
);
230 /* Initialize Interrupts */
231 void ml_install_interrupt_handler(
235 IOInterruptHandler handler
,
238 boolean_t current_state
;
240 current_state
= ml_get_interrupts_enabled();
242 PE_install_interrupt_handler(nub
, source
, target
,
243 (IOInterruptHandler
) handler
, refCon
);
245 (void) ml_set_interrupts_enabled(current_state
);
247 initialize_screen(0, kPEAcquireScreen
);
254 cpu_core_t
*my_core
= cpu_core();
258 * We halt this cpu thread
259 * unless kernel param idlehalt is false and no other thread
260 * in the same core is active - if so, don't halt so that this
261 * core doesn't go into a low-power mode.
262 * For 4/4, we set a null "active cr3" while idle.
264 others_active
= !atomic_decl_and_test(
265 (long *) &my_core
->active_threads
, 1);
266 if (idlehalt
|| others_active
) {
267 DBGLOG(cpu_handle
, cpu_number(), MP_IDLE
);
268 MARK_CPU_IDLE(cpu_number());
269 machine_idle_cstate();
270 MARK_CPU_ACTIVE(cpu_number());
271 DBGLOG(cpu_handle
, cpu_number(), MP_UNIDLE
);
273 __asm__
volatile("sti");
275 atomic_incl((long *) &my_core
->active_threads
, 1);
280 processor_t processor
)
282 cpu_interrupt(PROCESSOR_DATA(processor
, slot_num
));
286 machine_processor_shutdown(
288 void (*doshutdown
)(processor_t
),
289 processor_t processor
)
291 fpu_save_context(thread
);
292 return(Shutdown_context(thread
, doshutdown
, processor
));
296 ml_processor_register(
299 processor_t
*processor_out
,
300 ipi_handler_t
*ipi_handler
,
304 cpu_data_t
*this_cpu_datap
;
306 this_cpu_datap
= cpu_data_alloc(boot_cpu
);
307 if (this_cpu_datap
== NULL
) {
310 target_cpu
= this_cpu_datap
->cpu_number
;
311 assert((boot_cpu
&& (target_cpu
== 0)) ||
312 (!boot_cpu
&& (target_cpu
!= 0)));
314 lapic_cpu_map(lapic_id
, target_cpu
);
316 this_cpu_datap
->cpu_id
= cpu_id
;
317 this_cpu_datap
->cpu_phys_number
= lapic_id
;
319 this_cpu_datap
->cpu_console_buf
= console_cpu_alloc(boot_cpu
);
320 if (this_cpu_datap
->cpu_console_buf
== NULL
)
323 this_cpu_datap
->cpu_chud
= chudxnu_cpu_alloc(boot_cpu
);
324 if (this_cpu_datap
->cpu_chud
== NULL
)
328 this_cpu_datap
->cpu_core
= cpu_thread_alloc(target_cpu
);
330 this_cpu_datap
->cpu_pmap
= pmap_cpu_alloc(boot_cpu
);
331 if (this_cpu_datap
->cpu_pmap
== NULL
)
334 this_cpu_datap
->cpu_processor
= cpu_processor_alloc(boot_cpu
);
335 if (this_cpu_datap
->cpu_processor
== NULL
)
337 processor_init(this_cpu_datap
->cpu_processor
, target_cpu
);
340 *processor_out
= this_cpu_datap
->cpu_processor
;
346 cpu_processor_free(this_cpu_datap
->cpu_processor
);
347 pmap_cpu_free(this_cpu_datap
->cpu_pmap
);
348 chudxnu_cpu_free(this_cpu_datap
->cpu_chud
);
349 console_cpu_free(this_cpu_datap
->cpu_console_buf
);
354 ml_cpu_get_info(ml_cpu_info_t
*cpu_infop
)
356 boolean_t os_supports_sse
;
357 i386_cpu_info_t
*cpuid_infop
;
359 if (cpu_infop
== NULL
)
363 * Are we supporting MMX/SSE/SSE2/SSE3?
364 * As distinct from whether the cpu has these capabilities.
366 os_supports_sse
= get_cr4() & CR4_XMM
;
367 if ((cpuid_features() & CPUID_FEATURE_MNI
) && os_supports_sse
)
368 cpu_infop
->vector_unit
= 6;
369 else if ((cpuid_features() & CPUID_FEATURE_SSE3
) && os_supports_sse
)
370 cpu_infop
->vector_unit
= 5;
371 else if ((cpuid_features() & CPUID_FEATURE_SSE2
) && os_supports_sse
)
372 cpu_infop
->vector_unit
= 4;
373 else if ((cpuid_features() & CPUID_FEATURE_SSE
) && os_supports_sse
)
374 cpu_infop
->vector_unit
= 3;
375 else if (cpuid_features() & CPUID_FEATURE_MMX
)
376 cpu_infop
->vector_unit
= 2;
378 cpu_infop
->vector_unit
= 0;
380 cpuid_infop
= cpuid_info();
382 cpu_infop
->cache_line_size
= cpuid_infop
->cache_linesize
;
384 cpu_infop
->l1_icache_size
= cpuid_infop
->cache_size
[L1I
];
385 cpu_infop
->l1_dcache_size
= cpuid_infop
->cache_size
[L1D
];
387 if (cpuid_infop
->cache_size
[L2U
] > 0) {
388 cpu_infop
->l2_settings
= 1;
389 cpu_infop
->l2_cache_size
= cpuid_infop
->cache_size
[L2U
];
391 cpu_infop
->l2_settings
= 0;
392 cpu_infop
->l2_cache_size
= 0xFFFFFFFF;
395 if (cpuid_infop
->cache_size
[L3U
] > 0) {
396 cpu_infop
->l3_settings
= 1;
397 cpu_infop
->l3_cache_size
= cpuid_infop
->cache_size
[L3U
];
399 cpu_infop
->l3_settings
= 0;
400 cpu_infop
->l3_cache_size
= 0xFFFFFFFF;
405 ml_init_max_cpus(unsigned long max_cpus
)
407 boolean_t current_state
;
409 current_state
= ml_set_interrupts_enabled(FALSE
);
410 if (max_cpus_initialized
!= MAX_CPUS_SET
) {
411 if (max_cpus
> 0 && max_cpus
<= MAX_CPUS
) {
413 * Note: max_cpus is the number of enable processors
414 * that ACPI found; max_ncpus is the maximum number
415 * that the kernel supports or that the "cpus="
416 * boot-arg has set. Here we take int minimum.
418 machine_info
.max_cpus
= MIN(max_cpus
, max_ncpus
);
420 if (max_cpus_initialized
== MAX_CPUS_WAIT
)
421 wakeup((event_t
)&max_cpus_initialized
);
422 max_cpus_initialized
= MAX_CPUS_SET
;
424 (void) ml_set_interrupts_enabled(current_state
);
428 ml_get_max_cpus(void)
430 boolean_t current_state
;
432 current_state
= ml_set_interrupts_enabled(FALSE
);
433 if (max_cpus_initialized
!= MAX_CPUS_SET
) {
434 max_cpus_initialized
= MAX_CPUS_WAIT
;
435 assert_wait((event_t
)&max_cpus_initialized
, THREAD_UNINT
);
436 (void)thread_block(THREAD_CONTINUE_NULL
);
438 (void) ml_set_interrupts_enabled(current_state
);
439 return(machine_info
.max_cpus
);
443 * Routine: ml_init_lock_timeout
447 ml_init_lock_timeout(void)
453 * XXX As currently implemented for x86, LockTimeOut should be a
454 * cycle (tsc) count not an absolute time (nanoseconds) -
455 * but it's of the right order.
457 nanoseconds_to_absolutetime(NSEC_PER_SEC
>>2, &abstime
);
458 LockTimeOut
= (unsigned int)abstime
;
460 if (PE_parse_boot_arg("mtxspin", &mtxspin
)) {
461 if (mtxspin
> USEC_PER_SEC
>>4)
462 mtxspin
= USEC_PER_SEC
>>4;
463 nanoseconds_to_absolutetime(mtxspin
*NSEC_PER_USEC
, &abstime
);
465 nanoseconds_to_absolutetime(10*NSEC_PER_USEC
, &abstime
);
467 MutexSpin
= (unsigned int)abstime
;
471 * This is called from the machine-independent routine cpu_up()
472 * to perform machine-dependent info updates. Defer to cpu_thread_init().
481 * This is called from the machine-independent routine cpu_down()
482 * to perform machine-dependent info updates.
490 /* Stubs for pc tracing mechanism */
493 int pc_trace_cnt
= 0;
514 * The following are required for parts of the kernel
515 * that cannot resolve these functions as inlines:
517 extern thread_t
current_act(void);
521 return(current_thread_fast());
524 #undef current_thread
525 extern thread_t
current_thread(void);
529 return(current_thread_fast());
533 * Set the worst-case time for the C4 to C2 transition.
534 * The maxdelay parameter is in nanoseconds.
538 ml_set_maxsnoop(uint32_t maxdelay
)
540 C4C2SnoopDelay
= maxdelay
; /* Set the transition time */
541 machine_nap_policy(); /* Adjust the current nap state */
546 * Get the worst-case time for the C4 to C2 transition. Returns nanoseconds.
550 ml_get_maxsnoop(void)
552 return C4C2SnoopDelay
; /* Set the transition time */
557 ml_get_maxbusdelay(void)
563 * Set the maximum delay time allowed for snoop on the bus.
565 * Note that this value will be compared to the amount of time that it takes
566 * to transition from a non-snooping power state (C4) to a snooping state (C2).
567 * If maxBusDelay is less than C4C2SnoopDelay,
568 * we will not enter the lowest power state.
572 ml_set_maxbusdelay(uint32_t mdelay
)
574 maxBusDelay
= mdelay
; /* Set the delay */
575 machine_nap_policy(); /* Adjust the current nap state */
579 boolean_t
ml_is64bit(void) {
581 return (cpu_mode_is64bit());
585 boolean_t
ml_thread_is64bit(thread_t thread
) {
587 return (thread_is_64bit(thread
));
591 boolean_t
ml_state_is64bit(void *saved_state
) {
593 return is_saved_state64(saved_state
);
596 void ml_cpu_set_ldt(int selector
)
599 * Avoid loading the LDT
600 * if we're setting the KERNEL LDT and it's already set.
602 if (selector
== KERNEL_LDT
&&
603 current_cpu_datap()->cpu_ldt
== KERNEL_LDT
)
607 * If 64bit this requires a mode switch (and back).
609 if (cpu_mode_is64bit())
610 ml_64bit_lldt(selector
);
613 current_cpu_datap()->cpu_ldt
= selector
;
616 void ml_fp_setvalid(boolean_t value
)
624 * Display the global msrs
629 db_msr(__unused db_expr_t addr
,
630 __unused
int have_addr
,
631 __unused db_expr_t count
,
632 __unused
char *modif
)
635 uint32_t i
, msrlow
, msrhigh
;
637 /* Try all of the first 4096 msrs */
638 for (i
= 0; i
< 4096; i
++) {
639 if (!rdmsr_carefully(i
, &msrlow
, &msrhigh
)) {
640 db_printf("%08X - %08X.%08X\n", i
, msrhigh
, msrlow
);
644 /* Try all of the 4096 msrs at 0x0C000000 */
645 for (i
= 0; i
< 4096; i
++) {
646 if (!rdmsr_carefully(0x0C000000 | i
, &msrlow
, &msrhigh
)) {
647 db_printf("%08X - %08X.%08X\n",
648 0x0C000000 | i
, msrhigh
, msrlow
);
652 /* Try all of the 4096 msrs at 0xC0000000 */
653 for (i
= 0; i
< 4096; i
++) {
654 if (!rdmsr_carefully(0xC0000000 | i
, &msrlow
, &msrhigh
)) {
655 db_printf("%08X - %08X.%08X\n",
656 0xC0000000 | i
, msrhigh
, msrlow
);