X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/91447636331957f3d9b5ca5b508f07c526b0074d..36401178fd6817c043cc00b0c00c7f723e58efae:/osfmk/i386/machine_routines.c diff --git a/osfmk/i386/machine_routines.c b/osfmk/i386/machine_routines.c index fa5002a29..4ffb4ddc3 100644 --- a/osfmk/i386/machine_routines.c +++ b/osfmk/i386/machine_routines.c @@ -1,28 +1,36 @@ /* - * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * The contents of this file constitute Original Code as defined in and - * are subject to the Apple Public Source License Version 1.1 (the - * "License"). You may not use this file except in compliance with the - * License. Please obtain a copy of the License at - * http://www.apple.com/publicsource and read it before using this file. + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. * - * This Original Code and all software distributed under the License are - * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the - * License for the specific language governing rights and limitations - * under the License. + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ + #include #include #include #include +#include #include #include #include @@ -32,18 +40,40 @@ #include #include #include -#include #include #include +#include +#include +#include +#include #include - -#define MIN(a,b) ((a)<(b)? (a) : (b)) - -extern void initialize_screen(Boot_Video *, unsigned int); +#if MACH_KDB +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +#if DEBUG +#define DBG(x...) kprintf("DBG: " x) +#else +#define DBG(x...) +#endif + +extern thread_t Shutdown_context(thread_t thread, void (*doshutdown)(processor_t),processor_t processor); extern void wakeup(void *); +extern unsigned KernelRelocOffset; static int max_cpus_initialized = 0; +unsigned int LockTimeOut; +unsigned int LockTimeOutTSC; +unsigned int MutexSpin; + #define MAX_CPUS_SET 0x1 #define MAX_CPUS_WAIT 0x2 @@ -54,7 +84,7 @@ vm_offset_t ml_io_map( vm_offset_t phys_addr, vm_size_t size) { - return(io_map(phys_addr,size)); + return(io_map(phys_addr,size,VM_WIMG_IO)); } /* boot memory allocation */ @@ -64,6 +94,21 @@ vm_offset_t ml_static_malloc( return((vm_offset_t)NULL); } + +void ml_get_bouncepool_info(vm_offset_t *phys_addr, vm_size_t *size) +{ + *phys_addr = bounce_pool_base; + *size = bounce_pool_size; +} + + +vm_offset_t +ml_boot_ptovirt( + vm_offset_t paddr) +{ + return (vm_offset_t)((paddr-KernelRelocOffset) | LINEAR_KERNEL_ADDRESS); +} + vm_offset_t ml_static_ptovirt( vm_offset_t paddr) @@ -84,7 +129,7 @@ ml_static_mfree( vm_offset_t vaddr_cur; ppnum_t ppn; - if (vaddr < VM_MIN_KERNEL_ADDRESS) return; +// if (vaddr < VM_MIN_KERNEL_ADDRESS) return; assert((vaddr & (PAGE_SIZE-1)) == 0); /* must be page aligned */ @@ -93,6 +138,12 @@ ml_static_mfree( vaddr_cur += PAGE_SIZE) { ppn = pmap_find_phys(kernel_pmap, (addr64_t)vaddr_cur); if (ppn != (vm_offset_t)NULL) { + kernel_pmap->stats.resident_count++; + if (kernel_pmap->stats.resident_count > + kernel_pmap->stats.resident_max) { + kernel_pmap->stats.resident_max = + kernel_pmap->stats.resident_count; + } pmap_remove(kernel_pmap, (addr64_t)vaddr_cur, (addr64_t)(vaddr_cur+PAGE_SIZE)); vm_page_create(ppn,(ppn+1)); vm_page_wire_count--; @@ -100,6 +151,7 @@ ml_static_mfree( } } + /* virtual to physical on wired pages */ vm_offset_t ml_vtophys( vm_offset_t vaddr) @@ -107,6 +159,46 @@ vm_offset_t ml_vtophys( return kvtophys(vaddr); } +/* + * Routine: ml_nofault_copy + * Function: Perform a physical mode copy if the source and + * destination have valid translations in the kernel pmap. + * If translations are present, they are assumed to + * be wired; i.e. no attempt is made to guarantee that the + * translations obtained remained valid for + * the duration of the copy process. + */ + +vm_size_t ml_nofault_copy( + vm_offset_t virtsrc, vm_offset_t virtdst, vm_size_t size) +{ + addr64_t cur_phys_dst, cur_phys_src; + uint32_t count, nbytes = 0; + + while (size > 0) { + if (!(cur_phys_src = kvtophys(virtsrc))) + break; + if (!(cur_phys_dst = kvtophys(virtdst))) + break; + if (!pmap_valid_page(i386_btop(cur_phys_dst)) || !pmap_valid_page(i386_btop(cur_phys_src))) + break; + count = PAGE_SIZE - (cur_phys_src & PAGE_MASK); + if (count > (PAGE_SIZE - (cur_phys_dst & PAGE_MASK))) + count = PAGE_SIZE - (cur_phys_dst & PAGE_MASK); + if (count > size) + count = size; + + bcopy_phys(cur_phys_src, cur_phys_dst, count); + + nbytes += count; + virtsrc += count; + virtdst += count; + size -= count; + } + + return nbytes; +} + /* Interrupt handling */ /* Initialize Interrupts */ @@ -131,10 +223,21 @@ boolean_t ml_set_interrupts_enabled(boolean_t enable) __asm__ volatile("pushf; popl %0" : "=r" (flags)); - if (enable) + if (enable) { + ast_t *myast; + + myast = ast_pending(); + + if ( (get_preemption_level() == 0) && (*myast & AST_URGENT) ) { __asm__ volatile("sti"); - else + __asm__ volatile ("int $0xff"); + } else { + __asm__ volatile ("sti"); + } + } + else { __asm__ volatile("cli"); + } return (flags & EFL_IF) != 0; } @@ -153,12 +256,9 @@ void ml_cause_interrupt(void) void ml_thread_policy( thread_t thread, - unsigned policy_id, +__unused unsigned policy_id, unsigned policy_info) { - if (policy_id == MACHINE_GROUP) - thread_bind(thread, master_processor); - if (policy_info & MACHINE_NETWORK_WORKLOOP) { spl_t s = splsched(); @@ -188,20 +288,15 @@ void ml_install_interrupt_handler( (void) ml_set_interrupts_enabled(current_state); - initialize_screen(0, kPEAcquireScreen); + initialize_screen(NULL, kPEAcquireScreen); } -static void -cpu_idle(void) -{ - __asm__ volatile("sti; hlt": : :"memory"); -} -void (*cpu_idle_handler)(void) = cpu_idle; void machine_idle(void) { - cpu_core_t *my_core = cpu_core(); + x86_core_t *my_core = x86_core(); + cpu_data_t *my_cpu = current_cpu_datap(); int others_active; /* @@ -209,17 +304,25 @@ machine_idle(void) * unless kernel param idlehalt is false and no other thread * in the same core is active - if so, don't halt so that this * core doesn't go into a low-power mode. + * For 4/4, we set a null "active cr3" while idle. */ + if (my_core == NULL || my_cpu == NULL) + goto out; + others_active = !atomic_decl_and_test( - (long *) &my_core->active_threads, 1); + (long *) &my_core->active_lcpus, 1); + my_cpu->lcpu.idle = TRUE; if (idlehalt || others_active) { DBGLOG(cpu_handle, cpu_number(), MP_IDLE); - cpu_idle_handler(); + MARK_CPU_IDLE(cpu_number()); + machine_idle_cstate(FALSE); + MARK_CPU_ACTIVE(cpu_number()); DBGLOG(cpu_handle, cpu_number(), MP_UNIDLE); - } else { - __asm__ volatile("sti"); } - atomic_incl((long *) &my_core->active_threads, 1); + my_cpu->lcpu.idle = FALSE; + atomic_incl((long *) &my_core->active_lcpus, 1); + out: + __asm__ volatile("sti"); } void @@ -229,6 +332,17 @@ machine_signal_idle( cpu_interrupt(PROCESSOR_DATA(processor, slot_num)); } +thread_t +machine_processor_shutdown( + thread_t thread, + void (*doshutdown)(processor_t), + processor_t processor) +{ + vmx_suspend(); + fpu_save_context(thread); + return(Shutdown_context(thread, doshutdown, processor)); +} + kern_return_t ml_processor_register( cpu_id_t cpu_id, @@ -257,7 +371,17 @@ ml_processor_register( if (this_cpu_datap->cpu_console_buf == NULL) goto failed; + this_cpu_datap->cpu_chud = chudxnu_cpu_alloc(boot_cpu); + if (this_cpu_datap->cpu_chud == NULL) + goto failed; + if (!boot_cpu) { + this_cpu_datap->lcpu.core = cpu_thread_alloc(this_cpu_datap->cpu_number); + if (this_cpu_datap->lcpu.core == NULL) + goto failed; + + pmCPUStateInit(); + this_cpu_datap->cpu_pmap = pmap_cpu_alloc(boot_cpu); if (this_cpu_datap->cpu_pmap == NULL) goto failed; @@ -265,17 +389,31 @@ ml_processor_register( this_cpu_datap->cpu_processor = cpu_processor_alloc(boot_cpu); if (this_cpu_datap->cpu_processor == NULL) goto failed; - processor_init(this_cpu_datap->cpu_processor, target_cpu); + /* + * processor_init() deferred to topology start + * because "slot numbers" a.k.a. logical processor numbers + * are not yet finalized. + */ } *processor_out = this_cpu_datap->cpu_processor; *ipi_handler = NULL; + if (target_cpu == machine_info.max_cpus - 1) { + /* + * All processors are now registered but not started (except + * for this "in-limbo" boot processor). We call to the machine + * topology code to finalize and activate the topology. + */ + cpu_topology_start(); + } + return KERN_SUCCESS; failed: cpu_processor_free(this_cpu_datap->cpu_processor); pmap_cpu_free(this_cpu_datap->cpu_pmap); + chudxnu_cpu_free(this_cpu_datap->cpu_chud); console_cpu_free(this_cpu_datap->cpu_console_buf); return KERN_FAILURE; } @@ -290,11 +428,19 @@ ml_cpu_get_info(ml_cpu_info_t *cpu_infop) return; /* - * Are we supporting XMM/SSE/SSE2? + * Are we supporting MMX/SSE/SSE2/SSE3? * As distinct from whether the cpu has these capabilities. */ os_supports_sse = get_cr4() & CR4_XMM; - if ((cpuid_features() & CPUID_FEATURE_SSE2) && os_supports_sse) + if ((cpuid_features() & CPUID_FEATURE_SSE4_2) && os_supports_sse) + cpu_infop->vector_unit = 8; + else if ((cpuid_features() & CPUID_FEATURE_SSE4_1) && os_supports_sse) + cpu_infop->vector_unit = 7; + else if ((cpuid_features() & CPUID_FEATURE_SSSE3) && os_supports_sse) + cpu_infop->vector_unit = 6; + else if ((cpuid_features() & CPUID_FEATURE_SSE3) && os_supports_sse) + cpu_infop->vector_unit = 5; + else if ((cpuid_features() & CPUID_FEATURE_SSE2) && os_supports_sse) cpu_infop->vector_unit = 4; else if ((cpuid_features() & CPUID_FEATURE_SSE) && os_supports_sse) cpu_infop->vector_unit = 3; @@ -319,8 +465,8 @@ ml_cpu_get_info(ml_cpu_info_t *cpu_infop) } if (cpuid_infop->cache_size[L3U] > 0) { - cpu_infop->l2_settings = 1; - cpu_infop->l2_cache_size = cpuid_infop->cache_size[L3U]; + cpu_infop->l3_settings = 1; + cpu_infop->l3_cache_size = cpuid_infop->cache_size[L3U]; } else { cpu_infop->l3_settings = 0; cpu_infop->l3_cache_size = 0xFFFFFFFF; @@ -336,7 +482,7 @@ ml_init_max_cpus(unsigned long max_cpus) if (max_cpus_initialized != MAX_CPUS_SET) { if (max_cpus > 0 && max_cpus <= MAX_CPUS) { /* - * Note: max_cpus is the number of enable processors + * Note: max_cpus is the number of enabled processors * that ACPI found; max_ncpus is the maximum number * that the kernel supports or that the "cpus=" * boot-arg has set. Here we take int minimum. @@ -365,6 +511,31 @@ ml_get_max_cpus(void) return(machine_info.max_cpus); } +/* + * Routine: ml_init_lock_timeout + * Function: + */ +void +ml_init_lock_timeout(void) +{ + uint64_t abstime; + uint32_t mtxspin; + + /* LockTimeOut is absolutetime, LockTimeOutTSC is in TSC ticks */ + nanoseconds_to_absolutetime(NSEC_PER_SEC>>2, &abstime); + LockTimeOut = (uint32_t) abstime; + LockTimeOutTSC = (uint32_t) tmrCvt(abstime, tscFCvtn2t); + + if (PE_parse_boot_arg("mtxspin", &mtxspin)) { + if (mtxspin > USEC_PER_SEC>>4) + mtxspin = USEC_PER_SEC>>4; + nanoseconds_to_absolutetime(mtxspin*NSEC_PER_USEC, &abstime); + } else { + nanoseconds_to_absolutetime(10*NSEC_PER_USEC, &abstime); + } + MutexSpin = (unsigned int)abstime; +} + /* * This is called from the machine-independent routine cpu_up() * to perform machine-dependent info updates. Defer to cpu_thread_init(). @@ -385,29 +556,6 @@ ml_cpu_down(void) return; } -/* Stubs for pc tracing mechanism */ - -int *pc_trace_buf; -int pc_trace_cnt = 0; - -int -set_be_bit(void) -{ - return(0); -} - -int -clr_be_bit(void) -{ - return(0); -} - -int -be_tracing(void) -{ - return(0); -} - /* * The following are required for parts of the kernel * that cannot resolve these functions as inlines: @@ -426,3 +574,94 @@ current_thread(void) { return(current_thread_fast()); } + + +boolean_t ml_is64bit(void) { + + return (cpu_mode_is64bit()); +} + + +boolean_t ml_thread_is64bit(thread_t thread) { + + return (thread_is_64bit(thread)); +} + + +boolean_t ml_state_is64bit(void *saved_state) { + + return is_saved_state64(saved_state); +} + +void ml_cpu_set_ldt(int selector) +{ + /* + * Avoid loading the LDT + * if we're setting the KERNEL LDT and it's already set. + */ + if (selector == KERNEL_LDT && + current_cpu_datap()->cpu_ldt == KERNEL_LDT) + return; + + /* + * If 64bit this requires a mode switch (and back). + */ + if (cpu_mode_is64bit()) + ml_64bit_lldt(selector); + else + lldt(selector); + current_cpu_datap()->cpu_ldt = selector; +} + +void ml_fp_setvalid(boolean_t value) +{ + fp_setvalid(value); +} + +uint64_t ml_cpu_int_event_time(void) +{ + return current_cpu_datap()->cpu_int_event_time; +} + + +#if MACH_KDB + +/* + * Display the global msrs + * * + * ms + */ +void +db_msr(__unused db_expr_t addr, + __unused int have_addr, + __unused db_expr_t count, + __unused char *modif) +{ + + uint32_t i, msrlow, msrhigh; + + /* Try all of the first 4096 msrs */ + for (i = 0; i < 4096; i++) { + if (!rdmsr_carefully(i, &msrlow, &msrhigh)) { + db_printf("%08X - %08X.%08X\n", i, msrhigh, msrlow); + } + } + + /* Try all of the 4096 msrs at 0x0C000000 */ + for (i = 0; i < 4096; i++) { + if (!rdmsr_carefully(0x0C000000 | i, &msrlow, &msrhigh)) { + db_printf("%08X - %08X.%08X\n", + 0x0C000000 | i, msrhigh, msrlow); + } + } + + /* Try all of the 4096 msrs at 0xC0000000 */ + for (i = 0; i < 4096; i++) { + if (!rdmsr_carefully(0xC0000000 | i, &msrlow, &msrhigh)) { + db_printf("%08X - %08X.%08X\n", + 0xC0000000 | i, msrhigh, msrlow); + } + } +} + +#endif