X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/316670eb35587141e969394ae8537d66b9211e80..eb6b6ca394357805f2bdba989abae309f718b4d8:/osfmk/kern/startup.c diff --git a/osfmk/kern/startup.c b/osfmk/kern/startup.c index b20629ffa..36a97ec7d 100644 --- a/osfmk/kern/startup.c +++ b/osfmk/kern/startup.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2000-2010 Apple Inc. All rights reserved. + * Copyright (c) 2000-2018 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,34 +22,34 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ -/* +/* * Mach Operating System * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University * All Rights Reserved. - * + * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. - * + * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. - * + * * Carnegie Mellon requests users of this software to return to - * + * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 - * + * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ @@ -67,7 +67,6 @@ */ #include -#include #include #include @@ -80,19 +79,30 @@ #include #include #include +#include #include +#include #include #include #include +#include #include +#include +#if CONFIG_SCHED_SFI +#include +#endif #include #include #include #include -#include -#include +#if CONFIG_TELEMETRY +#include +#endif #include #include +#include +#include +#include #include #include #include @@ -104,7 +114,37 @@ #include #include #include +#include #include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#if CONFIG_XNUPOST +#include +#include +#endif + +#if CONFIG_ATM +#include +#endif + +#if CONFIG_CSR +#include +#endif + +#include + +#if ALTERNATE_DEBUGGER +#include +#endif #if MACH_KDP #include @@ -112,27 +152,57 @@ #if CONFIG_MACF #include +#if CONFIG_VNGUARD +extern void vnguard_policy_init(void); +#endif +#endif + +#if KPC +#include +#endif + +#if HYPERVISOR +#include #endif -#if CONFIG_COUNTERS -#include +#include + +#if defined(__arm__) || defined(__arm64__) +#include // for arm_vm_prot_finalize #endif -static void kernel_bootstrap_thread(void); +#include +static void kernel_bootstrap_thread(void); -static void load_context( - thread_t thread); +static void load_context( + thread_t thread); #if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0 extern void cpu_userwindow_init(int); extern void cpu_physwindow_init(int); #endif +#if CONFIG_ECC_LOGGING +#include +#endif + +#if (defined(__i386__) || defined(__x86_64__)) && CONFIG_VMX +#include +#endif + +#if CONFIG_DTRACE +extern void dtrace_early_init(void); +extern void sdt_early_init(void); +#endif + // libkern/OSKextLib.cpp -extern void OSKextRemoveKextBootstrap(void); +extern void OSKextRemoveKextBootstrap(void); void scale_setup(void); extern void bsd_scale_setup(int); extern unsigned int semaphore_max; +extern void stackshot_init(void); +extern void ktrace_init(void); +extern void oslog_init(void); /* * Running in virtual memory, on the interrupt stack. @@ -142,51 +212,73 @@ extern int serverperfmode; /* size of kernel trace buffer, disabled by default */ unsigned int new_nkdbufs = 0; +unsigned int wake_nkdbufs = 0; +unsigned int write_trace_on_panic = 0; +unsigned int trace_wrap = 0; +boolean_t trace_serial = FALSE; +boolean_t early_boot_complete = FALSE; /* mach leak logging */ int log_leaks = 0; -int turn_on_log_leaks = 0; +static inline void +kernel_bootstrap_log(const char *message) +{ +// kprintf("kernel_bootstrap: %s\n", message); + kernel_debug_string_early(message); +} + +static inline void +kernel_bootstrap_thread_log(const char *message) +{ +// kprintf("kernel_bootstrap_thread: %s\n", message); + kernel_debug_string_early(message); +} void kernel_early_bootstrap(void) { + /* serverperfmode is needed by timer setup */ + if (PE_parse_boot_argn("serverperfmode", &serverperfmode, sizeof(serverperfmode))) { + serverperfmode = 1; + } - lck_mod_init(); - +#if CONFIG_SCHED_SFI /* - * Initialize the timer callout world + * Configure SFI classes */ - timer_call_initialize(); + sfi_early_init(); +#endif } void kernel_bootstrap(void) { - kern_return_t result; - thread_t thread; - char namep[16]; + kern_return_t result; + thread_t thread; + char namep[16]; printf("%s\n", version); /* log kernel version */ -#define kernel_bootstrap_kprintf(x...) /* kprintf("kernel_bootstrap: " x) */ - - if (PE_parse_boot_argn("-l", namep, sizeof (namep))) /* leaks logging */ - turn_on_log_leaks = 1; + if (PE_parse_boot_argn("-l", namep, sizeof(namep))) { /* leaks logging */ + log_leaks = 1; + } - PE_parse_boot_argn("trace", &new_nkdbufs, sizeof (new_nkdbufs)); + PE_parse_boot_argn("trace", &new_nkdbufs, sizeof(new_nkdbufs)); + PE_parse_boot_argn("trace_wake", &wake_nkdbufs, sizeof(wake_nkdbufs)); + PE_parse_boot_argn("trace_panic", &write_trace_on_panic, sizeof(write_trace_on_panic)); + PE_parse_boot_argn("trace_wrap", &trace_wrap, sizeof(trace_wrap)); - /* i386_vm_init already checks for this ; do it aagin anyway */ - if (PE_parse_boot_argn("serverperfmode", &serverperfmode, sizeof (serverperfmode))) { - serverperfmode = 1; - } scale_setup(); - kernel_bootstrap_kprintf("calling vm_mem_bootstrap\n"); + kernel_bootstrap_log("vm_mem_bootstrap"); vm_mem_bootstrap(); - kernel_bootstrap_kprintf("calling vm_mem_init\n"); + kernel_bootstrap_log("cs_init"); + cs_init(); + + kernel_bootstrap_log("vm_mem_init"); vm_mem_init(); machine_info.memory_size = (uint32_t)mem_size; @@ -194,35 +286,76 @@ kernel_bootstrap(void) machine_info.major_version = version_major; machine_info.minor_version = version_minor; - kernel_bootstrap_kprintf("calling sched_init\n"); + oslog_init(); + +#if KASAN + kernel_bootstrap_log("kasan_late_init"); + kasan_late_init(); +#endif + +#if CONFIG_TELEMETRY + kernel_bootstrap_log("telemetry_init"); + telemetry_init(); +#endif + +#if CONFIG_CSR + kernel_bootstrap_log("csr_init"); + csr_init(); +#endif + + if (PE_i_can_has_debugger(NULL)) { + if (PE_parse_boot_argn("-show_pointers", &namep, sizeof(namep))) { + doprnt_hide_pointers = FALSE; + } + if (PE_parse_boot_argn("-no_slto_panic", &namep, sizeof(namep))) { + extern boolean_t spinlock_timeout_panic; + spinlock_timeout_panic = FALSE; + } + } + + kernel_bootstrap_log("console_init"); + console_init(); + + kernel_bootstrap_log("stackshot_init"); + stackshot_init(); + + kernel_bootstrap_log("sched_init"); sched_init(); - kernel_bootstrap_kprintf("calling wait_queue_bootstrap\n"); - wait_queue_bootstrap(); + kernel_bootstrap_log("ltable_bootstrap"); + ltable_bootstrap(); - kernel_bootstrap_kprintf("calling ipc_bootstrap\n"); + kernel_bootstrap_log("waitq_bootstrap"); + waitq_bootstrap(); + + kernel_bootstrap_log("ipc_bootstrap"); ipc_bootstrap(); #if CONFIG_MACF + kernel_bootstrap_log("mac_policy_init"); mac_policy_init(); #endif - kernel_bootstrap_kprintf("calling ipc_init\n"); + + kernel_bootstrap_log("ipc_init"); ipc_init(); /* * As soon as the virtual memory system is up, we record * that this CPU is using the kernel pmap. */ - kernel_bootstrap_kprintf("calling PMAP_ACTIVATE_KERNEL\n"); + kernel_bootstrap_log("PMAP_ACTIVATE_KERNEL"); PMAP_ACTIVATE_KERNEL(master_cpu); - kernel_bootstrap_kprintf("calling mapping_free_prime\n"); - mapping_free_prime(); /* Load up with temporary mapping blocks */ + kernel_bootstrap_log("mapping_free_prime"); + mapping_free_prime(); /* Load up with temporary mapping blocks */ - kernel_bootstrap_kprintf("calling machine_init\n"); + kernel_bootstrap_log("machine_init"); machine_init(); - kernel_bootstrap_kprintf("calling clock_init\n"); + kernel_bootstrap_log("thread_machine_init_template"); + thread_machine_init_template(); + + kernel_bootstrap_log("clock_init"); clock_init(); ledger_init(); @@ -230,24 +363,78 @@ kernel_bootstrap(void) /* * Initialize the IPC, task, and thread subsystems. */ - kernel_bootstrap_kprintf("calling task_init\n"); + +#if CONFIG_COALITIONS + kernel_bootstrap_log("coalitions_init"); + coalitions_init(); +#endif + + kernel_bootstrap_log("task_init"); task_init(); - kernel_bootstrap_kprintf("calling thread_init\n"); + kernel_bootstrap_log("thread_init"); thread_init(); - + + kernel_bootstrap_log("restartable_init"); + restartable_init(); + + kernel_bootstrap_log("workq_init"); + workq_init(); + + kernel_bootstrap_log("turnstiles_init"); + turnstiles_init(); + +#if CONFIG_ATM + /* Initialize the Activity Trace Resource Manager. */ + kernel_bootstrap_log("atm_init"); + atm_init(); +#endif + kernel_bootstrap_log("mach_init_activity_id"); + mach_init_activity_id(); + + /* Initialize the BANK Manager. */ + kernel_bootstrap_log("bank_init"); + bank_init(); + + kernel_bootstrap_log("ipc_pthread_priority_init"); + ipc_pthread_priority_init(); + + /* initialize the corpse config based on boot-args */ + corpses_init(); + + /* initialize host_statistics */ + host_statistics_init(); + + /* initialize exceptions */ + exception_init(); + +#if CONFIG_SCHED_SFI + kernel_bootstrap_log("sfi_init"); + sfi_init(); +#endif + /* * Create a kernel thread to execute the kernel bootstrap. */ - kernel_bootstrap_kprintf("calling kernel_thread_create\n"); + + kernel_bootstrap_log("kernel_thread_create"); result = kernel_thread_create((thread_continue_t)kernel_bootstrap_thread, NULL, MAXPRI_KERNEL, &thread); - if (result != KERN_SUCCESS) panic("kernel_bootstrap: result = %08X\n", result); + if (result != KERN_SUCCESS) { + panic("kernel_bootstrap: result = %08X\n", result); + } + + /* The static init_thread is re-used as the bootstrap thread */ + assert(thread == current_thread()); + /* TODO: do a proper thread_start() (without the thread_setrun()) */ thread->state = TH_RUN; + thread->last_made_runnable_time = mach_absolute_time(); + thread_set_thread_name(thread, "kernel_bootstrap_thread"); + thread_deallocate(thread); - kernel_bootstrap_kprintf("calling load_context - done\n"); + kernel_bootstrap_log("load_context - done"); load_context(thread); /*NOTREACHED*/ } @@ -255,6 +442,10 @@ kernel_bootstrap(void) int kth_started = 0; vm_offset_t vm_kernel_addrperm; +vm_offset_t buf_kernel_addrperm; +vm_offset_t vm_kernel_addrperm_ext; +uint64_t vm_kernel_addrhash_salt; +uint64_t vm_kernel_addrhash_salt_ext; /* * Now running in a thread. Kick off other services, @@ -263,10 +454,10 @@ vm_offset_t vm_kernel_addrperm; static void kernel_bootstrap_thread(void) { - processor_t processor = current_processor(); + processor_t processor = current_processor(); #define kernel_bootstrap_thread_kprintf(x...) /* kprintf("kernel_bootstrap_thread: " x) */ - kernel_bootstrap_thread_kprintf("calling idle_thread_create\n"); + kernel_bootstrap_thread_log("idle_thread_create"); /* * Create the idle processor thread. */ @@ -278,38 +469,47 @@ kernel_bootstrap_thread(void) * * Start up the scheduler services. */ - kernel_bootstrap_thread_kprintf("calling sched_startup\n"); + kernel_bootstrap_thread_log("sched_startup"); sched_startup(); /* * Thread lifecycle maintenance (teardown, stack allocation) */ - kernel_bootstrap_thread_kprintf("calling thread_daemon_init\n"); + kernel_bootstrap_thread_log("thread_daemon_init"); thread_daemon_init(); - + + /* Create kernel map entry reserve */ + vm_kernel_reserved_entry_init(); + /* * Thread callout service. */ - kernel_bootstrap_thread_kprintf("calling thread_call_initialize\n"); + kernel_bootstrap_thread_log("thread_call_initialize"); thread_call_initialize(); - + /* * Remain on current processor as * additional processors come online. */ - kernel_bootstrap_thread_kprintf("calling thread_bind\n"); + kernel_bootstrap_thread_log("thread_bind"); thread_bind(processor); + /* + * Initialize ipc thread call support. + */ + kernel_bootstrap_thread_log("ipc_thread_call_init"); + ipc_thread_call_init(); + /* * Kick off memory mapping adjustments. */ - kernel_bootstrap_thread_kprintf("calling mapping_adjust\n"); + kernel_bootstrap_thread_log("mapping_adjust"); mapping_adjust(); /* * Create the clock service. */ - kernel_bootstrap_thread_kprintf("calling clock_service_create\n"); + kernel_bootstrap_thread_log("clock_service_create"); clock_service_create(); /* @@ -318,39 +518,82 @@ kernel_bootstrap_thread(void) device_service_create(); kth_started = 1; - + #if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0 /* * Create and initialize the physical copy window for processor 0 - * This is required before starting kicking off IOKit. + * This is required before starting kicking off IOKit. */ cpu_physwindow_init(0); #endif - vm_kernel_reserved_entry_init(); - + phys_carveout_init(); + #if MACH_KDP - kernel_bootstrap_kprintf("calling kdp_init\n"); + kernel_bootstrap_log("kdp_init"); kdp_init(); #endif -#if CONFIG_COUNTERS - pmc_bootstrap(); +#if ALTERNATE_DEBUGGER + alternate_debugger_init(); #endif -#if (defined(__i386__) || defined(__x86_64__)) - if (turn_on_log_leaks && !new_nkdbufs) - new_nkdbufs = 200000; - start_kern_tracing(new_nkdbufs); - if (turn_on_log_leaks) - log_leaks = 1; +#if KPC + kpc_init(); +#endif + +#if CONFIG_ECC_LOGGING + ecc_log_init(); +#endif + +#if HYPERVISOR + hv_support_init(); +#endif + +#if CONFIG_TELEMETRY + kernel_bootstrap_log("bootprofile_init"); + bootprofile_init(); #endif -#ifdef IOKIT +#if (defined(__i386__) || defined(__x86_64__)) && CONFIG_VMX + vmx_init(); +#endif + + kernel_bootstrap_thread_log("ktrace_init"); + ktrace_init(); + + char trace_typefilter[256] = {}; + PE_parse_boot_arg_str("trace_typefilter", trace_typefilter, + sizeof(trace_typefilter)); + kdebug_init(new_nkdbufs, trace_typefilter, trace_wrap); + +#ifdef MACH_BSD + kernel_bootstrap_log("bsd_early_init"); + bsd_early_init(); +#endif + +#if defined(__arm64__) + ml_lockdown_init(); +#endif + +#ifdef IOKIT + kernel_bootstrap_log("PE_init_iokit"); PE_init_iokit(); #endif - - (void) spllo(); /* Allow interruptions */ + + assert(ml_get_interrupts_enabled() == FALSE); + + /* + * Past this point, kernel subsystems that expect to operate with + * interrupts or preemption enabled may begin enforcement. + */ + early_boot_complete = TRUE; + +#if INTERRUPT_MASKED_DEBUG + // Reset interrupts masked timeout before we enable interrupts + ml_spin_debug_clear_self(); +#endif + (void) spllo(); /* Allow interruptions */ #if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0 /* @@ -363,14 +606,6 @@ kernel_bootstrap_thread(void) cpu_userwindow_init(0); #endif -#if (!defined(__i386__) && !defined(__x86_64__)) - if (turn_on_log_leaks && !new_nkdbufs) - new_nkdbufs = 200000; - start_kern_tracing(new_nkdbufs); - if (turn_on_log_leaks) - log_leaks = 1; -#endif - /* * Initialize the shared region module. */ @@ -379,37 +614,97 @@ kernel_bootstrap_thread(void) vm_commpage_text_init(); #if CONFIG_MACF + kernel_bootstrap_log("mac_policy_initmach"); mac_policy_initmach(); +#if CONFIG_VNGUARD + vnguard_policy_init(); +#endif +#endif + +#if CONFIG_DTRACE + dtrace_early_init(); + sdt_early_init(); +#endif + + + /* + * Get rid of segments used to bootstrap kext loading. This removes + * the KLD, PRELINK symtab, LINKEDIT, and symtab segments/load commands. + * Must be done prior to lockdown so that we can free (and possibly relocate) + * the static KVA mappings used for the jettisoned bootstrap segments. + */ + OSKextRemoveKextBootstrap(); +#if defined(__arm__) || defined(__arm64__) +#if CONFIG_KERNEL_INTEGRITY + machine_lockdown_preflight(); +#endif + /* + * Finalize protections on statically mapped pages now that comm page mapping is established. + */ + arm_vm_prot_finalize(PE_state.bootArgs); #endif /* - * Initialize the global used for permuting kernel + * Initialize the globals used for permuting kernel * addresses that may be exported to userland as tokens - * using VM_KERNEL_ADDRPERM(). Force the random number - * to be odd to avoid mapping a non-zero + * using VM_KERNEL_ADDRPERM()/VM_KERNEL_ADDRPERM_EXTERNAL(). + * Force the random number to be odd to avoid mapping a non-zero * word-aligned address to zero via addition. + * Note: at this stage we can use the cryptographically secure PRNG + * rather than early_random(). */ - vm_kernel_addrperm = (vm_offset_t)early_random() | 1; + read_random(&vm_kernel_addrperm, sizeof(vm_kernel_addrperm)); + vm_kernel_addrperm |= 1; + read_random(&buf_kernel_addrperm, sizeof(buf_kernel_addrperm)); + buf_kernel_addrperm |= 1; + read_random(&vm_kernel_addrperm_ext, sizeof(vm_kernel_addrperm_ext)); + vm_kernel_addrperm_ext |= 1; + read_random(&vm_kernel_addrhash_salt, sizeof(vm_kernel_addrhash_salt)); + read_random(&vm_kernel_addrhash_salt_ext, sizeof(vm_kernel_addrhash_salt_ext)); + + vm_set_restrictions(); + + +#ifdef CONFIG_XNUPOST + kern_return_t result = kernel_list_tests(); + result = kernel_do_post(); + if (result != KERN_SUCCESS) { + panic("kernel_do_post: Tests failed with result = 0x%08x\n", result); + } + kernel_bootstrap_log("kernel_do_post - done"); +#endif /* CONFIG_XNUPOST */ + /* * Start the user bootstrap. */ -#ifdef MACH_BSD +#ifdef MACH_BSD bsd_init(); #endif - /* - * Get rid of segments used to bootstrap kext loading. This removes - * the KLD, PRELINK symtab, LINKEDIT, and symtab segments/load commands. - */ - OSKextRemoveKextBootstrap(); +#if defined (__x86_64__) + x86_64_protect_data_const(); +#endif - serial_keyboard_init(); /* Start serial keyboard if wanted */ + + /* + * Get rid of pages used for early boot tracing. + */ + kdebug_free_early_buf(); + + serial_keyboard_init(); /* Start serial keyboard if wanted */ vm_page_init_local_q(); thread_bind(PROCESSOR_NULL); + /* + * Now that all CPUs are available to run threads, this is essentially + * a background thread. Take this opportunity to initialize and free + * any remaining vm_pages that were delayed earlier by pmap_startup(). + */ + vm_free_delayed_pages(); + /* * Become the pageout daemon. */ @@ -421,25 +716,32 @@ kernel_bootstrap_thread(void) * slave_main: * * Load the first thread to start a processor. + * This path will also be used by the master processor + * after being offlined. */ void slave_main(void *machine_param) { - processor_t processor = current_processor(); - thread_t thread; + processor_t processor = current_processor(); + thread_t thread; /* * Use the idle processor thread if there * is no dedicated start up thread. */ - if (processor->next_thread == THREAD_NULL) { + if (processor->processor_offlined == true) { + /* Return to the saved processor_offline context */ + assert(processor->startup_thread == THREAD_NULL); + thread = processor->idle_thread; - thread->continuation = (thread_continue_t)processor_start_thread; thread->parameter = machine_param; - } - else { - thread = processor->next_thread; - processor->next_thread = THREAD_NULL; + } else if (processor->startup_thread) { + thread = processor->startup_thread; + processor->startup_thread = THREAD_NULL; + } else { + thread = processor->idle_thread; + thread->continuation = processor_start_thread; + thread->parameter = machine_param; } load_context(thread); @@ -454,10 +756,11 @@ slave_main(void *machine_param) * Called at splsched. */ void -processor_start_thread(void *machine_param) +processor_start_thread(void *machine_param, + __unused wait_result_t result) { - processor_t processor = current_processor(); - thread_t self = current_thread(); + processor_t processor = current_processor(); + thread_t self = current_thread(); slave_machine_init(machine_param); @@ -465,8 +768,9 @@ processor_start_thread(void *machine_param) * If running the idle processor thread, * reenter the idle loop, else terminate. */ - if (self == processor->idle_thread) - thread_block((thread_continue_t)idle_thread); + if (self == processor->idle_thread) { + thread_block(idle_thread); + } thread_terminate(self); /*NOTREACHED*/ @@ -476,21 +780,22 @@ processor_start_thread(void *machine_param) * load_context: * * Start the first thread on a processor. + * This may be the first thread ever run on a processor, or + * it could be a processor that was previously offlined. */ -static void +static void __attribute__((noreturn)) load_context( - thread_t thread) + thread_t thread) { - processor_t processor = current_processor(); + processor_t processor = current_processor(); #define load_context_kprintf(x...) /* kprintf("load_context: " x) */ - load_context_kprintf("calling machine_set_current_thread\n"); + load_context_kprintf("machine_set_current_thread\n"); machine_set_current_thread(thread); - load_context_kprintf("calling processor_up\n"); - processor_up(processor); + load_context_kprintf("processor_up\n"); PMAP_ACTIVATE_KERNEL(processor->cpu_id); @@ -499,27 +804,34 @@ load_context( * should never occur since the thread is expected * to have reserved stack. */ - load_context_kprintf("thread %p, stack %x, stackptr %x\n", thread, - thread->kernel_stack, thread->machine.kstackptr); + load_context_kprintf("thread %p, stack %lx, stackptr %lx\n", thread, + thread->kernel_stack, thread->machine.kstackptr); if (!thread->kernel_stack) { - load_context_kprintf("calling stack_alloc_try\n"); - if (!stack_alloc_try(thread)) + load_context_kprintf("stack_alloc_try\n"); + if (!stack_alloc_try(thread)) { panic("load_context"); + } } /* * The idle processor threads are not counted as * running for load calculations. */ - if (!(thread->state & TH_IDLE)) - sched_run_incr(); + if (!(thread->state & TH_IDLE)) { + SCHED(run_count_incr)(thread); + } processor->active_thread = thread; - processor->current_pri = thread->sched_pri; - processor->current_thmode = thread->sched_mode; + processor_state_update_explicit(processor, thread->sched_pri, + SFI_CLASS_KERNEL, PSET_SMP, thread_get_perfcontrol_class(thread), THREAD_URGENCY_NONE); + processor->current_is_bound = thread->bound_processor != PROCESSOR_NULL; + processor->current_is_NO_SMT = false; + processor->starting_pri = thread->sched_pri; processor->deadline = UINT64_MAX; thread->last_processor = processor; + processor_up(processor); + processor->last_dispatch = mach_absolute_time(); timer_start(&thread->system_timer, processor->last_dispatch); PROCESSOR_DATA(processor, thread_timer) = PROCESSOR_DATA(processor, kernel_timer) = &thread->system_timer; @@ -527,9 +839,20 @@ load_context( timer_start(&PROCESSOR_DATA(processor, system_state), processor->last_dispatch); PROCESSOR_DATA(processor, current_state) = &PROCESSOR_DATA(processor, system_state); + + cpu_quiescent_counter_join(processor->last_dispatch); + PMAP_ACTIVATE_USER(thread, processor->cpu_id); - load_context_kprintf("calling machine_load_context\n"); + load_context_kprintf("machine_load_context\n"); + +#if __arm__ || __arm64__ +#if __SMP__ + /* TODO: Should this be ordered? */ + thread->machine.machine_thread_flags |= MACHINE_THREAD_FLAGS_ON_CPU; +#endif /* __SMP__ */ +#endif /* __arm__ || __arm64__ */ + machine_load_context(thread); /*NOTREACHED*/ } @@ -542,29 +865,36 @@ scale_setup() typeof(task_max) task_max_base = task_max; /* Raise limits for servers with >= 16G */ - if ((serverperfmode != 0) && ((uint64_t)sane_size >= (uint64_t)(16 * 1024 * 1024 *1024ULL))) { - scale = (int)((uint64_t)sane_size / (uint64_t)(8 * 1024 * 1024 *1024ULL)); + if ((serverperfmode != 0) && ((uint64_t)sane_size >= (uint64_t)(16 * 1024 * 1024 * 1024ULL))) { + scale = (int)((uint64_t)sane_size / (uint64_t)(8 * 1024 * 1024 * 1024ULL)); /* limit to 128 G */ - if (scale > 16) + if (scale > 16) { scale = 16; + } task_max_base = 2500; - } else if ((uint64_t)sane_size >= (uint64_t)(3 * 1024 * 1024 *1024ULL)) - scale = 2; + /* Raise limits for machines with >= 3GB */ + } else if ((uint64_t)sane_size >= (uint64_t)(3 * 1024 * 1024 * 1024ULL)) { + if ((uint64_t)sane_size < (uint64_t)(8 * 1024 * 1024 * 1024ULL)) { + scale = 2; + } else { + /* limit to 64GB */ + scale = MIN(16, (int)((uint64_t)sane_size / (uint64_t)(4 * 1024 * 1024 * 1024ULL))); + } + } task_max = MAX(task_max, task_max_base * scale); if (scale != 0) { task_threadmax = task_max; - thread_max = task_max * 5; + thread_max = task_max * 5; } #endif bsd_scale_setup(scale); - + ipc_space_max = SPACE_MAX; ipc_port_max = PORT_MAX; ipc_pset_max = SET_MAX; semaphore_max = SEMAPHORE_MAX; } -