X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/2d21ac55c334faf3a56e5634905ed6987fc787d4..39236c6e673c41db228275375ab7fdb0f837b292:/osfmk/kern/startup.c

diff --git a/osfmk/kern/startup.c b/osfmk/kern/startup.c
index 3c132bb74..30d2bbff2 100644
--- a/osfmk/kern/startup.c
+++ b/osfmk/kern/startup.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2010 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -69,7 +69,6 @@
 #include <debug.h>
 #include <xpr_debug.h>
 #include <mach_kdp.h>
-#include <norma_vm.h>
 
 #include <mach/boolean.h>
 #include <mach/machine.h>
@@ -78,6 +77,7 @@
 #include <mach/vm_param.h>
 #include <ipc/ipc_init.h>
 #include <kern/assert.h>
+#include <kern/mach_param.h>
 #include <kern/misc_protos.h>
 #include <kern/clock.h>
 #include <kern/cpu_number.h>
@@ -89,6 +89,10 @@
 #include <kern/task.h>
 #include <kern/thread.h>
 #include <kern/timer.h>
+#if CONFIG_TELEMETRY
+#include <kern/telemetry.h>
+#endif
+#include <kern/wait_queue.h>
 #include <kern/xpr.h>
 #include <kern/zalloc.h>
 #include <kern/locks.h>
@@ -103,60 +107,131 @@
 #include <machine/pmap.h>
 #include <machine/commpage.h>
 #include <libkern/version.h>
+#include <sys/codesign.h>
+#include <sys/kdebug.h>
+
+#if MACH_KDP
+#include <kdp/kdp.h>
+#endif
 
 #if CONFIG_MACF
 #include <security/mac_mach_internal.h>
 #endif
 
-#ifdef __ppc__
-#include <ppc/Firmware.h>
-#include <ppc/mappings.h>
+#if CONFIG_COUNTERS
+#include <pmc/pmc.h>
+#endif
+
+#if KPC
+#include <kern/kpc.h>
+#endif
+
+#if KPERF
+#include <kperf/kperf.h>
 #endif
 
+
+#include <i386/pmCPU.h>
 static void		kernel_bootstrap_thread(void);
 
 static void		load_context(
 					thread_t	thread);
-#ifdef i386
+#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
 extern void cpu_userwindow_init(int);
 extern void cpu_physwindow_init(int);
 #endif
 
-#ifdef CONFIG_JETTISON_KERNEL_LINKER
-extern void	jettison_kernel_linker(void);
-#endif
+// libkern/OSKextLib.cpp
+extern void	OSKextRemoveKextBootstrap(void);
+
+void scale_setup(void);
+extern void bsd_scale_setup(int);
+extern unsigned int semaphore_max;
+extern void stackshot_lock_init(void);
 
 /*
  *	Running in virtual memory, on the interrupt stack.
  */
 
+extern int serverperfmode;
+
+/* size of kernel trace buffer, disabled by default */
+unsigned int new_nkdbufs = 0;
+unsigned int wake_nkdbufs = 0;
+
+/* mach leak logging */
+int log_leaks = 0;
+int turn_on_log_leaks = 0;
+
+
+void
+kernel_early_bootstrap(void)
+{
+
+	lck_mod_init();
+
+	/*
+	 * Initialize the timer callout world
+	 */
+	timer_call_init();
+}
+
+
 void
 kernel_bootstrap(void)
 {
 	kern_return_t	result;
-	thread_t		thread;
+	thread_t	thread;
+	char		namep[16];
 
 	printf("%s\n", version); /* log kernel version */
 
 #define kernel_bootstrap_kprintf(x...) /* kprintf("kernel_bootstrap: " x) */
 
-	kernel_bootstrap_kprintf("calling lck_mod_init\n");
-	lck_mod_init();
+	if (PE_parse_boot_argn("-l", namep, sizeof (namep))) /* leaks logging */
+		turn_on_log_leaks = 1;
 
-	kernel_bootstrap_kprintf("calling sched_init\n");
-	sched_init();
+	PE_parse_boot_argn("trace", &new_nkdbufs, sizeof (new_nkdbufs));
+
+	PE_parse_boot_argn("trace_wake", &wake_nkdbufs, sizeof (wake_nkdbufs));
+
+	/* i386_vm_init already checks for this ; do it aagin anyway */
+        if (PE_parse_boot_argn("serverperfmode", &serverperfmode, sizeof (serverperfmode))) {
+                serverperfmode = 1;
+        }
+	scale_setup();
 
 	kernel_bootstrap_kprintf("calling vm_mem_bootstrap\n");
 	vm_mem_bootstrap();
 
-	kernel_bootstrap_kprintf("calling ipc_bootstrap\n");
-	ipc_bootstrap();
+	kernel_bootstrap_kprintf("calling cs_init\n");
+	cs_init();
 
 	kernel_bootstrap_kprintf("calling vm_mem_init\n");
 	vm_mem_init();
 
-	kernel_bootstrap_kprintf("calling kmod_init\n");
-	kmod_init();
+	machine_info.memory_size = (uint32_t)mem_size;
+	machine_info.max_mem = max_mem;
+	machine_info.major_version = version_major;
+	machine_info.minor_version = version_minor;
+
+#if CONFIG_TELEMETRY
+	kernel_bootstrap_kprintf("calling telemetry_init\n");
+	telemetry_init();
+#endif
+
+	kernel_bootstrap_kprintf("calling stackshot_lock_init\n");	
+	stackshot_lock_init();
+
+	kernel_bootstrap_kprintf("calling sched_init\n");
+	sched_init();
+
+	kernel_bootstrap_kprintf("calling wait_queue_bootstrap\n");
+	wait_queue_bootstrap();
+
+	kernel_bootstrap_kprintf("calling ipc_bootstrap\n");
+	ipc_bootstrap();
+
 #if CONFIG_MACF
 	mac_policy_init();
 #endif
@@ -179,17 +254,11 @@ kernel_bootstrap(void)
 	kernel_bootstrap_kprintf("calling clock_init\n");
 	clock_init();
 
-	machine_info.memory_size = mem_size;
-	machine_info.max_mem = max_mem;
-	machine_info.major_version = version_major;
-	machine_info.minor_version = version_minor;
+	ledger_init();
 
 	/*
 	 *	Initialize the IPC, task, and thread subsystems.
 	 */
-	kernel_bootstrap_kprintf("calling ledger_init\n");
-	ledger_init();
-
 	kernel_bootstrap_kprintf("calling task_init\n");
 	task_init();
 
@@ -214,6 +283,9 @@ kernel_bootstrap(void)
 
 int kth_started = 0;
 
+vm_offset_t vm_kernel_addrperm;
+vm_offset_t buf_kernel_addrperm;
+
 /*
  * Now running in a thread.  Kick off other services,
  * invoke user bootstrap, enter pageout loop.
@@ -239,6 +311,21 @@ kernel_bootstrap_thread(void)
 	kernel_bootstrap_thread_kprintf("calling sched_startup\n");
 	sched_startup();
 
+	/*
+	 * Thread lifecycle maintenance (teardown, stack allocation)
+	 */
+	kernel_bootstrap_thread_kprintf("calling thread_daemon_init\n");
+	thread_daemon_init();
+
+	/* Create kernel map entry reserve */
+	vm_kernel_reserved_entry_init();
+
+	/*
+	 * Thread callout service.
+	 */
+	kernel_bootstrap_thread_kprintf("calling thread_call_initialize\n");
+	thread_call_initialize();
+	
 	/*
 	 * Remain on current processor as
 	 * additional processors come online.
@@ -264,8 +351,8 @@ kernel_bootstrap_thread(void)
 	device_service_create();
 
 	kth_started = 1;
-
-#ifdef i386
+		
+#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
 	/*
 	 * Create and initialize the physical copy window for processor 0
 	 * This is required before starting kicking off  IOKit.
@@ -273,13 +360,50 @@ kernel_bootstrap_thread(void)
 	cpu_physwindow_init(0);
 #endif
 
+
+	
+#if MACH_KDP
+	kernel_bootstrap_kprintf("calling kdp_init\n");
+	kdp_init();
+#endif
+
+#if ALTERNATE_DEBUGGER
+	alternate_debugger_init();
+#endif
+
+#if CONFIG_COUNTERS
+	pmc_bootstrap();
+#endif
+
+#if KPC
+	kpc_init();
+#endif
+
+#if KPERF
+	kperf_bootstrap();
+#endif
+
+#if CONFIG_TELEMETRY
+	kernel_bootstrap_kprintf("calling bootprofile_init\n");
+	bootprofile_init();
+#endif
+
+#if (defined(__i386__) || defined(__x86_64__))
+	if (turn_on_log_leaks && !new_nkdbufs)
+		new_nkdbufs = 200000;
+	start_kern_tracing(new_nkdbufs, FALSE);
+	if (turn_on_log_leaks)
+		log_leaks = 1;
+
+#endif
+
 #ifdef	IOKIT
 	PE_init_iokit();
 #endif
 	
 	(void) spllo();		/* Allow interruptions */
 
-#ifdef i386
+#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
 	/*
 	 * Create and initialize the copy window for processor 0
 	 * This also allocates window space for all other processors.
@@ -290,16 +414,36 @@ kernel_bootstrap_thread(void)
 	cpu_userwindow_init(0);
 #endif
 
+#if (!defined(__i386__) && !defined(__x86_64__))
+	if (turn_on_log_leaks && !new_nkdbufs)
+		new_nkdbufs = 200000;
+	start_kern_tracing(new_nkdbufs, FALSE);
+	if (turn_on_log_leaks)
+		log_leaks = 1;
+#endif
+
 	/*
 	 *	Initialize the shared region module.
 	 */
 	vm_shared_region_init();
 	vm_commpage_init();
+	vm_commpage_text_init();
+
 
 #if CONFIG_MACF
 	mac_policy_initmach();
 #endif
 
+	/*
+	 * Initialize the global used for permuting kernel
+	 * addresses that may be exported to userland as tokens
+	 * using VM_KERNEL_ADDRPERM(). Force the random number
+	 * to be odd to avoid mapping a non-zero
+	 * word-aligned address to zero via addition.
+	 */
+	vm_kernel_addrperm = (vm_offset_t)early_random() | 1;
+	buf_kernel_addrperm = (vm_offset_t)early_random() | 1;
+
 	/*
 	 *	Start the user bootstrap.
 	 */
@@ -307,13 +451,16 @@ kernel_bootstrap_thread(void)
 	bsd_init();
 #endif
 
-#ifdef CONFIG_JETTISON_KERNEL_LINKER
-	/* We do not run kextd, so get rid of the kernel linker now */
-	jettison_kernel_linker();
-#endif
+    /*
+     * Get rid of segments used to bootstrap kext loading. This removes
+     * the KLD, PRELINK symtab, LINKEDIT, and symtab segments/load commands.
+     */
+	OSKextRemoveKextBootstrap();
 
 	serial_keyboard_init();		/* Start serial keyboard if wanted */
 
+	vm_page_init_local_q();
+	
 	thread_bind(PROCESSOR_NULL);
 
 	/*
@@ -329,7 +476,7 @@ kernel_bootstrap_thread(void)
  *	Load the first thread to start a processor.
  */
 void
-slave_main(void)
+slave_main(void *machine_param)
 {
 	processor_t		processor = current_processor();
 	thread_t		thread;
@@ -341,7 +488,7 @@ slave_main(void)
 	if (processor->next_thread == THREAD_NULL) {
 		thread = processor->idle_thread;
 		thread->continuation = (thread_continue_t)processor_start_thread;
-		thread->parameter = NULL;
+		thread->parameter = machine_param;
 	}
 	else {
 		thread = processor->next_thread;
@@ -360,12 +507,12 @@ slave_main(void)
  *	Called at splsched.
  */
 void
-processor_start_thread(void)
+processor_start_thread(void *machine_param)
 {
 	processor_t		processor = current_processor();
 	thread_t		self = current_thread();
 
-	slave_machine_init();
+	slave_machine_init(machine_param);
 
 	/*
 	 *	If running the idle processor thread,
@@ -398,15 +545,15 @@ load_context(
 	load_context_kprintf("calling processor_up\n");
 	processor_up(processor);
 
-	PMAP_ACTIVATE_KERNEL(PROCESSOR_DATA(processor, slot_num));
+	PMAP_ACTIVATE_KERNEL(processor->cpu_id);
 
 	/*
 	 * Acquire a stack if none attached.  The panic
 	 * should never occur since the thread is expected
 	 * to have reserved stack.
 	 */
-	load_context_kprintf("stack %x, stackptr %x\n", 
-			      thread->kernel_stack, thread->machine.kstackptr);
+	load_context_kprintf("thread %p, stack %lx, stackptr %lx\n", thread,
+			     thread->kernel_stack, thread->machine.kstackptr);
 	if (!thread->kernel_stack) {
 		load_context_kprintf("calling stack_alloc_try\n");
 		if (!stack_alloc_try(thread))
@@ -422,6 +569,7 @@ load_context(
 
 	processor->active_thread = thread;
 	processor->current_pri = thread->sched_pri;
+	processor->current_thmode = thread->sched_mode;
 	processor->deadline = UINT64_MAX;
 	thread->last_processor = processor;
 
@@ -432,9 +580,44 @@ load_context(
 	timer_start(&PROCESSOR_DATA(processor, system_state), processor->last_dispatch);
 	PROCESSOR_DATA(processor, current_state) = &PROCESSOR_DATA(processor, system_state);
 
-	PMAP_ACTIVATE_USER(thread, PROCESSOR_DATA(processor, slot_num));
+	PMAP_ACTIVATE_USER(thread, processor->cpu_id);
 
 	load_context_kprintf("calling machine_load_context\n");
 	machine_load_context(thread);
 	/*NOTREACHED*/
 }
+
+void
+scale_setup()
+{
+	int scale = 0;
+#if defined(__LP64__)
+	typeof(task_max) task_max_base = task_max;
+
+	/* Raise limits for servers with >= 16G */
+	if ((serverperfmode != 0) && ((uint64_t)sane_size >= (uint64_t)(16 * 1024 * 1024 *1024ULL))) {
+		scale = (int)((uint64_t)sane_size / (uint64_t)(8 * 1024 * 1024 *1024ULL));
+		/* limit to 128 G */
+		if (scale > 16)
+			scale = 16;
+		task_max_base = 2500;
+	} else if ((uint64_t)sane_size >= (uint64_t)(3 * 1024 * 1024 *1024ULL))
+		scale = 2;
+
+	task_max = MAX(task_max, task_max_base * scale);
+
+	if (scale != 0) {
+		task_threadmax = task_max;
+		thread_max = task_max * 5; 
+	}
+
+#endif
+
+	bsd_scale_setup(scale);
+	
+	ipc_space_max = SPACE_MAX;
+	ipc_port_max = PORT_MAX;
+	ipc_pset_max = SET_MAX;
+	semaphore_max = SEMAPHORE_MAX;
+}
+