2  * Copyright (c) 2000-2010 Apple Inc. All rights reserved. 
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 
   6  * This file contains Original Code and/or Modifications of Original Code 
   7  * as defined in and that are subject to the Apple Public Source License 
   8  * Version 2.0 (the 'License'). You may not use this file except in 
   9  * compliance with the License. The rights granted to you under the License 
  10  * may not be used to create, or enable the creation or redistribution of, 
  11  * unlawful or unlicensed copies of an Apple operating system, or to 
  12  * circumvent, violate, or enable the circumvention or violation of, any 
  13  * terms of an Apple operating system software license agreement. 
  15  * Please obtain a copy of the License at 
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file. 
  18  * The Original Code and all software distributed under the License are 
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 
  23  * Please see the License for the specific language governing rights and 
  24  * limitations under the License. 
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 
  32  * Mach Operating System 
  33  * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University 
  34  * All Rights Reserved. 
  36  * Permission to use, copy, modify and distribute this software and its 
  37  * documentation is hereby granted, provided that both the copyright 
  38  * notice and this permission notice appear in all copies of the 
  39  * software, derivative works or modified versions, and any portions 
  40  * thereof, and that both notices appear in supporting documentation. 
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 
  46  * Carnegie Mellon requests users of this software to return to 
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU 
  49  *  School of Computer Science 
  50  *  Carnegie Mellon University 
  51  *  Pittsburgh PA 15213-3890 
  53  * any improvements or extensions that they make and grant Carnegie Mellon 
  54  * the rights to redistribute these changes. 
  57  * NOTICE: This file was modified by McAfee Research in 2004 to introduce 
  58  * support for mandatory and extensible security protections.  This notice 
  59  * is included in support of clause 2.2 (b) of the Apple Public License, 
  66  *      Mach kernel startup. 
  70 #include <xpr_debug.h> 
  73 #include <mach/boolean.h> 
  74 #include <mach/machine.h> 
  75 #include <mach/thread_act.h> 
  76 #include <mach/task_special_ports.h> 
  77 #include <mach/vm_param.h> 
  78 #include <ipc/ipc_init.h> 
  79 #include <kern/assert.h> 
  80 #include <kern/mach_param.h> 
  81 #include <kern/misc_protos.h> 
  82 #include <kern/clock.h> 
  83 #include <kern/coalition.h> 
  84 #include <kern/cpu_number.h> 
  85 #include <kern/ledger.h> 
  86 #include <kern/machine.h> 
  87 #include <kern/processor.h> 
  88 #include <kern/sched_prim.h> 
  90 #include <kern/startup.h> 
  91 #include <kern/task.h> 
  92 #include <kern/thread.h> 
  93 #include <kern/timer.h> 
  95 #include <kern/telemetry.h> 
  97 #include <kern/wait_queue.h> 
  99 #include <kern/zalloc.h> 
 100 #include <kern/locks.h> 
 101 #include <prng/random.h> 
 102 #include <console/serial_protos.h> 
 103 #include <vm/vm_kern.h> 
 104 #include <vm/vm_init.h> 
 105 #include <vm/vm_map.h> 
 106 #include <vm/vm_object.h> 
 107 #include <vm/vm_page.h> 
 108 #include <vm/vm_pageout.h> 
 109 #include <vm/vm_shared_region.h> 
 110 #include <machine/pmap.h> 
 111 #include <machine/commpage.h> 
 112 #include <libkern/version.h> 
 113 #include <sys/codesign.h> 
 114 #include <sys/kdebug.h> 
 115 #include <sys/random.h> 
 118 #include <atm/atm_internal.h> 
 126 #include <bank/bank_internal.h> 
 129 #if ALTERNATE_DEBUGGER 
 130 #include <arm64/alternate_debugger.h> 
 138 #include <security/mac_mach_internal.h> 
 146 #include <kern/kpc.h> 
 150 #include <kperf/kperf.h> 
 154 #include <kern/hv_support.h> 
 158 #include <i386/pmCPU.h> 
 159 static void             kernel_bootstrap_thread(void); 
 161 static void             load_context( 
 163 #if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0 
 164 extern void cpu_userwindow_init(int); 
 165 extern void cpu_physwindow_init(int); 
 168 #if CONFIG_ECC_LOGGING 
 169 #include <kern/ecc.h> 
 172 #if (defined(__i386__) || defined(__x86_64__)) && CONFIG_VMX 
 173 #include <i386/vmx/vmx_cpu.h> 
 176 // libkern/OSKextLib.cpp 
 177 extern void     OSKextRemoveKextBootstrap(void); 
 179 void scale_setup(void); 
 180 extern void bsd_scale_setup(int); 
 181 extern unsigned int semaphore_max
; 
 182 extern void stackshot_lock_init(void); 
 185  *      Running in virtual memory, on the interrupt stack. 
 188 extern int serverperfmode
; 
 190 /* size of kernel trace buffer, disabled by default */ 
 191 unsigned int new_nkdbufs 
= 0; 
 192 unsigned int wake_nkdbufs 
= 0; 
 193 unsigned int write_trace_on_panic 
= 0; 
 194 unsigned int trace_typefilter 
= 0; 
 195 boolean_t    trace_serial 
= FALSE
; 
 197 /* mach leak logging */ 
 199 int turn_on_log_leaks 
= 0; 
 202 kernel_bootstrap_log(const char *message
) 
 204 //      kprintf("kernel_bootstrap: %s\n", message); 
 205         kernel_debug_string(message
); 
 209 kernel_bootstrap_thread_log(const char *message
) 
 211 //      kprintf("kernel_bootstrap_thread: %s\n", message); 
 212         kernel_debug_string(message
); 
 216 kernel_early_bootstrap(void) 
 218         /* serverperfmode is needed by timer setup */ 
 219         if (PE_parse_boot_argn("serverperfmode", &serverperfmode
, sizeof (serverperfmode
))) { 
 226          * Initialize the timer callout world 
 231          * Configure SFI classes 
 238 kernel_bootstrap(void) 
 240         kern_return_t   result
; 
 244         printf("%s\n", version
); /* log kernel version */ 
 246         if (PE_parse_boot_argn("-l", namep
, sizeof (namep
))) /* leaks logging */ 
 247                 turn_on_log_leaks 
= 1; 
 249         PE_parse_boot_argn("trace", &new_nkdbufs
, sizeof (new_nkdbufs
)); 
 250         PE_parse_boot_argn("trace_wake", &wake_nkdbufs
, sizeof (wake_nkdbufs
)); 
 251         PE_parse_boot_argn("trace_panic", &write_trace_on_panic
, sizeof(write_trace_on_panic
)); 
 252         PE_parse_boot_argn("trace_typefilter", &trace_typefilter
, sizeof(trace_typefilter
)); 
 256         kernel_bootstrap_log("vm_mem_bootstrap"); 
 259         kernel_bootstrap_log("cs_init"); 
 262         kernel_bootstrap_log("vm_mem_init"); 
 265         machine_info
.memory_size 
= (uint32_t)mem_size
; 
 266         machine_info
.max_mem 
= max_mem
; 
 267         machine_info
.major_version 
= version_major
; 
 268         machine_info
.minor_version 
= version_minor
; 
 271         kernel_bootstrap_log("telemetry_init"); 
 275         kernel_bootstrap_log("stackshot_lock_init");     
 276         stackshot_lock_init(); 
 278         kernel_bootstrap_log("sched_init"); 
 281         kernel_bootstrap_log("wait_queue_bootstrap"); 
 282         wait_queue_bootstrap(); 
 284         kernel_bootstrap_log("ipc_bootstrap"); 
 290         kernel_bootstrap_log("ipc_init"); 
 294          * As soon as the virtual memory system is up, we record 
 295          * that this CPU is using the kernel pmap. 
 297         kernel_bootstrap_log("PMAP_ACTIVATE_KERNEL"); 
 298         PMAP_ACTIVATE_KERNEL(master_cpu
); 
 300         kernel_bootstrap_log("mapping_free_prime"); 
 301         mapping_free_prime();                                           /* Load up with temporary mapping blocks */ 
 303         kernel_bootstrap_log("machine_init"); 
 306         kernel_bootstrap_log("clock_init"); 
 312          *      Initialize the IPC, task, and thread subsystems. 
 314 #if CONFIG_COALITIONS 
 315         kernel_bootstrap_log("coalition_init"); 
 319         kernel_bootstrap_log("task_init"); 
 322         kernel_bootstrap_log("thread_init"); 
 326         /* Initialize the Activity Trace Resource Manager. */ 
 327         kernel_bootstrap_log("atm_init"); 
 332         kernel_bootstrap_log("csr_init"); 
 337         /* Initialize the BANK Manager. */ 
 338         kernel_bootstrap_log("bank_init"); 
 343          *      Create a kernel thread to execute the kernel bootstrap. 
 345         kernel_bootstrap_log("kernel_thread_create"); 
 346         result 
= kernel_thread_create((thread_continue_t
)kernel_bootstrap_thread
, NULL
, MAXPRI_KERNEL
, &thread
); 
 348         if (result 
!= KERN_SUCCESS
) panic("kernel_bootstrap: result = %08X\n", result
); 
 350         thread
->state 
= TH_RUN
; 
 351         thread_deallocate(thread
); 
 353         kernel_bootstrap_log("load_context - done"); 
 354         load_context(thread
); 
 360 vm_offset_t vm_kernel_addrperm
; 
 361 vm_offset_t buf_kernel_addrperm
; 
 364  * Now running in a thread.  Kick off other services, 
 365  * invoke user bootstrap, enter pageout loop. 
 368 kernel_bootstrap_thread(void) 
 370         processor_t             processor 
= current_processor(); 
 372 #define kernel_bootstrap_thread_kprintf(x...) /* kprintf("kernel_bootstrap_thread: " x) */ 
 373         kernel_bootstrap_thread_log("idle_thread_create"); 
 375          * Create the idle processor thread. 
 377         idle_thread_create(processor
); 
 380          * N.B. Do not stick anything else 
 383          * Start up the scheduler services. 
 385         kernel_bootstrap_thread_log("sched_startup"); 
 389          * Thread lifecycle maintenance (teardown, stack allocation) 
 391         kernel_bootstrap_thread_log("thread_daemon_init"); 
 392         thread_daemon_init(); 
 394         /* Create kernel map entry reserve */ 
 395         vm_kernel_reserved_entry_init(); 
 398          * Thread callout service. 
 400         kernel_bootstrap_thread_log("thread_call_initialize"); 
 401         thread_call_initialize(); 
 404          * Remain on current processor as 
 405          * additional processors come online. 
 407         kernel_bootstrap_thread_log("thread_bind"); 
 408         thread_bind(processor
); 
 411          * Initialize ipc thread call support. 
 413         kernel_bootstrap_thread_log("ipc_thread_call_init"); 
 414         ipc_thread_call_init(); 
 417          * Kick off memory mapping adjustments. 
 419         kernel_bootstrap_thread_log("mapping_adjust"); 
 423          *      Create the clock service. 
 425         kernel_bootstrap_thread_log("clock_service_create"); 
 426         clock_service_create(); 
 429          *      Create the device service. 
 431         device_service_create(); 
 435 #if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0 
 437          * Create and initialize the physical copy window for processor 0 
 438          * This is required before starting kicking off  IOKit. 
 440         cpu_physwindow_init(0); 
 446         kernel_bootstrap_log("kdp_init"); 
 450 #if ALTERNATE_DEBUGGER 
 451         alternate_debugger_init(); 
 462 #if CONFIG_ECC_LOGGING 
 475         kernel_bootstrap_log("bootprofile_init"); 
 479 #if (defined(__i386__) || defined(__x86_64__)) && CONFIG_VMX 
 483 #if (defined(__i386__) || defined(__x86_64__)) 
 486                 if (trace_typefilter 
== 0) 
 487                         trace_typefilter 
= 1; 
 489         if (turn_on_log_leaks 
&& !new_nkdbufs
) 
 490                 new_nkdbufs 
= 200000; 
 491         if (trace_typefilter
) 
 492                 start_kern_tracing_with_typefilter(new_nkdbufs
, 
 496                 start_kern_tracing(new_nkdbufs
, FALSE
); 
 497         if (turn_on_log_leaks
) 
 502         kernel_bootstrap_log("prng_init"); 
 503         prng_cpu_init(master_cpu
); 
 509         assert(ml_get_interrupts_enabled() == FALSE
); 
 510         (void) spllo();         /* Allow interruptions */ 
 512 #if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0 
 514          * Create and initialize the copy window for processor 0 
 515          * This also allocates window space for all other processors. 
 516          * However, this is dependent on the number of processors - so this call 
 517          * must be after IOKit has been started because IOKit performs processor 
 520         cpu_userwindow_init(0); 
 523 #if (!defined(__i386__) && !defined(__x86_64__)) 
 524         if (turn_on_log_leaks 
&& !new_nkdbufs
) 
 525                 new_nkdbufs 
= 200000; 
 526         if (trace_typefilter
) 
 527                 start_kern_tracing_with_typefilter(new_nkdbufs
, FALSE
, trace_typefilter
); 
 529                 start_kern_tracing(new_nkdbufs
, FALSE
); 
 530         if (turn_on_log_leaks
) 
 535          *      Initialize the shared region module. 
 537         vm_shared_region_init(); 
 539         vm_commpage_text_init(); 
 543         kernel_bootstrap_log("mac_policy_initmach"); 
 544         mac_policy_initmach(); 
 547         kernel_bootstrap_log("sfi_init"); 
 551          * Initialize the global used for permuting kernel 
 552          * addresses that may be exported to userland as tokens 
 553          * using VM_KERNEL_ADDRPERM(). Force the random number 
 554          * to be odd to avoid mapping a non-zero 
 555          * word-aligned address to zero via addition. 
 556          * Note: at this stage we can use the cryptographically secure PRNG 
 557          * rather than early_random(). 
 559         read_random(&vm_kernel_addrperm
, sizeof(vm_kernel_addrperm
)); 
 560         vm_kernel_addrperm 
|= 1; 
 561         read_random(&buf_kernel_addrperm
, sizeof(buf_kernel_addrperm
)); 
 562         buf_kernel_addrperm 
|= 1; 
 565          *      Start the user bootstrap. 
 572      * Get rid of segments used to bootstrap kext loading. This removes 
 573      * the KLD, PRELINK symtab, LINKEDIT, and symtab segments/load commands. 
 575         OSKextRemoveKextBootstrap(); 
 577         serial_keyboard_init();         /* Start serial keyboard if wanted */ 
 579         vm_page_init_local_q(); 
 581         thread_bind(PROCESSOR_NULL
); 
 584          *      Become the pageout daemon. 
 593  *      Load the first thread to start a processor. 
 596 slave_main(void *machine_param
) 
 598         processor_t             processor 
= current_processor(); 
 602          *      Use the idle processor thread if there 
 603          *      is no dedicated start up thread. 
 605         if (processor
->next_thread 
== THREAD_NULL
) { 
 606                 thread 
= processor
->idle_thread
; 
 607                 thread
->continuation 
= (thread_continue_t
)processor_start_thread
; 
 608                 thread
->parameter 
= machine_param
; 
 611                 thread 
= processor
->next_thread
; 
 612                 processor
->next_thread 
= THREAD_NULL
; 
 615         load_context(thread
); 
 620  *      processor_start_thread: 
 622  *      First thread to execute on a started processor. 
 624  *      Called at splsched. 
 627 processor_start_thread(void *machine_param
) 
 629         processor_t             processor 
= current_processor(); 
 630         thread_t                self 
= current_thread(); 
 632         slave_machine_init(machine_param
); 
 635          *      If running the idle processor thread, 
 636          *      reenter the idle loop, else terminate. 
 638         if (self 
== processor
->idle_thread
) 
 639                 thread_block((thread_continue_t
)idle_thread
); 
 641         thread_terminate(self
); 
 648  *      Start the first thread on a processor. 
 654         processor_t             processor 
= current_processor(); 
 657 #define load_context_kprintf(x...) /* kprintf("load_context: " x) */ 
 659         load_context_kprintf("machine_set_current_thread\n"); 
 660         machine_set_current_thread(thread
); 
 662         load_context_kprintf("processor_up\n"); 
 663         processor_up(processor
); 
 665         PMAP_ACTIVATE_KERNEL(processor
->cpu_id
); 
 668          * Acquire a stack if none attached.  The panic 
 669          * should never occur since the thread is expected 
 670          * to have reserved stack. 
 672         load_context_kprintf("thread %p, stack %lx, stackptr %lx\n", thread
, 
 673                              thread
->kernel_stack
, thread
->machine
.kstackptr
); 
 674         if (!thread
->kernel_stack
) { 
 675                 load_context_kprintf("stack_alloc_try\n"); 
 676                 if (!stack_alloc_try(thread
)) 
 677                         panic("load_context"); 
 681          * The idle processor threads are not counted as 
 682          * running for load calculations. 
 684         if (!(thread
->state 
& TH_IDLE
)) 
 685                 sched_run_incr(thread
); 
 687         processor
->active_thread 
= thread
; 
 688         processor
->current_pri 
= thread
->sched_pri
; 
 689         processor
->current_thmode 
= thread
->sched_mode
; 
 690         processor
->deadline 
= UINT64_MAX
; 
 691         thread
->last_processor 
= processor
; 
 693         processor
->last_dispatch 
= mach_absolute_time(); 
 694         timer_start(&thread
->system_timer
, processor
->last_dispatch
); 
 695         PROCESSOR_DATA(processor
, thread_timer
) = PROCESSOR_DATA(processor
, kernel_timer
) = &thread
->system_timer
; 
 697         timer_start(&PROCESSOR_DATA(processor
, system_state
), processor
->last_dispatch
); 
 698         PROCESSOR_DATA(processor
, current_state
) = &PROCESSOR_DATA(processor
, system_state
); 
 700         PMAP_ACTIVATE_USER(thread
, processor
->cpu_id
); 
 702         load_context_kprintf("machine_load_context\n"); 
 703         machine_load_context(thread
); 
 711 #if defined(__LP64__) 
 712         typeof(task_max
) task_max_base 
= task_max
; 
 714         /* Raise limits for servers with >= 16G */ 
 715         if ((serverperfmode 
!= 0) && ((uint64_t)sane_size 
>= (uint64_t)(16 * 1024 * 1024 *1024ULL))) { 
 716                 scale 
= (int)((uint64_t)sane_size 
/ (uint64_t)(8 * 1024 * 1024 *1024ULL)); 
 720                 task_max_base 
= 2500; 
 721         } else if ((uint64_t)sane_size 
>= (uint64_t)(3 * 1024 * 1024 *1024ULL)) 
 724         task_max 
= MAX(task_max
, task_max_base 
* scale
); 
 727                 task_threadmax 
= task_max
; 
 728                 thread_max 
= task_max 
* 5;  
 733         bsd_scale_setup(scale
); 
 735         ipc_space_max 
= SPACE_MAX
; 
 736         ipc_port_max 
= PORT_MAX
; 
 737         ipc_pset_max 
= SET_MAX
; 
 738         semaphore_max 
= SEMAPHORE_MAX
;