+
+extern struct startup_entry startup_entries[]
+__SECTION_START_SYM(STARTUP_HOOK_SEGMENT, STARTUP_HOOK_SECTION);
+
+extern struct startup_entry startup_entries_end[]
+__SECTION_END_SYM(STARTUP_HOOK_SEGMENT, STARTUP_HOOK_SECTION);
+
+static struct startup_entry *__startup_data startup_entry_cur = startup_entries;
+
+SECURITY_READ_ONLY_LATE(startup_subsystem_id_t) startup_phase = STARTUP_SUB_NONE;
+
+extern int serverperfmode;
+
+#if DEBUG || DEVELOPMENT
+TUNABLE(startup_debug_t, startup_debug, "startup_debug", 0);
+#endif
+
+/* size of kernel trace buffer, disabled by default */
+TUNABLE(unsigned int, new_nkdbufs, "trace", 0);
+TUNABLE(unsigned int, wake_nkdbufs, "trace_wake", 0);
+TUNABLE(unsigned int, write_trace_on_panic, "trace_panic", 0);
+TUNABLE(unsigned int, trace_wrap, "trace_wrap", 0);
+
+/* mach leak logging */
+TUNABLE(int, log_leaks, "-l", 0);
+
+static inline void
+kernel_bootstrap_log(const char *message)
+{
+ if ((startup_debug & STARTUP_DEBUG_VERBOSE) &&
+ startup_phase >= STARTUP_SUB_KPRINTF) {
+ kprintf("kernel_bootstrap: %s\n", message);
+ }
+ kernel_debug_string_early(message);
+}
+
+static inline void
+kernel_bootstrap_thread_log(const char *message)
+{
+ if ((startup_debug & STARTUP_DEBUG_VERBOSE) &&
+ startup_phase >= STARTUP_SUB_KPRINTF) {
+ kprintf("kernel_bootstrap_thread: %s\n", message);
+ }
+ kernel_debug_string_early(message);
+}
+
+extern void
+qsort(void *a, size_t n, size_t es, int (*cmp)(const void *, const void *));
+
+__startup_func
+static int
+startup_entry_cmp(const void *e1, const void *e2)
+{
+ const struct startup_entry *a = e1;
+ const struct startup_entry *b = e2;
+ if (a->subsystem == b->subsystem) {
+ if (a->rank == b->rank) {
+ return 0;
+ }
+ return a->rank > b->rank ? 1 : -1;
+ }
+ return a->subsystem > b->subsystem ? 1 : -1;
+}
+
+__startup_func
+void
+kernel_startup_bootstrap(void)
+{
+ /*
+ * Sort the various STARTUP() entries by subsystem/rank.
+ */
+ size_t n = startup_entries_end - startup_entries;
+
+ if (n == 0) {
+ panic("Section %s,%s missing",
+ STARTUP_HOOK_SEGMENT, STARTUP_HOOK_SECTION);
+ }
+ if (((uintptr_t)startup_entries_end - (uintptr_t)startup_entries) %
+ sizeof(struct startup_entry)) {
+ panic("Section %s,%s has invalid size",
+ STARTUP_HOOK_SEGMENT, STARTUP_HOOK_SECTION);
+ }
+
+ qsort(startup_entries, n, sizeof(struct startup_entry), startup_entry_cmp);
+
+ /*
+ * Then initialize all tunables, and early locks
+ */
+ kernel_startup_initialize_upto(STARTUP_SUB_LOCKS_EARLY);
+}
+
+__startup_func
+extern void
+kernel_startup_tunable_init(const struct startup_tunable_spec *spec)
+{
+ if (PE_parse_boot_argn(spec->name, spec->var_addr, spec->var_len)) {
+ if (spec->var_is_bool) {
+ /* make sure bool's are valued in {0, 1} */
+ *(bool *)spec->var_addr = *(uint8_t *)spec->var_addr;
+ }
+ }
+}
+
+static void
+kernel_startup_log(startup_subsystem_id_t subsystem)
+{
+ static const char *names[] = {
+ [STARTUP_SUB_TUNABLES] = "tunables",
+ [STARTUP_SUB_LOCKS_EARLY] = "locks_early",
+ [STARTUP_SUB_KPRINTF] = "kprintf",
+
+ [STARTUP_SUB_PMAP_STEAL] = "pmap_steal",
+ [STARTUP_SUB_VM_KERNEL] = "vm_kernel",
+ [STARTUP_SUB_KMEM] = "kmem",
+ [STARTUP_SUB_KMEM_ALLOC] = "kmem_alloc",
+ [STARTUP_SUB_ZALLOC] = "zalloc",
+ [STARTUP_SUB_PERCPU] = "percpu",
+ [STARTUP_SUB_LOCKS] = "locks",
+
+ [STARTUP_SUB_CODESIGNING] = "codesigning",
+ [STARTUP_SUB_OSLOG] = "oslog",
+ [STARTUP_SUB_MACH_IPC] = "mach_ipc",
+ [STARTUP_SUB_SYSCTL] = "sysctl",
+ [STARTUP_SUB_EARLY_BOOT] = "early_boot",
+
+ /* LOCKDOWN is special and its value won't fit here. */
+ };
+ static startup_subsystem_id_t logged = STARTUP_SUB_NONE;
+
+ if (subsystem <= logged) {
+ return;
+ }
+
+ if (subsystem < sizeof(names) / sizeof(names[0]) && names[subsystem]) {
+ kernel_bootstrap_log(names[subsystem]);
+ }
+ logged = subsystem;
+}
+
+__startup_func
+void
+kernel_startup_initialize_upto(startup_subsystem_id_t upto)
+{
+ struct startup_entry *cur = startup_entry_cur;
+
+ assert(startup_phase < upto);
+
+ while (cur < startup_entries_end && cur->subsystem <= upto) {
+ if ((startup_debug & STARTUP_DEBUG_VERBOSE) &&
+ startup_phase >= STARTUP_SUB_KPRINTF) {
+ kprintf("%s[%d, rank %d]: %p(%p)\n", __func__,
+ cur->subsystem, cur->rank, cur->func, cur->arg);
+ }
+ startup_phase = cur->subsystem - 1;
+ kernel_startup_log(cur->subsystem);
+ cur->func(cur->arg);
+ startup_entry_cur = ++cur;
+ }
+ kernel_startup_log(upto);
+
+ if ((startup_debug & STARTUP_DEBUG_VERBOSE) &&
+ upto >= STARTUP_SUB_KPRINTF) {
+ kprintf("%s: reached phase %d\n", __func__, upto);
+ }
+ startup_phase = upto;
+}
+