X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/0a7de7458d150b5d4dffc935ba399be265ef0a1a..2a1bd2d3eef5c7a7bb14f4bb9fdbca9a96ee4752:/osfmk/arm64/cpu.c diff --git a/osfmk/arm64/cpu.c b/osfmk/arm64/cpu.c index 483d4673b..ad3f9a685 100644 --- a/osfmk/arm64/cpu.c +++ b/osfmk/arm64/cpu.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -67,6 +68,14 @@ #include #endif /* MONOTONIC */ +#if HIBERNATION +#include +#include +#endif /* HIBERNATION */ + + +#include + extern boolean_t idle_enable; extern uint64_t wake_abstime; @@ -82,14 +91,17 @@ extern uintptr_t start_cpu; extern void exc_vectors_table; #endif /* __ARM_KERNEL_PROTECT__ */ -extern void __attribute__((noreturn)) arm64_prepare_for_sleep(void); +extern void __attribute__((noreturn)) arm64_prepare_for_sleep(boolean_t deep_sleep); extern void arm64_force_wfi_clock_gate(void); -#if (defined(APPLECYCLONE) || defined(APPLETYPHOON)) -// CPU1 Stuck in WFIWT Because of MMU Prefetch -extern void cyclone_typhoon_prepare_for_wfi(void); -extern void cyclone_typhoon_return_from_wfi(void); +#if defined(APPLETYPHOON) +// +extern void typhoon_prepare_for_wfi(void); +extern void typhoon_return_from_wfi(void); #endif +#if HAS_RETENTION_STATE +extern void arm64_retention_wfi(void); +#endif vm_address_t start_cpu_paddr; @@ -115,6 +127,14 @@ static int wfi_flags = 0; static uint64_t wfi_delay = 0; #endif /* DEVELOPMENT || DEBUG */ +#if DEVELOPMENT || DEBUG +static bool idle_proximate_timer_wfe = true; +static bool idle_proximate_io_wfe = true; +#define CPUPM_IDLE_WFE 0x5310300 +#else +static const bool idle_proximate_timer_wfe = true; +static const bool idle_proximate_io_wfe = true; +#endif #if __ARM_GLOBAL_SLEEP_BIT__ volatile boolean_t arm64_stall_sleep = TRUE; @@ -136,6 +156,7 @@ static boolean_t coresight_debug_enabled = FALSE; #if defined(CONFIG_XNUPOST) void arm64_ipi_test_callback(void *); +void arm64_immediate_ipi_test_callback(void *); void arm64_ipi_test_callback(void *parm) @@ -148,12 +169,23 @@ arm64_ipi_test_callback(void *parm) *ipi_test_data = cpu_data->cpu_number; } -uint64_t arm64_ipi_test_data[MAX_CPUS]; +void +arm64_immediate_ipi_test_callback(void *parm) +{ + volatile uint64_t *ipi_test_data = parm; + cpu_data_t *cpu_data; + + cpu_data = getCpuDatap(); + + *ipi_test_data = cpu_data->cpu_number + MAX_CPUS; +} + +uint64_t arm64_ipi_test_data[MAX_CPUS * 2]; void arm64_ipi_test() { - volatile uint64_t *ipi_test_data; + volatile uint64_t *ipi_test_data, *immediate_ipi_test_data; uint32_t timeout_ms = 100; uint64_t then, now, delta; int current_cpu_number = getCpuDatap()->cpu_number; @@ -167,21 +199,37 @@ arm64_ipi_test() return; } - for (unsigned int i = 0; i < MAX_CPUS; ++i) { + const unsigned int max_cpu_id = ml_get_max_cpu_number(); + for (unsigned int i = 0; i <= max_cpu_id; ++i) { ipi_test_data = &arm64_ipi_test_data[i]; + immediate_ipi_test_data = &arm64_ipi_test_data[i + MAX_CPUS]; *ipi_test_data = ~i; kern_return_t error = cpu_xcall((int)i, (void *)arm64_ipi_test_callback, (void *)(uintptr_t)ipi_test_data); if (error != KERN_SUCCESS) { panic("CPU %d was unable to IPI CPU %u: error %d", current_cpu_number, i, error); } + while ((error = cpu_immediate_xcall((int)i, (void *)arm64_immediate_ipi_test_callback, + (void *)(uintptr_t)immediate_ipi_test_data)) == KERN_ALREADY_WAITING) { + now = mach_absolute_time(); + absolutetime_to_nanoseconds(now - then, &delta); + if ((delta / NSEC_PER_MSEC) > timeout_ms) { + panic("CPU %d was unable to immediate-IPI CPU %u within %dms", current_cpu_number, i, timeout_ms); + } + } + + if (error != KERN_SUCCESS) { + panic("CPU %d was unable to immediate-IPI CPU %u: error %d", current_cpu_number, i, error); + } + then = mach_absolute_time(); - while (*ipi_test_data != i) { + while ((*ipi_test_data != i) || (*immediate_ipi_test_data != (i + MAX_CPUS))) { now = mach_absolute_time(); absolutetime_to_nanoseconds(now - then, &delta); if ((delta / NSEC_PER_MSEC) > timeout_ms) { - panic("CPU %d tried to IPI CPU %d but didn't get correct response within %dms, respose: %llx", current_cpu_number, i, timeout_ms, *ipi_test_data); + panic("CPU %d tried to IPI CPU %d but didn't get correct responses within %dms, responses: %llx, %llx", + current_cpu_number, i, timeout_ms, *ipi_test_data, *immediate_ipi_test_data); } } } @@ -191,25 +239,19 @@ arm64_ipi_test() static void configure_coresight_registers(cpu_data_t *cdp) { - uint64_t addr; int i; assert(cdp); + vm_offset_t coresight_regs = ml_get_topology_info()->cpus[cdp->cpu_number].coresight_regs; /* * ARMv8 coresight registers are optional. If the device tree did not - * provide cpu_regmap_paddr, assume that coresight registers are not - * supported. + * provide either cpu_regmap_paddr (from the legacy "reg-private" EDT property) + * or coresight_regs (from the new "coresight-reg" property), assume that + * coresight registers are not supported. */ - if (cdp->cpu_regmap_paddr) { + if (cdp->cpu_regmap_paddr || coresight_regs) { for (i = 0; i < CORESIGHT_REGIONS; ++i) { - /* Skip CTI; these registers are debug-only (they are - * not present on production hardware), and there is - * at least one known Cyclone errata involving CTI - * (rdar://12802966). We have no known clients that - * need the kernel to unlock CTI, so it is safer - * to avoid doing the access. - */ if (i == CORESIGHT_CTI) { continue; } @@ -219,8 +261,12 @@ configure_coresight_registers(cpu_data_t *cdp) } if (!cdp->coresight_base[i]) { - addr = cdp->cpu_regmap_paddr + CORESIGHT_OFFSET(i); - cdp->coresight_base[i] = (vm_offset_t)ml_io_map(addr, CORESIGHT_SIZE); + if (coresight_regs) { + cdp->coresight_base[i] = coresight_regs + CORESIGHT_OFFSET(i); + } else { + uint64_t addr = cdp->cpu_regmap_paddr + CORESIGHT_OFFSET(i); + cdp->coresight_base[i] = (vm_offset_t)ml_io_map(addr, CORESIGHT_SIZE); + } /* * At this point, failing to io map the @@ -271,7 +317,76 @@ cpu_sleep(void) CleanPoC_Dcache(); +#if USE_APPLEARMSMP + if (ml_is_quiescing()) { + PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id); + } else { + bool deep_sleep = PE_cpu_down(cpu_data_ptr->cpu_id); + cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH; + // hang CPU on spurious wakeup + cpu_data_ptr->cpu_reset_handler = (uintptr_t)0; + __builtin_arm_dsb(DSB_ISH); + CleanPoU_Dcache(); + arm64_prepare_for_sleep(deep_sleep); + } +#else PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id); +#endif + /*NOTREACHED*/ +} + +/* + * Routine: cpu_interrupt_is_pending + * Function: Returns the value of ISR. Due to how this register is + * is implemented, this returns 0 if there are no + * interrupts pending, so it can be used as a boolean test. + */ +int +cpu_interrupt_is_pending(void) +{ + uint64_t isr_value; + isr_value = __builtin_arm_rsr64("ISR_EL1"); + return (int)isr_value; +} + +static bool +cpu_proximate_timer(void) +{ + return !SetIdlePop(); +} + +static bool +wfe_to_deadline_or_interrupt(uint32_t cid, uint64_t wfe_deadline, __unused cpu_data_t *cdp) +{ + bool ipending = false; + while ((ipending = (cpu_interrupt_is_pending() != 0)) == false) { + /* Assumes event stream enablement + * TODO: evaluate temporarily stretching the per-CPU event + * interval to a larger value for possible efficiency + * improvements. + */ + __builtin_arm_wfe(); +#if DEVELOPMENT || DEBUG + cdp->wfe_count++; +#endif + if (wfe_deadline != ~0ULL) { +#if DEVELOPMENT || DEBUG + cdp->wfe_deadline_checks++; +#endif + /* Check if the WFE recommendation has expired. + * We do not recompute the deadline here. + */ + if ((ml_cluster_wfe_timeout(cid) == 0) || + mach_absolute_time() >= wfe_deadline) { +#if DEVELOPMENT || DEBUG + cdp->wfe_terminations++; +#endif + break; + } + } + } + /* TODO: worth refreshing pending interrupt status? */ + return ipending; } /* @@ -283,27 +398,71 @@ cpu_idle(void) { cpu_data_t *cpu_data_ptr = getCpuDatap(); uint64_t new_idle_timeout_ticks = 0x0ULL, lastPop; + bool idle_disallowed = false; + + if (__improbable((!idle_enable))) { + idle_disallowed = true; + } else if (__improbable(cpu_data_ptr->cpu_signal & SIGPdisabled)) { + idle_disallowed = true; + } - if ((!idle_enable) || (cpu_data_ptr->cpu_signal & SIGPdisabled)) { + if (__improbable(idle_disallowed)) { Idle_load_context(); } - if (!SetIdlePop()) { + + bool ipending = false; + uint32_t cid = ~0U; + + if (__probable(idle_proximate_io_wfe == true)) { + uint64_t wfe_deadline = 0; + /* Check for an active perf. controller generated + * WFE recommendation for this cluster. + */ + cid = cpu_data_ptr->cpu_cluster_id; + uint64_t wfe_ttd = 0; + if ((wfe_ttd = ml_cluster_wfe_timeout(cid)) != 0) { + wfe_deadline = mach_absolute_time() + wfe_ttd; + } + + if (wfe_deadline != 0) { + /* Poll issuing event-bounded WFEs until an interrupt + * arrives or the WFE recommendation expires + */ + ipending = wfe_to_deadline_or_interrupt(cid, wfe_deadline, cpu_data_ptr); +#if DEVELOPMENT || DEBUG + KDBG(CPUPM_IDLE_WFE, ipending, cpu_data_ptr->wfe_count, wfe_deadline, 0); +#endif + if (ipending == true) { + /* Back to machine_idle() */ + Idle_load_context(); + } + } + } + + if (__improbable(cpu_proximate_timer())) { + if (idle_proximate_timer_wfe == true) { + /* Poll issuing WFEs until the expected + * timer FIQ arrives. + */ + ipending = wfe_to_deadline_or_interrupt(cid, ~0ULL, cpu_data_ptr); + assert(ipending == true); + } Idle_load_context(); } + lastPop = cpu_data_ptr->rtcPop; - pmap_switch_user_ttb(kernel_pmap); cpu_data_ptr->cpu_active_thread = current_thread(); if (cpu_data_ptr->cpu_user_debug) { arm_debug_set(NULL); } cpu_data_ptr->cpu_user_debug = NULL; - if (cpu_data_ptr->cpu_idle_notify) { - ((processor_idle_t) cpu_data_ptr->cpu_idle_notify)(cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks); + if (wfi && (cpu_data_ptr->cpu_idle_notify != NULL)) { + cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks); } - if (cpu_data_ptr->idle_timer_notify != 0) { + if (cpu_data_ptr->idle_timer_notify != NULL) { if (new_idle_timeout_ticks == 0x0ULL) { /* turn off the idle timer */ cpu_data_ptr->idle_timer_deadline = 0x0ULL; @@ -325,7 +484,9 @@ cpu_idle(void) #endif /* MONOTONIC */ if (wfi) { +#if !defined(APPLE_ARM64_ARCH_FAMILY) platform_cache_idle_enter(); +#endif #if DEVELOPMENT || DEBUG // When simulating wfi overhead, @@ -335,16 +496,20 @@ cpu_idle(void) } #endif /* DEVELOPMENT || DEBUG */ -#if defined(APPLECYCLONE) || defined(APPLETYPHOON) +#if defined(APPLETYPHOON) // CPU1 Stuck in WFIWT Because of MMU Prefetch - cyclone_typhoon_prepare_for_wfi(); + typhoon_prepare_for_wfi(); #endif __builtin_arm_dsb(DSB_SY); +#if HAS_RETENTION_STATE + arm64_retention_wfi(); +#else __builtin_arm_wfi(); +#endif -#if defined(APPLECYCLONE) || defined(APPLETYPHOON) +#if defined(APPLETYPHOON) // CPU1 Stuck in WFIWT Because of MMU Prefetch - cyclone_typhoon_return_from_wfi(); + typhoon_return_from_wfi(); #endif #if DEVELOPMENT || DEBUG @@ -370,8 +535,9 @@ cpu_idle(void) clock_delay_until(deadline); } #endif /* DEVELOPMENT || DEBUG */ - +#if !defined(APPLE_ARM64_ARCH_FAMILY) platform_cache_idle_exit(); +#endif } ClearIdlePop(TRUE); @@ -404,13 +570,11 @@ cpu_idle_exit(boolean_t from_reset) mt_cpu_run(cpu_data_ptr); #endif /* MONOTONIC */ - pmap_switch_user_ttb(cpu_data_ptr->cpu_active_thread->map->pmap); - - if (cpu_data_ptr->cpu_idle_notify) { - ((processor_idle_t) cpu_data_ptr->cpu_idle_notify)(cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks); + if (wfi && (cpu_data_ptr->cpu_idle_notify != NULL)) { + cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks); } - if (cpu_data_ptr->idle_timer_notify != 0) { + if (cpu_data_ptr->idle_timer_notify != NULL) { if (new_idle_timeout_ticks == 0x0ULL) { /* turn off the idle timer */ cpu_data_ptr->idle_timer_deadline = 0x0ULL; @@ -459,6 +623,9 @@ cpu_init(void) case CPU_ARCH_ARMv8: cdp->cpu_subtype = CPU_SUBTYPE_ARM64_V8; break; + case CPU_ARCH_ARMv8E: + cdp->cpu_subtype = CPU_SUBTYPE_ARM64E; + break; default: //cdp->cpu_subtype = CPU_SUBTYPE_ARM64_ALL; /* this panic doesn't work this early in startup */ @@ -470,8 +637,9 @@ cpu_init(void) } cdp->cpu_stat.irq_ex_cnt_wake = 0; cdp->cpu_stat.ipi_cnt_wake = 0; - cdp->cpu_stat.timer_cnt_wake = 0; +#if MONOTONIC cdp->cpu_stat.pmi_cnt_wake = 0; +#endif /* MONOTONIC */ cdp->cpu_running = TRUE; cdp->cpu_sleep_token_last = cdp->cpu_sleep_token; cdp->cpu_sleep_token = 0x0UL; @@ -517,14 +685,19 @@ cpu_stack_alloc(cpu_data_t *cpu_data_ptr) void cpu_data_free(cpu_data_t *cpu_data_ptr) { - if (cpu_data_ptr == &BootCpuData) { + if ((cpu_data_ptr == NULL) || (cpu_data_ptr == &BootCpuData)) { return; } - cpu_processor_free( cpu_data_ptr->cpu_processor); + int cpu_number = cpu_data_ptr->cpu_number; + + if (CpuDataEntries[cpu_number].cpu_data_vaddr == cpu_data_ptr) { + CpuDataEntries[cpu_number].cpu_data_vaddr = NULL; + CpuDataEntries[cpu_number].cpu_data_paddr = 0; + __builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu array are visible + } (kfree)((void *)(cpu_data_ptr->intstack_top - INTSTACK_SIZE), INTSTACK_SIZE); (kfree)((void *)(cpu_data_ptr->excepstack_top - EXCEPSTACK_SIZE), EXCEPSTACK_SIZE); - kmem_free(kernel_map, (vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t)); } void @@ -533,17 +706,16 @@ cpu_data_init(cpu_data_t *cpu_data_ptr) uint32_t i; cpu_data_ptr->cpu_flags = 0; - cpu_data_ptr->interrupts_enabled = 0; cpu_data_ptr->cpu_int_state = 0; cpu_data_ptr->cpu_pending_ast = AST_NONE; - cpu_data_ptr->cpu_cache_dispatch = (void *) 0; + cpu_data_ptr->cpu_cache_dispatch = NULL; cpu_data_ptr->rtcPop = EndOfAllTime; cpu_data_ptr->rtclock_datap = &RTClockData; cpu_data_ptr->cpu_user_debug = NULL; cpu_data_ptr->cpu_base_timebase = 0; - cpu_data_ptr->cpu_idle_notify = (void *) 0; + cpu_data_ptr->cpu_idle_notify = NULL; cpu_data_ptr->cpu_idle_latency = 0x0ULL; cpu_data_ptr->cpu_idle_pop = 0x0ULL; cpu_data_ptr->cpu_reset_type = 0x0UL; @@ -561,12 +733,6 @@ cpu_data_init(cpu_data_t *cpu_data_ptr) cpu_data_ptr->cpu_signal = SIGPdisabled; -#if DEBUG || DEVELOPMENT - cpu_data_ptr->failed_xcall = NULL; - cpu_data_ptr->failed_signal = 0; - cpu_data_ptr->failed_signal_count = 0; -#endif - cpu_data_ptr->cpu_get_fiq_handler = NULL; cpu_data_ptr->cpu_tbd_hardware_addr = NULL; cpu_data_ptr->cpu_tbd_hardware_val = NULL; @@ -576,24 +742,34 @@ cpu_data_init(cpu_data_t *cpu_data_ptr) cpu_data_ptr->cpu_sleep_token_last = 0x00000000UL; cpu_data_ptr->cpu_xcall_p0 = NULL; cpu_data_ptr->cpu_xcall_p1 = NULL; + cpu_data_ptr->cpu_imm_xcall_p0 = NULL; + cpu_data_ptr->cpu_imm_xcall_p1 = NULL; for (i = 0; i < CORESIGHT_REGIONS; ++i) { cpu_data_ptr->coresight_base[i] = 0; } +#if !XNU_MONITOR pmap_cpu_data_t * pmap_cpu_data_ptr = &cpu_data_ptr->cpu_pmap_cpu_data; pmap_cpu_data_ptr->cpu_nested_pmap = (struct pmap *) NULL; pmap_cpu_data_ptr->cpu_number = PMAP_INVALID_CPU_NUM; + pmap_cpu_data_ptr->pv_free.list = NULL; + pmap_cpu_data_ptr->pv_free.count = 0; + pmap_cpu_data_ptr->pv_free_tail = NULL; - for (i = 0; i < (sizeof(pmap_cpu_data_ptr->cpu_asid_high_bits) / sizeof(*pmap_cpu_data_ptr->cpu_asid_high_bits)); i++) { - pmap_cpu_data_ptr->cpu_asid_high_bits[i] = 0; - } + bzero(&(pmap_cpu_data_ptr->cpu_sw_asids[0]), sizeof(pmap_cpu_data_ptr->cpu_sw_asids)); +#endif cpu_data_ptr->halt_status = CPU_NOT_HALTED; #if __ARM_KERNEL_PROTECT__ cpu_data_ptr->cpu_exc_vectors = (vm_offset_t)&exc_vectors_table; #endif /* __ARM_KERNEL_PROTECT__ */ +#if defined(HAS_APPLE_PAC) + cpu_data_ptr->rop_key = 0; + cpu_data_ptr->jop_key = ml_default_jop_pid(); +#endif + } kern_return_t @@ -607,11 +783,29 @@ cpu_data_register(cpu_data_t *cpu_data_ptr) } #endif + __builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu data are visible CpuDataEntries[cpu].cpu_data_vaddr = cpu_data_ptr; CpuDataEntries[cpu].cpu_data_paddr = (void *)ml_vtophys((vm_offset_t)cpu_data_ptr); return KERN_SUCCESS; } +#if defined(KERNEL_INTEGRITY_CTRR) +/* Hibernation needs to reset this state, so data and text are in the hib segment; + * this allows them be accessed and executed early. + */ +LCK_GRP_DECLARE(ctrr_cpu_start_lock_grp, "ctrr_cpu_start_lock"); +LCK_SPIN_DECLARE(ctrr_cpu_start_lck, &ctrr_cpu_start_lock_grp); +enum ctrr_cluster_states ctrr_cluster_locked[MAX_CPU_CLUSTERS] MARK_AS_HIBERNATE_DATA; + +MARK_AS_HIBERNATE_TEXT +void +init_ctrr_cluster_states(void) +{ + for (int i = 0; i < MAX_CPU_CLUSTERS; i++) { + ctrr_cluster_locked[i] = CTRR_UNLOCKED; + } +} +#endif kern_return_t cpu_start(int cpu) @@ -625,23 +819,50 @@ cpu_start(int cpu) configure_coresight_registers(cpu_data_ptr); } else { thread_t first_thread; + processor_t processor; cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr; +#if !XNU_MONITOR cpu_data_ptr->cpu_pmap_cpu_data.cpu_nested_pmap = NULL; +#endif - if (cpu_data_ptr->cpu_processor->next_thread != THREAD_NULL) { - first_thread = cpu_data_ptr->cpu_processor->next_thread; + processor = PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data_ptr); + if (processor->startup_thread != THREAD_NULL) { + first_thread = processor->startup_thread; } else { - first_thread = cpu_data_ptr->cpu_processor->idle_thread; + first_thread = processor->idle_thread; } cpu_data_ptr->cpu_active_thread = first_thread; first_thread->machine.CpuDatap = cpu_data_ptr; + first_thread->machine.pcpu_data_base = + (vm_address_t)cpu_data_ptr - __PERCPU_ADDR(cpu_data); configure_coresight_registers(cpu_data_ptr); flush_dcache((vm_offset_t)&CpuDataEntries[cpu], sizeof(cpu_data_entry_t), FALSE); flush_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE); +#if defined(KERNEL_INTEGRITY_CTRR) + + /* First CPU being started within a cluster goes ahead to lock CTRR for cluster; + * other CPUs block until cluster is locked. */ + lck_spin_lock(&ctrr_cpu_start_lck); + switch (ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id]) { + case CTRR_UNLOCKED: + ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] = CTRR_LOCKING; + lck_spin_unlock(&ctrr_cpu_start_lck); + break; + case CTRR_LOCKING: + assert_wait(&ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id], THREAD_UNINT); + lck_spin_unlock(&ctrr_cpu_start_lck); + thread_block(THREAD_CONTINUE_NULL); + assert(ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] != CTRR_LOCKING); + break; + default: // CTRR_LOCKED + lck_spin_unlock(&ctrr_cpu_start_lck); + break; + } +#endif (void) PE_cpu_start(cpu_data_ptr->cpu_id, (vm_offset_t)NULL, (vm_offset_t)NULL); } @@ -675,6 +896,9 @@ cpu_timebase_init(boolean_t from_boot) * This ensures that mach_absolute_time() stops ticking across sleep. */ rtclock_base_abstime = wake_abstime - ml_get_hwclock(); + } else if (from_boot) { + /* On initial boot, initialize time_since_reset to CNTPCT_EL0. */ + ml_set_reset_time(ml_get_hwclock()); } cdp->cpu_decrementer = 0x7FFFFFFFUL; @@ -717,6 +941,7 @@ ml_arm_sleep(void) * the abstime value we'll use when we resume. */ wake_abstime = ml_get_timebase(); + ml_set_reset_time(UINT64_MAX); } else { CleanPoU_Dcache(); } @@ -754,11 +979,23 @@ ml_arm_sleep(void) *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGPRCR) = 0; } +#if HIBERNATION + uint32_t mode = hibernate_write_image(); + if (mode == kIOHibernatePostWriteHalt) { + HIBLOG("powering off after writing hibernation image\n"); + int halt_result = -1; + if (PE_halt_restart) { + halt_result = (*PE_halt_restart)(kPEHaltCPU); + } + panic("can't shutdown: PE_halt_restart returned %d", halt_result); + } +#endif /* HIBERNATION */ + #if MONOTONIC mt_sleep(); #endif /* MONOTONIC */ /* ARM64-specific preparation */ - arm64_prepare_for_sleep(); + arm64_prepare_for_sleep(true); } else { #if __ARM_GLOBAL_SLEEP_BIT__ /* @@ -788,7 +1025,7 @@ ml_arm_sleep(void) } /* ARM64-specific preparation */ - arm64_prepare_for_sleep(); + arm64_prepare_for_sleep(true); } } @@ -799,11 +1036,12 @@ cpu_machine_idle_init(boolean_t from_boot) cpu_data_t *cpu_data_ptr = getCpuDatap(); if (from_boot) { - unsigned long jtag = 0; int wfi_tmp = 1; uint32_t production = 1; DTEntry entry; + unsigned long jtag = 0; + if (PE_parse_boot_argn("jtag", &jtag, sizeof(jtag))) { if (jtag != 0) { idle_enable = FALSE; @@ -814,6 +1052,13 @@ cpu_machine_idle_init(boolean_t from_boot) idle_enable = TRUE; } +#if DEVELOPMENT || DEBUG + uint32_t wfe_mode = 0; + if (PE_parse_boot_argn("wfe_mode", &wfe_mode, sizeof(wfe_mode))) { + idle_proximate_timer_wfe = ((wfe_mode & 1) == 1); + idle_proximate_io_wfe = ((wfe_mode & 2) == 2); + } +#endif PE_parse_boot_argn("wfi", &wfi_tmp, sizeof(wfi_tmp)); // bits 7..0 give the wfi type @@ -851,11 +1096,11 @@ cpu_machine_idle_init(boolean_t from_boot) #endif /* MONITOR */ // Determine if we are on production or debug chip - if (kSuccess == DTLookupEntry(NULL, "/chosen", &entry)) { + if (kSuccess == SecureDTLookupEntry(NULL, "/chosen", &entry)) { unsigned int size; - void *prop; + void const *prop; - if (kSuccess == DTGetProperty(entry, "effective-production-status-ap", &prop, &size)) { + if (kSuccess == SecureDTGetProperty(entry, "effective-production-status-ap", &prop, &size)) { if (size == 4) { bcopy(prop, &production, size); } @@ -898,9 +1143,9 @@ void machine_track_platform_idle(boolean_t entry) { if (entry) { - (void)__c11_atomic_fetch_add(&cpu_idle_count, 1, __ATOMIC_RELAXED); + os_atomic_inc(&cpu_idle_count, relaxed); } else { - (void)__c11_atomic_fetch_sub(&cpu_idle_count, 1, __ATOMIC_RELAXED); + os_atomic_dec(&cpu_idle_count, relaxed); } } @@ -911,20 +1156,20 @@ sleep_token_buffer_init(void) cpu_data_t *cpu_data_ptr = getCpuDatap(); DTEntry entry; size_t size; - void **prop; + void const * const *prop; if ((cpu_data_ptr == &BootCpuData) && (sleepTokenBuffer == (vm_offset_t) NULL)) { /* Find the stpage node in the device tree */ - if (kSuccess != DTLookupEntry(0, "stram", &entry)) { + if (kSuccess != SecureDTLookupEntry(0, "stram", &entry)) { return; } - if (kSuccess != DTGetProperty(entry, "reg", (void **)&prop, (unsigned int *)&size)) { + if (kSuccess != SecureDTGetProperty(entry, "reg", (const void **)&prop, (unsigned int *)&size)) { return; } /* Map the page into the kernel space */ - sleepTokenBuffer = ml_io_map(((vm_offset_t *)prop)[0], ((vm_size_t *)prop)[1]); + sleepTokenBuffer = ml_io_map(((vm_offset_t const *)prop)[0], ((vm_size_t const *)prop)[1]); } } #endif