#include <kern/kalloc.h>
#include <kern/machine.h>
#include <kern/cpu_number.h>
+#include <kern/percpu.h>
#include <kern/thread.h>
#include <kern/timer_queue.h>
#include <arm/cpu_data.h>
#include <kern/monotonic.h>
#endif /* MONOTONIC */
+#if HIBERNATION
+#include <IOKit/IOPlatformExpert.h>
+#include <IOKit/IOHibernatePrivate.h>
+#endif /* HIBERNATION */
+
+
+#include <libkern/section_keywords.h>
+
extern boolean_t idle_enable;
extern uint64_t wake_abstime;
extern void exc_vectors_table;
#endif /* __ARM_KERNEL_PROTECT__ */
-extern void __attribute__((noreturn)) arm64_prepare_for_sleep(void);
+extern void __attribute__((noreturn)) arm64_prepare_for_sleep(boolean_t deep_sleep);
extern void arm64_force_wfi_clock_gate(void);
-#if (defined(APPLECYCLONE) || defined(APPLETYPHOON))
-// <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
-extern void cyclone_typhoon_prepare_for_wfi(void);
-extern void cyclone_typhoon_return_from_wfi(void);
+#if defined(APPLETYPHOON)
+// <rdar://problem/15827409>
+extern void typhoon_prepare_for_wfi(void);
+extern void typhoon_return_from_wfi(void);
#endif
+#if HAS_RETENTION_STATE
+extern void arm64_retention_wfi(void);
+#endif
vm_address_t start_cpu_paddr;
static uint64_t wfi_delay = 0;
#endif /* DEVELOPMENT || DEBUG */
+#if DEVELOPMENT || DEBUG
+static bool idle_proximate_timer_wfe = true;
+static bool idle_proximate_io_wfe = true;
+#define CPUPM_IDLE_WFE 0x5310300
+#else
+static const bool idle_proximate_timer_wfe = true;
+static const bool idle_proximate_io_wfe = true;
+#endif
#if __ARM_GLOBAL_SLEEP_BIT__
volatile boolean_t arm64_stall_sleep = TRUE;
#if defined(CONFIG_XNUPOST)
void arm64_ipi_test_callback(void *);
+void arm64_immediate_ipi_test_callback(void *);
void
arm64_ipi_test_callback(void *parm)
*ipi_test_data = cpu_data->cpu_number;
}
-uint64_t arm64_ipi_test_data[MAX_CPUS];
+void
+arm64_immediate_ipi_test_callback(void *parm)
+{
+ volatile uint64_t *ipi_test_data = parm;
+ cpu_data_t *cpu_data;
+
+ cpu_data = getCpuDatap();
+
+ *ipi_test_data = cpu_data->cpu_number + MAX_CPUS;
+}
+
+uint64_t arm64_ipi_test_data[MAX_CPUS * 2];
void
arm64_ipi_test()
{
- volatile uint64_t *ipi_test_data;
+ volatile uint64_t *ipi_test_data, *immediate_ipi_test_data;
uint32_t timeout_ms = 100;
uint64_t then, now, delta;
int current_cpu_number = getCpuDatap()->cpu_number;
return;
}
- for (unsigned int i = 0; i < MAX_CPUS; ++i) {
+ const unsigned int max_cpu_id = ml_get_max_cpu_number();
+ for (unsigned int i = 0; i <= max_cpu_id; ++i) {
ipi_test_data = &arm64_ipi_test_data[i];
+ immediate_ipi_test_data = &arm64_ipi_test_data[i + MAX_CPUS];
*ipi_test_data = ~i;
kern_return_t error = cpu_xcall((int)i, (void *)arm64_ipi_test_callback, (void *)(uintptr_t)ipi_test_data);
if (error != KERN_SUCCESS) {
panic("CPU %d was unable to IPI CPU %u: error %d", current_cpu_number, i, error);
}
+ while ((error = cpu_immediate_xcall((int)i, (void *)arm64_immediate_ipi_test_callback,
+ (void *)(uintptr_t)immediate_ipi_test_data)) == KERN_ALREADY_WAITING) {
+ now = mach_absolute_time();
+ absolutetime_to_nanoseconds(now - then, &delta);
+ if ((delta / NSEC_PER_MSEC) > timeout_ms) {
+ panic("CPU %d was unable to immediate-IPI CPU %u within %dms", current_cpu_number, i, timeout_ms);
+ }
+ }
+
+ if (error != KERN_SUCCESS) {
+ panic("CPU %d was unable to immediate-IPI CPU %u: error %d", current_cpu_number, i, error);
+ }
+
then = mach_absolute_time();
- while (*ipi_test_data != i) {
+ while ((*ipi_test_data != i) || (*immediate_ipi_test_data != (i + MAX_CPUS))) {
now = mach_absolute_time();
absolutetime_to_nanoseconds(now - then, &delta);
if ((delta / NSEC_PER_MSEC) > timeout_ms) {
- panic("CPU %d tried to IPI CPU %d but didn't get correct response within %dms, respose: %llx", current_cpu_number, i, timeout_ms, *ipi_test_data);
+ panic("CPU %d tried to IPI CPU %d but didn't get correct responses within %dms, responses: %llx, %llx",
+ current_cpu_number, i, timeout_ms, *ipi_test_data, *immediate_ipi_test_data);
}
}
}
static void
configure_coresight_registers(cpu_data_t *cdp)
{
- uint64_t addr;
int i;
assert(cdp);
+ vm_offset_t coresight_regs = ml_get_topology_info()->cpus[cdp->cpu_number].coresight_regs;
/*
* ARMv8 coresight registers are optional. If the device tree did not
- * provide cpu_regmap_paddr, assume that coresight registers are not
- * supported.
+ * provide either cpu_regmap_paddr (from the legacy "reg-private" EDT property)
+ * or coresight_regs (from the new "coresight-reg" property), assume that
+ * coresight registers are not supported.
*/
- if (cdp->cpu_regmap_paddr) {
+ if (cdp->cpu_regmap_paddr || coresight_regs) {
for (i = 0; i < CORESIGHT_REGIONS; ++i) {
- /* Skip CTI; these registers are debug-only (they are
- * not present on production hardware), and there is
- * at least one known Cyclone errata involving CTI
- * (rdar://12802966). We have no known clients that
- * need the kernel to unlock CTI, so it is safer
- * to avoid doing the access.
- */
if (i == CORESIGHT_CTI) {
continue;
}
}
if (!cdp->coresight_base[i]) {
- addr = cdp->cpu_regmap_paddr + CORESIGHT_OFFSET(i);
- cdp->coresight_base[i] = (vm_offset_t)ml_io_map(addr, CORESIGHT_SIZE);
+ if (coresight_regs) {
+ cdp->coresight_base[i] = coresight_regs + CORESIGHT_OFFSET(i);
+ } else {
+ uint64_t addr = cdp->cpu_regmap_paddr + CORESIGHT_OFFSET(i);
+ cdp->coresight_base[i] = (vm_offset_t)ml_io_map(addr, CORESIGHT_SIZE);
+ }
/*
* At this point, failing to io map the
CleanPoC_Dcache();
+#if USE_APPLEARMSMP
+ if (ml_is_quiescing()) {
+ PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id);
+ } else {
+ bool deep_sleep = PE_cpu_down(cpu_data_ptr->cpu_id);
+ cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
+ // hang CPU on spurious wakeup
+ cpu_data_ptr->cpu_reset_handler = (uintptr_t)0;
+ __builtin_arm_dsb(DSB_ISH);
+ CleanPoU_Dcache();
+ arm64_prepare_for_sleep(deep_sleep);
+ }
+#else
PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id);
+#endif
+ /*NOTREACHED*/
+}
+
+/*
+ * Routine: cpu_interrupt_is_pending
+ * Function: Returns the value of ISR. Due to how this register is
+ * is implemented, this returns 0 if there are no
+ * interrupts pending, so it can be used as a boolean test.
+ */
+int
+cpu_interrupt_is_pending(void)
+{
+ uint64_t isr_value;
+ isr_value = __builtin_arm_rsr64("ISR_EL1");
+ return (int)isr_value;
+}
+
+static bool
+cpu_proximate_timer(void)
+{
+ return !SetIdlePop();
+}
+
+static bool
+wfe_to_deadline_or_interrupt(uint32_t cid, uint64_t wfe_deadline, __unused cpu_data_t *cdp)
+{
+ bool ipending = false;
+ while ((ipending = (cpu_interrupt_is_pending() != 0)) == false) {
+ /* Assumes event stream enablement
+ * TODO: evaluate temporarily stretching the per-CPU event
+ * interval to a larger value for possible efficiency
+ * improvements.
+ */
+ __builtin_arm_wfe();
+#if DEVELOPMENT || DEBUG
+ cdp->wfe_count++;
+#endif
+ if (wfe_deadline != ~0ULL) {
+#if DEVELOPMENT || DEBUG
+ cdp->wfe_deadline_checks++;
+#endif
+ /* Check if the WFE recommendation has expired.
+ * We do not recompute the deadline here.
+ */
+ if ((ml_cluster_wfe_timeout(cid) == 0) ||
+ mach_absolute_time() >= wfe_deadline) {
+#if DEVELOPMENT || DEBUG
+ cdp->wfe_terminations++;
+#endif
+ break;
+ }
+ }
+ }
+ /* TODO: worth refreshing pending interrupt status? */
+ return ipending;
}
/*
{
cpu_data_t *cpu_data_ptr = getCpuDatap();
uint64_t new_idle_timeout_ticks = 0x0ULL, lastPop;
+ bool idle_disallowed = false;
+
+ if (__improbable((!idle_enable))) {
+ idle_disallowed = true;
+ } else if (__improbable(cpu_data_ptr->cpu_signal & SIGPdisabled)) {
+ idle_disallowed = true;
+ }
- if ((!idle_enable) || (cpu_data_ptr->cpu_signal & SIGPdisabled)) {
+ if (__improbable(idle_disallowed)) {
Idle_load_context();
}
- if (!SetIdlePop()) {
+
+ bool ipending = false;
+ uint32_t cid = ~0U;
+
+ if (__probable(idle_proximate_io_wfe == true)) {
+ uint64_t wfe_deadline = 0;
+ /* Check for an active perf. controller generated
+ * WFE recommendation for this cluster.
+ */
+ cid = cpu_data_ptr->cpu_cluster_id;
+ uint64_t wfe_ttd = 0;
+ if ((wfe_ttd = ml_cluster_wfe_timeout(cid)) != 0) {
+ wfe_deadline = mach_absolute_time() + wfe_ttd;
+ }
+
+ if (wfe_deadline != 0) {
+ /* Poll issuing event-bounded WFEs until an interrupt
+ * arrives or the WFE recommendation expires
+ */
+ ipending = wfe_to_deadline_or_interrupt(cid, wfe_deadline, cpu_data_ptr);
+#if DEVELOPMENT || DEBUG
+ KDBG(CPUPM_IDLE_WFE, ipending, cpu_data_ptr->wfe_count, wfe_deadline, 0);
+#endif
+ if (ipending == true) {
+ /* Back to machine_idle() */
+ Idle_load_context();
+ }
+ }
+ }
+
+ if (__improbable(cpu_proximate_timer())) {
+ if (idle_proximate_timer_wfe == true) {
+ /* Poll issuing WFEs until the expected
+ * timer FIQ arrives.
+ */
+ ipending = wfe_to_deadline_or_interrupt(cid, ~0ULL, cpu_data_ptr);
+ assert(ipending == true);
+ }
Idle_load_context();
}
+
lastPop = cpu_data_ptr->rtcPop;
- pmap_switch_user_ttb(kernel_pmap);
cpu_data_ptr->cpu_active_thread = current_thread();
if (cpu_data_ptr->cpu_user_debug) {
arm_debug_set(NULL);
}
cpu_data_ptr->cpu_user_debug = NULL;
- if (cpu_data_ptr->cpu_idle_notify) {
- ((processor_idle_t) cpu_data_ptr->cpu_idle_notify)(cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks);
+ if (wfi && (cpu_data_ptr->cpu_idle_notify != NULL)) {
+ cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks);
}
- if (cpu_data_ptr->idle_timer_notify != 0) {
+ if (cpu_data_ptr->idle_timer_notify != NULL) {
if (new_idle_timeout_ticks == 0x0ULL) {
/* turn off the idle timer */
cpu_data_ptr->idle_timer_deadline = 0x0ULL;
#endif /* MONOTONIC */
if (wfi) {
+#if !defined(APPLE_ARM64_ARCH_FAMILY)
platform_cache_idle_enter();
+#endif
#if DEVELOPMENT || DEBUG
// When simulating wfi overhead,
}
#endif /* DEVELOPMENT || DEBUG */
-#if defined(APPLECYCLONE) || defined(APPLETYPHOON)
+#if defined(APPLETYPHOON)
// <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
- cyclone_typhoon_prepare_for_wfi();
+ typhoon_prepare_for_wfi();
#endif
__builtin_arm_dsb(DSB_SY);
+#if HAS_RETENTION_STATE
+ arm64_retention_wfi();
+#else
__builtin_arm_wfi();
+#endif
-#if defined(APPLECYCLONE) || defined(APPLETYPHOON)
+#if defined(APPLETYPHOON)
// <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
- cyclone_typhoon_return_from_wfi();
+ typhoon_return_from_wfi();
#endif
#if DEVELOPMENT || DEBUG
clock_delay_until(deadline);
}
#endif /* DEVELOPMENT || DEBUG */
-
+#if !defined(APPLE_ARM64_ARCH_FAMILY)
platform_cache_idle_exit();
+#endif
}
ClearIdlePop(TRUE);
mt_cpu_run(cpu_data_ptr);
#endif /* MONOTONIC */
- pmap_switch_user_ttb(cpu_data_ptr->cpu_active_thread->map->pmap);
-
- if (cpu_data_ptr->cpu_idle_notify) {
- ((processor_idle_t) cpu_data_ptr->cpu_idle_notify)(cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks);
+ if (wfi && (cpu_data_ptr->cpu_idle_notify != NULL)) {
+ cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks);
}
- if (cpu_data_ptr->idle_timer_notify != 0) {
+ if (cpu_data_ptr->idle_timer_notify != NULL) {
if (new_idle_timeout_ticks == 0x0ULL) {
/* turn off the idle timer */
cpu_data_ptr->idle_timer_deadline = 0x0ULL;
case CPU_ARCH_ARMv8:
cdp->cpu_subtype = CPU_SUBTYPE_ARM64_V8;
break;
+ case CPU_ARCH_ARMv8E:
+ cdp->cpu_subtype = CPU_SUBTYPE_ARM64E;
+ break;
default:
//cdp->cpu_subtype = CPU_SUBTYPE_ARM64_ALL;
/* this panic doesn't work this early in startup */
}
cdp->cpu_stat.irq_ex_cnt_wake = 0;
cdp->cpu_stat.ipi_cnt_wake = 0;
- cdp->cpu_stat.timer_cnt_wake = 0;
+#if MONOTONIC
cdp->cpu_stat.pmi_cnt_wake = 0;
+#endif /* MONOTONIC */
cdp->cpu_running = TRUE;
cdp->cpu_sleep_token_last = cdp->cpu_sleep_token;
cdp->cpu_sleep_token = 0x0UL;
void
cpu_data_free(cpu_data_t *cpu_data_ptr)
{
- if (cpu_data_ptr == &BootCpuData) {
+ if ((cpu_data_ptr == NULL) || (cpu_data_ptr == &BootCpuData)) {
return;
}
- cpu_processor_free( cpu_data_ptr->cpu_processor);
+ int cpu_number = cpu_data_ptr->cpu_number;
+
+ if (CpuDataEntries[cpu_number].cpu_data_vaddr == cpu_data_ptr) {
+ CpuDataEntries[cpu_number].cpu_data_vaddr = NULL;
+ CpuDataEntries[cpu_number].cpu_data_paddr = 0;
+ __builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu array are visible
+ }
(kfree)((void *)(cpu_data_ptr->intstack_top - INTSTACK_SIZE), INTSTACK_SIZE);
(kfree)((void *)(cpu_data_ptr->excepstack_top - EXCEPSTACK_SIZE), EXCEPSTACK_SIZE);
- kmem_free(kernel_map, (vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t));
}
void
uint32_t i;
cpu_data_ptr->cpu_flags = 0;
- cpu_data_ptr->interrupts_enabled = 0;
cpu_data_ptr->cpu_int_state = 0;
cpu_data_ptr->cpu_pending_ast = AST_NONE;
- cpu_data_ptr->cpu_cache_dispatch = (void *) 0;
+ cpu_data_ptr->cpu_cache_dispatch = NULL;
cpu_data_ptr->rtcPop = EndOfAllTime;
cpu_data_ptr->rtclock_datap = &RTClockData;
cpu_data_ptr->cpu_user_debug = NULL;
cpu_data_ptr->cpu_base_timebase = 0;
- cpu_data_ptr->cpu_idle_notify = (void *) 0;
+ cpu_data_ptr->cpu_idle_notify = NULL;
cpu_data_ptr->cpu_idle_latency = 0x0ULL;
cpu_data_ptr->cpu_idle_pop = 0x0ULL;
cpu_data_ptr->cpu_reset_type = 0x0UL;
cpu_data_ptr->cpu_signal = SIGPdisabled;
-#if DEBUG || DEVELOPMENT
- cpu_data_ptr->failed_xcall = NULL;
- cpu_data_ptr->failed_signal = 0;
- cpu_data_ptr->failed_signal_count = 0;
-#endif
-
cpu_data_ptr->cpu_get_fiq_handler = NULL;
cpu_data_ptr->cpu_tbd_hardware_addr = NULL;
cpu_data_ptr->cpu_tbd_hardware_val = NULL;
cpu_data_ptr->cpu_sleep_token_last = 0x00000000UL;
cpu_data_ptr->cpu_xcall_p0 = NULL;
cpu_data_ptr->cpu_xcall_p1 = NULL;
+ cpu_data_ptr->cpu_imm_xcall_p0 = NULL;
+ cpu_data_ptr->cpu_imm_xcall_p1 = NULL;
for (i = 0; i < CORESIGHT_REGIONS; ++i) {
cpu_data_ptr->coresight_base[i] = 0;
}
+#if !XNU_MONITOR
pmap_cpu_data_t * pmap_cpu_data_ptr = &cpu_data_ptr->cpu_pmap_cpu_data;
pmap_cpu_data_ptr->cpu_nested_pmap = (struct pmap *) NULL;
pmap_cpu_data_ptr->cpu_number = PMAP_INVALID_CPU_NUM;
+ pmap_cpu_data_ptr->pv_free.list = NULL;
+ pmap_cpu_data_ptr->pv_free.count = 0;
+ pmap_cpu_data_ptr->pv_free_tail = NULL;
- for (i = 0; i < (sizeof(pmap_cpu_data_ptr->cpu_asid_high_bits) / sizeof(*pmap_cpu_data_ptr->cpu_asid_high_bits)); i++) {
- pmap_cpu_data_ptr->cpu_asid_high_bits[i] = 0;
- }
+ bzero(&(pmap_cpu_data_ptr->cpu_sw_asids[0]), sizeof(pmap_cpu_data_ptr->cpu_sw_asids));
+#endif
cpu_data_ptr->halt_status = CPU_NOT_HALTED;
#if __ARM_KERNEL_PROTECT__
cpu_data_ptr->cpu_exc_vectors = (vm_offset_t)&exc_vectors_table;
#endif /* __ARM_KERNEL_PROTECT__ */
+#if defined(HAS_APPLE_PAC)
+ cpu_data_ptr->rop_key = 0;
+ cpu_data_ptr->jop_key = ml_default_jop_pid();
+#endif
+
}
kern_return_t
}
#endif
+ __builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu data are visible
CpuDataEntries[cpu].cpu_data_vaddr = cpu_data_ptr;
CpuDataEntries[cpu].cpu_data_paddr = (void *)ml_vtophys((vm_offset_t)cpu_data_ptr);
return KERN_SUCCESS;
}
+#if defined(KERNEL_INTEGRITY_CTRR)
+/* Hibernation needs to reset this state, so data and text are in the hib segment;
+ * this allows them be accessed and executed early.
+ */
+LCK_GRP_DECLARE(ctrr_cpu_start_lock_grp, "ctrr_cpu_start_lock");
+LCK_SPIN_DECLARE(ctrr_cpu_start_lck, &ctrr_cpu_start_lock_grp);
+enum ctrr_cluster_states ctrr_cluster_locked[MAX_CPU_CLUSTERS] MARK_AS_HIBERNATE_DATA;
+
+MARK_AS_HIBERNATE_TEXT
+void
+init_ctrr_cluster_states(void)
+{
+ for (int i = 0; i < MAX_CPU_CLUSTERS; i++) {
+ ctrr_cluster_locked[i] = CTRR_UNLOCKED;
+ }
+}
+#endif
kern_return_t
cpu_start(int cpu)
configure_coresight_registers(cpu_data_ptr);
} else {
thread_t first_thread;
+ processor_t processor;
cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr;
+#if !XNU_MONITOR
cpu_data_ptr->cpu_pmap_cpu_data.cpu_nested_pmap = NULL;
+#endif
- if (cpu_data_ptr->cpu_processor->next_thread != THREAD_NULL) {
- first_thread = cpu_data_ptr->cpu_processor->next_thread;
+ processor = PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data_ptr);
+ if (processor->startup_thread != THREAD_NULL) {
+ first_thread = processor->startup_thread;
} else {
- first_thread = cpu_data_ptr->cpu_processor->idle_thread;
+ first_thread = processor->idle_thread;
}
cpu_data_ptr->cpu_active_thread = first_thread;
first_thread->machine.CpuDatap = cpu_data_ptr;
+ first_thread->machine.pcpu_data_base =
+ (vm_address_t)cpu_data_ptr - __PERCPU_ADDR(cpu_data);
configure_coresight_registers(cpu_data_ptr);
flush_dcache((vm_offset_t)&CpuDataEntries[cpu], sizeof(cpu_data_entry_t), FALSE);
flush_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
+#if defined(KERNEL_INTEGRITY_CTRR)
+
+ /* First CPU being started within a cluster goes ahead to lock CTRR for cluster;
+ * other CPUs block until cluster is locked. */
+ lck_spin_lock(&ctrr_cpu_start_lck);
+ switch (ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id]) {
+ case CTRR_UNLOCKED:
+ ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] = CTRR_LOCKING;
+ lck_spin_unlock(&ctrr_cpu_start_lck);
+ break;
+ case CTRR_LOCKING:
+ assert_wait(&ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id], THREAD_UNINT);
+ lck_spin_unlock(&ctrr_cpu_start_lck);
+ thread_block(THREAD_CONTINUE_NULL);
+ assert(ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] != CTRR_LOCKING);
+ break;
+ default: // CTRR_LOCKED
+ lck_spin_unlock(&ctrr_cpu_start_lck);
+ break;
+ }
+#endif
(void) PE_cpu_start(cpu_data_ptr->cpu_id, (vm_offset_t)NULL, (vm_offset_t)NULL);
}
* This ensures that mach_absolute_time() stops ticking across sleep.
*/
rtclock_base_abstime = wake_abstime - ml_get_hwclock();
+ } else if (from_boot) {
+ /* On initial boot, initialize time_since_reset to CNTPCT_EL0. */
+ ml_set_reset_time(ml_get_hwclock());
}
cdp->cpu_decrementer = 0x7FFFFFFFUL;
* the abstime value we'll use when we resume.
*/
wake_abstime = ml_get_timebase();
+ ml_set_reset_time(UINT64_MAX);
} else {
CleanPoU_Dcache();
}
*(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGPRCR) = 0;
}
+#if HIBERNATION
+ uint32_t mode = hibernate_write_image();
+ if (mode == kIOHibernatePostWriteHalt) {
+ HIBLOG("powering off after writing hibernation image\n");
+ int halt_result = -1;
+ if (PE_halt_restart) {
+ halt_result = (*PE_halt_restart)(kPEHaltCPU);
+ }
+ panic("can't shutdown: PE_halt_restart returned %d", halt_result);
+ }
+#endif /* HIBERNATION */
+
#if MONOTONIC
mt_sleep();
#endif /* MONOTONIC */
/* ARM64-specific preparation */
- arm64_prepare_for_sleep();
+ arm64_prepare_for_sleep(true);
} else {
#if __ARM_GLOBAL_SLEEP_BIT__
/*
}
/* ARM64-specific preparation */
- arm64_prepare_for_sleep();
+ arm64_prepare_for_sleep(true);
}
}
cpu_data_t *cpu_data_ptr = getCpuDatap();
if (from_boot) {
- unsigned long jtag = 0;
int wfi_tmp = 1;
uint32_t production = 1;
DTEntry entry;
+ unsigned long jtag = 0;
+
if (PE_parse_boot_argn("jtag", &jtag, sizeof(jtag))) {
if (jtag != 0) {
idle_enable = FALSE;
idle_enable = TRUE;
}
+#if DEVELOPMENT || DEBUG
+ uint32_t wfe_mode = 0;
+ if (PE_parse_boot_argn("wfe_mode", &wfe_mode, sizeof(wfe_mode))) {
+ idle_proximate_timer_wfe = ((wfe_mode & 1) == 1);
+ idle_proximate_io_wfe = ((wfe_mode & 2) == 2);
+ }
+#endif
PE_parse_boot_argn("wfi", &wfi_tmp, sizeof(wfi_tmp));
// bits 7..0 give the wfi type
#endif /* MONITOR */
// Determine if we are on production or debug chip
- if (kSuccess == DTLookupEntry(NULL, "/chosen", &entry)) {
+ if (kSuccess == SecureDTLookupEntry(NULL, "/chosen", &entry)) {
unsigned int size;
- void *prop;
+ void const *prop;
- if (kSuccess == DTGetProperty(entry, "effective-production-status-ap", &prop, &size)) {
+ if (kSuccess == SecureDTGetProperty(entry, "effective-production-status-ap", &prop, &size)) {
if (size == 4) {
bcopy(prop, &production, size);
}
machine_track_platform_idle(boolean_t entry)
{
if (entry) {
- (void)__c11_atomic_fetch_add(&cpu_idle_count, 1, __ATOMIC_RELAXED);
+ os_atomic_inc(&cpu_idle_count, relaxed);
} else {
- (void)__c11_atomic_fetch_sub(&cpu_idle_count, 1, __ATOMIC_RELAXED);
+ os_atomic_dec(&cpu_idle_count, relaxed);
}
}
cpu_data_t *cpu_data_ptr = getCpuDatap();
DTEntry entry;
size_t size;
- void **prop;
+ void const * const *prop;
if ((cpu_data_ptr == &BootCpuData) && (sleepTokenBuffer == (vm_offset_t) NULL)) {
/* Find the stpage node in the device tree */
- if (kSuccess != DTLookupEntry(0, "stram", &entry)) {
+ if (kSuccess != SecureDTLookupEntry(0, "stram", &entry)) {
return;
}
- if (kSuccess != DTGetProperty(entry, "reg", (void **)&prop, (unsigned int *)&size)) {
+ if (kSuccess != SecureDTGetProperty(entry, "reg", (const void **)&prop, (unsigned int *)&size)) {
return;
}
/* Map the page into the kernel space */
- sleepTokenBuffer = ml_io_map(((vm_offset_t *)prop)[0], ((vm_size_t *)prop)[1]);
+ sleepTokenBuffer = ml_io_map(((vm_offset_t const *)prop)[0], ((vm_size_t const *)prop)[1]);
}
}
#endif
projects / apple / xnu.git / blobdiff