#include <arm/io_map_entries.h>
#include <arm/cpu_data.h>
#include <arm/cpu_data_internal.h>
+#include <arm/machine_routines.h>
#include <arm/misc_protos.h>
#include <arm/rtclock.h>
#include <arm/caches_internal.h>
#include <machine/atomic.h>
#include <vm/pmap.h>
#include <vm/vm_page.h>
+#include <vm/vm_map.h>
#include <sys/kdebug.h>
#include <kern/coalition.h>
#include <pexpert/device_tree.h>
#include <arm/cpuid_internal.h>
+#include <arm/cpu_capabilities.h>
#include <IOKit/IOPlatformExpert.h>
#include <kern/kpc.h>
#endif
-static int max_cpus_initialized = 0;
-#define MAX_CPUS_SET 0x1
-#define MAX_CPUS_WAIT 0x2
-
-static unsigned int avail_cpus = 0;
+/* arm32 only supports a highly simplified topology, fixed at 1 cluster */
+static ml_topology_cpu_t topology_cpu_array[MAX_CPUS];
+static ml_topology_cluster_t topology_cluster = {
+ .cluster_id = 0,
+ .cluster_type = CLUSTER_TYPE_SMP,
+ .first_cpu_id = 0,
+};
+static ml_topology_info_t topology_info = {
+ .version = CPU_TOPOLOGY_VERSION,
+ .num_clusters = 1,
+ .max_cluster_id = 0,
+ .cpus = topology_cpu_array,
+ .clusters = &topology_cluster,
+ .boot_cpu = &topology_cpu_array[0],
+ .boot_cluster = &topology_cluster,
+};
uint32_t LockTimeOut;
uint32_t LockTimeOutUsec;
uint64_t TLockTimeOut;
uint64_t MutexSpin;
-boolean_t is_clock_configured = FALSE;
-
-extern int mach_assert;
-extern volatile uint32_t debug_enabled;
-
-void machine_conf(void);
+extern uint32_t lockdown_done;
+uint64_t low_MutexSpin;
+int64_t high_MutexSpin;
void
machine_startup(__unused boot_args * args)
{
- int boot_arg;
-
- PE_parse_boot_argn("assert", &mach_assert, sizeof(mach_assert));
-
- if (PE_parse_boot_argn("preempt", &boot_arg, sizeof(boot_arg))) {
- default_preemption_rate = boot_arg;
- }
- if (PE_parse_boot_argn("bg_preempt", &boot_arg, sizeof(boot_arg))) {
- default_bg_preemption_rate = boot_arg;
- }
-
machine_conf();
/*
return PE_boot_args();
}
-void
-machine_conf(void)
-{
- machine_info.memory_size = mem_size;
-}
-
-void
-machine_init(void)
-{
- debug_log_init();
- clock_config();
- is_clock_configured = TRUE;
- if (debug_enabled) {
- pmap_map_globals();
- }
-}
-
void
slave_machine_init(__unused void *param)
{
return Shutdown_context(doshutdown, processor);
}
-/*
- * Routine: ml_init_max_cpus
- * Function:
- */
-void
-ml_init_max_cpus(unsigned int max_cpus)
-{
- boolean_t current_state;
-
- current_state = ml_set_interrupts_enabled(FALSE);
- if (max_cpus_initialized != MAX_CPUS_SET) {
- machine_info.max_cpus = max_cpus;
- machine_info.physical_cpu_max = max_cpus;
- machine_info.logical_cpu_max = max_cpus;
- if (max_cpus_initialized == MAX_CPUS_WAIT) {
- thread_wakeup((event_t) &max_cpus_initialized);
- }
- max_cpus_initialized = MAX_CPUS_SET;
- }
- (void) ml_set_interrupts_enabled(current_state);
-}
-
-/*
- * Routine: ml_get_max_cpus
- * Function:
- */
-unsigned int
-ml_get_max_cpus(void)
-{
- boolean_t current_state;
-
- current_state = ml_set_interrupts_enabled(FALSE);
- if (max_cpus_initialized != MAX_CPUS_SET) {
- max_cpus_initialized = MAX_CPUS_WAIT;
- assert_wait((event_t) &max_cpus_initialized, THREAD_UNINT);
- (void) thread_block(THREAD_CONTINUE_NULL);
- }
- (void) ml_set_interrupts_enabled(current_state);
- return machine_info.max_cpus;
-}
-
/*
* Routine: ml_init_lock_timeout
* Function:
nanoseconds_to_absolutetime(10 * NSEC_PER_USEC, &abstime);
}
MutexSpin = abstime;
+ low_MutexSpin = MutexSpin;
+ /*
+ * high_MutexSpin should be initialized as low_MutexSpin * real_ncpus, but
+ * real_ncpus is not set at this time
+ *
+ * NOTE: active spinning is disabled in arm. It can be activated
+ * by setting high_MutexSpin through the sysctl.
+ */
+ high_MutexSpin = low_MutexSpin;
+}
+
+/*
+ * This is called when all of the ml_processor_info_t structures have been
+ * initialized and all the processors have been started through processor_start().
+ *
+ * Required by the scheduler subsystem.
+ */
+void
+ml_cpu_init_completed(void)
+{
}
/*
void
ml_cpu_up(void)
{
- hw_atomic_add(&machine_info.physical_cpu, 1);
- hw_atomic_add(&machine_info.logical_cpu, 1);
+ os_atomic_inc(&machine_info.physical_cpu, relaxed);
+ os_atomic_inc(&machine_info.logical_cpu, relaxed);
}
/*
{
cpu_data_t *cpu_data_ptr;
- hw_atomic_sub(&machine_info.physical_cpu, 1);
- hw_atomic_sub(&machine_info.logical_cpu, 1);
+ os_atomic_dec(&machine_info.physical_cpu, relaxed);
+ os_atomic_dec(&machine_info.logical_cpu, relaxed);
/*
* If we want to deal with outstanding IPIs, we need to
return pmap_max_offset(is64, pmap_max_offset_option);
}
-boolean_t
-ml_wants_panic_trap_to_debugger(void)
-{
- return FALSE;
-}
-
void
ml_panic_trap_to_debugger(__unused const char *panic_format_str,
__unused va_list *panic_args,
cpu_data_ptr->interrupt_handler = handler;
cpu_data_ptr->interrupt_refCon = refCon;
- cpu_data_ptr->interrupts_enabled = TRUE;
(void) ml_set_interrupts_enabled(current_state);
-
- initialize_screen(NULL, kPEAcquireScreen);
}
/*
}
}
-void
-fiq_context_bootstrap(boolean_t enable_fiq)
-{
- fiq_context_init(enable_fiq);
-}
-
void
ml_parse_cpu_topology(void)
{
uint32_t cpu_boot_arg;
int err;
- err = DTLookupEntry(NULL, "/cpus", &entry);
+ err = SecureDTLookupEntry(NULL, "/cpus", &entry);
assert(err == kSuccess);
- err = DTInitEntryIterator(entry, &iter);
+ err = SecureDTInitEntryIterator(entry, &iter);
assert(err == kSuccess);
- while (kSuccess == DTIterateEntries(&iter, &child)) {
+ cpu_boot_arg = MAX_CPUS;
+ PE_parse_boot_argn("cpus", &cpu_boot_arg, sizeof(cpu_boot_arg));
+
+ ml_topology_cluster_t *cluster = &topology_info.clusters[0];
+ unsigned int cpu_id = 0;
+ while (kSuccess == SecureDTIterateEntries(&iter, &child)) {
#if MACH_ASSERT
unsigned int propSize;
- void *prop = NULL;
- if (avail_cpus == 0) {
- if (kSuccess != DTGetProperty(child, "state", &prop, &propSize)) {
- panic("unable to retrieve state for cpu %u", avail_cpus);
+ void const *prop = NULL;
+ if (cpu_id == 0) {
+ if (kSuccess != SecureDTGetProperty(child, "state", &prop, &propSize)) {
+ panic("unable to retrieve state for cpu %u", cpu_id);
}
- if (strncmp((char*)prop, "running", propSize) != 0) {
+ if (strncmp((char const *)prop, "running", propSize) != 0) {
panic("cpu 0 has not been marked as running!");
}
}
- assert(kSuccess == DTGetProperty(child, "reg", &prop, &propSize));
- assert(avail_cpus == *((uint32_t*)prop));
+ assert(kSuccess == SecureDTGetProperty(child, "reg", &prop, &propSize));
+ assert(cpu_id == *((uint32_t const *)prop));
#endif
- ++avail_cpus;
- }
+ if (cpu_id >= cpu_boot_arg) {
+ break;
+ }
+
+ ml_topology_cpu_t *cpu = &topology_info.cpus[cpu_id];
+
+ cpu->cpu_id = cpu_id;
+ cpu->phys_id = cpu_id;
+ cpu->cluster_type = cluster->cluster_type;
+
+ cluster->num_cpus++;
+ cluster->cpu_mask |= 1ULL << cpu_id;
+
+ topology_info.num_cpus++;
+ topology_info.max_cpu_id = cpu_id;
- cpu_boot_arg = avail_cpus;
- if (PE_parse_boot_argn("cpus", &cpu_boot_arg, sizeof(cpu_boot_arg)) &&
- (avail_cpus > cpu_boot_arg)) {
- avail_cpus = cpu_boot_arg;
+ cpu_id++;
}
- if (avail_cpus == 0) {
+ if (cpu_id == 0) {
panic("No cpus found!");
}
}
+const ml_topology_info_t *
+ml_get_topology_info(void)
+{
+ return &topology_info;
+}
+
unsigned int
ml_get_cpu_count(void)
{
- return avail_cpus;
+ return topology_info.num_cpus;
+}
+
+unsigned int
+ml_get_cluster_count(void)
+{
+ return topology_info.num_clusters;
}
int
int
ml_get_cpu_number(uint32_t phys_id)
{
+ if (phys_id > (uint32_t)ml_get_max_cpu_number()) {
+ return -1;
+ }
+
return (int)phys_id;
}
+int
+ml_get_cluster_number(__unused uint32_t phys_id)
+{
+ return 0;
+}
+
int
ml_get_max_cpu_number(void)
{
- return avail_cpus - 1;
+ return topology_info.num_cpus - 1;
+}
+
+int
+ml_get_max_cluster_number(void)
+{
+ return topology_info.max_cluster_id;
+}
+
+unsigned int
+ml_get_first_cpu_id(unsigned int cluster_id)
+{
+ return topology_info.clusters[cluster_id].first_cpu_id;
}
kern_return_t
cpu_data_t *this_cpu_datap;
boolean_t is_boot_cpu;
- if (in_processor_info->phys_id >= MAX_CPUS) {
+ const unsigned int max_cpu_id = ml_get_max_cpu_number();
+ if (in_processor_info->phys_id > max_cpu_id) {
/*
* The physical CPU ID indicates that we have more CPUs than
* this xnu build support. This probably means we have an
* is simply a convenient way to catch bugs in the pexpert
* headers.
*/
- panic("phys_id %u is too large for MAX_CPUS (%u)", in_processor_info->phys_id, MAX_CPUS);
+ panic("phys_id %u is too large for max_cpu_id (%u)", in_processor_info->phys_id, max_cpu_id);
}
/* Fail the registration if the number of CPUs has been limited by boot-arg. */
- if ((in_processor_info->phys_id >= avail_cpus) ||
+ if ((in_processor_info->phys_id >= topology_info.num_cpus) ||
(in_processor_info->log_id > (uint32_t)ml_get_max_cpu_number())) {
return KERN_FAILURE;
}
}
}
- this_cpu_datap->cpu_idle_notify = (void *) in_processor_info->processor_idle;
- this_cpu_datap->cpu_cache_dispatch = in_processor_info->platform_cache_dispatch;
+ this_cpu_datap->cpu_idle_notify = in_processor_info->processor_idle;
+ this_cpu_datap->cpu_cache_dispatch = (cache_dispatch_t) in_processor_info->platform_cache_dispatch;
nanoseconds_to_absolutetime((uint64_t) in_processor_info->powergate_latency, &this_cpu_datap->cpu_idle_latency);
this_cpu_datap->cpu_reset_assist = kvtophys(in_processor_info->powergate_stub_addr);
- this_cpu_datap->idle_timer_notify = (void *) in_processor_info->idle_timer;
+ this_cpu_datap->idle_timer_notify = in_processor_info->idle_timer;
this_cpu_datap->idle_timer_refcon = in_processor_info->idle_timer_refcon;
- this_cpu_datap->platform_error_handler = (void *) in_processor_info->platform_error_handler;
+ this_cpu_datap->platform_error_handler = in_processor_info->platform_error_handler;
this_cpu_datap->cpu_regmap_paddr = in_processor_info->regmap_paddr;
this_cpu_datap->cpu_phys_id = in_processor_info->phys_id;
this_cpu_datap->cpu_l2_access_penalty = in_processor_info->l2_access_penalty;
+ processor_t processor = PERCPU_GET_RELATIVE(processor, cpu_data, this_cpu_datap);
if (!is_boot_cpu) {
- processor_init((struct processor *)this_cpu_datap->cpu_processor,
- this_cpu_datap->cpu_number, processor_pset(master_processor));
+ processor_init(processor, this_cpu_datap->cpu_number,
+ processor_pset(master_processor));
if (this_cpu_datap->cpu_l2_access_penalty) {
/*
* scheduler, so that threads use the cores with better L2
* preferentially.
*/
- processor_set_primary(this_cpu_datap->cpu_processor,
- master_processor);
+ processor_set_primary(processor, master_processor);
}
}
- *processor_out = this_cpu_datap->cpu_processor;
+ *processor_out = processor;
*ipi_handler_out = cpu_signal_handler;
*pmi_handler_out = NULL;
if (in_processor_info->idle_tickle != (idle_tickle_t *) NULL) {
#endif
if (!is_boot_cpu) {
- early_random_cpu_init(this_cpu_datap->cpu_number);
+ random_cpu_init(this_cpu_datap->cpu_number);
}
return KERN_SUCCESS;
return io_map(phys_addr, size, VM_WIMG_IO);
}
+/* Map memory map IO space (with protections specified) */
+vm_offset_t
+ml_io_map_with_prot(
+ vm_offset_t phys_addr,
+ vm_size_t size,
+ vm_prot_t prot)
+{
+ return io_map_with_prot(phys_addr, size, VM_WIMG_IO, prot);
+}
+
vm_offset_t
ml_io_map_wcomb(
vm_offset_t phys_addr,
return io_map(phys_addr, size, VM_WIMG_WCOMB);
}
+void
+ml_io_unmap(vm_offset_t addr, vm_size_t sz)
+{
+ pmap_remove(kernel_pmap, addr, addr + sz);
+ kmem_free(kernel_map, addr, sz);
+}
+
/* boot memory allocation */
vm_offset_t
ml_static_malloc(
ml_static_vtop(
vm_offset_t vaddr)
{
- if (((vm_address_t)(vaddr) - gVirtBase) >= gPhysSize) {
- panic("ml_static_ptovirt(): illegal vaddr: %p\n", (void*)vaddr);
- }
+ assertf(((vm_address_t)(vaddr) - gVirtBase) < gPhysSize, "%s: illegal vaddr: %p", __func__, (void*)vaddr);
return (vm_address_t)(vaddr) - gVirtBase + gPhysBase;
}
+/*
+ * Return the maximum contiguous KVA range that can be accessed from this
+ * physical address. For arm64, we employ a segmented physical aperture
+ * relocation table which can limit the available range for a given PA to
+ * something less than the extent of physical memory. But here, we still
+ * have a flat physical aperture, so no such requirement exists.
+ */
+vm_map_address_t
+phystokv_range(pmap_paddr_t pa, vm_size_t *max_len)
+{
+ vm_size_t len = gPhysSize - (pa - gPhysBase);
+ if (*max_len > len) {
+ *max_len = len;
+ }
+ assertf((pa - gPhysBase) < gPhysSize, "%s: illegal PA: 0x%lx", __func__, (unsigned long)pa);
+ return pa - gPhysBase + gVirtBase;
+}
+
vm_offset_t
ml_static_slide(
vm_offset_t vaddr)
return VM_KERNEL_SLIDE(vaddr);
}
+kern_return_t
+ml_static_verify_page_protections(
+ uint64_t base, uint64_t size, vm_prot_t prot)
+{
+ /* XXX Implement Me */
+ (void)base;
+ (void)size;
+ (void)prot;
+ return KERN_FAILURE;
+}
+
+
vm_offset_t
ml_static_unslide(
vm_offset_t vaddr)
if ((new_prot & VM_PROT_WRITE) && (new_prot & VM_PROT_EXECUTE)) {
panic("ml_static_protect(): WX request on %p", (void *) vaddr);
}
+ if (lockdown_done && (new_prot & VM_PROT_EXECUTE)) {
+ panic("ml_static_protect(): attempt to inject executable mapping on %p", (void *) vaddr);
+ }
/* Set up the protection bits, and block bits so we can validate block mappings. */
if (new_prot & VM_PROT_WRITE) {
ptmp = (ptmp & ~(ARM_PTE_APMASK | ARM_PTE_NX_MASK)) | arm_prot;
*pte_p = ptmp;
-#ifndef __ARM_L1_PTW__
- FlushPoC_DcacheRegion((vm_offset_t) pte_p, sizeof(*pte_p));
-#endif
}
}
vm_offset_t vaddr_cur;
ppnum_t ppn;
uint32_t freed_pages = 0;
+ uint32_t freed_kernelcache_pages = 0;
/* It is acceptable (if bad) to fail to free. */
if (vaddr < VM_MIN_KERNEL_ADDRESS) {
if (ml_static_protect(vaddr_cur, PAGE_SIZE, VM_PROT_WRITE | VM_PROT_READ) != KERN_SUCCESS) {
panic("Failed ml_static_mfree on %p", (void *) vaddr_cur);
}
-#if 0
- /*
- * Must NOT tear down the "V==P" mapping for vaddr_cur as the zone alias scheme
- * relies on the persistence of these mappings for all time.
- */
- // pmap_remove(kernel_pmap, (addr64_t) vaddr_cur, (addr64_t) (vaddr_cur + PAGE_SIZE));
-#endif
vm_page_create(ppn, (ppn + 1));
freed_pages++;
+ if (vaddr_cur >= segLOWEST && vaddr_cur < end_kern) {
+ freed_kernelcache_pages++;
+ }
}
}
vm_page_lockspin_queues();
vm_page_wire_count -= freed_pages;
vm_page_wire_count_initial -= freed_pages;
+ vm_page_kernelcache_count -= freed_kernelcache_pages;
vm_page_unlock_queues();
#if DEBUG
kprintf("ml_static_mfree: Released 0x%x pages at VA %p, size:0x%llx, last ppn: 0x%x\n", freed_pages, (void *)vaddr, (uint64_t)size, ppn);
void
ml_gpu_stat_update(__unused uint64_t gpu_ns_delta)
{
-#if CONFIG_EMBEDDED
/*
* For now: update the resource coalition stats of the
* current thread's coalition
*/
task_coalition_update_gpu_stats(current_task(), gpu_ns_delta);
-#endif
}
uint64_t
return;
}
- processor_data_t *pd = &getCpuDatap()->cpu_processor->processor_data;
+ processor_t pd = current_processor();
uint64_t now = ml_get_timebase();
timer_stop(pd->current_state, now);
return FALSE;
}
-boolean_t
-user_timebase_allowed(void)
+uint8_t
+user_timebase_type(void)
{
#if __ARM_TIME__
- return TRUE;
+ return USER_TIMEBASE_SPEC;
#else
- return FALSE;
+ return USER_TIMEBASE_NONE;
#endif
}
* The following are required for parts of the kernel
* that cannot resolve these functions as inlines:
*/
-extern thread_t current_act(void);
+extern thread_t current_act(void) __attribute__((const));
thread_t
current_act(void)
{
}
#undef current_thread
-extern thread_t current_thread(void);
+extern thread_t current_thread(void) __attribute__((const));
thread_t
current_thread(void)
{
}
}
#endif // __ARM_USER_PROTECT__
+
+void
+machine_lockdown(void)
+{
+ arm_vm_prot_finalize(PE_state.bootArgs);
+ lockdown_done = 1;
+}
+
+void
+ml_lockdown_init(void)
+{
+}
+
+void
+ml_hibernate_active_pre(void)
+{
+}
+
+void
+ml_hibernate_active_post(void)
+{
+}
+
+size_t
+ml_get_vm_reserved_regions(bool vm_is64bit, struct vm_reserved_region **regions)
+{
+#pragma unused(vm_is64bit)
+ assert(regions != NULL);
+
+ *regions = NULL;
+ return 0;
+}
projects / apple / xnu.git / blobdiff