X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/7ddcb079202367355dddccdfa4318e57d50318be..refs/heads/master:/osfmk/i386/cpu_topology.c

diff --git a/osfmk/i386/cpu_topology.c b/osfmk/i386/cpu_topology.c
index 6be77e6ff..1ed64e3df 100644
--- a/osfmk/i386/cpu_topology.c
+++ b/osfmk/i386/cpu_topology.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2007-2010 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
+ *
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
- * 
+ *
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
+ *
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
@@ -22,7 +22,7 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
- * 
+ *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
@@ -33,39 +33,61 @@
 #include <i386/cpu_topology.h>
 #include <i386/cpu_threads.h>
 #include <i386/machine_cpu.h>
-#include <i386/lock.h>
+#include <i386/bit_routines.h>
 #include <i386/cpu_data.h>
 #include <i386/lapic.h>
 #include <i386/machine_routines.h>
+#include <stddef.h>
 
 __private_extern__ void qsort(
-    void * array,
-    size_t nmembers,
-    size_t member_size,
-    int (*)(const void *, const void *));
+	void * array,
+	size_t nmembers,
+	size_t member_size,
+	int (*)(const void *, const void *));
 
 static int lapicid_cmp(const void *x, const void *y);
 static x86_affinity_set_t *find_cache_affinity(x86_cpu_cache_t *L2_cachep);
 
-x86_affinity_set_t	*x86_affinities = NULL;
-static int		x86_affinity_count = 0;
+x86_affinity_set_t      *x86_affinities = NULL;
+static int              x86_affinity_count = 0;
+
+extern cpu_data_t cpshadows[];
+
+#if DEVELOPMENT || DEBUG
+void iotrace_init(int ncpus);
+void traptrace_init(int ncpus);
+#endif /* DEVELOPMENT || DEBUG */
+
+
+/* Re-sort double-mapped CPU data shadows after topology discovery sorts the
+ * primary CPU data structures by physical/APIC CPU ID.
+ */
+static void
+cpu_shadow_sort(int ncpus)
+{
+	for (int i = 0; i < ncpus; i++) {
+		cpu_data_t      *cpup = cpu_datap(i);
+		ptrdiff_t       coff = cpup - cpu_datap(0);
+
+		cpup->cd_shadow = &cpshadows[coff];
+	}
+}
 
 /*
- * cpu_topology_sort() is called after all processors have been registered
- * but before any non-boot processor id started.
- * We establish canonical logical processor numbering - logical cpus must be
- * contiguous, zero-based and assigned in physical (local apic id) order.
- * This step is required because the discovery/registration order is
- * non-deterministic - cores are registered in differing orders over boots.
- * Enforcing canonical numbering simplifies identification
- * of processors - in particular, for stopping/starting from CHUD.
- */ 
+ * cpu_topology_sort() is called after all processors have been registered but
+ * before any non-boot processor is started.  We establish canonical logical
+ * processor numbering - logical cpus must be contiguous, zero-based and
+ * assigned in physical (local apic id) order.  This step is required because
+ * the discovery/registration order is non-deterministic - cores are registered
+ * in differing orders over boots.  Enforcing canonical numbering simplifies
+ * identification of processors.
+ */
 void
 cpu_topology_sort(int ncpus)
 {
-	int		i;
-	boolean_t	istate;
-	processor_t		lprim = NULL;
+	int             i;
+	boolean_t       istate;
+	processor_t             lprim = NULL;
 
 	assert(machine_info.physical_cpu == 1);
 	assert(machine_info.logical_cpu == 1);
@@ -73,16 +95,22 @@ cpu_topology_sort(int ncpus)
 	assert(cpu_number() == 0);
 	assert(cpu_datap(0)->cpu_number == 0);
 
+	uint32_t cpus_per_pset = 0;
+
+#if DEVELOPMENT || DEBUG
+	PE_parse_boot_argn("cpus_per_pset", &cpus_per_pset, sizeof(cpus_per_pset));
+#endif
+
 	/* Lights out for this */
 	istate = ml_set_interrupts_enabled(FALSE);
 
 	if (topo_dbg) {
 		TOPO_DBG("cpu_topology_start() %d cpu%s registered\n",
-			ncpus, (ncpus > 1) ? "s" : "");
+		    ncpus, (ncpus > 1) ? "s" : "");
 		for (i = 0; i < ncpus; i++) {
-			cpu_data_t	*cpup = cpu_datap(i);
+			cpu_data_t      *cpup = cpu_datap(i);
 			TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",
-				i, (void *) cpup, cpup->cpu_phys_number);
+			    i, (void *) cpup, cpup->cpu_phys_number);
 		}
 	}
 
@@ -92,53 +120,47 @@ cpu_topology_sort(int ncpus)
 	 */
 	if (ncpus > 1) {
 		qsort((void *) &cpu_data_ptr[1],
-			ncpus - 1,
-			sizeof(cpu_data_t *),
-			lapicid_cmp);
+		    ncpus - 1,
+		    sizeof(cpu_data_t *),
+		    lapicid_cmp);
 	}
 	if (topo_dbg) {
 		TOPO_DBG("cpu_topology_start() after sorting:\n");
 		for (i = 0; i < ncpus; i++) {
-			cpu_data_t	*cpup = cpu_datap(i);
+			cpu_data_t      *cpup = cpu_datap(i);
 			TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",
-				i, (void *) cpup, cpup->cpu_phys_number);
+			    i, (void *) cpup, cpup->cpu_phys_number);
 		}
 	}
 
 	/*
-	 * Fix up logical numbers and reset the map kept by the lapic code.
+	 * Finalize logical numbers and map kept by the lapic code.
 	 */
-	for (i = 1; i < ncpus; i++) {
-		cpu_data_t	*cpup = cpu_datap(i);
-		x86_core_t	*core = cpup->lcpu.core;
-		x86_die_t	*die  = cpup->lcpu.die;
-		x86_pkg_t	*pkg  = cpup->lcpu.package;
-
-		assert(core != NULL);
-		assert(die != NULL);
-		assert(pkg != NULL);
+	for (i = 0; i < ncpus; i++) {
+		cpu_data_t      *cpup = cpu_datap(i);
 
 		if (cpup->cpu_number != i) {
 			kprintf("cpu_datap(%d):%p local apic id 0x%x "
-				"remapped from %d\n",
-				i, cpup, cpup->cpu_phys_number,
-				cpup->cpu_number);
+			    "remapped from %d\n",
+			    i, cpup, cpup->cpu_phys_number,
+			    cpup->cpu_number);
 		}
 		cpup->cpu_number = i;
-		cpup->lcpu.cpu_num = i;
-		cpup->lcpu.pnum = cpup->cpu_phys_number;
 		lapic_cpu_map(cpup->cpu_phys_number, i);
-		x86_set_lcpu_numbers(&cpup->lcpu);
-		x86_set_core_numbers(core, &cpup->lcpu);
-		x86_set_die_numbers(die, &cpup->lcpu);
-		x86_set_pkg_numbers(pkg, &cpup->lcpu);
+		x86_set_logical_topology(&cpup->lcpu, cpup->cpu_phys_number, i);
 	}
 
-	validate_topology();
+	cpu_shadow_sort(ncpus);
+	x86_validate_topology();
 
 	ml_set_interrupts_enabled(istate);
 	TOPO_DBG("cpu_topology_start() LLC is L%d\n", topoParms.LLCDepth + 1);
 
+#if DEVELOPMENT || DEBUG
+	iotrace_init(ncpus);
+	traptrace_init(ncpus);
+#endif /* DEVELOPMENT || DEBUG */
+
 	/*
 	 * Let the CPU Power Management know that the topology is stable.
 	 */
@@ -150,46 +172,61 @@ cpu_topology_sort(int ncpus)
 	 * for their LLC cache. Each affinity set possesses a processor set
 	 * into which each logical processor is added.
 	 */
-	TOPO_DBG("cpu_topology_start() creating affinity sets:\n");
-	for (i = 0; i < ncpus; i++) {
-		cpu_data_t		*cpup = cpu_datap(i);
-		x86_lcpu_t		*lcpup = cpu_to_lcpu(i);
-		x86_cpu_cache_t		*LLC_cachep;
-		x86_affinity_set_t	*aset;
+	TOPO_DBG("cpu_topology_start() creating affinity sets:ncpus=%d max_cpus=%d\n", ncpus, machine_info.max_cpus);
+	for (i = 0; i < machine_info.max_cpus; i++) {
+		cpu_data_t              *cpup = cpu_datap(i);
+		x86_lcpu_t              *lcpup = cpu_to_lcpu(i);
+		x86_cpu_cache_t         *LLC_cachep;
+		x86_affinity_set_t      *aset;
 
 		LLC_cachep = lcpup->caches[topoParms.LLCDepth];
 		assert(LLC_cachep->type == CPU_CACHE_TYPE_UNIF);
-		aset = find_cache_affinity(LLC_cachep); 
-		if (aset == NULL) {
+		aset = find_cache_affinity(LLC_cachep);
+		if ((aset == NULL) || ((cpus_per_pset != 0) && (i % cpus_per_pset) == 0)) {
 			aset = (x86_affinity_set_t *) kalloc(sizeof(*aset));
-			if (aset == NULL)
+			if (aset == NULL) {
 				panic("cpu_topology_start() failed aset alloc");
+			}
 			aset->next = x86_affinities;
 			x86_affinities = aset;
 			aset->num = x86_affinity_count++;
 			aset->cache = LLC_cachep;
 			aset->pset = (i == master_cpu) ?
-					processor_pset(master_processor) :
-					pset_create(pset_node_root());
-			if (aset->pset == PROCESSOR_SET_NULL)
+			    processor_pset(master_processor) :
+			    pset_create(pset_node_root());
+			if (aset->pset == PROCESSOR_SET_NULL) {
 				panic("cpu_topology_start: pset_create");
+			}
 			TOPO_DBG("\tnew set %p(%d) pset %p for cache %p\n",
-				aset, aset->num, aset->pset, aset->cache);
+			    aset, aset->num, aset->pset, aset->cache);
 		}
 
 		TOPO_DBG("\tprocessor_init set %p(%d) lcpup %p(%d) cpu %p processor %p\n",
-			aset, aset->num, lcpup, lcpup->cpu_num, cpup, cpup->cpu_processor);
+		    aset, aset->num, lcpup, lcpup->cpu_num, cpup, cpup->cpu_processor);
 
-		if (i != master_cpu)
+		if (i != master_cpu) {
 			processor_init(cpup->cpu_processor, i, aset->pset);
+		}
 
 		if (lcpup->core->num_lcpus > 1) {
-			if (lcpup->lnum == 0)
+			if (lcpup->lnum == 0) {
 				lprim = cpup->cpu_processor;
+			}
 
-			processor_meta_init(cpup->cpu_processor, lprim);
+			processor_set_primary(cpup->cpu_processor, lprim);
 		}
 	}
+
+	if (machine_info.max_cpus < machine_info.logical_cpu_max) {
+		/* boot-args cpus=n is set, so adjust max numbers to match */
+		int logical_max = machine_info.max_cpus;
+		int physical_max = logical_max;
+		if (machine_info.logical_cpu_max != machine_info.physical_cpu_max) {
+			physical_max = (logical_max + 1) / 2;
+		}
+		machine_info.logical_cpu_max = logical_max;
+		machine_info.physical_cpu_max = physical_max;
+	}
 }
 
 /* We got a request to start a CPU. Check that this CPU is within the
@@ -198,46 +235,48 @@ cpu_topology_sort(int ncpus)
 kern_return_t
 cpu_topology_start_cpu( int cpunum )
 {
-	int		ncpus = machine_info.max_cpus;
-	int		i = cpunum;
+	int             ncpus = machine_info.max_cpus;
+	int             i = cpunum;
 
 	/* Decide whether to start a CPU, and actually start it */
 	TOPO_DBG("cpu_topology_start() processor_start():\n");
-	if( i < ncpus)
-	{
+	if (i < ncpus) {
 		TOPO_DBG("\tlcpu %d\n", cpu_datap(i)->cpu_number);
-		processor_start(cpu_datap(i)->cpu_processor); 
+		processor_start(cpu_datap(i)->cpu_processor);
 		return KERN_SUCCESS;
+	} else {
+		return KERN_FAILURE;
 	}
-	else
-	    return KERN_FAILURE;
 }
 
 static int
 lapicid_cmp(const void *x, const void *y)
 {
-	cpu_data_t	*cpu_x = *((cpu_data_t **)(uintptr_t)x);
-	cpu_data_t	*cpu_y = *((cpu_data_t **)(uintptr_t)y);
+	cpu_data_t      *cpu_x = *((cpu_data_t **)(uintptr_t)x);
+	cpu_data_t      *cpu_y = *((cpu_data_t **)(uintptr_t)y);
 
 	TOPO_DBG("lapicid_cmp(%p,%p) (%d,%d)\n",
-		x, y, cpu_x->cpu_phys_number, cpu_y->cpu_phys_number);
-	if (cpu_x->cpu_phys_number < cpu_y->cpu_phys_number)
+	    x, y, cpu_x->cpu_phys_number, cpu_y->cpu_phys_number);
+	if (cpu_x->cpu_phys_number < cpu_y->cpu_phys_number) {
 		return -1;
-	if (cpu_x->cpu_phys_number == cpu_y->cpu_phys_number)
+	}
+	if (cpu_x->cpu_phys_number == cpu_y->cpu_phys_number) {
 		return 0;
+	}
 	return 1;
 }
 
 static x86_affinity_set_t *
 find_cache_affinity(x86_cpu_cache_t *l2_cachep)
 {
-	x86_affinity_set_t	*aset;
+	x86_affinity_set_t      *aset;
 
 	for (aset = x86_affinities; aset != NULL; aset = aset->next) {
-		if (l2_cachep == aset->cache)
+		if (l2_cachep == aset->cache) {
 			break;
+		}
 	}
-	return aset;			
+	return aset;
 }
 
 int
@@ -247,13 +286,14 @@ ml_get_max_affinity_sets(void)
 }
 
 processor_set_t
-ml_affinity_to_pset(uint32_t affinity_num) 
+ml_affinity_to_pset(uint32_t affinity_num)
 {
-	x86_affinity_set_t	*aset;
+	x86_affinity_set_t      *aset;
 
 	for (aset = x86_affinities; aset != NULL; aset = aset->next) {
-		if (affinity_num == aset->num)
+		if (affinity_num == aset->num) {
 			break;
+		}
 	}
 	return (aset == NULL) ? PROCESSOR_SET_NULL : aset->pset;
 }
@@ -261,12 +301,12 @@ ml_affinity_to_pset(uint32_t affinity_num)
 uint64_t
 ml_cpu_cache_size(unsigned int level)
 {
-	x86_cpu_cache_t	*cachep;
+	x86_cpu_cache_t *cachep;
 
 	if (level == 0) {
 		return machine_info.max_mem;
-	} else if ( 1 <= level && level <= MAX_CACHE_DEPTH) {
-		cachep = current_cpu_datap()->lcpu.caches[level-1];
+	} else if (1 <= level && level <= MAX_CACHE_DEPTH) {
+		cachep = current_cpu_datap()->lcpu.caches[level - 1];
 		return cachep ? cachep->cache_size : 0;
 	} else {
 		return 0;
@@ -276,15 +316,136 @@ ml_cpu_cache_size(unsigned int level)
 uint64_t
 ml_cpu_cache_sharing(unsigned int level)
 {
-	x86_cpu_cache_t	*cachep;
+	x86_cpu_cache_t *cachep;
 
 	if (level == 0) {
 		return machine_info.max_cpus;
-	} else if ( 1 <= level && level <= MAX_CACHE_DEPTH) {
-		cachep = current_cpu_datap()->lcpu.caches[level-1];
+	} else if (1 <= level && level <= MAX_CACHE_DEPTH) {
+		cachep = current_cpu_datap()->lcpu.caches[level - 1];
 		return cachep ? cachep->nlcpus : 0;
 	} else {
 		return 0;
 	}
 }
 
+#if     DEVELOPMENT || DEBUG
+volatile int mmiotrace_enabled = 1;
+int iotrace_generators = 0;
+int iotrace_entries_per_cpu = 0;
+int *iotrace_next;
+iotrace_entry_t **iotrace_ring;
+
+volatile int traptrace_enabled = 1;
+int traptrace_generators = 0;
+int traptrace_entries_per_cpu = 0;
+int *traptrace_next;
+traptrace_entry_t **traptrace_ring;
+
+static void
+init_trace_bufs(int cpucnt, int entries_per_cpu, void ***ring, int entry_size,
+    int **next_array, int *allocated_entries_per_cpu, int *allocated_generator_count)
+{
+	int i;
+
+	*next_array = kalloc_tag(cpucnt * sizeof(int), VM_KERN_MEMORY_DIAG);
+	if (__improbable(*next_array == NULL)) {
+		*allocated_generator_count = 0;
+		return;
+	} else {
+		bzero(*next_array, cpucnt * sizeof(int));
+	}
+
+	*ring = kalloc_tag(cpucnt * sizeof(void *), VM_KERN_MEMORY_DIAG);
+	if (__improbable(*ring == NULL)) {
+		kfree(*next_array, cpucnt * sizeof(int));
+		*next_array = NULL;
+		*allocated_generator_count = 0;
+		return;
+	}
+	for (i = 0; i < cpucnt; i++) {
+		(*ring)[i] = kalloc_tag(entries_per_cpu * entry_size, VM_KERN_MEMORY_DIAG);
+		if (__improbable((*ring)[i] == NULL)) {
+			kfree(*next_array, cpucnt * sizeof(int));
+			*next_array = NULL;
+			for (int j = 0; j < i; j++) {
+				kfree((*ring)[j], entries_per_cpu * entry_size);
+			}
+			kfree(*ring, cpucnt * sizeof(void *));
+			*ring = NULL;
+			return;
+		}
+		bzero((*ring)[i], entries_per_cpu * entry_size);
+	}
+
+	*allocated_entries_per_cpu = entries_per_cpu;
+	*allocated_generator_count = cpucnt;
+}
+
+
+static void
+init_iotrace_bufs(int cpucnt, int entries_per_cpu)
+{
+	init_trace_bufs(cpucnt, entries_per_cpu, (void ***)&iotrace_ring, sizeof(iotrace_entry_t),
+	    &iotrace_next, &iotrace_entries_per_cpu, &iotrace_generators);
+}
+
+static void
+init_traptrace_bufs(int cpucnt, int entries_per_cpu)
+{
+	init_trace_bufs(cpucnt, entries_per_cpu, (void ***)&traptrace_ring, sizeof(traptrace_entry_t),
+	    &traptrace_next, &traptrace_entries_per_cpu, &traptrace_generators);
+}
+
+static void
+gentrace_configure_from_bootargs(const char *ena_prop, int *ena_valp, const char *epc_prop,
+    int *epcp, int max_epc, int def_epc, int override)
+{
+	if (kern_feature_override(override)) {
+		*ena_valp = 0;
+	}
+
+	(void) PE_parse_boot_argn(ena_prop, ena_valp, sizeof(*ena_valp));
+
+	if (*ena_valp == 0) {
+		return;
+	}
+
+	if (PE_parse_boot_argn(epc_prop, epcp, sizeof(*epcp)) &&
+	    (*epcp < 1 || *epcp > max_epc)) {
+		*epcp = def_epc;
+	}
+}
+
+void
+iotrace_init(int ncpus)
+{
+	int entries_per_cpu = DEFAULT_IOTRACE_ENTRIES_PER_CPU;
+	int enable = mmiotrace_enabled;
+
+	gentrace_configure_from_bootargs("iotrace", &enable, "iotrace_epc", &entries_per_cpu,
+	    IOTRACE_MAX_ENTRIES_PER_CPU, DEFAULT_IOTRACE_ENTRIES_PER_CPU, KF_IOTRACE_OVRD);
+
+	mmiotrace_enabled = enable;
+
+	if (mmiotrace_enabled) {
+		init_iotrace_bufs(ncpus, entries_per_cpu);
+	}
+}
+
+void
+traptrace_init(int ncpus)
+{
+	int entries_per_cpu = DEFAULT_TRAPTRACE_ENTRIES_PER_CPU;
+	int enable = traptrace_enabled;
+
+	gentrace_configure_from_bootargs("traptrace", &enable, "traptrace_epc", &entries_per_cpu,
+	    TRAPTRACE_MAX_ENTRIES_PER_CPU, DEFAULT_TRAPTRACE_ENTRIES_PER_CPU, KF_TRAPTRACE_OVRD);
+
+	traptrace_enabled = enable;
+
+	if (traptrace_enabled) {
+		init_traptrace_bufs(ncpus, entries_per_cpu);
+	}
+}
+
+#endif /* DEVELOPMENT || DEBUG */