/*
- * Copyright (c) 2003-2008 Apple Inc. All rights reserved.
+ * Copyright (c) 2003-2016 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
*/
#include <vm/vm_kern.h>
#include <kern/kalloc.h>
+#include <kern/timer_queue.h>
#include <mach/machine.h>
#include <i386/cpu_threads.h>
#include <i386/cpuid.h>
#include <i386/machine_cpu.h>
-#include <i386/lock.h>
-#include <i386/perfmon.h>
#include <i386/pmCPU.h>
+#include <i386/bit_routines.h>
-//#define TOPO_DEBUG 1
-#if TOPO_DEBUG
-void debug_topology_print(void);
-#define DBG(x...) kprintf("DBG: " x)
-#else
-#define DBG(x...)
-#endif /* TOPO_DEBUG */
+#define DIVISOR_GUARD(denom) \
+ if ((denom) == 0) { \
+ kprintf("%s: %d Zero divisor: " #denom, \
+ __FILE__, __LINE__); \
+ }
+
+static void debug_topology_print(void);
-#define bitmask(h,l) ((bit(h)|(bit(h)-1)) & ~(bit(l)-1))
-#define bitfield(x,h,l) (((x) & bitmask(h,l)) >> l)
+boolean_t topo_dbg = FALSE;
x86_pkg_t *x86_pkgs = NULL;
uint32_t num_Lx_caches[MAX_CACHE_DEPTH] = { 0 };
decl_simple_lock_data(, x86_topo_lock);
+static struct cpu_cache {
+ int level; int type;
+} cpu_caches [LCACHE_MAX] = {
+ [L1D] = { 1, CPU_CACHE_TYPE_DATA },
+ [L1I] = { 1, CPU_CACHE_TYPE_INST },
+ [L2U] = { 2, CPU_CACHE_TYPE_UNIF },
+ [L3U] = { 3, CPU_CACHE_TYPE_UNIF },
+};
+
static boolean_t
cpu_is_hyperthreaded(void)
{
static void
x86_LLC_info(void)
{
- uint32_t index;
- uint32_t cache_info[4];
- uint32_t cache_level = 0;
+ int cache_level = 0;
uint32_t nCPUsSharing = 1;
i386_cpu_info_t *cpuinfo;
+ struct cpu_cache *cachep;
+ int i;
cpuinfo = cpuid_info();
- do_cpuid(0, cache_info);
-
- if (cache_info[eax] < 4) {
- /*
- * Processor does not support deterministic
- * cache information. Set LLC sharing to 1, since
- * we have no better information.
- */
- if (cpu_is_hyperthreaded()) {
- topoParms.nCoresSharingLLC = 1;
- topoParms.nLCPUsSharingLLC = 2;
- topoParms.maxSharingLLC = 2;
- } else {
- topoParms.nCoresSharingLLC = 1;
- topoParms.nLCPUsSharingLLC = 1;
- topoParms.maxSharingLLC = 1;
- }
- return;
- }
-
- for (index = 0; ; index += 1) {
- uint32_t this_level;
-
- cache_info[eax] = 4;
- cache_info[ecx] = index;
- cache_info[ebx] = 0;
- cache_info[edx] = 0;
-
- cpuid(cache_info);
-
- /*
- * See if all levels have been queried.
- */
- if (bitfield(cache_info[eax], 4, 0) == 0)
- break;
+ for (i = 0, cachep = &cpu_caches[0]; i < LCACHE_MAX; i++, cachep++) {
- /*
- * Get the current level.
- */
- this_level = bitfield(cache_info[eax], 7, 5);
+ if (cachep->type == 0 || cpuid_info()->cache_size[i] == 0)
+ continue;
/*
* Only worry about it if it's a deeper level than
* what we've seen before.
*/
- if (this_level > cache_level) {
- cache_level = this_level;
+ if (cachep->level > cache_level) {
+ cache_level = cachep->level;
/*
* Save the number of CPUs sharing this cache.
*/
- nCPUsSharing = bitfield(cache_info[eax], 25, 14) + 1;
+ nCPUsSharing = cpuinfo->cache_sharing[i];
}
}
*/
topoParms.maxSharingLLC = nCPUsSharing;
- topoParms.nCoresSharingLLC = nCPUsSharing;
+ topoParms.nCoresSharingLLC = nCPUsSharing / (cpuinfo->thread_count /
+ cpuinfo->core_count);
topoParms.nLCPUsSharingLLC = nCPUsSharing;
/*
topoParms.nCoresSharingLLC = cpuinfo->core_count;
if (nCPUsSharing > cpuinfo->thread_count)
topoParms.nLCPUsSharingLLC = cpuinfo->thread_count;
-
-
- if (nCPUsSharing > cpuinfo->thread_count)
- topoParms.maxSharingLLC = cpuinfo->thread_count;
}
static void
{
i386_cpu_info_t *cpuinfo;
+ topoParms.stable = FALSE;
+
cpuinfo = cpuid_info();
+ PE_parse_boot_argn("-topo", &topo_dbg, sizeof(topo_dbg));
+
/*
* We need to start with getting the LLC information correct.
*/
/*
* Compute the number of threads (logical CPUs) per core.
*/
+ DIVISOR_GUARD(cpuinfo->core_count);
topoParms.nLThreadsPerCore = cpuinfo->thread_count / cpuinfo->core_count;
+ DIVISOR_GUARD(cpuinfo->cpuid_cores_per_package);
topoParms.nPThreadsPerCore = cpuinfo->cpuid_logical_per_package / cpuinfo->cpuid_cores_per_package;
/*
* Compute the number of dies per package.
*/
+ DIVISOR_GUARD(topoParms.nCoresSharingLLC);
topoParms.nLDiesPerPackage = cpuinfo->core_count / topoParms.nCoresSharingLLC;
+ DIVISOR_GUARD(topoParms.nPThreadsPerCore);
+ DIVISOR_GUARD(topoParms.maxSharingLLC / topoParms.nPThreadsPerCore);
topoParms.nPDiesPerPackage = cpuinfo->cpuid_cores_per_package / (topoParms.maxSharingLLC / topoParms.nPThreadsPerCore);
+
/*
* Compute the number of cores per die.
*/
topoParms.nLThreadsPerPackage = topoParms.nLThreadsPerCore * topoParms.nLCoresPerPackage;
topoParms.nPThreadsPerPackage = topoParms.nPThreadsPerCore * topoParms.nPCoresPerPackage;
- DBG("\nLogical Topology Parameters:\n");
- DBG("\tThreads per Core: %d\n", topoParms.nLThreadsPerCore);
- DBG("\tCores per Die: %d\n", topoParms.nLCoresPerDie);
- DBG("\tThreads per Die: %d\n", topoParms.nLThreadsPerDie);
- DBG("\tDies per Package: %d\n", topoParms.nLDiesPerPackage);
- DBG("\tCores per Package: %d\n", topoParms.nLCoresPerPackage);
- DBG("\tThreads per Package: %d\n", topoParms.nLThreadsPerPackage);
-
- DBG("\nPhysical Topology Parameters:\n");
- DBG("\tThreads per Core: %d\n", topoParms.nPThreadsPerCore);
- DBG("\tCores per Die: %d\n", topoParms.nPCoresPerDie);
- DBG("\tThreads per Die: %d\n", topoParms.nPThreadsPerDie);
- DBG("\tDies per Package: %d\n", topoParms.nPDiesPerPackage);
- DBG("\tCores per Package: %d\n", topoParms.nPCoresPerPackage);
- DBG("\tThreads per Package: %d\n", topoParms.nPThreadsPerPackage);
+ TOPO_DBG("\nCache Topology Parameters:\n");
+ TOPO_DBG("\tLLC Depth: %d\n", topoParms.LLCDepth);
+ TOPO_DBG("\tCores Sharing LLC: %d\n", topoParms.nCoresSharingLLC);
+ TOPO_DBG("\tThreads Sharing LLC: %d\n", topoParms.nLCPUsSharingLLC);
+ TOPO_DBG("\tmax Sharing of LLC: %d\n", topoParms.maxSharingLLC);
+
+ TOPO_DBG("\nLogical Topology Parameters:\n");
+ TOPO_DBG("\tThreads per Core: %d\n", topoParms.nLThreadsPerCore);
+ TOPO_DBG("\tCores per Die: %d\n", topoParms.nLCoresPerDie);
+ TOPO_DBG("\tThreads per Die: %d\n", topoParms.nLThreadsPerDie);
+ TOPO_DBG("\tDies per Package: %d\n", topoParms.nLDiesPerPackage);
+ TOPO_DBG("\tCores per Package: %d\n", topoParms.nLCoresPerPackage);
+ TOPO_DBG("\tThreads per Package: %d\n", topoParms.nLThreadsPerPackage);
+
+ TOPO_DBG("\nPhysical Topology Parameters:\n");
+ TOPO_DBG("\tThreads per Core: %d\n", topoParms.nPThreadsPerCore);
+ TOPO_DBG("\tCores per Die: %d\n", topoParms.nPCoresPerDie);
+ TOPO_DBG("\tThreads per Die: %d\n", topoParms.nPThreadsPerDie);
+ TOPO_DBG("\tDies per Package: %d\n", topoParms.nPDiesPerPackage);
+ TOPO_DBG("\tCores per Package: %d\n", topoParms.nPCoresPerPackage);
+ TOPO_DBG("\tThreads per Package: %d\n", topoParms.nPThreadsPerPackage);
topoParmsInited = TRUE;
}
x86_cpu_cache_t *root = NULL;
x86_cpu_cache_t *cur = NULL;
x86_cpu_cache_t *last = NULL;
- uint32_t index;
- uint32_t cache_info[4];
- uint32_t nsets;
-
- do_cpuid(0, cache_info);
-
- if (cache_info[eax] < 4) {
- /*
- * Processor does not support deterministic
- * cache information. Don't report anything
- */
- return NULL;
- }
-
- for (index = 0; ; index += 1) {
- cache_info[eax] = 4;
- cache_info[ecx] = index;
- cache_info[ebx] = 0;
- cache_info[edx] = 0;
-
- cpuid(cache_info);
+ struct cpu_cache *cachep;
+ int i;
- /*
- * See if all levels have been queried.
- */
- if (bitfield(cache_info[eax], 4, 0) == 0)
- break;
+ /*
+ * Cons up a list driven not by CPUID leaf 4 (deterministic cache params)
+ * but by the table above plus parameters already cracked from cpuid...
+ */
+ for (i = 0, cachep = &cpu_caches[0]; i < LCACHE_MAX; i++, cachep++) {
+ if (cachep->type == 0 || cpuid_info()->cache_size[i] == 0)
+ continue;
+
cur = x86_cache_alloc();
- if (cur == NULL) {
+ if (cur == NULL)
break;
- }
- cur->type = bitfield(cache_info[eax], 4, 0);
- cur->level = bitfield(cache_info[eax], 7, 5);
- cur->maxcpus = (bitfield(cache_info[eax], 25, 14) + 1);
- cur->line_size = bitfield(cache_info[ebx], 11, 0) + 1;
- cur->partitions = bitfield(cache_info[ebx], 21, 12) + 1;
- cur->ways = bitfield(cache_info[ebx], 31, 22) + 1;
- nsets = bitfield(cache_info[ecx], 31, 0) + 1;
- cur->cache_size = cur->line_size * cur->ways * cur->partitions * nsets;
+ cur->type = cachep->type;
+ cur->level = cachep->level;
+ cur->nlcpus = 0;
+ cur->maxcpus = cpuid_info()->cache_sharing[i];
+ cur->partitions = cpuid_info()->cache_partitions[i];
+ cur->cache_size = cpuid_info()->cache_size[i];
+ cur->line_size = cpuid_info()->cache_linesize;
if (last == NULL) {
root = cur;
last->next = cur;
last = cur;
}
-
- cur->nlcpus = 0;
num_Lx_caches[cur->level - 1] += 1;
}
-
- return(root);
+ return root;
}
+
static x86_cpu_cache_t *
x86_match_cache(x86_cpu_cache_t *list, x86_cpu_cache_t *matcher)
{
x86_cpu_cache_t *cur_cache;
-
+
cur_cache = list;
while (cur_cache != NULL) {
if (cur_cache->maxcpus == matcher->maxcpus
&& cur_cache->type == matcher->type
&& cur_cache->level == matcher->level
- && cur_cache->ways == matcher->ways
&& cur_cache->partitions == matcher->partitions
&& cur_cache->line_size == matcher->line_size
&& cur_cache->cache_size == matcher->cache_size)
lcpu->state = LCPU_OFF;
for (i = 0; i < MAX_CACHE_DEPTH; i += 1)
lcpu->caches[i] = NULL;
-
- lcpu->master = (lcpu->cpu_num == (unsigned int) master_cpu);
- lcpu->primary = (lcpu->pnum % topoParms.nPThreadsPerPackage) == 0;
}
static x86_core_t *
}
void
-x86_set_lcpu_numbers(x86_lcpu_t *lcpu)
+x86_set_logical_topology(x86_lcpu_t *lcpu, int pnum, int lnum)
{
- lcpu->lnum = lcpu->cpu_num % topoParms.nLThreadsPerCore;
-}
+ x86_core_t *core = lcpu->core;
+ x86_die_t *die = lcpu->die;
+ x86_pkg_t *pkg = lcpu->package;
+
+ assert(core != NULL);
+ assert(die != NULL);
+ assert(pkg != NULL);
-void
-x86_set_core_numbers(x86_core_t *core, x86_lcpu_t *lcpu)
-{
- core->pcore_num = lcpu->cpu_num / topoParms.nLThreadsPerCore;
+ lcpu->cpu_num = lnum;
+ lcpu->pnum = pnum;
+ lcpu->master = (lnum == master_cpu);
+ lcpu->primary = (lnum % topoParms.nLThreadsPerPackage) == 0;
+
+ lcpu->lnum = lnum % topoParms.nLThreadsPerCore;
+
+ core->pcore_num = lnum / topoParms.nLThreadsPerCore;
core->lcore_num = core->pcore_num % topoParms.nLCoresPerDie;
-}
-void
-x86_set_die_numbers(x86_die_t *die, x86_lcpu_t *lcpu)
-{
- die->pdie_num = lcpu->cpu_num / (topoParms.nLThreadsPerCore * topoParms.nLCoresPerDie);
+ die->pdie_num = lnum / (topoParms.nLThreadsPerCore*topoParms.nLCoresPerDie);
die->ldie_num = die->pdie_num % topoParms.nLDiesPerPackage;
-}
-void
-x86_set_pkg_numbers(x86_pkg_t *pkg, x86_lcpu_t *lcpu)
-{
- pkg->ppkg_num = lcpu->cpu_num / topoParms.nLThreadsPerPackage;
+ pkg->ppkg_num = lnum / topoParms.nLThreadsPerPackage;
pkg->lpkg_num = pkg->ppkg_num;
+
}
static x86_die_t *
x86_lcpu_init(cpu);
- /*
- * Allocate performance counter structure.
- */
- simple_unlock(&x86_topo_lock);
- cpup->lcpu.pmc = pmc_alloc();
- simple_lock(&x86_topo_lock);
-
/*
* Assume that all cpus have the same features.
*/
simple_unlock(&x86_topo_lock);
pmCPUMarkRunning(cpup);
- etimer_resync_deadlines();
+ timer_resync_deadlines();
}
/*
* Called for a cpu to halt permanently
* (as opposed to halting and expecting an interrupt to awaken it).
*/
+__attribute__((noreturn))
void
cpu_thread_halt(void)
{
/* NOT REACHED */
}
-#if TOPO_DEBUG
/*
- * Prints out the topology
+ * Validates that the topology was built correctly. Must be called only
+ * after the complete topology is built and no other changes are being made.
*/
void
+x86_validate_topology(void)
+{
+ x86_pkg_t *pkg;
+ x86_die_t *die;
+ x86_core_t *core;
+ x86_lcpu_t *lcpu;
+ uint32_t nDies;
+ uint32_t nCores;
+ uint32_t nCPUs;
+
+ if (topo_dbg)
+ debug_topology_print();
+
+ /*
+ * XXX
+ *
+ * Right now this only works if the number of CPUs started is the total
+ * number of CPUs. However, when specifying cpus=n the topology is only
+ * partially constructed and the checks below will fail.
+ *
+ * We should *always* build the complete topology and only start the CPUs
+ * indicated by cpus=n. Until that happens, this code will not check the
+ * topology if the number of cpus defined is < that described the the
+ * topology parameters.
+ */
+ nCPUs = topoParms.nPackages * topoParms.nLThreadsPerPackage;
+ if (nCPUs > real_ncpus)
+ return;
+
+ pkg = x86_pkgs;
+ while (pkg != NULL) {
+ /*
+ * Make sure that the package has the correct number of dies.
+ */
+ nDies = 0;
+ die = pkg->dies;
+ while (die != NULL) {
+ if (die->package == NULL)
+ panic("Die(%d)->package is NULL",
+ die->pdie_num);
+ if (die->package != pkg)
+ panic("Die %d points to package %d, should be %d",
+ die->pdie_num, die->package->lpkg_num, pkg->lpkg_num);
+
+ TOPO_DBG("Die(%d)->package %d\n",
+ die->pdie_num, pkg->lpkg_num);
+
+ /*
+ * Make sure that the die has the correct number of cores.
+ */
+ TOPO_DBG("Die(%d)->cores: ", die->pdie_num);
+ nCores = 0;
+ core = die->cores;
+ while (core != NULL) {
+ if (core->die == NULL)
+ panic("Core(%d)->die is NULL",
+ core->pcore_num);
+ if (core->die != die)
+ panic("Core %d points to die %d, should be %d",
+ core->pcore_num, core->die->pdie_num, die->pdie_num);
+ nCores += 1;
+ TOPO_DBG("%d ", core->pcore_num);
+ core = core->next_in_die;
+ }
+ TOPO_DBG("\n");
+
+ if (nCores != topoParms.nLCoresPerDie)
+ panic("Should have %d Cores, but only found %d for Die %d",
+ topoParms.nLCoresPerDie, nCores, die->pdie_num);
+
+ /*
+ * Make sure that the die has the correct number of CPUs.
+ */
+ TOPO_DBG("Die(%d)->lcpus: ", die->pdie_num);
+ nCPUs = 0;
+ lcpu = die->lcpus;
+ while (lcpu != NULL) {
+ if (lcpu->die == NULL)
+ panic("CPU(%d)->die is NULL",
+ lcpu->cpu_num);
+ if (lcpu->die != die)
+ panic("CPU %d points to die %d, should be %d",
+ lcpu->cpu_num, lcpu->die->pdie_num, die->pdie_num);
+ nCPUs += 1;
+ TOPO_DBG("%d ", lcpu->cpu_num);
+ lcpu = lcpu->next_in_die;
+ }
+ TOPO_DBG("\n");
+
+ if (nCPUs != topoParms.nLThreadsPerDie)
+ panic("Should have %d Threads, but only found %d for Die %d",
+ topoParms.nLThreadsPerDie, nCPUs, die->pdie_num);
+
+ nDies += 1;
+ die = die->next_in_pkg;
+ }
+
+ if (nDies != topoParms.nLDiesPerPackage)
+ panic("Should have %d Dies, but only found %d for package %d",
+ topoParms.nLDiesPerPackage, nDies, pkg->lpkg_num);
+
+ /*
+ * Make sure that the package has the correct number of cores.
+ */
+ nCores = 0;
+ core = pkg->cores;
+ while (core != NULL) {
+ if (core->package == NULL)
+ panic("Core(%d)->package is NULL",
+ core->pcore_num);
+ if (core->package != pkg)
+ panic("Core %d points to package %d, should be %d",
+ core->pcore_num, core->package->lpkg_num, pkg->lpkg_num);
+ TOPO_DBG("Core(%d)->package %d\n",
+ core->pcore_num, pkg->lpkg_num);
+
+ /*
+ * Make sure that the core has the correct number of CPUs.
+ */
+ nCPUs = 0;
+ lcpu = core->lcpus;
+ TOPO_DBG("Core(%d)->lcpus: ", core->pcore_num);
+ while (lcpu != NULL) {
+ if (lcpu->core == NULL)
+ panic("CPU(%d)->core is NULL",
+ lcpu->cpu_num);
+ if (lcpu->core != core)
+ panic("CPU %d points to core %d, should be %d",
+ lcpu->cpu_num, lcpu->core->pcore_num, core->pcore_num);
+ TOPO_DBG("%d ", lcpu->cpu_num);
+ nCPUs += 1;
+ lcpu = lcpu->next_in_core;
+ }
+ TOPO_DBG("\n");
+
+ if (nCPUs != topoParms.nLThreadsPerCore)
+ panic("Should have %d Threads, but only found %d for Core %d",
+ topoParms.nLThreadsPerCore, nCPUs, core->pcore_num);
+ nCores += 1;
+ core = core->next_in_pkg;
+ }
+
+ if (nCores != topoParms.nLCoresPerPackage)
+ panic("Should have %d Cores, but only found %d for package %d",
+ topoParms.nLCoresPerPackage, nCores, pkg->lpkg_num);
+
+ /*
+ * Make sure that the package has the correct number of CPUs.
+ */
+ nCPUs = 0;
+ lcpu = pkg->lcpus;
+ while (lcpu != NULL) {
+ if (lcpu->package == NULL)
+ panic("CPU(%d)->package is NULL",
+ lcpu->cpu_num);
+ if (lcpu->package != pkg)
+ panic("CPU %d points to package %d, should be %d",
+ lcpu->cpu_num, lcpu->package->lpkg_num, pkg->lpkg_num);
+ TOPO_DBG("CPU(%d)->package %d\n",
+ lcpu->cpu_num, pkg->lpkg_num);
+ nCPUs += 1;
+ lcpu = lcpu->next_in_pkg;
+ }
+
+ if (nCPUs != topoParms.nLThreadsPerPackage)
+ panic("Should have %d Threads, but only found %d for package %d",
+ topoParms.nLThreadsPerPackage, nCPUs, pkg->lpkg_num);
+
+ pkg = pkg->next;
+ }
+}
+
+/*
+ * Prints out the topology
+ */
+static void
debug_topology_print(void)
{
x86_pkg_t *pkg;
pkg = pkg->next;
}
}
-#endif /* TOPO_DEBUG */
projects / apple / xnu.git / blobdiff