X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/593a1d5fd87cdf5b46dd5fcb84467b432cea0f91..4d15aeb193b2c68f1d38666c317f8d3734f5f083:/osfmk/i386/cpu_threads.c?ds=inline diff --git a/osfmk/i386/cpu_threads.c b/osfmk/i386/cpu_threads.c index 7727eb7ea..e58a9369e 100644 --- a/osfmk/i386/cpu_threads.c +++ b/osfmk/i386/cpu_threads.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003-2008 Apple Inc. All rights reserved. + * Copyright (c) 2003-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -27,24 +27,23 @@ */ #include #include +#include #include #include #include #include -#include -#include #include +#include -//#define TOPO_DEBUG 1 -#if TOPO_DEBUG -void debug_topology_print(void); -#define DBG(x...) kprintf("DBG: " x) -#else -#define DBG(x...) -#endif /* TOPO_DEBUG */ +#define DIVISOR_GUARD(denom) \ + if ((denom) == 0) { \ + kprintf("%s: %d Zero divisor: " #denom, \ + __FILE__, __LINE__); \ + } + +static void debug_topology_print(void); -#define bitmask(h,l) ((bit(h)|(bit(h)-1)) & ~(bit(l)-1)) -#define bitfield(x,h,l) (((x) & bitmask(h,l)) >> l) +boolean_t topo_dbg = FALSE; x86_pkg_t *x86_pkgs = NULL; uint32_t num_Lx_caches[MAX_CACHE_DEPTH] = { 0 }; @@ -62,6 +61,15 @@ x86_topology_parameters_t topoParms; decl_simple_lock_data(, x86_topo_lock); +static struct cpu_cache { + int level; int type; +} cpu_caches [LCACHE_MAX] = { + [L1D] = { 1, CPU_CACHE_TYPE_DATA }, + [L1I] = { 1, CPU_CACHE_TYPE_INST }, + [L2U] = { 2, CPU_CACHE_TYPE_UNIF }, + [L3U] = { 3, CPU_CACHE_TYPE_UNIF }, +}; + static boolean_t cpu_is_hyperthreaded(void) { @@ -102,66 +110,30 @@ x86_cache_alloc(void) static void x86_LLC_info(void) { - uint32_t index; - uint32_t cache_info[4]; - uint32_t cache_level = 0; + int cache_level = 0; uint32_t nCPUsSharing = 1; i386_cpu_info_t *cpuinfo; + struct cpu_cache *cachep; + int i; cpuinfo = cpuid_info(); - do_cpuid(0, cache_info); - - if (cache_info[eax] < 4) { - /* - * Processor does not support deterministic - * cache information. Set LLC sharing to 1, since - * we have no better information. - */ - if (cpu_is_hyperthreaded()) { - topoParms.nCoresSharingLLC = 1; - topoParms.nLCPUsSharingLLC = 2; - topoParms.maxSharingLLC = 2; - } else { - topoParms.nCoresSharingLLC = 1; - topoParms.nLCPUsSharingLLC = 1; - topoParms.maxSharingLLC = 1; - } - return; - } - - for (index = 0; ; index += 1) { - uint32_t this_level; - - cache_info[eax] = 4; - cache_info[ecx] = index; - cache_info[ebx] = 0; - cache_info[edx] = 0; - - cpuid(cache_info); - - /* - * See if all levels have been queried. - */ - if (bitfield(cache_info[eax], 4, 0) == 0) - break; + for (i = 0, cachep = &cpu_caches[0]; i < LCACHE_MAX; i++, cachep++) { - /* - * Get the current level. - */ - this_level = bitfield(cache_info[eax], 7, 5); + if (cachep->type == 0 || cpuid_info()->cache_size[i] == 0) + continue; /* * Only worry about it if it's a deeper level than * what we've seen before. */ - if (this_level > cache_level) { - cache_level = this_level; + if (cachep->level > cache_level) { + cache_level = cachep->level; /* * Save the number of CPUs sharing this cache. */ - nCPUsSharing = bitfield(cache_info[eax], 25, 14) + 1; + nCPUsSharing = cpuinfo->cache_sharing[i]; } } @@ -176,7 +148,8 @@ x86_LLC_info(void) */ topoParms.maxSharingLLC = nCPUsSharing; - topoParms.nCoresSharingLLC = nCPUsSharing; + topoParms.nCoresSharingLLC = nCPUsSharing / (cpuinfo->thread_count / + cpuinfo->core_count); topoParms.nLCPUsSharingLLC = nCPUsSharing; /* @@ -187,10 +160,6 @@ x86_LLC_info(void) topoParms.nCoresSharingLLC = cpuinfo->core_count; if (nCPUsSharing > cpuinfo->thread_count) topoParms.nLCPUsSharingLLC = cpuinfo->thread_count; - - - if (nCPUsSharing > cpuinfo->thread_count) - topoParms.maxSharingLLC = cpuinfo->thread_count; } static void @@ -198,8 +167,12 @@ initTopoParms(void) { i386_cpu_info_t *cpuinfo; + topoParms.stable = FALSE; + cpuinfo = cpuid_info(); + PE_parse_boot_argn("-topo", &topo_dbg, sizeof(topo_dbg)); + /* * We need to start with getting the LLC information correct. */ @@ -208,15 +181,21 @@ initTopoParms(void) /* * Compute the number of threads (logical CPUs) per core. */ + DIVISOR_GUARD(cpuinfo->core_count); topoParms.nLThreadsPerCore = cpuinfo->thread_count / cpuinfo->core_count; + DIVISOR_GUARD(cpuinfo->cpuid_cores_per_package); topoParms.nPThreadsPerCore = cpuinfo->cpuid_logical_per_package / cpuinfo->cpuid_cores_per_package; /* * Compute the number of dies per package. */ + DIVISOR_GUARD(topoParms.nCoresSharingLLC); topoParms.nLDiesPerPackage = cpuinfo->core_count / topoParms.nCoresSharingLLC; + DIVISOR_GUARD(topoParms.nPThreadsPerCore); + DIVISOR_GUARD(topoParms.maxSharingLLC / topoParms.nPThreadsPerCore); topoParms.nPDiesPerPackage = cpuinfo->cpuid_cores_per_package / (topoParms.maxSharingLLC / topoParms.nPThreadsPerCore); + /* * Compute the number of cores per die. */ @@ -241,21 +220,27 @@ initTopoParms(void) topoParms.nLThreadsPerPackage = topoParms.nLThreadsPerCore * topoParms.nLCoresPerPackage; topoParms.nPThreadsPerPackage = topoParms.nPThreadsPerCore * topoParms.nPCoresPerPackage; - DBG("\nLogical Topology Parameters:\n"); - DBG("\tThreads per Core: %d\n", topoParms.nLThreadsPerCore); - DBG("\tCores per Die: %d\n", topoParms.nLCoresPerDie); - DBG("\tThreads per Die: %d\n", topoParms.nLThreadsPerDie); - DBG("\tDies per Package: %d\n", topoParms.nLDiesPerPackage); - DBG("\tCores per Package: %d\n", topoParms.nLCoresPerPackage); - DBG("\tThreads per Package: %d\n", topoParms.nLThreadsPerPackage); - - DBG("\nPhysical Topology Parameters:\n"); - DBG("\tThreads per Core: %d\n", topoParms.nPThreadsPerCore); - DBG("\tCores per Die: %d\n", topoParms.nPCoresPerDie); - DBG("\tThreads per Die: %d\n", topoParms.nPThreadsPerDie); - DBG("\tDies per Package: %d\n", topoParms.nPDiesPerPackage); - DBG("\tCores per Package: %d\n", topoParms.nPCoresPerPackage); - DBG("\tThreads per Package: %d\n", topoParms.nPThreadsPerPackage); + TOPO_DBG("\nCache Topology Parameters:\n"); + TOPO_DBG("\tLLC Depth: %d\n", topoParms.LLCDepth); + TOPO_DBG("\tCores Sharing LLC: %d\n", topoParms.nCoresSharingLLC); + TOPO_DBG("\tThreads Sharing LLC: %d\n", topoParms.nLCPUsSharingLLC); + TOPO_DBG("\tmax Sharing of LLC: %d\n", topoParms.maxSharingLLC); + + TOPO_DBG("\nLogical Topology Parameters:\n"); + TOPO_DBG("\tThreads per Core: %d\n", topoParms.nLThreadsPerCore); + TOPO_DBG("\tCores per Die: %d\n", topoParms.nLCoresPerDie); + TOPO_DBG("\tThreads per Die: %d\n", topoParms.nLThreadsPerDie); + TOPO_DBG("\tDies per Package: %d\n", topoParms.nLDiesPerPackage); + TOPO_DBG("\tCores per Package: %d\n", topoParms.nLCoresPerPackage); + TOPO_DBG("\tThreads per Package: %d\n", topoParms.nLThreadsPerPackage); + + TOPO_DBG("\nPhysical Topology Parameters:\n"); + TOPO_DBG("\tThreads per Core: %d\n", topoParms.nPThreadsPerCore); + TOPO_DBG("\tCores per Die: %d\n", topoParms.nPCoresPerDie); + TOPO_DBG("\tThreads per Die: %d\n", topoParms.nPThreadsPerDie); + TOPO_DBG("\tDies per Package: %d\n", topoParms.nPDiesPerPackage); + TOPO_DBG("\tCores per Package: %d\n", topoParms.nPCoresPerPackage); + TOPO_DBG("\tThreads per Package: %d\n", topoParms.nPThreadsPerPackage); topoParmsInited = TRUE; } @@ -281,47 +266,29 @@ x86_cache_list(void) x86_cpu_cache_t *root = NULL; x86_cpu_cache_t *cur = NULL; x86_cpu_cache_t *last = NULL; - uint32_t index; - uint32_t cache_info[4]; - uint32_t nsets; - - do_cpuid(0, cache_info); - - if (cache_info[eax] < 4) { - /* - * Processor does not support deterministic - * cache information. Don't report anything - */ - return NULL; - } - - for (index = 0; ; index += 1) { - cache_info[eax] = 4; - cache_info[ecx] = index; - cache_info[ebx] = 0; - cache_info[edx] = 0; - - cpuid(cache_info); + struct cpu_cache *cachep; + int i; - /* - * See if all levels have been queried. - */ - if (bitfield(cache_info[eax], 4, 0) == 0) - break; + /* + * Cons up a list driven not by CPUID leaf 4 (deterministic cache params) + * but by the table above plus parameters already cracked from cpuid... + */ + for (i = 0, cachep = &cpu_caches[0]; i < LCACHE_MAX; i++, cachep++) { + if (cachep->type == 0 || cpuid_info()->cache_size[i] == 0) + continue; + cur = x86_cache_alloc(); - if (cur == NULL) { + if (cur == NULL) break; - } - cur->type = bitfield(cache_info[eax], 4, 0); - cur->level = bitfield(cache_info[eax], 7, 5); - cur->maxcpus = (bitfield(cache_info[eax], 25, 14) + 1); - cur->line_size = bitfield(cache_info[ebx], 11, 0) + 1; - cur->partitions = bitfield(cache_info[ebx], 21, 12) + 1; - cur->ways = bitfield(cache_info[ebx], 31, 22) + 1; - nsets = bitfield(cache_info[ecx], 31, 0) + 1; - cur->cache_size = cur->line_size * cur->ways * cur->partitions * nsets; + cur->type = cachep->type; + cur->level = cachep->level; + cur->nlcpus = 0; + cur->maxcpus = cpuid_info()->cache_sharing[i]; + cur->partitions = cpuid_info()->cache_partitions[i]; + cur->cache_size = cpuid_info()->cache_size[i]; + cur->line_size = cpuid_info()->cache_linesize; if (last == NULL) { root = cur; @@ -330,25 +297,22 @@ x86_cache_list(void) last->next = cur; last = cur; } - - cur->nlcpus = 0; num_Lx_caches[cur->level - 1] += 1; } - - return(root); + return root; } + static x86_cpu_cache_t * x86_match_cache(x86_cpu_cache_t *list, x86_cpu_cache_t *matcher) { x86_cpu_cache_t *cur_cache; - + cur_cache = list; while (cur_cache != NULL) { if (cur_cache->maxcpus == matcher->maxcpus && cur_cache->type == matcher->type && cur_cache->level == matcher->level - && cur_cache->ways == matcher->ways && cur_cache->partitions == matcher->partitions && cur_cache->line_size == matcher->line_size && cur_cache->cache_size == matcher->cache_size) @@ -384,9 +348,6 @@ x86_lcpu_init(int cpu) lcpu->state = LCPU_OFF; for (i = 0; i < MAX_CACHE_DEPTH; i += 1) lcpu->caches[i] = NULL; - - lcpu->master = (lcpu->cpu_num == (unsigned int) master_cpu); - lcpu->primary = (lcpu->pnum % topoParms.nPThreadsPerPackage) == 0; } static x86_core_t * @@ -504,30 +465,32 @@ x86_core_find(int cpu) } void -x86_set_lcpu_numbers(x86_lcpu_t *lcpu) +x86_set_logical_topology(x86_lcpu_t *lcpu, int pnum, int lnum) { - lcpu->lnum = lcpu->cpu_num % topoParms.nLThreadsPerCore; -} + x86_core_t *core = lcpu->core; + x86_die_t *die = lcpu->die; + x86_pkg_t *pkg = lcpu->package; + + assert(core != NULL); + assert(die != NULL); + assert(pkg != NULL); -void -x86_set_core_numbers(x86_core_t *core, x86_lcpu_t *lcpu) -{ - core->pcore_num = lcpu->cpu_num / topoParms.nLThreadsPerCore; + lcpu->cpu_num = lnum; + lcpu->pnum = pnum; + lcpu->master = (lnum == master_cpu); + lcpu->primary = (lnum % topoParms.nLThreadsPerPackage) == 0; + + lcpu->lnum = lnum % topoParms.nLThreadsPerCore; + + core->pcore_num = lnum / topoParms.nLThreadsPerCore; core->lcore_num = core->pcore_num % topoParms.nLCoresPerDie; -} -void -x86_set_die_numbers(x86_die_t *die, x86_lcpu_t *lcpu) -{ - die->pdie_num = lcpu->cpu_num / (topoParms.nLThreadsPerCore * topoParms.nLCoresPerDie); + die->pdie_num = lnum / (topoParms.nLThreadsPerCore*topoParms.nLCoresPerDie); die->ldie_num = die->pdie_num % topoParms.nLDiesPerPackage; -} -void -x86_set_pkg_numbers(x86_pkg_t *pkg, x86_lcpu_t *lcpu) -{ - pkg->ppkg_num = lcpu->cpu_num / topoParms.nLThreadsPerPackage; + pkg->ppkg_num = lnum / topoParms.nLThreadsPerPackage; pkg->lpkg_num = pkg->ppkg_num; + } static x86_die_t * @@ -863,13 +826,6 @@ cpu_thread_alloc(int cpu) x86_lcpu_init(cpu); - /* - * Allocate performance counter structure. - */ - simple_unlock(&x86_topo_lock); - cpup->lcpu.pmc = pmc_alloc(); - simple_lock(&x86_topo_lock); - /* * Assume that all cpus have the same features. */ @@ -1007,13 +963,14 @@ cpu_thread_init(void) simple_unlock(&x86_topo_lock); pmCPUMarkRunning(cpup); - etimer_resync_deadlines(); + timer_resync_deadlines(); } /* * Called for a cpu to halt permanently * (as opposed to halting and expecting an interrupt to awaken it). */ +__attribute__((noreturn)) void cpu_thread_halt(void) { @@ -1039,11 +996,187 @@ cpu_thread_halt(void) /* NOT REACHED */ } -#if TOPO_DEBUG /* - * Prints out the topology + * Validates that the topology was built correctly. Must be called only + * after the complete topology is built and no other changes are being made. */ void +x86_validate_topology(void) +{ + x86_pkg_t *pkg; + x86_die_t *die; + x86_core_t *core; + x86_lcpu_t *lcpu; + uint32_t nDies; + uint32_t nCores; + uint32_t nCPUs; + + if (topo_dbg) + debug_topology_print(); + + /* + * XXX + * + * Right now this only works if the number of CPUs started is the total + * number of CPUs. However, when specifying cpus=n the topology is only + * partially constructed and the checks below will fail. + * + * We should *always* build the complete topology and only start the CPUs + * indicated by cpus=n. Until that happens, this code will not check the + * topology if the number of cpus defined is < that described the the + * topology parameters. + */ + nCPUs = topoParms.nPackages * topoParms.nLThreadsPerPackage; + if (nCPUs > real_ncpus) + return; + + pkg = x86_pkgs; + while (pkg != NULL) { + /* + * Make sure that the package has the correct number of dies. + */ + nDies = 0; + die = pkg->dies; + while (die != NULL) { + if (die->package == NULL) + panic("Die(%d)->package is NULL", + die->pdie_num); + if (die->package != pkg) + panic("Die %d points to package %d, should be %d", + die->pdie_num, die->package->lpkg_num, pkg->lpkg_num); + + TOPO_DBG("Die(%d)->package %d\n", + die->pdie_num, pkg->lpkg_num); + + /* + * Make sure that the die has the correct number of cores. + */ + TOPO_DBG("Die(%d)->cores: ", die->pdie_num); + nCores = 0; + core = die->cores; + while (core != NULL) { + if (core->die == NULL) + panic("Core(%d)->die is NULL", + core->pcore_num); + if (core->die != die) + panic("Core %d points to die %d, should be %d", + core->pcore_num, core->die->pdie_num, die->pdie_num); + nCores += 1; + TOPO_DBG("%d ", core->pcore_num); + core = core->next_in_die; + } + TOPO_DBG("\n"); + + if (nCores != topoParms.nLCoresPerDie) + panic("Should have %d Cores, but only found %d for Die %d", + topoParms.nLCoresPerDie, nCores, die->pdie_num); + + /* + * Make sure that the die has the correct number of CPUs. + */ + TOPO_DBG("Die(%d)->lcpus: ", die->pdie_num); + nCPUs = 0; + lcpu = die->lcpus; + while (lcpu != NULL) { + if (lcpu->die == NULL) + panic("CPU(%d)->die is NULL", + lcpu->cpu_num); + if (lcpu->die != die) + panic("CPU %d points to die %d, should be %d", + lcpu->cpu_num, lcpu->die->pdie_num, die->pdie_num); + nCPUs += 1; + TOPO_DBG("%d ", lcpu->cpu_num); + lcpu = lcpu->next_in_die; + } + TOPO_DBG("\n"); + + if (nCPUs != topoParms.nLThreadsPerDie) + panic("Should have %d Threads, but only found %d for Die %d", + topoParms.nLThreadsPerDie, nCPUs, die->pdie_num); + + nDies += 1; + die = die->next_in_pkg; + } + + if (nDies != topoParms.nLDiesPerPackage) + panic("Should have %d Dies, but only found %d for package %d", + topoParms.nLDiesPerPackage, nDies, pkg->lpkg_num); + + /* + * Make sure that the package has the correct number of cores. + */ + nCores = 0; + core = pkg->cores; + while (core != NULL) { + if (core->package == NULL) + panic("Core(%d)->package is NULL", + core->pcore_num); + if (core->package != pkg) + panic("Core %d points to package %d, should be %d", + core->pcore_num, core->package->lpkg_num, pkg->lpkg_num); + TOPO_DBG("Core(%d)->package %d\n", + core->pcore_num, pkg->lpkg_num); + + /* + * Make sure that the core has the correct number of CPUs. + */ + nCPUs = 0; + lcpu = core->lcpus; + TOPO_DBG("Core(%d)->lcpus: ", core->pcore_num); + while (lcpu != NULL) { + if (lcpu->core == NULL) + panic("CPU(%d)->core is NULL", + lcpu->cpu_num); + if (lcpu->core != core) + panic("CPU %d points to core %d, should be %d", + lcpu->cpu_num, lcpu->core->pcore_num, core->pcore_num); + TOPO_DBG("%d ", lcpu->cpu_num); + nCPUs += 1; + lcpu = lcpu->next_in_core; + } + TOPO_DBG("\n"); + + if (nCPUs != topoParms.nLThreadsPerCore) + panic("Should have %d Threads, but only found %d for Core %d", + topoParms.nLThreadsPerCore, nCPUs, core->pcore_num); + nCores += 1; + core = core->next_in_pkg; + } + + if (nCores != topoParms.nLCoresPerPackage) + panic("Should have %d Cores, but only found %d for package %d", + topoParms.nLCoresPerPackage, nCores, pkg->lpkg_num); + + /* + * Make sure that the package has the correct number of CPUs. + */ + nCPUs = 0; + lcpu = pkg->lcpus; + while (lcpu != NULL) { + if (lcpu->package == NULL) + panic("CPU(%d)->package is NULL", + lcpu->cpu_num); + if (lcpu->package != pkg) + panic("CPU %d points to package %d, should be %d", + lcpu->cpu_num, lcpu->package->lpkg_num, pkg->lpkg_num); + TOPO_DBG("CPU(%d)->package %d\n", + lcpu->cpu_num, pkg->lpkg_num); + nCPUs += 1; + lcpu = lcpu->next_in_pkg; + } + + if (nCPUs != topoParms.nLThreadsPerPackage) + panic("Should have %d Threads, but only found %d for package %d", + topoParms.nLThreadsPerPackage, nCPUs, pkg->lpkg_num); + + pkg = pkg->next; + } +} + +/* + * Prints out the topology + */ +static void debug_topology_print(void) { x86_pkg_t *pkg; @@ -1096,4 +1229,3 @@ debug_topology_print(void) pkg = pkg->next; } } -#endif /* TOPO_DEBUG */