osfmk/i386/cpu_topology.c

   1 /*
   2  * Copyright (c) 2007-2010 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <mach/machine.h>
  30 #include <mach/processor.h>
  31 #include <kern/kalloc.h>
  32 #include <i386/cpu_affinity.h>
  33 #include <i386/cpu_topology.h>
  34 #include <i386/cpu_threads.h>
  35 #include <i386/machine_cpu.h>
  36 #include <i386/bit_routines.h>
  37 #include <i386/cpu_data.h>
  38 #include <i386/lapic.h>
  39 #include <i386/machine_routines.h>
  40 #include <stddef.h>
  41
  42 __private_extern__ void qsort(
  43         void * array,
  44         size_t nmembers,
  45         size_t member_size,
  46         int (*)(const void *, const void *));
  47
  48 static int lapicid_cmp(const void *x, const void *y);
  49 static x86_affinity_set_t *find_cache_affinity(x86_cpu_cache_t *L2_cachep);
  50
  51 x86_affinity_set_t      *x86_affinities = NULL;
  52 static int              x86_affinity_count = 0;
  53
  54 extern cpu_data_t cpshadows[];
  55
  56 #if DEVELOPMENT || DEBUG
  57 void iotrace_init(int ncpus);
  58 void traptrace_init(int ncpus);
  59 #endif /* DEVELOPMENT || DEBUG */
  60
  61
  62 /* Re-sort double-mapped CPU data shadows after topology discovery sorts the
  63  * primary CPU data structures by physical/APIC CPU ID.
  64  */
  65 static void
  66 cpu_shadow_sort(int ncpus)
  67 {
  68         for (int i = 0; i < ncpus; i++) {
  69                 cpu_data_t      *cpup = cpu_datap(i);
  70                 ptrdiff_t       coff = cpup - cpu_datap(0);
  71
  72                 cpup->cd_shadow = &cpshadows[coff];
  73         }
  74 }
  75
  76 /*
  77  * cpu_topology_sort() is called after all processors have been registered but
  78  * before any non-boot processor is started.  We establish canonical logical
  79  * processor numbering - logical cpus must be contiguous, zero-based and
  80  * assigned in physical (local apic id) order.  This step is required because
  81  * the discovery/registration order is non-deterministic - cores are registered
  82  * in differing orders over boots.  Enforcing canonical numbering simplifies
  83  * identification of processors.
  84  */
  85 void
  86 cpu_topology_sort(int ncpus)
  87 {
  88         int             i;
  89         boolean_t       istate;
  90         processor_t             lprim = NULL;
  91
  92         assert(machine_info.physical_cpu == 1);
  93         assert(machine_info.logical_cpu == 1);
  94         assert(master_cpu == 0);
  95         assert(cpu_number() == 0);
  96         assert(cpu_datap(0)->cpu_number == 0);
  97
  98         /* Lights out for this */
  99         istate = ml_set_interrupts_enabled(FALSE);
 100
 101         if (topo_dbg) {
 102                 TOPO_DBG("cpu_topology_start() %d cpu%s registered\n",
 103                     ncpus, (ncpus > 1) ? "s" : "");
 104                 for (i = 0; i < ncpus; i++) {
 105                         cpu_data_t      *cpup = cpu_datap(i);
 106                         TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",
 107                             i, (void *) cpup, cpup->cpu_phys_number);
 108                 }
 109         }
 110
 111         /*
 112          * Re-order the cpu_data_ptr vector sorting by physical id.
 113          * Skip the boot processor, it's required to be correct.
 114          */
 115         if (ncpus > 1) {
 116                 qsort((void *) &cpu_data_ptr[1],
 117                     ncpus - 1,
 118                     sizeof(cpu_data_t *),
 119                     lapicid_cmp);
 120         }
 121         if (topo_dbg) {
 122                 TOPO_DBG("cpu_topology_start() after sorting:\n");
 123                 for (i = 0; i < ncpus; i++) {
 124                         cpu_data_t      *cpup = cpu_datap(i);
 125                         TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",
 126                             i, (void *) cpup, cpup->cpu_phys_number);
 127                 }
 128         }
 129
 130         /*
 131          * Finalize logical numbers and map kept by the lapic code.
 132          */
 133         for (i = 0; i < ncpus; i++) {
 134                 cpu_data_t      *cpup = cpu_datap(i);
 135
 136                 if (cpup->cpu_number != i) {
 137                         kprintf("cpu_datap(%d):%p local apic id 0x%x "
 138                             "remapped from %d\n",
 139                             i, cpup, cpup->cpu_phys_number,
 140                             cpup->cpu_number);
 141                 }
 142                 cpup->cpu_number = i;
 143                 lapic_cpu_map(cpup->cpu_phys_number, i);
 144                 x86_set_logical_topology(&cpup->lcpu, cpup->cpu_phys_number, i);
 145         }
 146
 147         cpu_shadow_sort(ncpus);
 148         x86_validate_topology();
 149
 150         ml_set_interrupts_enabled(istate);
 151         TOPO_DBG("cpu_topology_start() LLC is L%d\n", topoParms.LLCDepth + 1);
 152
 153 #if DEVELOPMENT || DEBUG
 154         iotrace_init(ncpus);
 155         traptrace_init(ncpus);
 156 #endif /* DEVELOPMENT || DEBUG */
 157
 158         /*
 159          * Let the CPU Power Management know that the topology is stable.
 160          */
 161         topoParms.stable = TRUE;
 162         pmCPUStateInit();
 163
 164         /*
 165          * Iterate over all logical cpus finding or creating the affinity set
 166          * for their LLC cache. Each affinity set possesses a processor set
 167          * into which each logical processor is added.
 168          */
 169         TOPO_DBG("cpu_topology_start() creating affinity sets:\n");
 170         for (i = 0; i < ncpus; i++) {
 171                 cpu_data_t              *cpup = cpu_datap(i);
 172                 x86_lcpu_t              *lcpup = cpu_to_lcpu(i);
 173                 x86_cpu_cache_t         *LLC_cachep;
 174                 x86_affinity_set_t      *aset;
 175
 176                 LLC_cachep = lcpup->caches[topoParms.LLCDepth];
 177                 assert(LLC_cachep->type == CPU_CACHE_TYPE_UNIF);
 178                 aset = find_cache_affinity(LLC_cachep);
 179                 if (aset == NULL) {
 180                         aset = (x86_affinity_set_t *) kalloc(sizeof(*aset));
 181                         if (aset == NULL) {
 182                                 panic("cpu_topology_start() failed aset alloc");
 183                         }
 184                         aset->next = x86_affinities;
 185                         x86_affinities = aset;
 186                         aset->num = x86_affinity_count++;
 187                         aset->cache = LLC_cachep;
 188                         aset->pset = (i == master_cpu) ?
 189                             processor_pset(master_processor) :
 190                             pset_create(pset_node_root());
 191                         if (aset->pset == PROCESSOR_SET_NULL) {
 192                                 panic("cpu_topology_start: pset_create");
 193                         }
 194                         TOPO_DBG("\tnew set %p(%d) pset %p for cache %p\n",
 195                             aset, aset->num, aset->pset, aset->cache);
 196                 }
 197
 198                 TOPO_DBG("\tprocessor_init set %p(%d) lcpup %p(%d) cpu %p processor %p\n",
 199                     aset, aset->num, lcpup, lcpup->cpu_num, cpup, cpup->cpu_processor);
 200
 201                 if (i != master_cpu) {
 202                         processor_init(cpup->cpu_processor, i, aset->pset);
 203                 }
 204
 205                 if (lcpup->core->num_lcpus > 1) {
 206                         if (lcpup->lnum == 0) {
 207                                 lprim = cpup->cpu_processor;
 208                         }
 209
 210                         processor_set_primary(cpup->cpu_processor, lprim);
 211                 }
 212         }
 213 }
 214
 215 /* We got a request to start a CPU. Check that this CPU is within the
 216  * max cpu limit set before we do.
 217  */
 218 kern_return_t
 219 cpu_topology_start_cpu( int cpunum )
 220 {
 221         int             ncpus = machine_info.max_cpus;
 222         int             i = cpunum;
 223
 224         /* Decide whether to start a CPU, and actually start it */
 225         TOPO_DBG("cpu_topology_start() processor_start():\n");
 226         if (i < ncpus) {
 227                 TOPO_DBG("\tlcpu %d\n", cpu_datap(i)->cpu_number);
 228                 processor_start(cpu_datap(i)->cpu_processor);
 229                 return KERN_SUCCESS;
 230         } else {
 231                 return KERN_FAILURE;
 232         }
 233 }
 234
 235 static int
 236 lapicid_cmp(const void *x, const void *y)
 237 {
 238         cpu_data_t      *cpu_x = *((cpu_data_t **)(uintptr_t)x);
 239         cpu_data_t      *cpu_y = *((cpu_data_t **)(uintptr_t)y);
 240
 241         TOPO_DBG("lapicid_cmp(%p,%p) (%d,%d)\n",
 242             x, y, cpu_x->cpu_phys_number, cpu_y->cpu_phys_number);
 243         if (cpu_x->cpu_phys_number < cpu_y->cpu_phys_number) {
 244                 return -1;
 245         }
 246         if (cpu_x->cpu_phys_number == cpu_y->cpu_phys_number) {
 247                 return 0;
 248         }
 249         return 1;
 250 }
 251
 252 static x86_affinity_set_t *
 253 find_cache_affinity(x86_cpu_cache_t *l2_cachep)
 254 {
 255         x86_affinity_set_t      *aset;
 256
 257         for (aset = x86_affinities; aset != NULL; aset = aset->next) {
 258                 if (l2_cachep == aset->cache) {
 259                         break;
 260                 }
 261         }
 262         return aset;
 263 }
 264
 265 int
 266 ml_get_max_affinity_sets(void)
 267 {
 268         return x86_affinity_count;
 269 }
 270
 271 processor_set_t
 272 ml_affinity_to_pset(uint32_t affinity_num)
 273 {
 274         x86_affinity_set_t      *aset;
 275
 276         for (aset = x86_affinities; aset != NULL; aset = aset->next) {
 277                 if (affinity_num == aset->num) {
 278                         break;
 279                 }
 280         }
 281         return (aset == NULL) ? PROCESSOR_SET_NULL : aset->pset;
 282 }
 283
 284 uint64_t
 285 ml_cpu_cache_size(unsigned int level)
 286 {
 287         x86_cpu_cache_t *cachep;
 288
 289         if (level == 0) {
 290                 return machine_info.max_mem;
 291         } else if (1 <= level && level <= MAX_CACHE_DEPTH) {
 292                 cachep = current_cpu_datap()->lcpu.caches[level - 1];
 293                 return cachep ? cachep->cache_size : 0;
 294         } else {
 295                 return 0;
 296         }
 297 }
 298
 299 uint64_t
 300 ml_cpu_cache_sharing(unsigned int level)
 301 {
 302         x86_cpu_cache_t *cachep;
 303
 304         if (level == 0) {
 305                 return machine_info.max_cpus;
 306         } else if (1 <= level && level <= MAX_CACHE_DEPTH) {
 307                 cachep = current_cpu_datap()->lcpu.caches[level - 1];
 308                 return cachep ? cachep->nlcpus : 0;
 309         } else {
 310                 return 0;
 311         }
 312 }
 313
 314 #if     DEVELOPMENT || DEBUG
 315 volatile int mmiotrace_enabled = 1;
 316 int iotrace_generators = 0;
 317 int iotrace_entries_per_cpu = 0;
 318 int *iotrace_next;
 319 iotrace_entry_t **iotrace_ring;
 320
 321 volatile int traptrace_enabled = 1;
 322 int traptrace_generators = 0;
 323 int traptrace_entries_per_cpu = 0;
 324 int *traptrace_next;
 325 traptrace_entry_t **traptrace_ring;
 326
 327 static void
 328 init_trace_bufs(int cpucnt, int entries_per_cpu, void ***ring, int entry_size,
 329     int **next_array, int *allocated_entries_per_cpu, int *allocated_generator_count)
 330 {
 331         int i;
 332
 333         *next_array = kalloc_tag(cpucnt * sizeof(int), VM_KERN_MEMORY_DIAG);
 334         if (__improbable(*next_array == NULL)) {
 335                 *allocated_generator_count = 0;
 336                 return;
 337         } else {
 338                 bzero(*next_array, cpucnt * sizeof(int));
 339         }
 340
 341         *ring = kalloc_tag(cpucnt * sizeof(void *), VM_KERN_MEMORY_DIAG);
 342         if (__improbable(*ring == NULL)) {
 343                 kfree(*next_array, cpucnt * sizeof(int));
 344                 *next_array = NULL;
 345                 *allocated_generator_count = 0;
 346                 return;
 347         }
 348         for (i = 0; i < cpucnt; i++) {
 349                 (*ring)[i] = kalloc_tag(entries_per_cpu * entry_size, VM_KERN_MEMORY_DIAG);
 350                 if (__improbable((*ring)[i] == NULL)) {
 351                         kfree(*next_array, cpucnt * sizeof(int));
 352                         *next_array = NULL;
 353                         for (int j = 0; j < i; j++) {
 354                                 kfree((*ring)[j], entries_per_cpu * entry_size);
 355                         }
 356                         kfree(*ring, cpucnt * sizeof(void *));
 357                         *ring = NULL;
 358                         return;
 359                 }
 360                 bzero((*ring)[i], entries_per_cpu * entry_size);
 361         }
 362
 363         *allocated_entries_per_cpu = entries_per_cpu;
 364         *allocated_generator_count = cpucnt;
 365 }
 366
 367
 368 static void
 369 init_iotrace_bufs(int cpucnt, int entries_per_cpu)
 370 {
 371         init_trace_bufs(cpucnt, entries_per_cpu, (void ***)&iotrace_ring, sizeof(iotrace_entry_t),
 372             &iotrace_next, &iotrace_entries_per_cpu, &iotrace_generators);
 373 }
 374
 375 static void
 376 init_traptrace_bufs(int cpucnt, int entries_per_cpu)
 377 {
 378         init_trace_bufs(cpucnt, entries_per_cpu, (void ***)&traptrace_ring, sizeof(traptrace_entry_t),
 379             &traptrace_next, &traptrace_entries_per_cpu, &traptrace_generators);
 380 }
 381
 382 static void
 383 gentrace_configure_from_bootargs(const char *ena_prop, int *ena_valp, const char *epc_prop,
 384     int *epcp, int max_epc, int def_epc, int override)
 385 {
 386         if (kern_feature_override(override)) {
 387                 *ena_valp = 0;
 388         }
 389
 390         (void) PE_parse_boot_argn(ena_prop, ena_valp, sizeof(*ena_valp));
 391
 392         if (*ena_valp == 0) {
 393                 return;
 394         }
 395
 396         if (PE_parse_boot_argn(epc_prop, epcp, sizeof(*epcp)) &&
 397             (*epcp < 1 || *epcp > max_epc)) {
 398                 *epcp = def_epc;
 399         }
 400 }
 401
 402 void
 403 iotrace_init(int ncpus)
 404 {
 405         int entries_per_cpu = DEFAULT_IOTRACE_ENTRIES_PER_CPU;
 406         int enable = mmiotrace_enabled;
 407
 408         gentrace_configure_from_bootargs("iotrace", &enable, "iotrace_epc", &entries_per_cpu,
 409             IOTRACE_MAX_ENTRIES_PER_CPU, DEFAULT_IOTRACE_ENTRIES_PER_CPU, KF_IOTRACE_OVRD);
 410
 411         mmiotrace_enabled = enable;
 412
 413         if (mmiotrace_enabled) {
 414                 init_iotrace_bufs(ncpus, entries_per_cpu);
 415         }
 416 }
 417
 418 void
 419 traptrace_init(int ncpus)
 420 {
 421         int entries_per_cpu = DEFAULT_TRAPTRACE_ENTRIES_PER_CPU;
 422         int enable = traptrace_enabled;
 423
 424         gentrace_configure_from_bootargs("traptrace", &enable, "traptrace_epc", &entries_per_cpu,
 425             TRAPTRACE_MAX_ENTRIES_PER_CPU, DEFAULT_TRAPTRACE_ENTRIES_PER_CPU, KF_TRAPTRACE_OVRD);
 426
 427         traptrace_enabled = enable;
 428
 429         if (traptrace_enabled) {
 430                 init_traptrace_bufs(ncpus, entries_per_cpu);
 431         }
 432 }
 433
 434 #endif /* DEVELOPMENT || DEBUG */