[apple/xnu.git] / osfmk / i386 / cpu_topology.c

/*
 * Copyright (c) 2007-2010 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <mach/machine.h>
#include <mach/processor.h>
#include <kern/kalloc.h>
#include <i386/cpu_affinity.h>
#include <i386/cpu_topology.h>
#include <i386/cpu_threads.h>
#include <i386/machine_cpu.h>
#include <i386/bit_routines.h>
#include <i386/cpu_data.h>
#include <i386/lapic.h>
#include <i386/machine_routines.h>
#include <stddef.h>

__private_extern__ void qsort(
	void * array,
	size_t nmembers,
	size_t member_size,
	int (*)(const void *, const void *));

static int lapicid_cmp(const void *x, const void *y);
static x86_affinity_set_t *find_cache_affinity(x86_cpu_cache_t *L2_cachep);

x86_affinity_set_t      *x86_affinities = NULL;
static int              x86_affinity_count = 0;

extern cpu_data_t cpshadows[];

#if DEVELOPMENT || DEBUG
void iotrace_init(int ncpus);
#endif /* DEVELOPMENT || DEBUG */


/* Re-sort double-mapped CPU data shadows after topology discovery sorts the
 * primary CPU data structures by physical/APIC CPU ID.
 */
static void
cpu_shadow_sort(int ncpus)
{
	for (int i = 0; i < ncpus; i++) {
		cpu_data_t      *cpup = cpu_datap(i);
		ptrdiff_t       coff = cpup - cpu_datap(0);

		cpup->cd_shadow = &cpshadows[coff];
	}
}

/*
 * cpu_topology_sort() is called after all processors have been registered but
 * before any non-boot processor is started.  We establish canonical logical
 * processor numbering - logical cpus must be contiguous, zero-based and
 * assigned in physical (local apic id) order.  This step is required because
 * the discovery/registration order is non-deterministic - cores are registered
 * in differing orders over boots.  Enforcing canonical numbering simplifies
 * identification of processors.
 */
void
cpu_topology_sort(int ncpus)
{
	int             i;
	boolean_t       istate;
	processor_t             lprim = NULL;

	assert(machine_info.physical_cpu == 1);
	assert(machine_info.logical_cpu == 1);
	assert(master_cpu == 0);
	assert(cpu_number() == 0);
	assert(cpu_datap(0)->cpu_number == 0);

	/* Lights out for this */
	istate = ml_set_interrupts_enabled(FALSE);

	if (topo_dbg) {
		TOPO_DBG("cpu_topology_start() %d cpu%s registered\n",
		    ncpus, (ncpus > 1) ? "s" : "");
		for (i = 0; i < ncpus; i++) {
			cpu_data_t      *cpup = cpu_datap(i);
			TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",
			    i, (void *) cpup, cpup->cpu_phys_number);
		}
	}

	/*
	 * Re-order the cpu_data_ptr vector sorting by physical id.
	 * Skip the boot processor, it's required to be correct.
	 */
	if (ncpus > 1) {
		qsort((void *) &cpu_data_ptr[1],
		    ncpus - 1,
		    sizeof(cpu_data_t *),
		    lapicid_cmp);
	}
	if (topo_dbg) {
		TOPO_DBG("cpu_topology_start() after sorting:\n");
		for (i = 0; i < ncpus; i++) {
			cpu_data_t      *cpup = cpu_datap(i);
			TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",
			    i, (void *) cpup, cpup->cpu_phys_number);
		}
	}

	/*
	 * Finalize logical numbers and map kept by the lapic code.
	 */
	for (i = 0; i < ncpus; i++) {
		cpu_data_t      *cpup = cpu_datap(i);

		if (cpup->cpu_number != i) {
			kprintf("cpu_datap(%d):%p local apic id 0x%x "
			    "remapped from %d\n",
			    i, cpup, cpup->cpu_phys_number,
			    cpup->cpu_number);
		}
		cpup->cpu_number = i;
		lapic_cpu_map(cpup->cpu_phys_number, i);
		x86_set_logical_topology(&cpup->lcpu, cpup->cpu_phys_number, i);
	}

	cpu_shadow_sort(ncpus);
	x86_validate_topology();

	ml_set_interrupts_enabled(istate);
	TOPO_DBG("cpu_topology_start() LLC is L%d\n", topoParms.LLCDepth + 1);

#if DEVELOPMENT || DEBUG
	iotrace_init(ncpus);
#endif /* DEVELOPMENT || DEBUG */

	/*
	 * Let the CPU Power Management know that the topology is stable.
	 */
	topoParms.stable = TRUE;
	pmCPUStateInit();

	/*
	 * Iterate over all logical cpus finding or creating the affinity set
	 * for their LLC cache. Each affinity set possesses a processor set
	 * into which each logical processor is added.
	 */
	TOPO_DBG("cpu_topology_start() creating affinity sets:\n");
	for (i = 0; i < ncpus; i++) {
		cpu_data_t              *cpup = cpu_datap(i);
		x86_lcpu_t              *lcpup = cpu_to_lcpu(i);
		x86_cpu_cache_t         *LLC_cachep;
		x86_affinity_set_t      *aset;

		LLC_cachep = lcpup->caches[topoParms.LLCDepth];
		assert(LLC_cachep->type == CPU_CACHE_TYPE_UNIF);
		aset = find_cache_affinity(LLC_cachep);
		if (aset == NULL) {
			aset = (x86_affinity_set_t *) kalloc(sizeof(*aset));
			if (aset == NULL) {
				panic("cpu_topology_start() failed aset alloc");
			}
			aset->next = x86_affinities;
			x86_affinities = aset;
			aset->num = x86_affinity_count++;
			aset->cache = LLC_cachep;
			aset->pset = (i == master_cpu) ?
			    processor_pset(master_processor) :
			    pset_create(pset_node_root());
			if (aset->pset == PROCESSOR_SET_NULL) {
				panic("cpu_topology_start: pset_create");
			}
			TOPO_DBG("\tnew set %p(%d) pset %p for cache %p\n",
			    aset, aset->num, aset->pset, aset->cache);
		}

		TOPO_DBG("\tprocessor_init set %p(%d) lcpup %p(%d) cpu %p processor %p\n",
		    aset, aset->num, lcpup, lcpup->cpu_num, cpup, cpup->cpu_processor);

		if (i != master_cpu) {
			processor_init(cpup->cpu_processor, i, aset->pset);
		}

		if (lcpup->core->num_lcpus > 1) {
			if (lcpup->lnum == 0) {
				lprim = cpup->cpu_processor;
			}

			processor_set_primary(cpup->cpu_processor, lprim);
		}
	}
}

/* We got a request to start a CPU. Check that this CPU is within the
 * max cpu limit set before we do.
 */
kern_return_t
cpu_topology_start_cpu( int cpunum )
{
	int             ncpus = machine_info.max_cpus;
	int             i = cpunum;

	/* Decide whether to start a CPU, and actually start it */
	TOPO_DBG("cpu_topology_start() processor_start():\n");
	if (i < ncpus) {
		TOPO_DBG("\tlcpu %d\n", cpu_datap(i)->cpu_number);
		processor_start(cpu_datap(i)->cpu_processor);
		return KERN_SUCCESS;
	} else {
		return KERN_FAILURE;
	}
}

static int
lapicid_cmp(const void *x, const void *y)
{
	cpu_data_t      *cpu_x = *((cpu_data_t **)(uintptr_t)x);
	cpu_data_t      *cpu_y = *((cpu_data_t **)(uintptr_t)y);

	TOPO_DBG("lapicid_cmp(%p,%p) (%d,%d)\n",
	    x, y, cpu_x->cpu_phys_number, cpu_y->cpu_phys_number);
	if (cpu_x->cpu_phys_number < cpu_y->cpu_phys_number) {
		return -1;
	}
	if (cpu_x->cpu_phys_number == cpu_y->cpu_phys_number) {
		return 0;
	}
	return 1;
}

static x86_affinity_set_t *
find_cache_affinity(x86_cpu_cache_t *l2_cachep)
{
	x86_affinity_set_t      *aset;

	for (aset = x86_affinities; aset != NULL; aset = aset->next) {
		if (l2_cachep == aset->cache) {
			break;
		}
	}
	return aset;
}

int
ml_get_max_affinity_sets(void)
{
	return x86_affinity_count;
}

processor_set_t
ml_affinity_to_pset(uint32_t affinity_num)
{
	x86_affinity_set_t      *aset;

	for (aset = x86_affinities; aset != NULL; aset = aset->next) {
		if (affinity_num == aset->num) {
			break;
		}
	}
	return (aset == NULL) ? PROCESSOR_SET_NULL : aset->pset;
}

uint64_t
ml_cpu_cache_size(unsigned int level)
{
	x86_cpu_cache_t *cachep;

	if (level == 0) {
		return machine_info.max_mem;
	} else if (1 <= level && level <= MAX_CACHE_DEPTH) {
		cachep = current_cpu_datap()->lcpu.caches[level - 1];
		return cachep ? cachep->cache_size : 0;
	} else {
		return 0;
	}
}

uint64_t
ml_cpu_cache_sharing(unsigned int level)
{
	x86_cpu_cache_t *cachep;

	if (level == 0) {
		return machine_info.max_cpus;
	} else if (1 <= level && level <= MAX_CACHE_DEPTH) {
		cachep = current_cpu_datap()->lcpu.caches[level - 1];
		return cachep ? cachep->nlcpus : 0;
	} else {
		return 0;
	}
}

#if     DEVELOPMENT || DEBUG
volatile int mmiotrace_enabled = 1;
int iotrace_generators = 0;
int iotrace_entries_per_cpu = 0;
int *iotrace_next;
iotrace_entry_t **iotrace_ring;

void
init_iotrace_bufs(int cpucnt, int entries_per_cpu)
{
	int i;

	iotrace_next = kalloc_tag(cpucnt * sizeof(int), VM_KERN_MEMORY_DIAG);
	if (__improbable(iotrace_next == NULL)) {
		iotrace_generators = 0;
		return;
	} else {
		bzero(iotrace_next, cpucnt * sizeof(int));
	}

	iotrace_ring = kalloc_tag(cpucnt * sizeof(iotrace_entry_t *), VM_KERN_MEMORY_DIAG);
	if (__improbable(iotrace_ring == NULL)) {
		kfree(iotrace_next, cpucnt * sizeof(int));
		iotrace_generators = 0;
		return;
	}
	for (i = 0; i < cpucnt; i++) {
		iotrace_ring[i] = kalloc_tag(entries_per_cpu * sizeof(iotrace_entry_t), VM_KERN_MEMORY_DIAG);
		if (__improbable(iotrace_ring[i] == NULL)) {
			kfree(iotrace_next, cpucnt * sizeof(int));
			iotrace_next = NULL;
			for (int j = 0; j < i; j++) {
				kfree(iotrace_ring[j], entries_per_cpu * sizeof(iotrace_entry_t));
			}
			kfree(iotrace_ring, cpucnt * sizeof(iotrace_entry_t *));
			iotrace_ring = NULL;
			return;
		}
		bzero(iotrace_ring[i], entries_per_cpu * sizeof(iotrace_entry_t));
	}

	iotrace_entries_per_cpu = entries_per_cpu;
	iotrace_generators = cpucnt;
}

void
iotrace_init(int ncpus)
{
	int iot, epc;
	int entries_per_cpu;

	if (PE_parse_boot_argn("iotrace", &iot, sizeof(iot))) {
		mmiotrace_enabled = iot;
	}

	if (mmiotrace_enabled == 0) {
		return;
	}

	if (PE_parse_boot_argn("iotrace_epc", &epc, sizeof(epc)) &&
	    epc >= 1 && epc <= IOTRACE_MAX_ENTRIES_PER_CPU) {
		entries_per_cpu = epc;
	} else {
		entries_per_cpu = DEFAULT_IOTRACE_ENTRIES_PER_CPU;
	}

	init_iotrace_bufs(ncpus, entries_per_cpu);
}
#endif /* DEVELOPMENT || DEBUG */
Commit	Line	Data
2d21ac55	1	/*
d1ecb069	2	* Copyright (c) 2007-2010 Apple Inc. All rights reserved.
2d21ac55 A	3	*
2d21ac55 A	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0a7de745	5	*
2d21ac55 A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
0a7de745	14	*
2d21ac55 A	15	* Please obtain a copy of the License at
2d21ac55 A	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
0a7de745	17	*
2d21ac55 A	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
0a7de745	25	*
2d21ac55 A	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <mach/machine.h>
	30	#include <mach/processor.h>
	31	#include <kern/kalloc.h>
	32	#include <i386/cpu_affinity.h>
	33	#include <i386/cpu_topology.h>
2d21ac55 A	34	#include <i386/cpu_threads.h>
2d21ac55 A	35	#include <i386/machine_cpu.h>
fe8ab488	36	#include <i386/bit_routines.h>
b0d623f7	37	#include <i386/cpu_data.h>
593a1d5f	38	#include <i386/lapic.h>
b0d623f7	39	#include <i386/machine_routines.h>
a39ff7e2	40	#include <stddef.h>
2d21ac55	41
2d21ac55	42	__private_extern__ void qsort(
0a7de745 A	43	void * array,
	44	size_t nmembers,
	45	size_t member_size,
	46	int ()(const void , const void *));
2d21ac55 A	47
	48	static int lapicid_cmp(const void x, const void y);
	49	static x86_affinity_set_t find_cache_affinity(x86_cpu_cache_t L2_cachep);
	50
0a7de745 A	51	x86_affinity_set_t *x86_affinities = NULL;
0a7de745 A	52	static int x86_affinity_count = 0;
2d21ac55	53
5c9f4661	54	extern cpu_data_t cpshadows[];
0a7de745 A	55
	56	#if DEVELOPMENT \|\| DEBUG
	57	void iotrace_init(int ncpus);
	58	#endif /* DEVELOPMENT \|\| DEBUG */
	59
	60
5c9f4661 A	61	/* Re-sort double-mapped CPU data shadows after topology discovery sorts the
	62	* primary CPU data structures by physical/APIC CPU ID.
	63	*/
0a7de745 A	64	static void
	65	cpu_shadow_sort(int ncpus)
	66	{
5c9f4661	67	for (int i = 0; i < ncpus; i++) {
0a7de745 A	68	cpu_data_t *cpup = cpu_datap(i);
0a7de745 A	69	ptrdiff_t coff = cpup - cpu_datap(0);
5c9f4661 A	70
	71	cpup->cd_shadow = &cpshadows[coff];
	72	}
	73	}
	74
2d21ac55	75	/*
a39ff7e2	76	* cpu_topology_sort() is called after all processors have been registered but
0a7de745	77	* before any non-boot processor is started. We establish canonical logical
a39ff7e2 A	78	* processor numbering - logical cpus must be contiguous, zero-based and
	79	* assigned in physical (local apic id) order. This step is required because
	80	* the discovery/registration order is non-deterministic - cores are registered
	81	* in differing orders over boots. Enforcing canonical numbering simplifies
	82	* identification of processors.
	83	*/
2d21ac55	84	void
b0d623f7	85	cpu_topology_sort(int ncpus)
2d21ac55	86	{
0a7de745 A	87	int i;
	88	boolean_t istate;
	89	processor_t lprim = NULL;
2d21ac55 A	90
	91	assert(machine_info.physical_cpu == 1);
	92	assert(machine_info.logical_cpu == 1);
	93	assert(master_cpu == 0);
	94	assert(cpu_number() == 0);
	95	assert(cpu_datap(0)->cpu_number == 0);
b0d623f7	96
2d21ac55 A	97	/* Lights out for this */
	98	istate = ml_set_interrupts_enabled(FALSE);
	99
7ddcb079 A	100	if (topo_dbg) {
7ddcb079 A	101	TOPO_DBG("cpu_topology_start() %d cpu%s registered\n",
0a7de745	102	ncpus, (ncpus > 1) ? "s" : "");
7ddcb079	103	for (i = 0; i < ncpus; i++) {
0a7de745	104	cpu_data_t *cpup = cpu_datap(i);
7ddcb079	105	TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",
0a7de745	106	i, (void *) cpup, cpup->cpu_phys_number);
7ddcb079	107	}
2d21ac55	108	}
7ddcb079	109
2d21ac55 A	110	/*
	111	* Re-order the cpu_data_ptr vector sorting by physical id.
	112	* Skip the boot processor, it's required to be correct.
	113	*/
	114	if (ncpus > 1) {
	115	qsort((void *) &cpu_data_ptr[1],
0a7de745 A	116	ncpus - 1,
	117	sizeof(cpu_data_t *),
	118	lapicid_cmp);
2d21ac55	119	}
7ddcb079 A	120	if (topo_dbg) {
	121	TOPO_DBG("cpu_topology_start() after sorting:\n");
	122	for (i = 0; i < ncpus; i++) {
0a7de745	123	cpu_data_t *cpup = cpu_datap(i);
7ddcb079	124	TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",
0a7de745	125	i, (void *) cpup, cpup->cpu_phys_number);
7ddcb079	126	}
2d21ac55	127	}
2d21ac55 A	128
2d21ac55 A	129	/*
39236c6e	130	* Finalize logical numbers and map kept by the lapic code.
2d21ac55	131	*/
39236c6e	132	for (i = 0; i < ncpus; i++) {
0a7de745	133	cpu_data_t *cpup = cpu_datap(i);
2d21ac55 A	134
2d21ac55 A	135	if (cpup->cpu_number != i) {
b0d623f7	136	kprintf("cpu_datap(%d):%p local apic id 0x%x "
0a7de745 A	137	"remapped from %d\n",
	138	i, cpup, cpup->cpu_phys_number,
	139	cpup->cpu_number);
2d21ac55 A	140	}
2d21ac55 A	141	cpup->cpu_number = i;
2d21ac55	142	lapic_cpu_map(cpup->cpu_phys_number, i);
39236c6e	143	x86_set_logical_topology(&cpup->lcpu, cpup->cpu_phys_number, i);
2d21ac55 A	144	}
2d21ac55 A	145
5c9f4661	146	cpu_shadow_sort(ncpus);
39236c6e	147	x86_validate_topology();
593a1d5f	148
2d21ac55	149	ml_set_interrupts_enabled(istate);
7ddcb079	150	TOPO_DBG("cpu_topology_start() LLC is L%d\n", topoParms.LLCDepth + 1);
2d21ac55	151
0a7de745 A	152	#if DEVELOPMENT \|\| DEBUG
	153	iotrace_init(ncpus);
	154	#endif /* DEVELOPMENT \|\| DEBUG */
	155
d1ecb069 A	156	/*
	157	* Let the CPU Power Management know that the topology is stable.
	158	*/
	159	topoParms.stable = TRUE;
	160	pmCPUStateInit();
	161
2d21ac55 A	162	/*
2d21ac55 A	163	* Iterate over all logical cpus finding or creating the affinity set
593a1d5f	164	* for their LLC cache. Each affinity set possesses a processor set
2d21ac55 A	165	* into which each logical processor is added.
2d21ac55 A	166	*/
7ddcb079	167	TOPO_DBG("cpu_topology_start() creating affinity sets:\n");
2d21ac55	168	for (i = 0; i < ncpus; i++) {
0a7de745 A	169	cpu_data_t *cpup = cpu_datap(i);
	170	x86_lcpu_t *lcpup = cpu_to_lcpu(i);
	171	x86_cpu_cache_t *LLC_cachep;
	172	x86_affinity_set_t *aset;
2d21ac55	173
593a1d5f A	174	LLC_cachep = lcpup->caches[topoParms.LLCDepth];
593a1d5f A	175	assert(LLC_cachep->type == CPU_CACHE_TYPE_UNIF);
0a7de745	176	aset = find_cache_affinity(LLC_cachep);
2d21ac55 A	177	if (aset == NULL) {
2d21ac55 A	178	aset = (x86_affinity_set_t ) kalloc(sizeof(aset));
0a7de745	179	if (aset == NULL) {
2d21ac55	180	panic("cpu_topology_start() failed aset alloc");
0a7de745	181	}
2d21ac55 A	182	aset->next = x86_affinities;
	183	x86_affinities = aset;
	184	aset->num = x86_affinity_count++;
593a1d5f	185	aset->cache = LLC_cachep;
2d21ac55	186	aset->pset = (i == master_cpu) ?
0a7de745 A	187	processor_pset(master_processor) :
	188	pset_create(pset_node_root());
	189	if (aset->pset == PROCESSOR_SET_NULL) {
2d21ac55	190	panic("cpu_topology_start: pset_create");
0a7de745	191	}
7ddcb079	192	TOPO_DBG("\tnew set %p(%d) pset %p for cache %p\n",
0a7de745	193	aset, aset->num, aset->pset, aset->cache);
2d21ac55 A	194	}
2d21ac55 A	195
7ddcb079	196	TOPO_DBG("\tprocessor_init set %p(%d) lcpup %p(%d) cpu %p processor %p\n",
0a7de745	197	aset, aset->num, lcpup, lcpup->cpu_num, cpup, cpup->cpu_processor);
2d21ac55	198
0a7de745	199	if (i != master_cpu) {
2d21ac55	200	processor_init(cpup->cpu_processor, i, aset->pset);
0a7de745	201	}
b0d623f7 A	202
b0d623f7 A	203	if (lcpup->core->num_lcpus > 1) {
0a7de745	204	if (lcpup->lnum == 0) {
b0d623f7	205	lprim = cpup->cpu_processor;
0a7de745	206	}
b0d623f7	207
fe8ab488	208	processor_set_primary(cpup->cpu_processor, lprim);
b0d623f7	209	}
2d21ac55	210	}
b0d623f7	211	}
2d21ac55	212
b0d623f7 A	213	/* We got a request to start a CPU. Check that this CPU is within the
	214	* max cpu limit set before we do.
	215	*/
	216	kern_return_t
	217	cpu_topology_start_cpu( int cpunum )
	218	{
0a7de745 A	219	int ncpus = machine_info.max_cpus;
0a7de745 A	220	int i = cpunum;
b0d623f7 A	221
b0d623f7 A	222	/* Decide whether to start a CPU, and actually start it */
7ddcb079	223	TOPO_DBG("cpu_topology_start() processor_start():\n");
0a7de745	224	if (i < ncpus) {
7ddcb079	225	TOPO_DBG("\tlcpu %d\n", cpu_datap(i)->cpu_number);
0a7de745	226	processor_start(cpu_datap(i)->cpu_processor);
b0d623f7	227	return KERN_SUCCESS;
0a7de745 A	228	} else {
0a7de745 A	229	return KERN_FAILURE;
2d21ac55 A	230	}
	231	}
	232
	233	static int
	234	lapicid_cmp(const void x, const void y)
	235	{
0a7de745 A	236	cpu_data_t cpu_x = ((cpu_data_t **)(uintptr_t)x);
0a7de745 A	237	cpu_data_t cpu_y = ((cpu_data_t **)(uintptr_t)y);
2d21ac55	238
7ddcb079	239	TOPO_DBG("lapicid_cmp(%p,%p) (%d,%d)\n",
0a7de745 A	240	x, y, cpu_x->cpu_phys_number, cpu_y->cpu_phys_number);
0a7de745 A	241	if (cpu_x->cpu_phys_number < cpu_y->cpu_phys_number) {
2d21ac55	242	return -1;
0a7de745 A	243	}
0a7de745 A	244	if (cpu_x->cpu_phys_number == cpu_y->cpu_phys_number) {
2d21ac55	245	return 0;
0a7de745	246	}
2d21ac55 A	247	return 1;
	248	}
	249
	250	static x86_affinity_set_t *
	251	find_cache_affinity(x86_cpu_cache_t *l2_cachep)
	252	{
0a7de745	253	x86_affinity_set_t *aset;
2d21ac55 A	254
2d21ac55 A	255	for (aset = x86_affinities; aset != NULL; aset = aset->next) {
0a7de745	256	if (l2_cachep == aset->cache) {
2d21ac55	257	break;
0a7de745	258	}
2d21ac55	259	}
0a7de745	260	return aset;
2d21ac55 A	261	}
	262
	263	int
	264	ml_get_max_affinity_sets(void)
	265	{
	266	return x86_affinity_count;
	267	}
	268
	269	processor_set_t
0a7de745	270	ml_affinity_to_pset(uint32_t affinity_num)
2d21ac55	271	{
0a7de745	272	x86_affinity_set_t *aset;
2d21ac55 A	273
2d21ac55 A	274	for (aset = x86_affinities; aset != NULL; aset = aset->next) {
0a7de745	275	if (affinity_num == aset->num) {
2d21ac55	276	break;
0a7de745	277	}
2d21ac55	278	}
593a1d5f	279	return (aset == NULL) ? PROCESSOR_SET_NULL : aset->pset;
2d21ac55 A	280	}
	281
	282	uint64_t
	283	ml_cpu_cache_size(unsigned int level)
	284	{
0a7de745	285	x86_cpu_cache_t *cachep;
2d21ac55 A	286
	287	if (level == 0) {
	288	return machine_info.max_mem;
0a7de745 A	289	} else if (1 <= level && level <= MAX_CACHE_DEPTH) {
0a7de745 A	290	cachep = current_cpu_datap()->lcpu.caches[level - 1];
2d21ac55 A	291	return cachep ? cachep->cache_size : 0;
	292	} else {
	293	return 0;
	294	}
	295	}
	296
	297	uint64_t
	298	ml_cpu_cache_sharing(unsigned int level)
	299	{
0a7de745	300	x86_cpu_cache_t *cachep;
2d21ac55 A	301
	302	if (level == 0) {
	303	return machine_info.max_cpus;
0a7de745 A	304	} else if (1 <= level && level <= MAX_CACHE_DEPTH) {
0a7de745 A	305	cachep = current_cpu_datap()->lcpu.caches[level - 1];
2d21ac55 A	306	return cachep ? cachep->nlcpus : 0;
	307	} else {
	308	return 0;
	309	}
	310	}
	311
0a7de745 A	312	#if DEVELOPMENT \|\| DEBUG
	313	volatile int mmiotrace_enabled = 1;
	314	int iotrace_generators = 0;
	315	int iotrace_entries_per_cpu = 0;
	316	int *iotrace_next;
	317	iotrace_entry_t **iotrace_ring;
	318
	319	void
	320	init_iotrace_bufs(int cpucnt, int entries_per_cpu)
	321	{
	322	int i;
	323
	324	iotrace_next = kalloc_tag(cpucnt * sizeof(int), VM_KERN_MEMORY_DIAG);
	325	if (__improbable(iotrace_next == NULL)) {
	326	iotrace_generators = 0;
	327	return;
	328	} else {
	329	bzero(iotrace_next, cpucnt * sizeof(int));
	330	}
	331
	332	iotrace_ring = kalloc_tag(cpucnt * sizeof(iotrace_entry_t *), VM_KERN_MEMORY_DIAG);
	333	if (__improbable(iotrace_ring == NULL)) {
	334	kfree(iotrace_next, cpucnt * sizeof(int));
	335	iotrace_generators = 0;
	336	return;
	337	}
	338	for (i = 0; i < cpucnt; i++) {
	339	iotrace_ring[i] = kalloc_tag(entries_per_cpu * sizeof(iotrace_entry_t), VM_KERN_MEMORY_DIAG);
	340	if (__improbable(iotrace_ring[i] == NULL)) {
	341	kfree(iotrace_next, cpucnt * sizeof(int));
	342	iotrace_next = NULL;
	343	for (int j = 0; j < i; j++) {
	344	kfree(iotrace_ring[j], entries_per_cpu * sizeof(iotrace_entry_t));
	345	}
	346	kfree(iotrace_ring, cpucnt * sizeof(iotrace_entry_t *));
	347	iotrace_ring = NULL;
	348	return;
	349	}
	350	bzero(iotrace_ring[i], entries_per_cpu * sizeof(iotrace_entry_t));
	351	}
	352
	353	iotrace_entries_per_cpu = entries_per_cpu;
	354	iotrace_generators = cpucnt;
	355	}
	356
	357	void
	358	iotrace_init(int ncpus)
	359	{
	360	int iot, epc;
	361	int entries_per_cpu;
	362
	363	if (PE_parse_boot_argn("iotrace", &iot, sizeof(iot))) {
	364	mmiotrace_enabled = iot;
	365	}
	366
	367	if (mmiotrace_enabled == 0) {
	368	return;
	369	}
	370
	371	if (PE_parse_boot_argn("iotrace_epc", &epc, sizeof(epc)) &&
	372	epc >= 1 && epc <= IOTRACE_MAX_ENTRIES_PER_CPU) {
	373	entries_per_cpu = epc;
	374	} else {
	375	entries_per_cpu = DEFAULT_IOTRACE_ENTRIES_PER_CPU;
376	}
377
378	init_iotrace_bufs(ncpus, entries_per_cpu);
379	}
380	#endif /* DEVELOPMENT \|\| DEBUG */