[apple/xnu.git] / osfmk / i386 / cpu_topology.c

/*
 * Copyright (c) 2007-2010 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <mach/machine.h>
#include <mach/processor.h>
#include <kern/kalloc.h>
#include <i386/cpu_affinity.h>
#include <i386/cpu_topology.h>
#include <i386/cpu_threads.h>
#include <i386/machine_cpu.h>
#include <i386/bit_routines.h>
#include <i386/cpu_data.h>
#include <i386/lapic.h>
#include <i386/machine_routines.h>
#include <stddef.h>

__private_extern__ void qsort(
    void * array,
    size_t nmembers,
    size_t member_size,
    int (*)(const void *, const void *));

static int lapicid_cmp(const void *x, const void *y);
static x86_affinity_set_t *find_cache_affinity(x86_cpu_cache_t *L2_cachep);

x86_affinity_set_t	*x86_affinities = NULL;
static int		x86_affinity_count = 0;

extern cpu_data_t cpshadows[];
/* Re-sort double-mapped CPU data shadows after topology discovery sorts the
 * primary CPU data structures by physical/APIC CPU ID.
 */
static void cpu_shadow_sort(int ncpus) {
	for (int i = 0; i < ncpus; i++) {
		cpu_data_t	*cpup = cpu_datap(i);
		ptrdiff_t	coff = cpup - cpu_datap(0);

		cpup->cd_shadow = &cpshadows[coff];
	}
}

/*
 * cpu_topology_sort() is called after all processors have been registered but
 * before any non-boot processor id started.  We establish canonical logical
 * processor numbering - logical cpus must be contiguous, zero-based and
 * assigned in physical (local apic id) order.  This step is required because
 * the discovery/registration order is non-deterministic - cores are registered
 * in differing orders over boots.  Enforcing canonical numbering simplifies
 * identification of processors.
 */
void
cpu_topology_sort(int ncpus)
{
	int		i;
	boolean_t	istate;
	processor_t		lprim = NULL;

	assert(machine_info.physical_cpu == 1);
	assert(machine_info.logical_cpu == 1);
	assert(master_cpu == 0);
	assert(cpu_number() == 0);
	assert(cpu_datap(0)->cpu_number == 0);

	/* Lights out for this */
	istate = ml_set_interrupts_enabled(FALSE);

	if (topo_dbg) {
		TOPO_DBG("cpu_topology_start() %d cpu%s registered\n",
			ncpus, (ncpus > 1) ? "s" : "");
		for (i = 0; i < ncpus; i++) {
			cpu_data_t	*cpup = cpu_datap(i);
			TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",
				i, (void *) cpup, cpup->cpu_phys_number);
		}
	}

	/*
	 * Re-order the cpu_data_ptr vector sorting by physical id.
	 * Skip the boot processor, it's required to be correct.
	 */
	if (ncpus > 1) {
		qsort((void *) &cpu_data_ptr[1],
			ncpus - 1,
			sizeof(cpu_data_t *),
			lapicid_cmp);
	}
	if (topo_dbg) {
		TOPO_DBG("cpu_topology_start() after sorting:\n");
		for (i = 0; i < ncpus; i++) {
			cpu_data_t	*cpup = cpu_datap(i);
			TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",
				i, (void *) cpup, cpup->cpu_phys_number);
		}
	}

	/*
	 * Finalize logical numbers and map kept by the lapic code.
	 */
	for (i = 0; i < ncpus; i++) {
		cpu_data_t	*cpup = cpu_datap(i);

		if (cpup->cpu_number != i) {
			kprintf("cpu_datap(%d):%p local apic id 0x%x "
				"remapped from %d\n",
				i, cpup, cpup->cpu_phys_number,
				cpup->cpu_number);
		}
		cpup->cpu_number = i;
		lapic_cpu_map(cpup->cpu_phys_number, i);
		x86_set_logical_topology(&cpup->lcpu, cpup->cpu_phys_number, i);
	}

	cpu_shadow_sort(ncpus);
	x86_validate_topology();

	ml_set_interrupts_enabled(istate);
	TOPO_DBG("cpu_topology_start() LLC is L%d\n", topoParms.LLCDepth + 1);

	/*
	 * Let the CPU Power Management know that the topology is stable.
	 */
	topoParms.stable = TRUE;
	pmCPUStateInit();

	/*
	 * Iterate over all logical cpus finding or creating the affinity set
	 * for their LLC cache. Each affinity set possesses a processor set
	 * into which each logical processor is added.
	 */
	TOPO_DBG("cpu_topology_start() creating affinity sets:\n");
	for (i = 0; i < ncpus; i++) {
		cpu_data_t		*cpup = cpu_datap(i);
		x86_lcpu_t		*lcpup = cpu_to_lcpu(i);
		x86_cpu_cache_t		*LLC_cachep;
		x86_affinity_set_t	*aset;

		LLC_cachep = lcpup->caches[topoParms.LLCDepth];
		assert(LLC_cachep->type == CPU_CACHE_TYPE_UNIF);
		aset = find_cache_affinity(LLC_cachep); 
		if (aset == NULL) {
			aset = (x86_affinity_set_t *) kalloc(sizeof(*aset));
			if (aset == NULL)
				panic("cpu_topology_start() failed aset alloc");
			aset->next = x86_affinities;
			x86_affinities = aset;
			aset->num = x86_affinity_count++;
			aset->cache = LLC_cachep;
			aset->pset = (i == master_cpu) ?
					processor_pset(master_processor) :
					pset_create(pset_node_root());
			if (aset->pset == PROCESSOR_SET_NULL)
				panic("cpu_topology_start: pset_create");
			TOPO_DBG("\tnew set %p(%d) pset %p for cache %p\n",
				aset, aset->num, aset->pset, aset->cache);
		}

		TOPO_DBG("\tprocessor_init set %p(%d) lcpup %p(%d) cpu %p processor %p\n",
			aset, aset->num, lcpup, lcpup->cpu_num, cpup, cpup->cpu_processor);

		if (i != master_cpu)
			processor_init(cpup->cpu_processor, i, aset->pset);

		if (lcpup->core->num_lcpus > 1) {
			if (lcpup->lnum == 0)
				lprim = cpup->cpu_processor;

			processor_set_primary(cpup->cpu_processor, lprim);
		}
	}
}

/* We got a request to start a CPU. Check that this CPU is within the
 * max cpu limit set before we do.
 */
kern_return_t
cpu_topology_start_cpu( int cpunum )
{
	int		ncpus = machine_info.max_cpus;
	int		i = cpunum;

	/* Decide whether to start a CPU, and actually start it */
	TOPO_DBG("cpu_topology_start() processor_start():\n");
	if( i < ncpus)
	{
		TOPO_DBG("\tlcpu %d\n", cpu_datap(i)->cpu_number);
		processor_start(cpu_datap(i)->cpu_processor); 
		return KERN_SUCCESS;
	}
	else
	    return KERN_FAILURE;
}

static int
lapicid_cmp(const void *x, const void *y)
{
	cpu_data_t	*cpu_x = *((cpu_data_t **)(uintptr_t)x);
	cpu_data_t	*cpu_y = *((cpu_data_t **)(uintptr_t)y);

	TOPO_DBG("lapicid_cmp(%p,%p) (%d,%d)\n",
		x, y, cpu_x->cpu_phys_number, cpu_y->cpu_phys_number);
	if (cpu_x->cpu_phys_number < cpu_y->cpu_phys_number)
		return -1;
	if (cpu_x->cpu_phys_number == cpu_y->cpu_phys_number)
		return 0;
	return 1;
}

static x86_affinity_set_t *
find_cache_affinity(x86_cpu_cache_t *l2_cachep)
{
	x86_affinity_set_t	*aset;

	for (aset = x86_affinities; aset != NULL; aset = aset->next) {
		if (l2_cachep == aset->cache)
			break;
	}
	return aset;			
}

int
ml_get_max_affinity_sets(void)
{
	return x86_affinity_count;
}

processor_set_t
ml_affinity_to_pset(uint32_t affinity_num) 
{
	x86_affinity_set_t	*aset;

	for (aset = x86_affinities; aset != NULL; aset = aset->next) {
		if (affinity_num == aset->num)
			break;
	}
	return (aset == NULL) ? PROCESSOR_SET_NULL : aset->pset;
}

uint64_t
ml_cpu_cache_size(unsigned int level)
{
	x86_cpu_cache_t	*cachep;

	if (level == 0) {
		return machine_info.max_mem;
	} else if ( 1 <= level && level <= MAX_CACHE_DEPTH) {
		cachep = current_cpu_datap()->lcpu.caches[level-1];
		return cachep ? cachep->cache_size : 0;
	} else {
		return 0;
	}
}

uint64_t
ml_cpu_cache_sharing(unsigned int level)
{
	x86_cpu_cache_t	*cachep;

	if (level == 0) {
		return machine_info.max_cpus;
	} else if ( 1 <= level && level <= MAX_CACHE_DEPTH) {
		cachep = current_cpu_datap()->lcpu.caches[level-1];
		return cachep ? cachep->nlcpus : 0;
	} else {
		return 0;
	}
}
Commit	Line	Data
2d21ac55	1	/*
d1ecb069	2	* Copyright (c) 2007-2010 Apple Inc. All rights reserved.
2d21ac55 A	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <mach/machine.h>
	30	#include <mach/processor.h>
	31	#include <kern/kalloc.h>
	32	#include <i386/cpu_affinity.h>
	33	#include <i386/cpu_topology.h>
2d21ac55 A	34	#include <i386/cpu_threads.h>
2d21ac55 A	35	#include <i386/machine_cpu.h>
fe8ab488	36	#include <i386/bit_routines.h>
b0d623f7	37	#include <i386/cpu_data.h>
593a1d5f	38	#include <i386/lapic.h>
b0d623f7	39	#include <i386/machine_routines.h>
a39ff7e2	40	#include <stddef.h>
2d21ac55	41
2d21ac55 A	42	__private_extern__ void qsort(
	43	void * array,
	44	size_t nmembers,
	45	size_t member_size,
	46	int ()(const void , const void *));
	47
	48	static int lapicid_cmp(const void x, const void y);
	49	static x86_affinity_set_t find_cache_affinity(x86_cpu_cache_t L2_cachep);
	50
	51	x86_affinity_set_t *x86_affinities = NULL;
	52	static int x86_affinity_count = 0;
	53
5c9f4661 A	54	extern cpu_data_t cpshadows[];
	55	/* Re-sort double-mapped CPU data shadows after topology discovery sorts the
	56	* primary CPU data structures by physical/APIC CPU ID.
	57	*/
	58	static void cpu_shadow_sort(int ncpus) {
	59	for (int i = 0; i < ncpus; i++) {
	60	cpu_data_t *cpup = cpu_datap(i);
	61	ptrdiff_t coff = cpup - cpu_datap(0);
	62
	63	cpup->cd_shadow = &cpshadows[coff];
	64	}
	65	}
	66
2d21ac55	67	/*
a39ff7e2 A	68	* cpu_topology_sort() is called after all processors have been registered but
	69	* before any non-boot processor id started. We establish canonical logical
	70	* processor numbering - logical cpus must be contiguous, zero-based and
	71	* assigned in physical (local apic id) order. This step is required because
	72	* the discovery/registration order is non-deterministic - cores are registered
	73	* in differing orders over boots. Enforcing canonical numbering simplifies
	74	* identification of processors.
	75	*/
2d21ac55	76	void
b0d623f7	77	cpu_topology_sort(int ncpus)
2d21ac55	78	{
2d21ac55 A	79	int i;
2d21ac55 A	80	boolean_t istate;
b0d623f7	81	processor_t lprim = NULL;
2d21ac55 A	82
	83	assert(machine_info.physical_cpu == 1);
	84	assert(machine_info.logical_cpu == 1);
	85	assert(master_cpu == 0);
	86	assert(cpu_number() == 0);
	87	assert(cpu_datap(0)->cpu_number == 0);
b0d623f7	88
2d21ac55 A	89	/* Lights out for this */
	90	istate = ml_set_interrupts_enabled(FALSE);
	91
7ddcb079 A	92	if (topo_dbg) {
	93	TOPO_DBG("cpu_topology_start() %d cpu%s registered\n",
	94	ncpus, (ncpus > 1) ? "s" : "");
	95	for (i = 0; i < ncpus; i++) {
	96	cpu_data_t *cpup = cpu_datap(i);
	97	TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",
	98	i, (void *) cpup, cpup->cpu_phys_number);
	99	}
2d21ac55	100	}
7ddcb079	101
2d21ac55 A	102	/*
	103	* Re-order the cpu_data_ptr vector sorting by physical id.
	104	* Skip the boot processor, it's required to be correct.
	105	*/
	106	if (ncpus > 1) {
	107	qsort((void *) &cpu_data_ptr[1],
	108	ncpus - 1,
	109	sizeof(cpu_data_t *),
	110	lapicid_cmp);
	111	}
7ddcb079 A	112	if (topo_dbg) {
	113	TOPO_DBG("cpu_topology_start() after sorting:\n");
	114	for (i = 0; i < ncpus; i++) {
	115	cpu_data_t *cpup = cpu_datap(i);
	116	TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",
	117	i, (void *) cpup, cpup->cpu_phys_number);
	118	}
2d21ac55	119	}
2d21ac55 A	120
2d21ac55 A	121	/*
39236c6e	122	* Finalize logical numbers and map kept by the lapic code.
2d21ac55	123	*/
39236c6e	124	for (i = 0; i < ncpus; i++) {
2d21ac55 A	125	cpu_data_t *cpup = cpu_datap(i);
	126
	127	if (cpup->cpu_number != i) {
b0d623f7	128	kprintf("cpu_datap(%d):%p local apic id 0x%x "
2d21ac55	129	"remapped from %d\n",
b0d623f7	130	i, cpup, cpup->cpu_phys_number,
2d21ac55 A	131	cpup->cpu_number);
	132	}
	133	cpup->cpu_number = i;
2d21ac55	134	lapic_cpu_map(cpup->cpu_phys_number, i);
39236c6e	135	x86_set_logical_topology(&cpup->lcpu, cpup->cpu_phys_number, i);
2d21ac55 A	136	}
2d21ac55 A	137
5c9f4661	138	cpu_shadow_sort(ncpus);
39236c6e	139	x86_validate_topology();
593a1d5f	140
2d21ac55	141	ml_set_interrupts_enabled(istate);
7ddcb079	142	TOPO_DBG("cpu_topology_start() LLC is L%d\n", topoParms.LLCDepth + 1);
2d21ac55	143
d1ecb069 A	144	/*
	145	* Let the CPU Power Management know that the topology is stable.
	146	*/
	147	topoParms.stable = TRUE;
	148	pmCPUStateInit();
	149
2d21ac55 A	150	/*
2d21ac55 A	151	* Iterate over all logical cpus finding or creating the affinity set
593a1d5f	152	* for their LLC cache. Each affinity set possesses a processor set
2d21ac55 A	153	* into which each logical processor is added.
2d21ac55 A	154	*/
7ddcb079	155	TOPO_DBG("cpu_topology_start() creating affinity sets:\n");
2d21ac55 A	156	for (i = 0; i < ncpus; i++) {
	157	cpu_data_t *cpup = cpu_datap(i);
	158	x86_lcpu_t *lcpup = cpu_to_lcpu(i);
593a1d5f	159	x86_cpu_cache_t *LLC_cachep;
2d21ac55 A	160	x86_affinity_set_t *aset;
2d21ac55 A	161
593a1d5f A	162	LLC_cachep = lcpup->caches[topoParms.LLCDepth];
	163	assert(LLC_cachep->type == CPU_CACHE_TYPE_UNIF);
	164	aset = find_cache_affinity(LLC_cachep);
2d21ac55 A	165	if (aset == NULL) {
	166	aset = (x86_affinity_set_t ) kalloc(sizeof(aset));
	167	if (aset == NULL)
	168	panic("cpu_topology_start() failed aset alloc");
	169	aset->next = x86_affinities;
	170	x86_affinities = aset;
	171	aset->num = x86_affinity_count++;
593a1d5f	172	aset->cache = LLC_cachep;
2d21ac55 A	173	aset->pset = (i == master_cpu) ?
	174	processor_pset(master_processor) :
	175	pset_create(pset_node_root());
	176	if (aset->pset == PROCESSOR_SET_NULL)
	177	panic("cpu_topology_start: pset_create");
7ddcb079	178	TOPO_DBG("\tnew set %p(%d) pset %p for cache %p\n",
2d21ac55 A	179	aset, aset->num, aset->pset, aset->cache);
	180	}
	181
7ddcb079	182	TOPO_DBG("\tprocessor_init set %p(%d) lcpup %p(%d) cpu %p processor %p\n",
593a1d5f	183	aset, aset->num, lcpup, lcpup->cpu_num, cpup, cpup->cpu_processor);
2d21ac55 A	184
	185	if (i != master_cpu)
	186	processor_init(cpup->cpu_processor, i, aset->pset);
b0d623f7 A	187
	188	if (lcpup->core->num_lcpus > 1) {
	189	if (lcpup->lnum == 0)
	190	lprim = cpup->cpu_processor;
	191
fe8ab488	192	processor_set_primary(cpup->cpu_processor, lprim);
b0d623f7	193	}
2d21ac55	194	}
b0d623f7	195	}
2d21ac55	196
b0d623f7 A	197	/* We got a request to start a CPU. Check that this CPU is within the
	198	* max cpu limit set before we do.
	199	*/
	200	kern_return_t
	201	cpu_topology_start_cpu( int cpunum )
	202	{
	203	int ncpus = machine_info.max_cpus;
	204	int i = cpunum;
	205
	206	/* Decide whether to start a CPU, and actually start it */
7ddcb079	207	TOPO_DBG("cpu_topology_start() processor_start():\n");
b0d623f7 A	208	if( i < ncpus)
b0d623f7 A	209	{
7ddcb079	210	TOPO_DBG("\tlcpu %d\n", cpu_datap(i)->cpu_number);
2d21ac55	211	processor_start(cpu_datap(i)->cpu_processor);
b0d623f7	212	return KERN_SUCCESS;
2d21ac55	213	}
b0d623f7 A	214	else
b0d623f7 A	215	return KERN_FAILURE;
2d21ac55 A	216	}
	217
	218	static int
	219	lapicid_cmp(const void x, const void y)
	220	{
	221	cpu_data_t cpu_x = ((cpu_data_t **)(uintptr_t)x);
	222	cpu_data_t cpu_y = ((cpu_data_t **)(uintptr_t)y);
	223
7ddcb079	224	TOPO_DBG("lapicid_cmp(%p,%p) (%d,%d)\n",
2d21ac55 A	225	x, y, cpu_x->cpu_phys_number, cpu_y->cpu_phys_number);
	226	if (cpu_x->cpu_phys_number < cpu_y->cpu_phys_number)
	227	return -1;
	228	if (cpu_x->cpu_phys_number == cpu_y->cpu_phys_number)
	229	return 0;
	230	return 1;
	231	}
	232
	233	static x86_affinity_set_t *
	234	find_cache_affinity(x86_cpu_cache_t *l2_cachep)
	235	{
	236	x86_affinity_set_t *aset;
	237
	238	for (aset = x86_affinities; aset != NULL; aset = aset->next) {
	239	if (l2_cachep == aset->cache)
	240	break;
	241	}
	242	return aset;
	243	}
	244
	245	int
	246	ml_get_max_affinity_sets(void)
	247	{
	248	return x86_affinity_count;
	249	}
	250
	251	processor_set_t
	252	ml_affinity_to_pset(uint32_t affinity_num)
	253	{
	254	x86_affinity_set_t *aset;
	255
	256	for (aset = x86_affinities; aset != NULL; aset = aset->next) {
	257	if (affinity_num == aset->num)
	258	break;
	259	}
593a1d5f	260	return (aset == NULL) ? PROCESSOR_SET_NULL : aset->pset;
2d21ac55 A	261	}
	262
	263	uint64_t
	264	ml_cpu_cache_size(unsigned int level)
	265	{
	266	x86_cpu_cache_t *cachep;
	267
	268	if (level == 0) {
	269	return machine_info.max_mem;
593a1d5f	270	} else if ( 1 <= level && level <= MAX_CACHE_DEPTH) {
2d21ac55 A	271	cachep = current_cpu_datap()->lcpu.caches[level-1];
	272	return cachep ? cachep->cache_size : 0;
	273	} else {
	274	return 0;
	275	}
	276	}
	277
	278	uint64_t
	279	ml_cpu_cache_sharing(unsigned int level)
	280	{
	281	x86_cpu_cache_t *cachep;
	282
	283	if (level == 0) {
	284	return machine_info.max_cpus;
593a1d5f	285	} else if ( 1 <= level && level <= MAX_CACHE_DEPTH) {
2d21ac55 A	286	cachep = current_cpu_datap()->lcpu.caches[level-1];
	287	return cachep ? cachep->nlcpus : 0;
	288	} else {
	289	return 0;
	290	}
	291	}
	292