[apple/xnu.git] / osfmk / i386 / cpu_topology.c

/*
 * Copyright (c) 2007 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <mach/machine.h>
#include <mach/processor.h>
#include <kern/kalloc.h>
#include <i386/cpu_affinity.h>
#include <i386/cpu_topology.h>
#include <i386/cpu_data.h>
#include <i386/cpu_threads.h>
#include <i386/machine_cpu.h>
#include <i386/machine_routines.h>
#include <i386/lock.h>
#include <i386/mp.h>

//#define TOPO_DEBUG 1
#if TOPO_DEBUG
#define DBG(x...)	kprintf("DBG: " x)
#else
#define DBG(x...)
#endif

__private_extern__ void qsort(
    void * array,
    size_t nmembers,
    size_t member_size,
    int (*)(const void *, const void *));

static int lapicid_cmp(const void *x, const void *y);
static x86_affinity_set_t *find_cache_affinity(x86_cpu_cache_t *L2_cachep);

x86_affinity_set_t	*x86_affinities = NULL;
static int		x86_affinity_count = 0;

/*
 * cpu_topology_start() is called after all processors have been registered
 * but before any non-boot processor id started.
 * We establish canonical logical processor numbering - logical cpus must be
 * contiguous, zero-based and assigned in physical (local apic id) order.
 * This step is required because the discovery/registration order is
 * non-deterministic - cores are registered in differing orders over boots.
 * Enforcing canonical numbering simplifies identification
 * of processors - in particular, for stopping/starting from CHUD.
 */ 
void
cpu_topology_start(void)
{
	int		ncpus = machine_info.max_cpus;
	int		i;
	boolean_t	istate;

	assert(machine_info.physical_cpu == 1);
	assert(machine_info.logical_cpu == 1);
	assert(master_cpu == 0);
	assert(cpu_number() == 0);
	assert(cpu_datap(0)->cpu_number == 0);
	
	/* Lights out for this */
	istate = ml_set_interrupts_enabled(FALSE);

#ifdef TOPO_DEBUG
	DBG("cpu_topology_start() %d cpu%s registered\n",
		ncpus, (ncpus > 1) ? "s" : "");
	for (i = 0; i < ncpus; i++) {
		cpu_data_t	*cpup = cpu_datap(i);
		DBG("\tcpu_data[%d]:0x%08x local apic 0x%x\n",
			i, (unsigned) cpup, cpup->cpu_phys_number);
	}
#endif
	/*
	 * Re-order the cpu_data_ptr vector sorting by physical id.
	 * Skip the boot processor, it's required to be correct.
	 */
	if (ncpus > 1) {
		qsort((void *) &cpu_data_ptr[1],
			ncpus - 1,
			sizeof(cpu_data_t *),
			lapicid_cmp);
	}
#ifdef TOPO_DEBUG
	DBG("cpu_topology_start() after sorting:\n");
	for (i = 0; i < ncpus; i++) {
		cpu_data_t	*cpup = cpu_datap(i);
		DBG("\tcpu_data[%d]:0x%08x local apic 0x%x\n",
			i, (unsigned) cpup, cpup->cpu_phys_number);
	}
#endif

	/*
	 * Fix up logical numbers and reset the map kept by the lapic code.
	 */
	for (i = 1; i < ncpus; i++) {
		cpu_data_t	*cpup = cpu_datap(i);

		if (cpup->cpu_number != i) {
			kprintf("cpu_datap(%d):0x%08x local apic id 0x%x "
				"remapped from %d\n",
				i, (unsigned) cpup, cpup->cpu_phys_number,
				cpup->cpu_number);
		}
		cpup->cpu_number = i;
		cpup->lcpu.lnum = i;
		lapic_cpu_map(cpup->cpu_phys_number, i);
	}

	ml_set_interrupts_enabled(istate);

	/*
	 * Iterate over all logical cpus finding or creating the affinity set
	 * for their L2 cache. Each affinity set possesses a processor set
	 * into which each logical processor is added.
	 */
	DBG("cpu_topology_start() creating affinity sets:\n");
	for (i = 0; i < ncpus; i++) {
		cpu_data_t		*cpup = cpu_datap(i);
		x86_lcpu_t		*lcpup = cpu_to_lcpu(i);
		x86_cpu_cache_t		*L2_cachep;
		x86_affinity_set_t	*aset;

		L2_cachep = lcpup->caches[CPU_CACHE_DEPTH_L2];
		assert(L2_cachep->type == CPU_CACHE_TYPE_UNIF);
		aset = find_cache_affinity(L2_cachep); 
		if (aset == NULL) {
			aset = (x86_affinity_set_t *) kalloc(sizeof(*aset));
			if (aset == NULL)
				panic("cpu_topology_start() failed aset alloc");
			aset->next = x86_affinities;
			x86_affinities = aset;
			aset->num = x86_affinity_count++;
			aset->cache = L2_cachep;
			aset->pset = (i == master_cpu) ?
					processor_pset(master_processor) :
					pset_create(pset_node_root());
			if (aset->pset == PROCESSOR_SET_NULL)
				panic("cpu_topology_start: pset_create");
			DBG("\tnew set %p(%d) pset %p for cache %p\n",
				aset, aset->num, aset->pset, aset->cache);
		}

		DBG("\tprocessor_init set %p(%d) lcpup %p(%d) cpu %p processor %p\n",
			aset, aset->num, lcpup, lcpup->lnum, cpup, cpup->cpu_processor);

		if (i != master_cpu)
			processor_init(cpup->cpu_processor, i, aset->pset);
	}

	/*
	 * Finally we start all processors (including the boot cpu we're
	 * running on).
	 */
	DBG("cpu_topology_start() processor_start():\n");
	for (i = 0; i < ncpus; i++) {
		DBG("\tlcpu %d\n", cpu_datap(i)->cpu_number);
		processor_start(cpu_datap(i)->cpu_processor); 
	}
}

static int
lapicid_cmp(const void *x, const void *y)
{
	cpu_data_t	*cpu_x = *((cpu_data_t **)(uintptr_t)x);
	cpu_data_t	*cpu_y = *((cpu_data_t **)(uintptr_t)y);

	DBG("lapicid_cmp(%p,%p) (%d,%d)\n",
		x, y, cpu_x->cpu_phys_number, cpu_y->cpu_phys_number);
	if (cpu_x->cpu_phys_number < cpu_y->cpu_phys_number)
		return -1;
	if (cpu_x->cpu_phys_number == cpu_y->cpu_phys_number)
		return 0;
	return 1;
}

static x86_affinity_set_t *
find_cache_affinity(x86_cpu_cache_t *l2_cachep)
{
	x86_affinity_set_t	*aset;

	for (aset = x86_affinities; aset != NULL; aset = aset->next) {
		if (l2_cachep == aset->cache)
			break;
	}
	return aset;			
}

int
ml_get_max_affinity_sets(void)
{
	return x86_affinity_count;
}

processor_set_t
ml_affinity_to_pset(uint32_t affinity_num) 
{
	x86_affinity_set_t	*aset;

	for (aset = x86_affinities; aset != NULL; aset = aset->next) {
		if (affinity_num == aset->num)
			break;
	}
	return (aset == NULL) ? PROCESSOR_SET_NULL : aset->pset;			
	
}

uint64_t
ml_cpu_cache_size(unsigned int level)
{
	x86_cpu_cache_t	*cachep;

	if (level == 0) {
		return machine_info.max_mem;
	} else if ( 1 <= level && level <= 3) {
		cachep = current_cpu_datap()->lcpu.caches[level-1];
		return cachep ? cachep->cache_size : 0;
	} else {
		return 0;
	}
}

uint64_t
ml_cpu_cache_sharing(unsigned int level)
{
	x86_cpu_cache_t	*cachep;

	if (level == 0) {
		return machine_info.max_cpus;
	} else if ( 1 <= level && level <= 3) {
		cachep = current_cpu_datap()->lcpu.caches[level-1];
		return cachep ? cachep->nlcpus : 0;
	} else {
		return 0;
	}
}
Commit	Line	Data
2d21ac55 A	1	/*
	2	* Copyright (c) 2007 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <mach/machine.h>
	30	#include <mach/processor.h>
	31	#include <kern/kalloc.h>
	32	#include <i386/cpu_affinity.h>
	33	#include <i386/cpu_topology.h>
	34	#include <i386/cpu_data.h>
	35	#include <i386/cpu_threads.h>
	36	#include <i386/machine_cpu.h>
	37	#include <i386/machine_routines.h>
	38	#include <i386/lock.h>
	39	#include <i386/mp.h>
	40
	41	//#define TOPO_DEBUG 1
	42	#if TOPO_DEBUG
	43	#define DBG(x...) kprintf("DBG: " x)
	44	#else
	45	#define DBG(x...)
	46	#endif
	47
	48	__private_extern__ void qsort(
	49	void * array,
	50	size_t nmembers,
	51	size_t member_size,
	52	int ()(const void , const void *));
	53
	54	static int lapicid_cmp(const void x, const void y);
	55	static x86_affinity_set_t find_cache_affinity(x86_cpu_cache_t L2_cachep);
	56
	57	x86_affinity_set_t *x86_affinities = NULL;
	58	static int x86_affinity_count = 0;
	59
	60	/*
	61	* cpu_topology_start() is called after all processors have been registered
	62	* but before any non-boot processor id started.
	63	* We establish canonical logical processor numbering - logical cpus must be
	64	* contiguous, zero-based and assigned in physical (local apic id) order.
65	* This step is required because the discovery/registration order is
66	* non-deterministic - cores are registered in differing orders over boots.
67	* Enforcing canonical numbering simplifies identification
68	* of processors - in particular, for stopping/starting from CHUD.
69	*/
70	void
71	cpu_topology_start(void)
72	{
73	int ncpus = machine_info.max_cpus;
74	int i;
75	boolean_t istate;
76
77	assert(machine_info.physical_cpu == 1);
78	assert(machine_info.logical_cpu == 1);
79	assert(master_cpu == 0);
80	assert(cpu_number() == 0);
81	assert(cpu_datap(0)->cpu_number == 0);
82
83	/* Lights out for this */
84	istate = ml_set_interrupts_enabled(FALSE);
85
86	#ifdef TOPO_DEBUG
87	DBG("cpu_topology_start() %d cpu%s registered\n",
88	ncpus, (ncpus > 1) ? "s" : "");
89	for (i = 0; i < ncpus; i++) {
90	cpu_data_t *cpup = cpu_datap(i);
91	DBG("\tcpu_data[%d]:0x%08x local apic 0x%x\n",
92	i, (unsigned) cpup, cpup->cpu_phys_number);
93	}
94	#endif
95	/*
96	* Re-order the cpu_data_ptr vector sorting by physical id.
97	* Skip the boot processor, it's required to be correct.
98	*/
99	if (ncpus > 1) {
100	qsort((void *) &cpu_data_ptr[1],
101	ncpus - 1,
102	sizeof(cpu_data_t *),
103	lapicid_cmp);
104	}
105	#ifdef TOPO_DEBUG
106	DBG("cpu_topology_start() after sorting:\n");
107	for (i = 0; i < ncpus; i++) {
108	cpu_data_t *cpup = cpu_datap(i);
109	DBG("\tcpu_data[%d]:0x%08x local apic 0x%x\n",
110	i, (unsigned) cpup, cpup->cpu_phys_number);
111	}
112	#endif
113
114	/*
115	* Fix up logical numbers and reset the map kept by the lapic code.
116	*/
117	for (i = 1; i < ncpus; i++) {
118	cpu_data_t *cpup = cpu_datap(i);
119
120	if (cpup->cpu_number != i) {
121	kprintf("cpu_datap(%d):0x%08x local apic id 0x%x "
122	"remapped from %d\n",
123	i, (unsigned) cpup, cpup->cpu_phys_number,
124	cpup->cpu_number);
125	}
126	cpup->cpu_number = i;
127	cpup->lcpu.lnum = i;
128	lapic_cpu_map(cpup->cpu_phys_number, i);
129	}
130
131	ml_set_interrupts_enabled(istate);
132
133	/*
134	* Iterate over all logical cpus finding or creating the affinity set
135	* for their L2 cache. Each affinity set possesses a processor set
136	* into which each logical processor is added.
137	*/
138	DBG("cpu_topology_start() creating affinity sets:\n");
139	for (i = 0; i < ncpus; i++) {
140	cpu_data_t *cpup = cpu_datap(i);
141	x86_lcpu_t *lcpup = cpu_to_lcpu(i);
142	x86_cpu_cache_t *L2_cachep;
143	x86_affinity_set_t *aset;
144
145	L2_cachep = lcpup->caches[CPU_CACHE_DEPTH_L2];
146	assert(L2_cachep->type == CPU_CACHE_TYPE_UNIF);
147	aset = find_cache_affinity(L2_cachep);
148	if (aset == NULL) {
149	aset = (x86_affinity_set_t ) kalloc(sizeof(aset));
150	if (aset == NULL)
151	panic("cpu_topology_start() failed aset alloc");
152	aset->next = x86_affinities;
153	x86_affinities = aset;
154	aset->num = x86_affinity_count++;
155	aset->cache = L2_cachep;
156	aset->pset = (i == master_cpu) ?
157	processor_pset(master_processor) :
158	pset_create(pset_node_root());
159	if (aset->pset == PROCESSOR_SET_NULL)
160	panic("cpu_topology_start: pset_create");
161	DBG("\tnew set %p(%d) pset %p for cache %p\n",
162	aset, aset->num, aset->pset, aset->cache);
163	}
164
165	DBG("\tprocessor_init set %p(%d) lcpup %p(%d) cpu %p processor %p\n",
166	aset, aset->num, lcpup, lcpup->lnum, cpup, cpup->cpu_processor);
167
168	if (i != master_cpu)
169	processor_init(cpup->cpu_processor, i, aset->pset);
170	}
171
172	/*
173	* Finally we start all processors (including the boot cpu we're
174	* running on).
175	*/
176	DBG("cpu_topology_start() processor_start():\n");
177	for (i = 0; i < ncpus; i++) {
178	DBG("\tlcpu %d\n", cpu_datap(i)->cpu_number);
179	processor_start(cpu_datap(i)->cpu_processor);
180	}
181	}
182
183	static int
184	lapicid_cmp(const void x, const void y)
185	{
186	cpu_data_t cpu_x = ((cpu_data_t **)(uintptr_t)x);
187	cpu_data_t cpu_y = ((cpu_data_t **)(uintptr_t)y);
188
189	DBG("lapicid_cmp(%p,%p) (%d,%d)\n",
190	x, y, cpu_x->cpu_phys_number, cpu_y->cpu_phys_number);
191	if (cpu_x->cpu_phys_number < cpu_y->cpu_phys_number)
192	return -1;
193	if (cpu_x->cpu_phys_number == cpu_y->cpu_phys_number)
194	return 0;
195	return 1;
196	}
197
198	static x86_affinity_set_t *
199	find_cache_affinity(x86_cpu_cache_t *l2_cachep)
200	{
201	x86_affinity_set_t *aset;
202
203	for (aset = x86_affinities; aset != NULL; aset = aset->next) {
204	if (l2_cachep == aset->cache)
205	break;
206	}
207	return aset;
208	}
209
210	int
211	ml_get_max_affinity_sets(void)
212	{
213	return x86_affinity_count;
214	}
215
216	processor_set_t
217	ml_affinity_to_pset(uint32_t affinity_num)
218	{
219	x86_affinity_set_t *aset;
220
221	for (aset = x86_affinities; aset != NULL; aset = aset->next) {
222	if (affinity_num == aset->num)
223	break;
224	}
225	return (aset == NULL) ? PROCESSOR_SET_NULL : aset->pset;
226
227	}
228
229	uint64_t
230	ml_cpu_cache_size(unsigned int level)
231	{
232	x86_cpu_cache_t *cachep;
233
234	if (level == 0) {
235	return machine_info.max_mem;
236	} else if ( 1 <= level && level <= 3) {
237	cachep = current_cpu_datap()->lcpu.caches[level-1];
238	return cachep ? cachep->cache_size : 0;
239	} else {
240	return 0;
241	}
242	}
243
244	uint64_t
245	ml_cpu_cache_sharing(unsigned int level)
246	{
247	x86_cpu_cache_t *cachep;
248
249	if (level == 0) {
250	return machine_info.max_cpus;
251	} else if ( 1 <= level && level <= 3) {
252	cachep = current_cpu_datap()->lcpu.caches[level-1];
253	return cachep ? cachep->nlcpus : 0;
254	} else {
255	return 0;
256	}
257	}
258