[apple/xnu.git] / osfmk / i386 / pmap_x86_common.c

/*
 * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <i386/pmap_internal.h>
/*
 * The Intel platform can nest at the PDE level, so NBPDE (i.e. 2MB) at a time,
 * on a NBPDE boundary.
 */

/* These symbols may be referenced directly by VM */
uint64_t pmap_nesting_size_min = NBPDE;
uint64_t pmap_nesting_size_max = 0 - (uint64_t)NBPDE;

/*
 *	kern_return_t pmap_nest(grand, subord, va_start, size)
 *
 *	grand  = the pmap that we will nest subord into
 *	subord = the pmap that goes into the grand
 *	va_start  = start of range in pmap to be inserted
 *	nstart  = start of range in pmap nested pmap
 *	size   = Size of nest area (up to 16TB)
 *
 *	Inserts a pmap into another.  This is used to implement shared segments.
 *
 *	Note that we depend upon higher level VM locks to insure that things don't change while
 *	we are doing this.  For example, VM should not be doing any pmap enters while it is nesting
 *	or do 2 nests at once.
 */

/*
 * This routine can nest subtrees either at the PDPT level (1GiB) or at the
 * PDE level (2MiB). We currently disallow disparate offsets for the "subord"
 * container and the "grand" parent. A minor optimization to consider for the
 * future: make the "subord" truly a container rather than a full-fledged
 * pagetable hierarchy which can be unnecessarily sparse (DRK).
 */

kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t va_start, addr64_t nstart, uint64_t size) {
	vm_map_offset_t	vaddr, nvaddr;
	pd_entry_t	*pde,*npde;
	unsigned int	i;
	uint64_t	num_pde;

	if ((size & (pmap_nesting_size_min-1)) ||
	    (va_start & (pmap_nesting_size_min-1)) ||
	    (nstart & (pmap_nesting_size_min-1)) ||
	    ((size >> 28) > 65536))	/* Max size we can nest is 16TB */
		return KERN_INVALID_VALUE;

	if(size == 0) {
		panic("pmap_nest: size is invalid - %016llX\n", size);
	}

	if (va_start != nstart)
		panic("pmap_nest: va_start(0x%llx) != nstart(0x%llx)\n", va_start, nstart);

	PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_START,
	    (int) grand, (int) subord,
	    (int) (va_start>>32), (int) va_start, 0);

	nvaddr = (vm_map_offset_t)nstart;
	num_pde = size >> PDESHIFT;

	PMAP_LOCK(subord);

	subord->pm_shared = TRUE;

	for (i = 0; i < num_pde;) {
		if (((nvaddr & PDPTMASK) == 0) && (num_pde - i) >= NPDEPG && cpu_64bit) {

			npde = pmap64_pdpt(subord, nvaddr);

			while (0 == npde || ((*npde & INTEL_PTE_VALID) == 0)) {
				PMAP_UNLOCK(subord);
				pmap_expand_pdpt(subord, nvaddr);
				PMAP_LOCK(subord);
				npde = pmap64_pdpt(subord, nvaddr);
			}
			*npde |= INTEL_PDPTE_NESTED;
			nvaddr += NBPDPT;
			i += (uint32_t)NPDEPG;
		}
		else {
			npde = pmap_pde(subord, nvaddr);

			while (0 == npde || ((*npde & INTEL_PTE_VALID) == 0)) {
				PMAP_UNLOCK(subord);
				pmap_expand(subord, nvaddr);
				PMAP_LOCK(subord);
				npde = pmap_pde(subord, nvaddr);
			}
			nvaddr += NBPDE;
			i++;
		}
	}

	PMAP_UNLOCK(subord);

	vaddr = (vm_map_offset_t)va_start;

	PMAP_LOCK(grand);

	for (i = 0;i < num_pde;) {
		pd_entry_t tpde;

		if (((vaddr & PDPTMASK) == 0) && ((num_pde - i) >= NPDEPG) && cpu_64bit) {
			npde = pmap64_pdpt(subord, vaddr);
			if (npde == 0)
				panic("pmap_nest: no PDPT, subord %p nstart 0x%llx", subord, vaddr);
			tpde = *npde;
			pde = pmap64_pdpt(grand, vaddr);
			if (0 == pde) {
				PMAP_UNLOCK(grand);
				pmap_expand_pml4(grand, vaddr);
				PMAP_LOCK(grand);
				pde = pmap64_pdpt(grand, vaddr);
			}
			if (pde == 0)
				panic("pmap_nest: no PDPT, grand  %p vaddr 0x%llx", grand, vaddr);
			pmap_store_pte(pde, tpde);
			vaddr += NBPDPT;
			i += (uint32_t) NPDEPG;
		}
		else {
			npde = pmap_pde(subord, nstart);
			if (npde == 0)
				panic("pmap_nest: no npde, subord %p nstart 0x%llx", subord, nstart);
			tpde = *npde;
			nstart += NBPDE;
			pde = pmap_pde(grand, vaddr);
			if ((0 == pde) && cpu_64bit) {
				PMAP_UNLOCK(grand);
				pmap_expand_pdpt(grand, vaddr);
				PMAP_LOCK(grand);
				pde = pmap_pde(grand, vaddr);
			}

			if (pde == 0)
				panic("pmap_nest: no pde, grand  %p vaddr 0x%llx", grand, vaddr);
			vaddr += NBPDE;
			pmap_store_pte(pde, tpde);
			i++;
		}
	}

	PMAP_UNLOCK(grand);

	PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, 0, 0, 0, 0, 0);

	return KERN_SUCCESS;
}

/*
 *	kern_return_t pmap_unnest(grand, vaddr)
 *
 *	grand  = the pmap that we will un-nest subord from
 *	vaddr  = start of range in pmap to be unnested
 *
 *	Removes a pmap from another.  This is used to implement shared segments.
 */

kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) {
			
	pd_entry_t *pde;
	unsigned int i;
	uint64_t num_pde;
	addr64_t va_start, va_end;
	uint64_t npdpt = PMAP_INVALID_PDPTNUM;

	PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_START,
	    (int) grand, 
	    (int) (vaddr>>32), (int) vaddr, 0, 0);

	if ((size & (pmap_nesting_size_min-1)) ||
	    (vaddr & (pmap_nesting_size_min-1))) {
		panic("pmap_unnest(%p,0x%llx,0x%llx): unaligned...\n",
		    grand, vaddr, size);
	}

	/* align everything to PDE boundaries */
	va_start = vaddr & ~(NBPDE-1);
	va_end = (vaddr + size + NBPDE - 1) & ~(NBPDE-1);
	size = va_end - va_start;

	PMAP_LOCK(grand);

	num_pde = size >> PDESHIFT;
	vaddr = va_start;

	for (i = 0; i < num_pde; ) {
		if ((pdptnum(grand, vaddr) != npdpt) && cpu_64bit) {
			npdpt = pdptnum(grand, vaddr);
			pde = pmap64_pdpt(grand, vaddr);
			if (pde && (*pde & INTEL_PDPTE_NESTED)) {
				pmap_store_pte(pde, (pd_entry_t)0);
				i += (uint32_t) NPDEPG;
				vaddr += NBPDPT;
				continue;
			}
		}
		pde = pmap_pde(grand, (vm_map_offset_t)vaddr);
		if (pde == 0)
			panic("pmap_unnest: no pde, grand %p vaddr 0x%llx\n", grand, vaddr);
		pmap_store_pte(pde, (pd_entry_t)0);
		i++;
		vaddr += NBPDE;
	}

	PMAP_UPDATE_TLBS(grand, va_start, va_end);

	PMAP_UNLOCK(grand);
		
	PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_END, 0, 0, 0, 0, 0);

	return KERN_SUCCESS;
}

/* Invoked by the Mach VM to determine the platform specific unnest region */

boolean_t pmap_adjust_unnest_parameters(pmap_t p, vm_map_offset_t *s, vm_map_offset_t *e) {
	pd_entry_t *pdpte;
	boolean_t rval = FALSE;

	if (!cpu_64bit)
		return rval;

	PMAP_LOCK(p);

	pdpte = pmap64_pdpt(p, *s);
	if (pdpte && (*pdpte & INTEL_PDPTE_NESTED)) {
		*s &= ~(NBPDPT -1);
		rval = TRUE;
	}

	pdpte = pmap64_pdpt(p, *e);
	if (pdpte && (*pdpte & INTEL_PDPTE_NESTED)) {
		*e = ((*e + NBPDPT) & ~(NBPDPT -1));
		rval = TRUE;
	}

	PMAP_UNLOCK(p);

	return rval;
}

/*
 * pmap_find_phys returns the (4K) physical page number containing a
 * given virtual address in a given pmap.
 * Note that pmap_pte may return a pde if this virtual address is
 * mapped by a large page and this is taken into account in order
 * to return the correct page number in this case.
 */
ppnum_t
pmap_find_phys(pmap_t pmap, addr64_t va)
{
	pt_entry_t	*ptp;
	pd_entry_t	*pdep;
	ppnum_t		ppn = 0;
	pd_entry_t	pde;
	pt_entry_t	pte;

	mp_disable_preemption();

	/* This refcount test is a band-aid--several infrastructural changes
	 * are necessary to eliminate invocation of this routine from arbitrary
	 * contexts.
	 */
	
	if (!pmap->ref_count)
		goto pfp_exit;

	pdep = pmap_pde(pmap, va);

	if ((pdep != PD_ENTRY_NULL) && ((pde = *pdep) & INTEL_PTE_VALID)) {
		if (pde & INTEL_PTE_PS) {
			ppn = (ppnum_t) i386_btop(pte_to_pa(pde));
			ppn += (ppnum_t) ptenum(va);
		}
		else {
			ptp = pmap_pte(pmap, va);
			if ((PT_ENTRY_NULL != ptp) && (((pte = *ptp) & INTEL_PTE_VALID) != 0)) {
				ppn = (ppnum_t) i386_btop(pte_to_pa(pte));
			}
		}
	}
pfp_exit:	
	mp_enable_preemption();

        return ppn;
}
Commit	Line	Data
b0d623f7 A	1	/*
	2	* Copyright (c) 2000-2009 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28	#include <vm/pmap.h>
	29	#include <vm/vm_map.h>
	30	#include <i386/pmap_internal.h>
	31	/*
	32	* The Intel platform can nest at the PDE level, so NBPDE (i.e. 2MB) at a time,
	33	* on a NBPDE boundary.
	34	*/
	35
	36	/* These symbols may be referenced directly by VM */
	37	uint64_t pmap_nesting_size_min = NBPDE;
	38	uint64_t pmap_nesting_size_max = 0 - (uint64_t)NBPDE;
	39
	40	/*
	41	* kern_return_t pmap_nest(grand, subord, va_start, size)
	42	*
	43	* grand = the pmap that we will nest subord into
	44	* subord = the pmap that goes into the grand
	45	* va_start = start of range in pmap to be inserted
	46	* nstart = start of range in pmap nested pmap
	47	* size = Size of nest area (up to 16TB)
	48	*
	49	* Inserts a pmap into another. This is used to implement shared segments.
	50	*
	51	* Note that we depend upon higher level VM locks to insure that things don't change while
	52	* we are doing this. For example, VM should not be doing any pmap enters while it is nesting
	53	* or do 2 nests at once.
	54	*/
	55
	56	/*
	57	* This routine can nest subtrees either at the PDPT level (1GiB) or at the
	58	* PDE level (2MiB). We currently disallow disparate offsets for the "subord"
	59	* container and the "grand" parent. A minor optimization to consider for the
	60	* future: make the "subord" truly a container rather than a full-fledged
	61	* pagetable hierarchy which can be unnecessarily sparse (DRK).
	62	*/
	63
	64	kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t va_start, addr64_t nstart, uint64_t size) {
65	vm_map_offset_t vaddr, nvaddr;
66	pd_entry_t pde,npde;
67	unsigned int i;
68	uint64_t num_pde;
69
70	if ((size & (pmap_nesting_size_min-1)) \|\|
71	(va_start & (pmap_nesting_size_min-1)) \|\|
72	(nstart & (pmap_nesting_size_min-1)) \|\|
73	((size >> 28) > 65536)) /* Max size we can nest is 16TB */
74	return KERN_INVALID_VALUE;
75
76	if(size == 0) {
77	panic("pmap_nest: size is invalid - %016llX\n", size);
78	}
79
80	if (va_start != nstart)
81	panic("pmap_nest: va_start(0x%llx) != nstart(0x%llx)\n", va_start, nstart);
82
83	PMAP_TRACE(PMAP_CODE(PMAP__NEST) \| DBG_FUNC_START,
84	(int) grand, (int) subord,
85	(int) (va_start>>32), (int) va_start, 0);
86
87	nvaddr = (vm_map_offset_t)nstart;
88	num_pde = size >> PDESHIFT;
89
90	PMAP_LOCK(subord);
91
92	subord->pm_shared = TRUE;
93
94	for (i = 0; i < num_pde;) {
95	if (((nvaddr & PDPTMASK) == 0) && (num_pde - i) >= NPDEPG && cpu_64bit) {
96
97	npde = pmap64_pdpt(subord, nvaddr);
98
99	while (0 == npde \|\| ((*npde & INTEL_PTE_VALID) == 0)) {
100	PMAP_UNLOCK(subord);
101	pmap_expand_pdpt(subord, nvaddr);
102	PMAP_LOCK(subord);
103	npde = pmap64_pdpt(subord, nvaddr);
104	}
105	*npde \|= INTEL_PDPTE_NESTED;
106	nvaddr += NBPDPT;
107	i += (uint32_t)NPDEPG;
108	}
109	else {
110	npde = pmap_pde(subord, nvaddr);
111
112	while (0 == npde \|\| ((*npde & INTEL_PTE_VALID) == 0)) {
113	PMAP_UNLOCK(subord);
114	pmap_expand(subord, nvaddr);
115	PMAP_LOCK(subord);
116	npde = pmap_pde(subord, nvaddr);
117	}
118	nvaddr += NBPDE;
119	i++;
120	}
121	}
122
123	PMAP_UNLOCK(subord);
124
125	vaddr = (vm_map_offset_t)va_start;
126
127	PMAP_LOCK(grand);
128
129	for (i = 0;i < num_pde;) {
130	pd_entry_t tpde;
131
132	if (((vaddr & PDPTMASK) == 0) && ((num_pde - i) >= NPDEPG) && cpu_64bit) {
133	npde = pmap64_pdpt(subord, vaddr);
134	if (npde == 0)
135	panic("pmap_nest: no PDPT, subord %p nstart 0x%llx", subord, vaddr);
136	tpde = *npde;
137	pde = pmap64_pdpt(grand, vaddr);
138	if (0 == pde) {
139	PMAP_UNLOCK(grand);
140	pmap_expand_pml4(grand, vaddr);
141	PMAP_LOCK(grand);
142	pde = pmap64_pdpt(grand, vaddr);
143	}
144	if (pde == 0)
145	panic("pmap_nest: no PDPT, grand %p vaddr 0x%llx", grand, vaddr);
146	pmap_store_pte(pde, tpde);
147	vaddr += NBPDPT;
148	i += (uint32_t) NPDEPG;
149	}
150	else {
151	npde = pmap_pde(subord, nstart);
152	if (npde == 0)
153	panic("pmap_nest: no npde, subord %p nstart 0x%llx", subord, nstart);
154	tpde = *npde;
155	nstart += NBPDE;
156	pde = pmap_pde(grand, vaddr);
157	if ((0 == pde) && cpu_64bit) {
158	PMAP_UNLOCK(grand);
159	pmap_expand_pdpt(grand, vaddr);
160	PMAP_LOCK(grand);
161	pde = pmap_pde(grand, vaddr);
162	}
163
164	if (pde == 0)
165	panic("pmap_nest: no pde, grand %p vaddr 0x%llx", grand, vaddr);
166	vaddr += NBPDE;
167	pmap_store_pte(pde, tpde);
168	i++;
169	}
170	}
171
172	PMAP_UNLOCK(grand);
173
174	PMAP_TRACE(PMAP_CODE(PMAP__NEST) \| DBG_FUNC_END, 0, 0, 0, 0, 0);
175
176	return KERN_SUCCESS;
177	}
178
179	/*
180	* kern_return_t pmap_unnest(grand, vaddr)
181	*
182	* grand = the pmap that we will un-nest subord from
183	* vaddr = start of range in pmap to be unnested
184	*
185	* Removes a pmap from another. This is used to implement shared segments.
186	*/
187
188	kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) {
189
190	pd_entry_t *pde;
191	unsigned int i;
192	uint64_t num_pde;
193	addr64_t va_start, va_end;
194	uint64_t npdpt = PMAP_INVALID_PDPTNUM;
195
196	PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) \| DBG_FUNC_START,
197	(int) grand,
198	(int) (vaddr>>32), (int) vaddr, 0, 0);
199
200	if ((size & (pmap_nesting_size_min-1)) \|\|
201	(vaddr & (pmap_nesting_size_min-1))) {
202	panic("pmap_unnest(%p,0x%llx,0x%llx): unaligned...\n",
203	grand, vaddr, size);
204	}
205
206	/* align everything to PDE boundaries */
207	va_start = vaddr & ~(NBPDE-1);
208	va_end = (vaddr + size + NBPDE - 1) & ~(NBPDE-1);
209	size = va_end - va_start;
210
211	PMAP_LOCK(grand);
212
213	num_pde = size >> PDESHIFT;
214	vaddr = va_start;
215
216	for (i = 0; i < num_pde; ) {
217	if ((pdptnum(grand, vaddr) != npdpt) && cpu_64bit) {
218	npdpt = pdptnum(grand, vaddr);
219	pde = pmap64_pdpt(grand, vaddr);
220	if (pde && (*pde & INTEL_PDPTE_NESTED)) {
221	pmap_store_pte(pde, (pd_entry_t)0);
222	i += (uint32_t) NPDEPG;
223	vaddr += NBPDPT;
224	continue;
225	}
226	}
227	pde = pmap_pde(grand, (vm_map_offset_t)vaddr);
228	if (pde == 0)
229	panic("pmap_unnest: no pde, grand %p vaddr 0x%llx\n", grand, vaddr);
230	pmap_store_pte(pde, (pd_entry_t)0);
231	i++;
232	vaddr += NBPDE;
233	}
234
235	PMAP_UPDATE_TLBS(grand, va_start, va_end);
236
237	PMAP_UNLOCK(grand);
238
239	PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) \| DBG_FUNC_END, 0, 0, 0, 0, 0);
240
241	return KERN_SUCCESS;
242	}
243
244	/* Invoked by the Mach VM to determine the platform specific unnest region */
245
246	boolean_t pmap_adjust_unnest_parameters(pmap_t p, vm_map_offset_t s, vm_map_offset_t e) {
247	pd_entry_t *pdpte;
248	boolean_t rval = FALSE;
249
250	if (!cpu_64bit)
251	return rval;
252
253	PMAP_LOCK(p);
254
255	pdpte = pmap64_pdpt(p, *s);
256	if (pdpte && (*pdpte & INTEL_PDPTE_NESTED)) {
257	*s &= ~(NBPDPT -1);
258	rval = TRUE;
259	}
260
261	pdpte = pmap64_pdpt(p, *e);
262	if (pdpte && (*pdpte & INTEL_PDPTE_NESTED)) {
263	e = ((e + NBPDPT) & ~(NBPDPT -1));
264	rval = TRUE;
265	}
266
267	PMAP_UNLOCK(p);
268
269	return rval;
270	}
271
272	/*
273	* pmap_find_phys returns the (4K) physical page number containing a
274	* given virtual address in a given pmap.
275	* Note that pmap_pte may return a pde if this virtual address is
276	* mapped by a large page and this is taken into account in order
277	* to return the correct page number in this case.
278	*/
279	ppnum_t
280	pmap_find_phys(pmap_t pmap, addr64_t va)
281	{
282	pt_entry_t *ptp;
283	pd_entry_t *pdep;
284	ppnum_t ppn = 0;
285	pd_entry_t pde;
286	pt_entry_t pte;
287
288	mp_disable_preemption();
289
290	/* This refcount test is a band-aid--several infrastructural changes
291	* are necessary to eliminate invocation of this routine from arbitrary
292	* contexts.
293	*/
294
295	if (!pmap->ref_count)
296	goto pfp_exit;
297
298	pdep = pmap_pde(pmap, va);
299
300	if ((pdep != PD_ENTRY_NULL) && ((pde = *pdep) & INTEL_PTE_VALID)) {
301	if (pde & INTEL_PTE_PS) {
302	ppn = (ppnum_t) i386_btop(pte_to_pa(pde));
303	ppn += (ppnum_t) ptenum(va);
304	}
305	else {
306	ptp = pmap_pte(pmap, va);
307	if ((PT_ENTRY_NULL != ptp) && (((pte = *ptp) & INTEL_PTE_VALID) != 0)) {
308	ppn = (ppnum_t) i386_btop(pte_to_pa(pte));
309	}
310	}
311	}
312	pfp_exit:
313	mp_enable_preemption();
314
315	return ppn;
316	}
317