[apple/xnu.git] / osfmk / i386 / pmap_common.c

/*
 * Copyright (c) 2000-2010 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
#include <vm/pmap.h>
#include <i386/pmap_internal.h>

/*
 *	Each entry in the pv_head_table is locked by a bit in the
 *	pv_lock_table.  The lock bits are accessed by the physical
 *	address of the page they lock.
 */

char	*pv_lock_table;		/* pointer to array of bits */
char    *pv_hash_lock_table;

pv_rooted_entry_t	pv_head_table;		/* array of entries, one per
						 * page */
uint32_t			pv_hashed_free_count = 0;
uint32_t			pv_hashed_kern_free_count = 0;

pmap_pagetable_corruption_record_t pmap_pagetable_corruption_records[PMAP_PAGETABLE_CORRUPTION_MAX_LOG];
uint32_t pmap_pagetable_corruption_incidents;
uint64_t pmap_pagetable_corruption_last_abstime = (~(0ULL) >> 1);
uint64_t pmap_pagetable_corruption_interval_abstime;
thread_call_t 	pmap_pagetable_corruption_log_call;
static thread_call_data_t 	pmap_pagetable_corruption_log_call_data;
boolean_t pmap_pagetable_corruption_timeout = FALSE;

volatile uint32_t	mappingrecurse = 0;

uint32_t  pv_hashed_low_water_mark, pv_hashed_kern_low_water_mark, pv_hashed_alloc_chunk, pv_hashed_kern_alloc_chunk;

thread_t mapping_replenish_thread;
event_t	mapping_replenish_event, pmap_user_pv_throttle_event;

uint64_t pmap_pv_throttle_stat, pmap_pv_throttled_waiters;

unsigned int pmap_cache_attributes(ppnum_t pn) {
	if (pmap_get_cache_attributes(pn) & INTEL_PTE_NCACHE)
	        return (VM_WIMG_IO);
	else
		return (VM_WIMG_COPYBACK);
}

void	pmap_set_cache_attributes(ppnum_t pn, unsigned int cacheattr) {
	unsigned int current, template = 0;
	int pai;

	if (cacheattr & VM_MEM_NOT_CACHEABLE) {
		if(!(cacheattr & VM_MEM_GUARDED))
			template |= PHYS_PTA;
		template |= PHYS_NCACHE;
	}

	pmap_intr_assert();

	assert((pn != vm_page_fictitious_addr) && (pn != vm_page_guard_addr));

	pai = ppn_to_pai(pn);

	if (!IS_MANAGED_PAGE(pai)) {
		return;
	}

	/* override cache attributes for this phys page
	 * Does not walk through existing mappings to adjust,
	 * assumes page is disconnected
	 */

	LOCK_PVH(pai);

	pmap_update_cache_attributes_locked(pn, template);

	current = pmap_phys_attributes[pai] & PHYS_CACHEABILITY_MASK;
	pmap_phys_attributes[pai] &= ~PHYS_CACHEABILITY_MASK;
	pmap_phys_attributes[pai] |= template;

	UNLOCK_PVH(pai);

	if ((template & PHYS_NCACHE) && !(current & PHYS_NCACHE)) {
		pmap_sync_page_attributes_phys(pn);
	}
}

unsigned	pmap_get_cache_attributes(ppnum_t pn) {
	if (last_managed_page == 0)
		return 0;

	if (!IS_MANAGED_PAGE(ppn_to_pai(pn))) {
	    return INTEL_PTE_NCACHE;
	}

	/*
	 * The cache attributes are read locklessly for efficiency.
	 */
	unsigned int attr = pmap_phys_attributes[ppn_to_pai(pn)];
	unsigned int template = 0;
	
	if (attr & PHYS_PTA)
		template |= INTEL_PTE_PTA;
	if (attr & PHYS_NCACHE)
		template |= INTEL_PTE_NCACHE;
	return template;
}


boolean_t
pmap_is_noencrypt(ppnum_t pn)
{
	int		pai;

	pai = ppn_to_pai(pn);

	if (!IS_MANAGED_PAGE(pai))
		return (TRUE);

	if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT)
		return (TRUE);

	return (FALSE);
}


void
pmap_set_noencrypt(ppnum_t pn)
{
	int		pai;

	pai = ppn_to_pai(pn);

	if (IS_MANAGED_PAGE(pai)) {
		LOCK_PVH(pai);

		pmap_phys_attributes[pai] |= PHYS_NOENCRYPT;

		UNLOCK_PVH(pai);
	}
}


void
pmap_clear_noencrypt(ppnum_t pn)
{
	int		pai;

	pai = ppn_to_pai(pn);

	if (IS_MANAGED_PAGE(pai)) {
		LOCK_PVH(pai);

		pmap_phys_attributes[pai] &= ~PHYS_NOENCRYPT;

		UNLOCK_PVH(pai);
	}
}

void
compute_pmap_gc_throttle(void *arg __unused)
{
	
}


__private_extern__ void
pmap_pagetable_corruption_msg_log(int (*log_func)(const char * fmt, ...)__printflike(1,2)) {
	if (pmap_pagetable_corruption_incidents > 0) {
		int i, e = MIN(pmap_pagetable_corruption_incidents, PMAP_PAGETABLE_CORRUPTION_MAX_LOG);
		(*log_func)("%u pagetable corruption incident(s) detected, timeout: %u\n", pmap_pagetable_corruption_incidents, pmap_pagetable_corruption_timeout);
		for (i = 0; i < e; i++) {
			(*log_func)("Incident 0x%x, reason: 0x%x, action: 0x%x, time: 0x%llx\n", pmap_pagetable_corruption_records[i].incident,  pmap_pagetable_corruption_records[i].reason, pmap_pagetable_corruption_records[i].action, pmap_pagetable_corruption_records[i].abstime);
		}
	}
}

static inline void
pmap_pagetable_corruption_log_setup(void) {
	if (pmap_pagetable_corruption_log_call == NULL) {
		nanotime_to_absolutetime(PMAP_PAGETABLE_CORRUPTION_INTERVAL, 0, &pmap_pagetable_corruption_interval_abstime);
		thread_call_setup(&pmap_pagetable_corruption_log_call_data,
		    (thread_call_func_t) pmap_pagetable_corruption_msg_log,
		    (thread_call_param_t) &printf);
		pmap_pagetable_corruption_log_call = &pmap_pagetable_corruption_log_call_data;
	}
}

void
mapping_free_prime(void)
{
	unsigned		i;
	pv_hashed_entry_t	pvh_e;
	pv_hashed_entry_t	pvh_eh;
	pv_hashed_entry_t	pvh_et;
	int			pv_cnt;

	/* Scale based on DRAM size */
	pv_hashed_low_water_mark = MAX(PV_HASHED_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 2000);
	pv_hashed_low_water_mark = MIN(pv_hashed_low_water_mark, 16000);
	/* Alterable via sysctl */
	pv_hashed_kern_low_water_mark = MAX(PV_HASHED_KERN_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 1000);
	pv_hashed_kern_low_water_mark = MIN(pv_hashed_kern_low_water_mark, 16000);
	pv_hashed_kern_alloc_chunk = PV_HASHED_KERN_ALLOC_CHUNK_INITIAL;
	pv_hashed_alloc_chunk = PV_HASHED_ALLOC_CHUNK_INITIAL;

	pv_cnt = 0;
	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;

	for (i = 0; i < (5 * PV_HASHED_ALLOC_CHUNK_INITIAL); i++) {
		pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);

		pvh_e->qlink.next = (queue_entry_t)pvh_eh;
		pvh_eh = pvh_e;

		if (pvh_et == PV_HASHED_ENTRY_NULL)
		        pvh_et = pvh_e;
		pv_cnt++;
	}
	PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);

	pv_cnt = 0;
	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
	for (i = 0; i < PV_HASHED_KERN_ALLOC_CHUNK_INITIAL; i++) {
		pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);

		pvh_e->qlink.next = (queue_entry_t)pvh_eh;
		pvh_eh = pvh_e;

		if (pvh_et == PV_HASHED_ENTRY_NULL)
		        pvh_et = pvh_e;
		pv_cnt++;
	}
	PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
}

void mapping_replenish(void);

void mapping_adjust(void) {
	kern_return_t mres;

	pmap_pagetable_corruption_log_setup();

	mres = kernel_thread_start_priority((thread_continue_t)mapping_replenish, NULL, MAXPRI_KERNEL, &mapping_replenish_thread);
	if (mres != KERN_SUCCESS) {
		panic("pmap: mapping_replenish_thread creation failed");
	}
	thread_deallocate(mapping_replenish_thread);
}

unsigned pmap_mapping_thread_wakeups;	
unsigned pmap_kernel_reserve_replenish_stat;
unsigned pmap_user_reserve_replenish_stat;
unsigned pmap_kern_reserve_alloc_stat;

void mapping_replenish(void)
{
	pv_hashed_entry_t	pvh_e;
	pv_hashed_entry_t	pvh_eh;
	pv_hashed_entry_t	pvh_et;
	int			pv_cnt;
	unsigned             	i;

	/* We qualify for VM privileges...*/
	current_thread()->options |= TH_OPT_VMPRIV;

	for (;;) {

		while (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark) {
			pv_cnt = 0;
			pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;

			for (i = 0; i < pv_hashed_kern_alloc_chunk; i++) {
				pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
				pvh_e->qlink.next = (queue_entry_t)pvh_eh;
				pvh_eh = pvh_e;

				if (pvh_et == PV_HASHED_ENTRY_NULL)
					pvh_et = pvh_e;
				pv_cnt++;
			}
			pmap_kernel_reserve_replenish_stat += pv_cnt;
			PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
		}

		pv_cnt = 0;
		pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;

		if (pv_hashed_free_count < pv_hashed_low_water_mark) {
			for (i = 0; i < pv_hashed_alloc_chunk; i++) {
				pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);

				pvh_e->qlink.next = (queue_entry_t)pvh_eh;
				pvh_eh = pvh_e;

				if (pvh_et == PV_HASHED_ENTRY_NULL)
					pvh_et = pvh_e;
				pv_cnt++;
			}
			pmap_user_reserve_replenish_stat += pv_cnt;
			PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
		}
/* Wake threads throttled while the kernel reserve was being replenished.
 */
		if (pmap_pv_throttled_waiters) {
			pmap_pv_throttled_waiters = 0;
			thread_wakeup(&pmap_user_pv_throttle_event);
		}
		/* Check if the kernel pool has been depleted since the
		 * first pass, to reduce refill latency.
		 */
		if (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark)
			continue;
		/* Block sans continuation to avoid yielding kernel stack */
		assert_wait(&mapping_replenish_event, THREAD_UNINT);
		mappingrecurse = 0;
		thread_block(THREAD_CONTINUE_NULL);
		pmap_mapping_thread_wakeups++;
	}
}

/*
 *	Set specified attribute bits.
 */

void
phys_attribute_set(
	ppnum_t		pn,
	int		bits)
{
	int		pai;

	pmap_intr_assert();
	assert(pn != vm_page_fictitious_addr);
	if (pn == vm_page_guard_addr)
		return;

	pai = ppn_to_pai(pn);

	if (!IS_MANAGED_PAGE(pai)) {
		/* Not a managed page.  */
		return;
	}

	LOCK_PVH(pai);
	pmap_phys_attributes[pai] |= bits;
	UNLOCK_PVH(pai);
}

/*
 *	Set the modify bit on the specified physical page.
 */

void
pmap_set_modify(ppnum_t pn)
{
	phys_attribute_set(pn, PHYS_MODIFIED);
}

/*
 *	Clear the modify bits on the specified physical page.
 */

void
pmap_clear_modify(ppnum_t pn)
{
	phys_attribute_clear(pn, PHYS_MODIFIED);
}

/*
 *	pmap_is_modified:
 *
 *	Return whether or not the specified physical page is modified
 *	by any physical maps.
 */

boolean_t
pmap_is_modified(ppnum_t pn)
{
	if (phys_attribute_test(pn, PHYS_MODIFIED))
		return TRUE;
	return FALSE;
}


/*
 *	pmap_clear_reference:
 *
 *	Clear the reference bit on the specified physical page.
 */

void
pmap_clear_reference(ppnum_t pn)
{
	phys_attribute_clear(pn, PHYS_REFERENCED);
}

void
pmap_set_reference(ppnum_t pn)
{
	phys_attribute_set(pn, PHYS_REFERENCED);
}

/*
 *	pmap_is_referenced:
 *
 *	Return whether or not the specified physical page is referenced
 *	by any physical maps.
 */

boolean_t
pmap_is_referenced(ppnum_t pn)
{
        if (phys_attribute_test(pn, PHYS_REFERENCED))
		return TRUE;
	return FALSE;
}


/*
 * pmap_get_refmod(phys)
 *  returns the referenced and modified bits of the specified
 *  physical page.
 */
unsigned int
pmap_get_refmod(ppnum_t pn)
{
        int		refmod;
	unsigned int	retval = 0;

	refmod = phys_attribute_test(pn, PHYS_MODIFIED | PHYS_REFERENCED);

	if (refmod & PHYS_MODIFIED)
	        retval |= VM_MEM_MODIFIED;
	if (refmod & PHYS_REFERENCED)
	        retval |= VM_MEM_REFERENCED;

	return (retval);
}

/*
 * pmap_clear_refmod(phys, mask)
 *  clears the referenced and modified bits as specified by the mask
 *  of the specified physical page.
 */
void
pmap_clear_refmod(ppnum_t pn, unsigned int mask)
{
	unsigned int  x86Mask;

	x86Mask = (   ((mask &   VM_MEM_MODIFIED)?   PHYS_MODIFIED : 0)
	            | ((mask & VM_MEM_REFERENCED)? PHYS_REFERENCED : 0));
	phys_attribute_clear(pn, x86Mask);
}

/*
 *	Routine:
 *		pmap_disconnect
 *
 *	Function:
 *		Disconnect all mappings for this page and return reference and change status
 *		in generic format.
 *
 */
unsigned int
pmap_disconnect(ppnum_t pa)
{
	unsigned refmod, vmrefmod = 0;

	pmap_page_protect(pa, 0);		/* disconnect the page */

	pmap_assert(pa != vm_page_fictitious_addr);
	if ((pa == vm_page_guard_addr) || !IS_MANAGED_PAGE(pa))
		return 0;
	refmod = pmap_phys_attributes[pa] & (PHYS_MODIFIED | PHYS_REFERENCED);
	
	if (refmod & PHYS_MODIFIED)
	        vmrefmod |= VM_MEM_MODIFIED;
	if (refmod & PHYS_REFERENCED)
	        vmrefmod |= VM_MEM_REFERENCED;

	return vmrefmod;
}
Commit	Line	Data
6d2010ae A	1	/*
	2	* Copyright (c) 2000-2010 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28	#include <vm/pmap.h>
	29	#include <i386/pmap_internal.h>
	30
	31	/*
	32	* Each entry in the pv_head_table is locked by a bit in the
	33	* pv_lock_table. The lock bits are accessed by the physical
	34	* address of the page they lock.
	35	*/
	36
	37	char pv_lock_table; / pointer to array of bits */
	38	char *pv_hash_lock_table;
	39
	40	pv_rooted_entry_t pv_head_table; /* array of entries, one per
	41	* page */
	42	uint32_t pv_hashed_free_count = 0;
	43	uint32_t pv_hashed_kern_free_count = 0;
	44
	45	pmap_pagetable_corruption_record_t pmap_pagetable_corruption_records[PMAP_PAGETABLE_CORRUPTION_MAX_LOG];
	46	uint32_t pmap_pagetable_corruption_incidents;
	47	uint64_t pmap_pagetable_corruption_last_abstime = (~(0ULL) >> 1);
	48	uint64_t pmap_pagetable_corruption_interval_abstime;
	49	thread_call_t pmap_pagetable_corruption_log_call;
	50	static thread_call_data_t pmap_pagetable_corruption_log_call_data;
	51	boolean_t pmap_pagetable_corruption_timeout = FALSE;
	52
	53	volatile uint32_t mappingrecurse = 0;
	54
	55	uint32_t pv_hashed_low_water_mark, pv_hashed_kern_low_water_mark, pv_hashed_alloc_chunk, pv_hashed_kern_alloc_chunk;
	56
	57	thread_t mapping_replenish_thread;
	58	event_t mapping_replenish_event, pmap_user_pv_throttle_event;
	59
	60	uint64_t pmap_pv_throttle_stat, pmap_pv_throttled_waiters;
	61
	62	unsigned int pmap_cache_attributes(ppnum_t pn) {
	63	if (pmap_get_cache_attributes(pn) & INTEL_PTE_NCACHE)
	64	return (VM_WIMG_IO);
65	else
66	return (VM_WIMG_COPYBACK);
67	}
68
69	void pmap_set_cache_attributes(ppnum_t pn, unsigned int cacheattr) {
70	unsigned int current, template = 0;
71	int pai;
72
73	if (cacheattr & VM_MEM_NOT_CACHEABLE) {
74	if(!(cacheattr & VM_MEM_GUARDED))
75	template \|= PHYS_PTA;
76	template \|= PHYS_NCACHE;
77	}
78
79	pmap_intr_assert();
80
81	assert((pn != vm_page_fictitious_addr) && (pn != vm_page_guard_addr));
82
83	pai = ppn_to_pai(pn);
84
85	if (!IS_MANAGED_PAGE(pai)) {
86	return;
87	}
88
89	/* override cache attributes for this phys page
90	* Does not walk through existing mappings to adjust,
91	* assumes page is disconnected
92	*/
93
94	LOCK_PVH(pai);
95
96	pmap_update_cache_attributes_locked(pn, template);
97
98	current = pmap_phys_attributes[pai] & PHYS_CACHEABILITY_MASK;
99	pmap_phys_attributes[pai] &= ~PHYS_CACHEABILITY_MASK;
100	pmap_phys_attributes[pai] \|= template;
101
102	UNLOCK_PVH(pai);
103
104	if ((template & PHYS_NCACHE) && !(current & PHYS_NCACHE)) {
105	pmap_sync_page_attributes_phys(pn);
106	}
107	}
108
109	unsigned pmap_get_cache_attributes(ppnum_t pn) {
110	if (last_managed_page == 0)
111	return 0;
112
113	if (!IS_MANAGED_PAGE(ppn_to_pai(pn))) {
114	return INTEL_PTE_NCACHE;
115	}
116
117	/*
118	* The cache attributes are read locklessly for efficiency.
119	*/
120	unsigned int attr = pmap_phys_attributes[ppn_to_pai(pn)];
121	unsigned int template = 0;
122
123	if (attr & PHYS_PTA)
124	template \|= INTEL_PTE_PTA;
125	if (attr & PHYS_NCACHE)
126	template \|= INTEL_PTE_NCACHE;
127	return template;
128	}
129
130
131
132	boolean_t
133	pmap_is_noencrypt(ppnum_t pn)
134	{
135	int pai;
136
137	pai = ppn_to_pai(pn);
138
139	if (!IS_MANAGED_PAGE(pai))
140	return (TRUE);
141
142	if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT)
143	return (TRUE);
144
145	return (FALSE);
146	}
147
148
149	void
150	pmap_set_noencrypt(ppnum_t pn)
151	{
152	int pai;
153
154	pai = ppn_to_pai(pn);
155
156	if (IS_MANAGED_PAGE(pai)) {
157	LOCK_PVH(pai);
158
159	pmap_phys_attributes[pai] \|= PHYS_NOENCRYPT;
160
161	UNLOCK_PVH(pai);
162	}
163	}
164
165
166	void
167	pmap_clear_noencrypt(ppnum_t pn)
168	{
169	int pai;
170
171	pai = ppn_to_pai(pn);
172
173	if (IS_MANAGED_PAGE(pai)) {
174	LOCK_PVH(pai);
175
176	pmap_phys_attributes[pai] &= ~PHYS_NOENCRYPT;
177
178	UNLOCK_PVH(pai);
179	}
180	}
181
182	void
183	compute_pmap_gc_throttle(void *arg __unused)
184	{
185
186	}
187
188
189	__private_extern__ void
190	pmap_pagetable_corruption_msg_log(int (log_func)(const char fmt, ...)__printflike(1,2)) {
191	if (pmap_pagetable_corruption_incidents > 0) {
192	int i, e = MIN(pmap_pagetable_corruption_incidents, PMAP_PAGETABLE_CORRUPTION_MAX_LOG);
193	(*log_func)("%u pagetable corruption incident(s) detected, timeout: %u\n", pmap_pagetable_corruption_incidents, pmap_pagetable_corruption_timeout);
194	for (i = 0; i < e; i++) {
195	(*log_func)("Incident 0x%x, reason: 0x%x, action: 0x%x, time: 0x%llx\n", pmap_pagetable_corruption_records[i].incident, pmap_pagetable_corruption_records[i].reason, pmap_pagetable_corruption_records[i].action, pmap_pagetable_corruption_records[i].abstime);
196	}
197	}
198	}
199
200	static inline void
201	pmap_pagetable_corruption_log_setup(void) {
202	if (pmap_pagetable_corruption_log_call == NULL) {
203	nanotime_to_absolutetime(PMAP_PAGETABLE_CORRUPTION_INTERVAL, 0, &pmap_pagetable_corruption_interval_abstime);
204	thread_call_setup(&pmap_pagetable_corruption_log_call_data,
205	(thread_call_func_t) pmap_pagetable_corruption_msg_log,
206	(thread_call_param_t) &printf);
207	pmap_pagetable_corruption_log_call = &pmap_pagetable_corruption_log_call_data;
208	}
209	}
210
211	void
212	mapping_free_prime(void)
213	{
214	unsigned i;
215	pv_hashed_entry_t pvh_e;
216	pv_hashed_entry_t pvh_eh;
217	pv_hashed_entry_t pvh_et;
218	int pv_cnt;
219
220	/* Scale based on DRAM size */
221	pv_hashed_low_water_mark = MAX(PV_HASHED_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 2000);
222	pv_hashed_low_water_mark = MIN(pv_hashed_low_water_mark, 16000);
223	/* Alterable via sysctl */
224	pv_hashed_kern_low_water_mark = MAX(PV_HASHED_KERN_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 1000);
225	pv_hashed_kern_low_water_mark = MIN(pv_hashed_kern_low_water_mark, 16000);
226	pv_hashed_kern_alloc_chunk = PV_HASHED_KERN_ALLOC_CHUNK_INITIAL;
227	pv_hashed_alloc_chunk = PV_HASHED_ALLOC_CHUNK_INITIAL;
228
229	pv_cnt = 0;
230	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
231
232	for (i = 0; i < (5 * PV_HASHED_ALLOC_CHUNK_INITIAL); i++) {
233	pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
234
235	pvh_e->qlink.next = (queue_entry_t)pvh_eh;
236	pvh_eh = pvh_e;
237
238	if (pvh_et == PV_HASHED_ENTRY_NULL)
239	pvh_et = pvh_e;
240	pv_cnt++;
241	}
242	PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
243
244	pv_cnt = 0;
245	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
246	for (i = 0; i < PV_HASHED_KERN_ALLOC_CHUNK_INITIAL; i++) {
247	pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
248
249	pvh_e->qlink.next = (queue_entry_t)pvh_eh;
250	pvh_eh = pvh_e;
251
252	if (pvh_et == PV_HASHED_ENTRY_NULL)
253	pvh_et = pvh_e;
254	pv_cnt++;
255	}
256	PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
257	}
258
259	void mapping_replenish(void);
260
261	void mapping_adjust(void) {
262	kern_return_t mres;
263
264	pmap_pagetable_corruption_log_setup();
265
266	mres = kernel_thread_start_priority((thread_continue_t)mapping_replenish, NULL, MAXPRI_KERNEL, &mapping_replenish_thread);
267	if (mres != KERN_SUCCESS) {
268	panic("pmap: mapping_replenish_thread creation failed");
269	}
270	thread_deallocate(mapping_replenish_thread);
271	}
272
273	unsigned pmap_mapping_thread_wakeups;
274	unsigned pmap_kernel_reserve_replenish_stat;
275	unsigned pmap_user_reserve_replenish_stat;
276	unsigned pmap_kern_reserve_alloc_stat;
277
278	void mapping_replenish(void)
279	{
280	pv_hashed_entry_t pvh_e;
281	pv_hashed_entry_t pvh_eh;
282	pv_hashed_entry_t pvh_et;
283	int pv_cnt;
284	unsigned i;
285
286	/* We qualify for VM privileges...*/
287	current_thread()->options \|= TH_OPT_VMPRIV;
288
289	for (;;) {
290
291	while (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark) {
292	pv_cnt = 0;
293	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
294
295	for (i = 0; i < pv_hashed_kern_alloc_chunk; i++) {
296	pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
297	pvh_e->qlink.next = (queue_entry_t)pvh_eh;
298	pvh_eh = pvh_e;
299
300	if (pvh_et == PV_HASHED_ENTRY_NULL)
301	pvh_et = pvh_e;
302	pv_cnt++;
303	}
304	pmap_kernel_reserve_replenish_stat += pv_cnt;
305	PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
306	}
307
308	pv_cnt = 0;
309	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
310
311	if (pv_hashed_free_count < pv_hashed_low_water_mark) {
312	for (i = 0; i < pv_hashed_alloc_chunk; i++) {
313	pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
314
315	pvh_e->qlink.next = (queue_entry_t)pvh_eh;
316	pvh_eh = pvh_e;
317
318	if (pvh_et == PV_HASHED_ENTRY_NULL)
319	pvh_et = pvh_e;
320	pv_cnt++;
321	}
322	pmap_user_reserve_replenish_stat += pv_cnt;
323	PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
324	}
325	/* Wake threads throttled while the kernel reserve was being replenished.
326	*/
327	if (pmap_pv_throttled_waiters) {
328	pmap_pv_throttled_waiters = 0;
329	thread_wakeup(&pmap_user_pv_throttle_event);
330	}
331	/* Check if the kernel pool has been depleted since the
332	* first pass, to reduce refill latency.
333	*/
334	if (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark)
335	continue;
336	/* Block sans continuation to avoid yielding kernel stack */
337	assert_wait(&mapping_replenish_event, THREAD_UNINT);
338	mappingrecurse = 0;
339	thread_block(THREAD_CONTINUE_NULL);
340	pmap_mapping_thread_wakeups++;
341	}
342	}
343
344	/*
345	* Set specified attribute bits.
346	*/
347
348	void
349	phys_attribute_set(
350	ppnum_t pn,
351	int bits)
352	{
353	int pai;
354
355	pmap_intr_assert();
356	assert(pn != vm_page_fictitious_addr);
357	if (pn == vm_page_guard_addr)
358	return;
359
360	pai = ppn_to_pai(pn);
361
362	if (!IS_MANAGED_PAGE(pai)) {
363	/* Not a managed page. */
364	return;
365	}
366
367	LOCK_PVH(pai);
368	pmap_phys_attributes[pai] \|= bits;
369	UNLOCK_PVH(pai);
370	}
371
372	/*
373	* Set the modify bit on the specified physical page.
374	*/
375
376	void
377	pmap_set_modify(ppnum_t pn)
378	{
379	phys_attribute_set(pn, PHYS_MODIFIED);
380	}
381
382	/*
383	* Clear the modify bits on the specified physical page.
384	*/
385
386	void
387	pmap_clear_modify(ppnum_t pn)
388	{
389	phys_attribute_clear(pn, PHYS_MODIFIED);
390	}
391
392	/*
393	* pmap_is_modified:
394	*
395	* Return whether or not the specified physical page is modified
396	* by any physical maps.
397	*/
398
399	boolean_t
400	pmap_is_modified(ppnum_t pn)
401	{
402	if (phys_attribute_test(pn, PHYS_MODIFIED))
403	return TRUE;
404	return FALSE;
405	}
406
407
408	/*
409	* pmap_clear_reference:
410	*
411	* Clear the reference bit on the specified physical page.
412	*/
413
414	void
415	pmap_clear_reference(ppnum_t pn)
416	{
417	phys_attribute_clear(pn, PHYS_REFERENCED);
418	}
419
420	void
421	pmap_set_reference(ppnum_t pn)
422	{
423	phys_attribute_set(pn, PHYS_REFERENCED);
424	}
425
426	/*
427	* pmap_is_referenced:
428	*
429	* Return whether or not the specified physical page is referenced
430	* by any physical maps.
431	*/
432
433	boolean_t
434	pmap_is_referenced(ppnum_t pn)
435	{
436	if (phys_attribute_test(pn, PHYS_REFERENCED))
437	return TRUE;
438	return FALSE;
439	}
440
441
442	/*
443	* pmap_get_refmod(phys)
444	* returns the referenced and modified bits of the specified
445	* physical page.
446	*/
447	unsigned int
448	pmap_get_refmod(ppnum_t pn)
449	{
450	int refmod;
451	unsigned int retval = 0;
452
453	refmod = phys_attribute_test(pn, PHYS_MODIFIED \| PHYS_REFERENCED);
454
455	if (refmod & PHYS_MODIFIED)
456	retval \|= VM_MEM_MODIFIED;
457	if (refmod & PHYS_REFERENCED)
458	retval \|= VM_MEM_REFERENCED;
459
460	return (retval);
461	}
462
463	/*
464	* pmap_clear_refmod(phys, mask)
465	* clears the referenced and modified bits as specified by the mask
466	* of the specified physical page.
467	*/
468	void
469	pmap_clear_refmod(ppnum_t pn, unsigned int mask)
470	{
471	unsigned int x86Mask;
472
473	x86Mask = ( ((mask & VM_MEM_MODIFIED)? PHYS_MODIFIED : 0)
474	\| ((mask & VM_MEM_REFERENCED)? PHYS_REFERENCED : 0));
475	phys_attribute_clear(pn, x86Mask);
476	}
477
478	/*
479	* Routine:
480	* pmap_disconnect
481	*
482	* Function:
483	* Disconnect all mappings for this page and return reference and change status
484	* in generic format.
485	*
486	*/
487	unsigned int
488	pmap_disconnect(ppnum_t pa)
489	{
490	unsigned refmod, vmrefmod = 0;
491
492	pmap_page_protect(pa, 0); /* disconnect the page */
493
494	pmap_assert(pa != vm_page_fictitious_addr);
495	if ((pa == vm_page_guard_addr) \|\| !IS_MANAGED_PAGE(pa))
496	return 0;
497	refmod = pmap_phys_attributes[pa] & (PHYS_MODIFIED \| PHYS_REFERENCED);
498
499	if (refmod & PHYS_MODIFIED)
500	vmrefmod \|= VM_MEM_MODIFIED;
501	if (refmod & PHYS_REFERENCED)
502	vmrefmod \|= VM_MEM_REFERENCED;
503
504	return vmrefmod;
505	}