[apple/xnu.git] / osfmk / i386 / pmap_common.c

/*
 * Copyright (c) 2000-2010 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
#include <vm/pmap.h>
#include <i386/pmap_internal.h>

/*
 *	Each entry in the pv_head_table is locked by a bit in the
 *	pv_lock_table.  The lock bits are accessed by the physical
 *	address of the page they lock.
 */

char	*pv_lock_table;		/* pointer to array of bits */
char    *pv_hash_lock_table;

pv_rooted_entry_t	pv_head_table;		/* array of entries, one per
						 * page */
uint32_t			pv_hashed_free_count = 0;
uint32_t			pv_hashed_kern_free_count = 0;

pmap_pagetable_corruption_record_t pmap_pagetable_corruption_records[PMAP_PAGETABLE_CORRUPTION_MAX_LOG];
uint32_t pmap_pagetable_corruption_incidents;
uint64_t pmap_pagetable_corruption_last_abstime = (~(0ULL) >> 1);
uint64_t pmap_pagetable_corruption_interval_abstime;
thread_call_t 	pmap_pagetable_corruption_log_call;
static thread_call_data_t 	pmap_pagetable_corruption_log_call_data;
boolean_t pmap_pagetable_corruption_timeout = FALSE;

volatile uint32_t	mappingrecurse = 0;

uint32_t  pv_hashed_low_water_mark, pv_hashed_kern_low_water_mark, pv_hashed_alloc_chunk, pv_hashed_kern_alloc_chunk;

thread_t mapping_replenish_thread;
event_t	mapping_replenish_event, pmap_user_pv_throttle_event;

uint64_t pmap_pv_throttle_stat, pmap_pv_throttled_waiters;

unsigned int pmap_cache_attributes(ppnum_t pn) {
	if (pmap_get_cache_attributes(pn) & INTEL_PTE_NCACHE)
	        return (VM_WIMG_IO);
	else
		return (VM_WIMG_COPYBACK);
}

void	pmap_set_cache_attributes(ppnum_t pn, unsigned int cacheattr) {
	unsigned int current, template = 0;
	int pai;

	if (cacheattr & VM_MEM_NOT_CACHEABLE) {
		if(!(cacheattr & VM_MEM_GUARDED))
			template |= PHYS_PTA;
		template |= PHYS_NCACHE;
	}

	pmap_intr_assert();

	assert((pn != vm_page_fictitious_addr) && (pn != vm_page_guard_addr));

	pai = ppn_to_pai(pn);

	if (!IS_MANAGED_PAGE(pai)) {
		return;
	}

	/* override cache attributes for this phys page
	 * Does not walk through existing mappings to adjust,
	 * assumes page is disconnected
	 */

	LOCK_PVH(pai);

	pmap_update_cache_attributes_locked(pn, template);

	current = pmap_phys_attributes[pai] & PHYS_CACHEABILITY_MASK;
	pmap_phys_attributes[pai] &= ~PHYS_CACHEABILITY_MASK;
	pmap_phys_attributes[pai] |= template;

	UNLOCK_PVH(pai);

	if ((template & PHYS_NCACHE) && !(current & PHYS_NCACHE)) {
		pmap_sync_page_attributes_phys(pn);
	}
}

unsigned	pmap_get_cache_attributes(ppnum_t pn) {
	if (last_managed_page == 0)
		return 0;

	if (!IS_MANAGED_PAGE(ppn_to_pai(pn))) {
	    return INTEL_PTE_NCACHE;
	}

	/*
	 * The cache attributes are read locklessly for efficiency.
	 */
	unsigned int attr = pmap_phys_attributes[ppn_to_pai(pn)];
	unsigned int template = 0;
	
	if (attr & PHYS_PTA)
		template |= INTEL_PTE_PTA;
	if (attr & PHYS_NCACHE)
		template |= INTEL_PTE_NCACHE;
	return template;
}


boolean_t
pmap_is_noencrypt(ppnum_t pn)
{
	int		pai;

	pai = ppn_to_pai(pn);

	if (!IS_MANAGED_PAGE(pai))
		return (FALSE);

	if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT)
		return (TRUE);

	return (FALSE);
}


void
pmap_set_noencrypt(ppnum_t pn)
{
	int		pai;

	pai = ppn_to_pai(pn);

	if (IS_MANAGED_PAGE(pai)) {
		LOCK_PVH(pai);

		pmap_phys_attributes[pai] |= PHYS_NOENCRYPT;

		UNLOCK_PVH(pai);
	}
}


void
pmap_clear_noencrypt(ppnum_t pn)
{
	int		pai;

	pai = ppn_to_pai(pn);

	if (IS_MANAGED_PAGE(pai)) {
		/*
		 * synchronization at VM layer prevents PHYS_NOENCRYPT
		 * from changing state, so we don't need the lock to inspect
		 */
		if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT) {
			LOCK_PVH(pai);

			pmap_phys_attributes[pai] &= ~PHYS_NOENCRYPT;

			UNLOCK_PVH(pai);
		}
	}
}

void
compute_pmap_gc_throttle(void *arg __unused)
{
	
}


__private_extern__ void
pmap_pagetable_corruption_msg_log(int (*log_func)(const char * fmt, ...)__printflike(1,2)) {
	if (pmap_pagetable_corruption_incidents > 0) {
		int i, e = MIN(pmap_pagetable_corruption_incidents, PMAP_PAGETABLE_CORRUPTION_MAX_LOG);
		(*log_func)("%u pagetable corruption incident(s) detected, timeout: %u\n", pmap_pagetable_corruption_incidents, pmap_pagetable_corruption_timeout);
		for (i = 0; i < e; i++) {
			(*log_func)("Incident 0x%x, reason: 0x%x, action: 0x%x, time: 0x%llx\n", pmap_pagetable_corruption_records[i].incident,  pmap_pagetable_corruption_records[i].reason, pmap_pagetable_corruption_records[i].action, pmap_pagetable_corruption_records[i].abstime);
		}
	}
}

static inline void
pmap_pagetable_corruption_log_setup(void) {
	if (pmap_pagetable_corruption_log_call == NULL) {
		nanotime_to_absolutetime(PMAP_PAGETABLE_CORRUPTION_INTERVAL, 0, &pmap_pagetable_corruption_interval_abstime);
		thread_call_setup(&pmap_pagetable_corruption_log_call_data,
		    (thread_call_func_t) pmap_pagetable_corruption_msg_log,
		    (thread_call_param_t) &printf);
		pmap_pagetable_corruption_log_call = &pmap_pagetable_corruption_log_call_data;
	}
}

void
mapping_free_prime(void)
{
	unsigned		i;
	pv_hashed_entry_t	pvh_e;
	pv_hashed_entry_t	pvh_eh;
	pv_hashed_entry_t	pvh_et;
	int			pv_cnt;

	/* Scale based on DRAM size */
	pv_hashed_low_water_mark = MAX(PV_HASHED_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 2000);
	pv_hashed_low_water_mark = MIN(pv_hashed_low_water_mark, 16000);
	/* Alterable via sysctl */
	pv_hashed_kern_low_water_mark = MAX(PV_HASHED_KERN_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 1000);
	pv_hashed_kern_low_water_mark = MIN(pv_hashed_kern_low_water_mark, 16000);
	pv_hashed_kern_alloc_chunk = PV_HASHED_KERN_ALLOC_CHUNK_INITIAL;
	pv_hashed_alloc_chunk = PV_HASHED_ALLOC_CHUNK_INITIAL;

	pv_cnt = 0;
	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;

	for (i = 0; i < (5 * PV_HASHED_ALLOC_CHUNK_INITIAL); i++) {
		pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);

		pvh_e->qlink.next = (queue_entry_t)pvh_eh;
		pvh_eh = pvh_e;

		if (pvh_et == PV_HASHED_ENTRY_NULL)
		        pvh_et = pvh_e;
		pv_cnt++;
	}
	PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);

	pv_cnt = 0;
	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
	for (i = 0; i < PV_HASHED_KERN_ALLOC_CHUNK_INITIAL; i++) {
		pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);

		pvh_e->qlink.next = (queue_entry_t)pvh_eh;
		pvh_eh = pvh_e;

		if (pvh_et == PV_HASHED_ENTRY_NULL)
		        pvh_et = pvh_e;
		pv_cnt++;
	}
	PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
}

void mapping_replenish(void);

void mapping_adjust(void) {
	kern_return_t mres;

	pmap_pagetable_corruption_log_setup();

	mres = kernel_thread_start_priority((thread_continue_t)mapping_replenish, NULL, MAXPRI_KERNEL, &mapping_replenish_thread);
	if (mres != KERN_SUCCESS) {
		panic("pmap: mapping_replenish_thread creation failed");
	}
	thread_deallocate(mapping_replenish_thread);
}

unsigned pmap_mapping_thread_wakeups;	
unsigned pmap_kernel_reserve_replenish_stat;
unsigned pmap_user_reserve_replenish_stat;
unsigned pmap_kern_reserve_alloc_stat;

void mapping_replenish(void)
{
	pv_hashed_entry_t	pvh_e;
	pv_hashed_entry_t	pvh_eh;
	pv_hashed_entry_t	pvh_et;
	int			pv_cnt;
	unsigned             	i;

	/* We qualify for VM privileges...*/
	current_thread()->options |= TH_OPT_VMPRIV;

	for (;;) {

		while (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark) {
			pv_cnt = 0;
			pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;

			for (i = 0; i < pv_hashed_kern_alloc_chunk; i++) {
				pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
				pvh_e->qlink.next = (queue_entry_t)pvh_eh;
				pvh_eh = pvh_e;

				if (pvh_et == PV_HASHED_ENTRY_NULL)
					pvh_et = pvh_e;
				pv_cnt++;
			}
			pmap_kernel_reserve_replenish_stat += pv_cnt;
			PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
		}

		pv_cnt = 0;
		pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;

		if (pv_hashed_free_count < pv_hashed_low_water_mark) {
			for (i = 0; i < pv_hashed_alloc_chunk; i++) {
				pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);

				pvh_e->qlink.next = (queue_entry_t)pvh_eh;
				pvh_eh = pvh_e;

				if (pvh_et == PV_HASHED_ENTRY_NULL)
					pvh_et = pvh_e;
				pv_cnt++;
			}
			pmap_user_reserve_replenish_stat += pv_cnt;
			PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
		}
/* Wake threads throttled while the kernel reserve was being replenished.
 */
		if (pmap_pv_throttled_waiters) {
			pmap_pv_throttled_waiters = 0;
			thread_wakeup(&pmap_user_pv_throttle_event);
		}
		/* Check if the kernel pool has been depleted since the
		 * first pass, to reduce refill latency.
		 */
		if (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark)
			continue;
		/* Block sans continuation to avoid yielding kernel stack */
		assert_wait(&mapping_replenish_event, THREAD_UNINT);
		mappingrecurse = 0;
		thread_block(THREAD_CONTINUE_NULL);
		pmap_mapping_thread_wakeups++;
	}
}

/*
 *	Set specified attribute bits.
 */

void
phys_attribute_set(
	ppnum_t		pn,
	int		bits)
{
	int		pai;

	pmap_intr_assert();
	assert(pn != vm_page_fictitious_addr);
	if (pn == vm_page_guard_addr)
		return;

	pai = ppn_to_pai(pn);

	if (!IS_MANAGED_PAGE(pai)) {
		/* Not a managed page.  */
		return;
	}

	LOCK_PVH(pai);
	pmap_phys_attributes[pai] |= bits;
	UNLOCK_PVH(pai);
}

/*
 *	Set the modify bit on the specified physical page.
 */

void
pmap_set_modify(ppnum_t pn)
{
	phys_attribute_set(pn, PHYS_MODIFIED);
}

/*
 *	Clear the modify bits on the specified physical page.
 */

void
pmap_clear_modify(ppnum_t pn)
{
	phys_attribute_clear(pn, PHYS_MODIFIED);
}

/*
 *	pmap_is_modified:
 *
 *	Return whether or not the specified physical page is modified
 *	by any physical maps.
 */

boolean_t
pmap_is_modified(ppnum_t pn)
{
	if (phys_attribute_test(pn, PHYS_MODIFIED))
		return TRUE;
	return FALSE;
}


/*
 *	pmap_clear_reference:
 *
 *	Clear the reference bit on the specified physical page.
 */

void
pmap_clear_reference(ppnum_t pn)
{
	phys_attribute_clear(pn, PHYS_REFERENCED);
}

void
pmap_set_reference(ppnum_t pn)
{
	phys_attribute_set(pn, PHYS_REFERENCED);
}

/*
 *	pmap_is_referenced:
 *
 *	Return whether or not the specified physical page is referenced
 *	by any physical maps.
 */

boolean_t
pmap_is_referenced(ppnum_t pn)
{
        if (phys_attribute_test(pn, PHYS_REFERENCED))
		return TRUE;
	return FALSE;
}


/*
 * pmap_get_refmod(phys)
 *  returns the referenced and modified bits of the specified
 *  physical page.
 */
unsigned int
pmap_get_refmod(ppnum_t pn)
{
        int		refmod;
	unsigned int	retval = 0;

	refmod = phys_attribute_test(pn, PHYS_MODIFIED | PHYS_REFERENCED);

	if (refmod & PHYS_MODIFIED)
	        retval |= VM_MEM_MODIFIED;
	if (refmod & PHYS_REFERENCED)
	        retval |= VM_MEM_REFERENCED;

	return (retval);
}

/*
 * pmap_clear_refmod(phys, mask)
 *  clears the referenced and modified bits as specified by the mask
 *  of the specified physical page.
 */
void
pmap_clear_refmod(ppnum_t pn, unsigned int mask)
{
	unsigned int  x86Mask;

	x86Mask = (   ((mask &   VM_MEM_MODIFIED)?   PHYS_MODIFIED : 0)
	            | ((mask & VM_MEM_REFERENCED)? PHYS_REFERENCED : 0));
	phys_attribute_clear(pn, x86Mask);
}

/*
 *	Routine:
 *		pmap_disconnect
 *
 *	Function:
 *		Disconnect all mappings for this page and return reference and change status
 *		in generic format.
 *
 */
unsigned int
pmap_disconnect(ppnum_t pa)
{
	unsigned refmod, vmrefmod = 0;

	pmap_page_protect(pa, 0);		/* disconnect the page */

	pmap_assert(pa != vm_page_fictitious_addr);
	if ((pa == vm_page_guard_addr) || !IS_MANAGED_PAGE(pa))
		return 0;
	refmod = pmap_phys_attributes[pa] & (PHYS_MODIFIED | PHYS_REFERENCED);
	
	if (refmod & PHYS_MODIFIED)
	        vmrefmod |= VM_MEM_MODIFIED;
	if (refmod & PHYS_REFERENCED)
	        vmrefmod |= VM_MEM_REFERENCED;

	return vmrefmod;
}
Commit	Line	Data
6d2010ae A	1	/*
	2	* Copyright (c) 2000-2010 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28	#include <vm/pmap.h>
	29	#include <i386/pmap_internal.h>
	30
	31	/*
	32	* Each entry in the pv_head_table is locked by a bit in the
	33	* pv_lock_table. The lock bits are accessed by the physical
	34	* address of the page they lock.
	35	*/
	36
	37	char pv_lock_table; / pointer to array of bits */
	38	char *pv_hash_lock_table;
	39
	40	pv_rooted_entry_t pv_head_table; /* array of entries, one per
	41	* page */
	42	uint32_t pv_hashed_free_count = 0;
	43	uint32_t pv_hashed_kern_free_count = 0;
	44
	45	pmap_pagetable_corruption_record_t pmap_pagetable_corruption_records[PMAP_PAGETABLE_CORRUPTION_MAX_LOG];
	46	uint32_t pmap_pagetable_corruption_incidents;
	47	uint64_t pmap_pagetable_corruption_last_abstime = (~(0ULL) >> 1);
	48	uint64_t pmap_pagetable_corruption_interval_abstime;
	49	thread_call_t pmap_pagetable_corruption_log_call;
	50	static thread_call_data_t pmap_pagetable_corruption_log_call_data;
	51	boolean_t pmap_pagetable_corruption_timeout = FALSE;
	52
	53	volatile uint32_t mappingrecurse = 0;
	54
	55	uint32_t pv_hashed_low_water_mark, pv_hashed_kern_low_water_mark, pv_hashed_alloc_chunk, pv_hashed_kern_alloc_chunk;
	56
	57	thread_t mapping_replenish_thread;
	58	event_t mapping_replenish_event, pmap_user_pv_throttle_event;
	59
	60	uint64_t pmap_pv_throttle_stat, pmap_pv_throttled_waiters;
	61
	62	unsigned int pmap_cache_attributes(ppnum_t pn) {
	63	if (pmap_get_cache_attributes(pn) & INTEL_PTE_NCACHE)
	64	return (VM_WIMG_IO);
65	else
66	return (VM_WIMG_COPYBACK);
67	}
68
69	void pmap_set_cache_attributes(ppnum_t pn, unsigned int cacheattr) {
70	unsigned int current, template = 0;
71	int pai;
72
73	if (cacheattr & VM_MEM_NOT_CACHEABLE) {
74	if(!(cacheattr & VM_MEM_GUARDED))
75	template \|= PHYS_PTA;
76	template \|= PHYS_NCACHE;
77	}
78
79	pmap_intr_assert();
80
81	assert((pn != vm_page_fictitious_addr) && (pn != vm_page_guard_addr));
82
83	pai = ppn_to_pai(pn);
84
85	if (!IS_MANAGED_PAGE(pai)) {
86	return;
87	}
88
89	/* override cache attributes for this phys page
90	* Does not walk through existing mappings to adjust,
91	* assumes page is disconnected
92	*/
93
94	LOCK_PVH(pai);
95
96	pmap_update_cache_attributes_locked(pn, template);
97
98	current = pmap_phys_attributes[pai] & PHYS_CACHEABILITY_MASK;
99	pmap_phys_attributes[pai] &= ~PHYS_CACHEABILITY_MASK;
100	pmap_phys_attributes[pai] \|= template;
101
102	UNLOCK_PVH(pai);
103
104	if ((template & PHYS_NCACHE) && !(current & PHYS_NCACHE)) {
105	pmap_sync_page_attributes_phys(pn);
106	}
107	}
108
109	unsigned pmap_get_cache_attributes(ppnum_t pn) {
110	if (last_managed_page == 0)
111	return 0;
112
113	if (!IS_MANAGED_PAGE(ppn_to_pai(pn))) {
114	return INTEL_PTE_NCACHE;
115	}
116
117	/*
118	* The cache attributes are read locklessly for efficiency.
119	*/
120	unsigned int attr = pmap_phys_attributes[ppn_to_pai(pn)];
121	unsigned int template = 0;
122
123	if (attr & PHYS_PTA)
124	template \|= INTEL_PTE_PTA;
125	if (attr & PHYS_NCACHE)
126	template \|= INTEL_PTE_NCACHE;
127	return template;
128	}
129
130
131
132	boolean_t
133	pmap_is_noencrypt(ppnum_t pn)
134	{
135	int pai;
136
137	pai = ppn_to_pai(pn);
138
139	if (!IS_MANAGED_PAGE(pai))
7ddcb079	140	return (FALSE);
6d2010ae A	141
	142	if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT)
	143	return (TRUE);
	144
	145	return (FALSE);
	146	}
	147
	148
	149	void
	150	pmap_set_noencrypt(ppnum_t pn)
	151	{
	152	int pai;
	153
	154	pai = ppn_to_pai(pn);
	155
	156	if (IS_MANAGED_PAGE(pai)) {
	157	LOCK_PVH(pai);
	158
	159	pmap_phys_attributes[pai] \|= PHYS_NOENCRYPT;
	160
	161	UNLOCK_PVH(pai);
	162	}
	163	}
	164
	165
	166	void
	167	pmap_clear_noencrypt(ppnum_t pn)
	168	{
	169	int pai;
	170
	171	pai = ppn_to_pai(pn);
	172
	173	if (IS_MANAGED_PAGE(pai)) {
7ddcb079 A	174	/*
	175	* synchronization at VM layer prevents PHYS_NOENCRYPT
	176	* from changing state, so we don't need the lock to inspect
	177	*/
	178	if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT) {
	179	LOCK_PVH(pai);
6d2010ae	180
7ddcb079	181	pmap_phys_attributes[pai] &= ~PHYS_NOENCRYPT;
6d2010ae	182
7ddcb079 A	183	UNLOCK_PVH(pai);
7ddcb079 A	184	}
6d2010ae A	185	}
	186	}
	187
	188	void
	189	compute_pmap_gc_throttle(void *arg __unused)
	190	{
	191
	192	}
	193
	194
	195	__private_extern__ void
	196	pmap_pagetable_corruption_msg_log(int (log_func)(const char fmt, ...)__printflike(1,2)) {
	197	if (pmap_pagetable_corruption_incidents > 0) {
	198	int i, e = MIN(pmap_pagetable_corruption_incidents, PMAP_PAGETABLE_CORRUPTION_MAX_LOG);
	199	(*log_func)("%u pagetable corruption incident(s) detected, timeout: %u\n", pmap_pagetable_corruption_incidents, pmap_pagetable_corruption_timeout);
	200	for (i = 0; i < e; i++) {
	201	(*log_func)("Incident 0x%x, reason: 0x%x, action: 0x%x, time: 0x%llx\n", pmap_pagetable_corruption_records[i].incident, pmap_pagetable_corruption_records[i].reason, pmap_pagetable_corruption_records[i].action, pmap_pagetable_corruption_records[i].abstime);
	202	}
	203	}
	204	}
	205
	206	static inline void
	207	pmap_pagetable_corruption_log_setup(void) {
	208	if (pmap_pagetable_corruption_log_call == NULL) {
	209	nanotime_to_absolutetime(PMAP_PAGETABLE_CORRUPTION_INTERVAL, 0, &pmap_pagetable_corruption_interval_abstime);
	210	thread_call_setup(&pmap_pagetable_corruption_log_call_data,
	211	(thread_call_func_t) pmap_pagetable_corruption_msg_log,
	212	(thread_call_param_t) &printf);
	213	pmap_pagetable_corruption_log_call = &pmap_pagetable_corruption_log_call_data;
	214	}
	215	}
	216
	217	void
	218	mapping_free_prime(void)
	219	{
	220	unsigned i;
	221	pv_hashed_entry_t pvh_e;
	222	pv_hashed_entry_t pvh_eh;
	223	pv_hashed_entry_t pvh_et;
	224	int pv_cnt;
	225
	226	/* Scale based on DRAM size */
	227	pv_hashed_low_water_mark = MAX(PV_HASHED_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 2000);
	228	pv_hashed_low_water_mark = MIN(pv_hashed_low_water_mark, 16000);
	229	/* Alterable via sysctl */
	230	pv_hashed_kern_low_water_mark = MAX(PV_HASHED_KERN_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 1000);
	231	pv_hashed_kern_low_water_mark = MIN(pv_hashed_kern_low_water_mark, 16000);
	232	pv_hashed_kern_alloc_chunk = PV_HASHED_KERN_ALLOC_CHUNK_INITIAL;
	233	pv_hashed_alloc_chunk = PV_HASHED_ALLOC_CHUNK_INITIAL;
	234
	235	pv_cnt = 0;
	236	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
	237
	238	for (i = 0; i < (5 * PV_HASHED_ALLOC_CHUNK_INITIAL); i++) {
	239	pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
	240
	241	pvh_e->qlink.next = (queue_entry_t)pvh_eh;
	242	pvh_eh = pvh_e;
	243
	244	if (pvh_et == PV_HASHED_ENTRY_NULL)
	245	pvh_et = pvh_e;
	246	pv_cnt++;
	247	}
	248	PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
249
250	pv_cnt = 0;
251	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
252	for (i = 0; i < PV_HASHED_KERN_ALLOC_CHUNK_INITIAL; i++) {
253	pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
254
255	pvh_e->qlink.next = (queue_entry_t)pvh_eh;
256	pvh_eh = pvh_e;
257
258	if (pvh_et == PV_HASHED_ENTRY_NULL)
259	pvh_et = pvh_e;
260	pv_cnt++;
261	}
262	PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
263	}
264
265	void mapping_replenish(void);
266
267	void mapping_adjust(void) {
268	kern_return_t mres;
269
270	pmap_pagetable_corruption_log_setup();
271
272	mres = kernel_thread_start_priority((thread_continue_t)mapping_replenish, NULL, MAXPRI_KERNEL, &mapping_replenish_thread);
273	if (mres != KERN_SUCCESS) {
274	panic("pmap: mapping_replenish_thread creation failed");
275	}
276	thread_deallocate(mapping_replenish_thread);
277	}
278
279	unsigned pmap_mapping_thread_wakeups;
280	unsigned pmap_kernel_reserve_replenish_stat;
281	unsigned pmap_user_reserve_replenish_stat;
282	unsigned pmap_kern_reserve_alloc_stat;
283
284	void mapping_replenish(void)
285	{
286	pv_hashed_entry_t pvh_e;
287	pv_hashed_entry_t pvh_eh;
288	pv_hashed_entry_t pvh_et;
289	int pv_cnt;
290	unsigned i;
291
292	/* We qualify for VM privileges...*/
293	current_thread()->options \|= TH_OPT_VMPRIV;
294
295	for (;;) {
296
297	while (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark) {
298	pv_cnt = 0;
299	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
300
301	for (i = 0; i < pv_hashed_kern_alloc_chunk; i++) {
302	pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
303	pvh_e->qlink.next = (queue_entry_t)pvh_eh;
304	pvh_eh = pvh_e;
305
306	if (pvh_et == PV_HASHED_ENTRY_NULL)
307	pvh_et = pvh_e;
308	pv_cnt++;
309	}
310	pmap_kernel_reserve_replenish_stat += pv_cnt;
311	PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
312	}
313
314	pv_cnt = 0;
315	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
316
317	if (pv_hashed_free_count < pv_hashed_low_water_mark) {
318	for (i = 0; i < pv_hashed_alloc_chunk; i++) {
319	pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
320
321	pvh_e->qlink.next = (queue_entry_t)pvh_eh;
322	pvh_eh = pvh_e;
323
324	if (pvh_et == PV_HASHED_ENTRY_NULL)
325	pvh_et = pvh_e;
326	pv_cnt++;
327	}
328	pmap_user_reserve_replenish_stat += pv_cnt;
329	PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
330	}
331	/* Wake threads throttled while the kernel reserve was being replenished.
332	*/
333	if (pmap_pv_throttled_waiters) {
334	pmap_pv_throttled_waiters = 0;
335	thread_wakeup(&pmap_user_pv_throttle_event);
336	}
337	/* Check if the kernel pool has been depleted since the
338	* first pass, to reduce refill latency.
339	*/
340	if (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark)
341	continue;
342	/* Block sans continuation to avoid yielding kernel stack */
343	assert_wait(&mapping_replenish_event, THREAD_UNINT);
344	mappingrecurse = 0;
345	thread_block(THREAD_CONTINUE_NULL);
346	pmap_mapping_thread_wakeups++;
347	}
348	}
349
350	/*
351	* Set specified attribute bits.
352	*/
353
354	void
355	phys_attribute_set(
356	ppnum_t pn,
357	int bits)
358	{
359	int pai;
360
361	pmap_intr_assert();
362	assert(pn != vm_page_fictitious_addr);
363	if (pn == vm_page_guard_addr)
364	return;
365
366	pai = ppn_to_pai(pn);
367
368	if (!IS_MANAGED_PAGE(pai)) {
369	/* Not a managed page. */
370	return;
371	}
372
373	LOCK_PVH(pai);
374	pmap_phys_attributes[pai] \|= bits;
375	UNLOCK_PVH(pai);
376	}
377
378	/*
379	* Set the modify bit on the specified physical page.
380	*/
381
382	void
383	pmap_set_modify(ppnum_t pn)
384	{
385	phys_attribute_set(pn, PHYS_MODIFIED);
386	}
387
388	/*
389	* Clear the modify bits on the specified physical page.
390	*/
391
392	void
393	pmap_clear_modify(ppnum_t pn)
394	{
395	phys_attribute_clear(pn, PHYS_MODIFIED);
396	}
397
398	/*
399	* pmap_is_modified:
400	*
401	* Return whether or not the specified physical page is modified
402	* by any physical maps.
403	*/
404
405	boolean_t
406	pmap_is_modified(ppnum_t pn)
407	{
408	if (phys_attribute_test(pn, PHYS_MODIFIED))
409	return TRUE;
410	return FALSE;
411	}
412
413
414	/*
415	* pmap_clear_reference:
416	*
417	* Clear the reference bit on the specified physical page.
418	*/
419
420	void
421	pmap_clear_reference(ppnum_t pn)
422	{
423	phys_attribute_clear(pn, PHYS_REFERENCED);
424	}
425
426	void
427	pmap_set_reference(ppnum_t pn)
428	{
429	phys_attribute_set(pn, PHYS_REFERENCED);
430	}
431
432	/*
433	* pmap_is_referenced:
434	*
435	* Return whether or not the specified physical page is referenced
436	* by any physical maps.
437	*/
438
439	boolean_t
440	pmap_is_referenced(ppnum_t pn)
441	{
442	if (phys_attribute_test(pn, PHYS_REFERENCED))
443	return TRUE;
444	return FALSE;
445	}
446
447
448	/*
449	* pmap_get_refmod(phys)
450	* returns the referenced and modified bits of the specified
451	* physical page.
452	*/
453	unsigned int
454	pmap_get_refmod(ppnum_t pn)
455	{
456	int refmod;
457	unsigned int retval = 0;
458
459	refmod = phys_attribute_test(pn, PHYS_MODIFIED \| PHYS_REFERENCED);
460
461	if (refmod & PHYS_MODIFIED)
462	retval \|= VM_MEM_MODIFIED;
463	if (refmod & PHYS_REFERENCED)
464	retval \|= VM_MEM_REFERENCED;
465
466	return (retval);
467	}
468
469	/*
470	* pmap_clear_refmod(phys, mask)
471	* clears the referenced and modified bits as specified by the mask
472	* of the specified physical page.
473	*/
474	void
475	pmap_clear_refmod(ppnum_t pn, unsigned int mask)
476	{
477	unsigned int x86Mask;
478
479	x86Mask = ( ((mask & VM_MEM_MODIFIED)? PHYS_MODIFIED : 0)
480	\| ((mask & VM_MEM_REFERENCED)? PHYS_REFERENCED : 0));
481	phys_attribute_clear(pn, x86Mask);
482	}
483
484	/*
485	* Routine:
486	* pmap_disconnect
487	*
488	* Function:
489	* Disconnect all mappings for this page and return reference and change status
490	* in generic format.
491	*
492	*/
493	unsigned int
494	pmap_disconnect(ppnum_t pa)
495	{
496	unsigned refmod, vmrefmod = 0;
497
498	pmap_page_protect(pa, 0); /* disconnect the page */
499
500	pmap_assert(pa != vm_page_fictitious_addr);
501	if ((pa == vm_page_guard_addr) \|\| !IS_MANAGED_PAGE(pa))
502	return 0;
503	refmod = pmap_phys_attributes[pa] & (PHYS_MODIFIED \| PHYS_REFERENCED);
504
505	if (refmod & PHYS_MODIFIED)
506	vmrefmod \|= VM_MEM_MODIFIED;
507	if (refmod & PHYS_REFERENCED)
508	vmrefmod \|= VM_MEM_REFERENCED;
509
510	return vmrefmod;
511	}