[apple/xnu.git] / osfmk / i386 / pmap_common.c

/*
 * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
#include <vm/pmap.h>
#include <kern/ledger.h>
#include <i386/pmap_internal.h>


/*
 *	Each entry in the pv_head_table is locked by a bit in the
 *	pv_lock_table.  The lock bits are accessed by the physical
 *	address of the page they lock.
 */

char	*pv_lock_table;		/* pointer to array of bits */
char    *pv_hash_lock_table;

pv_rooted_entry_t	pv_head_table;		/* array of entries, one per
						 * page */
uint32_t			pv_hashed_free_count = 0;
uint32_t			pv_hashed_kern_free_count = 0;

pmap_pagetable_corruption_record_t pmap_pagetable_corruption_records[PMAP_PAGETABLE_CORRUPTION_MAX_LOG];
uint32_t pmap_pagetable_corruption_incidents;
uint64_t pmap_pagetable_corruption_last_abstime = (~(0ULL) >> 1);
uint64_t pmap_pagetable_corruption_interval_abstime;
thread_call_t 	pmap_pagetable_corruption_log_call;
static thread_call_data_t 	pmap_pagetable_corruption_log_call_data;
boolean_t pmap_pagetable_corruption_timeout = FALSE;

volatile uint32_t	mappingrecurse = 0;

uint32_t  pv_hashed_low_water_mark, pv_hashed_kern_low_water_mark, pv_hashed_alloc_chunk, pv_hashed_kern_alloc_chunk;

thread_t mapping_replenish_thread;
event_t	mapping_replenish_event, pmap_user_pv_throttle_event;

uint64_t pmap_pv_throttle_stat, pmap_pv_throttled_waiters;

int pmap_asserts_enabled = DEBUG;
int pmap_asserts_traced = 0;

unsigned int pmap_cache_attributes(ppnum_t pn) {
	if (pmap_get_cache_attributes(pn, FALSE) & INTEL_PTE_NCACHE)
	        return (VM_WIMG_IO);
	else
		return (VM_WIMG_COPYBACK);
}

void	pmap_set_cache_attributes(ppnum_t pn, unsigned int cacheattr) {
	unsigned int current, template = 0;
	int pai;

	if (cacheattr & VM_MEM_NOT_CACHEABLE) {
		if(!(cacheattr & VM_MEM_GUARDED))
			template |= PHYS_PTA;
		template |= PHYS_NCACHE;
	}

	pmap_intr_assert();

	assert((pn != vm_page_fictitious_addr) && (pn != vm_page_guard_addr));

	pai = ppn_to_pai(pn);

	if (!IS_MANAGED_PAGE(pai)) {
		return;
	}

	/* override cache attributes for this phys page
	 * Does not walk through existing mappings to adjust,
	 * assumes page is disconnected
	 */

	LOCK_PVH(pai);

	pmap_update_cache_attributes_locked(pn, template);

	current = pmap_phys_attributes[pai] & PHYS_CACHEABILITY_MASK;
	pmap_phys_attributes[pai] &= ~PHYS_CACHEABILITY_MASK;
	pmap_phys_attributes[pai] |= template;

	UNLOCK_PVH(pai);

	if ((template & PHYS_NCACHE) && !(current & PHYS_NCACHE)) {
		pmap_sync_page_attributes_phys(pn);
	}
}

unsigned	pmap_get_cache_attributes(ppnum_t pn, boolean_t is_ept) {
	if (last_managed_page == 0)
		return 0;

	if (!IS_MANAGED_PAGE(ppn_to_pai(pn)))
	    return PTE_NCACHE(is_ept);

	/*
	 * The cache attributes are read locklessly for efficiency.
	 */
	unsigned int attr = pmap_phys_attributes[ppn_to_pai(pn)];
	unsigned int template = 0;

	/*
	 * The PTA bit is currently unsupported for EPT PTEs.
	 */
	if ((attr & PHYS_PTA) && !is_ept)
		template |= INTEL_PTE_PTA;

	/*
	 * If the page isn't marked as NCACHE, the default for EPT entries
	 * is WB.
	 */
	if (attr & PHYS_NCACHE)
		template |= PTE_NCACHE(is_ept);
	else if (is_ept)
		template |= INTEL_EPT_WB;

	return template;
}

boolean_t 
pmap_has_managed_page(ppnum_t first, ppnum_t last)
{
	ppnum_t   pn;
    boolean_t result;

    assert(last_managed_page);
    assert(first <= last);

    for (result = FALSE, pn = first; 
    	!result 
    	  && (pn <= last)
    	  && (pn <= last_managed_page); 
    	 pn++)
    {
    	result = (0 != (pmap_phys_attributes[pn] & PHYS_MANAGED));
    }

	return (result);
}

boolean_t
pmap_is_noencrypt(ppnum_t pn)
{
	int		pai;

	pai = ppn_to_pai(pn);

	if (!IS_MANAGED_PAGE(pai))
		return (FALSE);

	if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT)
		return (TRUE);

	return (FALSE);
}


void
pmap_set_noencrypt(ppnum_t pn)
{
	int		pai;

	pai = ppn_to_pai(pn);

	if (IS_MANAGED_PAGE(pai)) {
		LOCK_PVH(pai);

		pmap_phys_attributes[pai] |= PHYS_NOENCRYPT;

		UNLOCK_PVH(pai);
	}
}


void
pmap_clear_noencrypt(ppnum_t pn)
{
	int		pai;

	pai = ppn_to_pai(pn);

	if (IS_MANAGED_PAGE(pai)) {
		/*
		 * synchronization at VM layer prevents PHYS_NOENCRYPT
		 * from changing state, so we don't need the lock to inspect
		 */
		if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT) {
			LOCK_PVH(pai);

			pmap_phys_attributes[pai] &= ~PHYS_NOENCRYPT;

			UNLOCK_PVH(pai);
		}
	}
}

void
compute_pmap_gc_throttle(void *arg __unused)
{
	
}


void
pmap_lock_phys_page(ppnum_t pn)
{
	int		pai;

	pai = ppn_to_pai(pn);

	if (IS_MANAGED_PAGE(pai)) {
		LOCK_PVH(pai);
	} else
		simple_lock(&phys_backup_lock);
}


void
pmap_unlock_phys_page(ppnum_t pn)
{
	int		pai;

	pai = ppn_to_pai(pn);

	if (IS_MANAGED_PAGE(pai)) {
		UNLOCK_PVH(pai);
	} else
		simple_unlock(&phys_backup_lock);
}


__private_extern__ void
pmap_pagetable_corruption_msg_log(int (*log_func)(const char * fmt, ...)__printflike(1,2)) {
	if (pmap_pagetable_corruption_incidents > 0) {
		int i, e = MIN(pmap_pagetable_corruption_incidents, PMAP_PAGETABLE_CORRUPTION_MAX_LOG);
		(*log_func)("%u pagetable corruption incident(s) detected, timeout: %u\n", pmap_pagetable_corruption_incidents, pmap_pagetable_corruption_timeout);
		for (i = 0; i < e; i++) {
			(*log_func)("Incident 0x%x, reason: 0x%x, action: 0x%x, time: 0x%llx\n", pmap_pagetable_corruption_records[i].incident,  pmap_pagetable_corruption_records[i].reason, pmap_pagetable_corruption_records[i].action, pmap_pagetable_corruption_records[i].abstime);
		}
	}
}

static inline void
pmap_pagetable_corruption_log_setup(void) {
	if (pmap_pagetable_corruption_log_call == NULL) {
		nanotime_to_absolutetime(PMAP_PAGETABLE_CORRUPTION_INTERVAL, 0, &pmap_pagetable_corruption_interval_abstime);
		thread_call_setup(&pmap_pagetable_corruption_log_call_data,
		    (thread_call_func_t) pmap_pagetable_corruption_msg_log,
		    (thread_call_param_t) &printf);
		pmap_pagetable_corruption_log_call = &pmap_pagetable_corruption_log_call_data;
	}
}

void
mapping_free_prime(void)
{
	unsigned		i;
	pv_hashed_entry_t	pvh_e;
	pv_hashed_entry_t	pvh_eh;
	pv_hashed_entry_t	pvh_et;
	int			pv_cnt;

	/* Scale based on DRAM size */
	pv_hashed_low_water_mark = MAX(PV_HASHED_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 2000);
	pv_hashed_low_water_mark = MIN(pv_hashed_low_water_mark, 16000);
	/* Alterable via sysctl */
	pv_hashed_kern_low_water_mark = MAX(PV_HASHED_KERN_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 1000);
	pv_hashed_kern_low_water_mark = MIN(pv_hashed_kern_low_water_mark, 16000);
	pv_hashed_kern_alloc_chunk = PV_HASHED_KERN_ALLOC_CHUNK_INITIAL;
	pv_hashed_alloc_chunk = PV_HASHED_ALLOC_CHUNK_INITIAL;

	pv_cnt = 0;
	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;

	for (i = 0; i < (5 * PV_HASHED_ALLOC_CHUNK_INITIAL); i++) {
		pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);

		pvh_e->qlink.next = (queue_entry_t)pvh_eh;
		pvh_eh = pvh_e;

		if (pvh_et == PV_HASHED_ENTRY_NULL)
		        pvh_et = pvh_e;
		pv_cnt++;
	}
	PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);

	pv_cnt = 0;
	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
	for (i = 0; i < PV_HASHED_KERN_ALLOC_CHUNK_INITIAL; i++) {
		pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);

		pvh_e->qlink.next = (queue_entry_t)pvh_eh;
		pvh_eh = pvh_e;

		if (pvh_et == PV_HASHED_ENTRY_NULL)
		        pvh_et = pvh_e;
		pv_cnt++;
	}
	PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
}

void mapping_replenish(void);

void mapping_adjust(void) {
	kern_return_t mres;

	pmap_pagetable_corruption_log_setup();

	mres = kernel_thread_start_priority((thread_continue_t)mapping_replenish, NULL, MAXPRI_KERNEL, &mapping_replenish_thread);
	if (mres != KERN_SUCCESS) {
		panic("pmap: mapping_replenish_thread creation failed");
	}
	thread_deallocate(mapping_replenish_thread);
}

unsigned pmap_mapping_thread_wakeups;	
unsigned pmap_kernel_reserve_replenish_stat;
unsigned pmap_user_reserve_replenish_stat;
unsigned pmap_kern_reserve_alloc_stat;

__attribute__((noreturn))
void
mapping_replenish(void)
{
	pv_hashed_entry_t	pvh_e;
	pv_hashed_entry_t	pvh_eh;
	pv_hashed_entry_t	pvh_et;
	int			pv_cnt;
	unsigned             	i;

	/* We qualify for VM privileges...*/
	current_thread()->options |= TH_OPT_VMPRIV;

	for (;;) {

		while (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark) {
			pv_cnt = 0;
			pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;

			for (i = 0; i < pv_hashed_kern_alloc_chunk; i++) {
				pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
				pvh_e->qlink.next = (queue_entry_t)pvh_eh;
				pvh_eh = pvh_e;

				if (pvh_et == PV_HASHED_ENTRY_NULL)
					pvh_et = pvh_e;
				pv_cnt++;
			}
			pmap_kernel_reserve_replenish_stat += pv_cnt;
			PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
		}

		pv_cnt = 0;
		pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;

		if (pv_hashed_free_count < pv_hashed_low_water_mark) {
			for (i = 0; i < pv_hashed_alloc_chunk; i++) {
				pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);

				pvh_e->qlink.next = (queue_entry_t)pvh_eh;
				pvh_eh = pvh_e;

				if (pvh_et == PV_HASHED_ENTRY_NULL)
					pvh_et = pvh_e;
				pv_cnt++;
			}
			pmap_user_reserve_replenish_stat += pv_cnt;
			PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
		}
/* Wake threads throttled while the kernel reserve was being replenished.
 */
		if (pmap_pv_throttled_waiters) {
			pmap_pv_throttled_waiters = 0;
			thread_wakeup(&pmap_user_pv_throttle_event);
		}
		/* Check if the kernel pool has been depleted since the
		 * first pass, to reduce refill latency.
		 */
		if (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark)
			continue;
		/* Block sans continuation to avoid yielding kernel stack */
		assert_wait(&mapping_replenish_event, THREAD_UNINT);
		mappingrecurse = 0;
		thread_block(THREAD_CONTINUE_NULL);
		pmap_mapping_thread_wakeups++;
	}
}

/*
 *	Set specified attribute bits.
 */

void
phys_attribute_set(
	ppnum_t		pn,
	int		bits)
{
	int		pai;

	pmap_intr_assert();
	assert(pn != vm_page_fictitious_addr);
	if (pn == vm_page_guard_addr)
		return;

	pai = ppn_to_pai(pn);

	if (!IS_MANAGED_PAGE(pai)) {
		/* Not a managed page.  */
		return;
	}

	LOCK_PVH(pai);
	pmap_phys_attributes[pai] |= bits;
	UNLOCK_PVH(pai);
}

/*
 *	Set the modify bit on the specified physical page.
 */

void
pmap_set_modify(ppnum_t pn)
{
	phys_attribute_set(pn, PHYS_MODIFIED);
}

/*
 *	Clear the modify bits on the specified physical page.
 */

void
pmap_clear_modify(ppnum_t pn)
{
	phys_attribute_clear(pn, PHYS_MODIFIED, 0, NULL);
}

/*
 *	pmap_is_modified:
 *
 *	Return whether or not the specified physical page is modified
 *	by any physical maps.
 */

boolean_t
pmap_is_modified(ppnum_t pn)
{
	if (phys_attribute_test(pn, PHYS_MODIFIED))
		return TRUE;
	return FALSE;
}


/*
 *	pmap_clear_reference:
 *
 *	Clear the reference bit on the specified physical page.
 */

void
pmap_clear_reference(ppnum_t pn)
{
	phys_attribute_clear(pn, PHYS_REFERENCED, 0, NULL);
}

void
pmap_set_reference(ppnum_t pn)
{
	phys_attribute_set(pn, PHYS_REFERENCED);
}

/*
 *	pmap_is_referenced:
 *
 *	Return whether or not the specified physical page is referenced
 *	by any physical maps.
 */

boolean_t
pmap_is_referenced(ppnum_t pn)
{
        if (phys_attribute_test(pn, PHYS_REFERENCED))
		return TRUE;
	return FALSE;
}


/*
 * pmap_get_refmod(phys)
 *  returns the referenced and modified bits of the specified
 *  physical page.
 */
unsigned int
pmap_get_refmod(ppnum_t pn)
{
        int		refmod;
	unsigned int	retval = 0;

	refmod = phys_attribute_test(pn, PHYS_MODIFIED | PHYS_REFERENCED);

	if (refmod & PHYS_MODIFIED)
	        retval |= VM_MEM_MODIFIED;
	if (refmod & PHYS_REFERENCED)
	        retval |= VM_MEM_REFERENCED;

	return (retval);
}


void
pmap_clear_refmod_options(ppnum_t pn, unsigned int mask, unsigned int options, void *arg)
{
        unsigned int  x86Mask;

        x86Mask = (   ((mask &   VM_MEM_MODIFIED)?   PHYS_MODIFIED : 0)
		      | ((mask & VM_MEM_REFERENCED)? PHYS_REFERENCED : 0));

        phys_attribute_clear(pn, x86Mask, options, arg);
}

/*
 * pmap_clear_refmod(phys, mask)
 *  clears the referenced and modified bits as specified by the mask
 *  of the specified physical page.
 */
void
pmap_clear_refmod(ppnum_t pn, unsigned int mask)
{
	unsigned int  x86Mask;

	x86Mask = (   ((mask &   VM_MEM_MODIFIED)?   PHYS_MODIFIED : 0)
	            | ((mask & VM_MEM_REFERENCED)? PHYS_REFERENCED : 0));

	phys_attribute_clear(pn, x86Mask, 0, NULL);
}

unsigned int
pmap_disconnect(ppnum_t pa)
{
	return (pmap_disconnect_options(pa, 0, NULL));
}

/*
 *	Routine:
 *		pmap_disconnect_options
 *
 *	Function:
 *		Disconnect all mappings for this page and return reference and change status
 *		in generic format.
 *
 */
unsigned int
pmap_disconnect_options(ppnum_t pa, unsigned int options, void *arg)
{
	unsigned refmod, vmrefmod = 0;

	pmap_page_protect_options(pa, 0, options, arg);		/* disconnect the page */

	pmap_assert(pa != vm_page_fictitious_addr);
	if ((pa == vm_page_guard_addr) || !IS_MANAGED_PAGE(pa) || (options & PMAP_OPTIONS_NOREFMOD))
		return 0;
	refmod = pmap_phys_attributes[pa] & (PHYS_MODIFIED | PHYS_REFERENCED);
	
	if (refmod & PHYS_MODIFIED)
	        vmrefmod |= VM_MEM_MODIFIED;
	if (refmod & PHYS_REFERENCED)
	        vmrefmod |= VM_MEM_REFERENCED;

	return vmrefmod;
}
Commit	Line	Data
6d2010ae	1	/*
39037602	2	* Copyright (c) 2000-2016 Apple Inc. All rights reserved.
6d2010ae A	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28	#include <vm/pmap.h>
316670eb	29	#include <kern/ledger.h>
6d2010ae A	30	#include <i386/pmap_internal.h>
6d2010ae A	31
316670eb	32
6d2010ae A	33	/*
	34	* Each entry in the pv_head_table is locked by a bit in the
	35	* pv_lock_table. The lock bits are accessed by the physical
	36	* address of the page they lock.
	37	*/
	38
	39	char pv_lock_table; / pointer to array of bits */
	40	char *pv_hash_lock_table;
	41
	42	pv_rooted_entry_t pv_head_table; /* array of entries, one per
	43	* page */
	44	uint32_t pv_hashed_free_count = 0;
	45	uint32_t pv_hashed_kern_free_count = 0;
	46
	47	pmap_pagetable_corruption_record_t pmap_pagetable_corruption_records[PMAP_PAGETABLE_CORRUPTION_MAX_LOG];
	48	uint32_t pmap_pagetable_corruption_incidents;
	49	uint64_t pmap_pagetable_corruption_last_abstime = (~(0ULL) >> 1);
	50	uint64_t pmap_pagetable_corruption_interval_abstime;
	51	thread_call_t pmap_pagetable_corruption_log_call;
	52	static thread_call_data_t pmap_pagetable_corruption_log_call_data;
	53	boolean_t pmap_pagetable_corruption_timeout = FALSE;
	54
	55	volatile uint32_t mappingrecurse = 0;
	56
	57	uint32_t pv_hashed_low_water_mark, pv_hashed_kern_low_water_mark, pv_hashed_alloc_chunk, pv_hashed_kern_alloc_chunk;
	58
	59	thread_t mapping_replenish_thread;
	60	event_t mapping_replenish_event, pmap_user_pv_throttle_event;
	61
	62	uint64_t pmap_pv_throttle_stat, pmap_pv_throttled_waiters;
	63
39037602 A	64	int pmap_asserts_enabled = DEBUG;
	65	int pmap_asserts_traced = 0;
	66
6d2010ae	67	unsigned int pmap_cache_attributes(ppnum_t pn) {
3e170ce0	68	if (pmap_get_cache_attributes(pn, FALSE) & INTEL_PTE_NCACHE)
6d2010ae A	69	return (VM_WIMG_IO);
	70	else
	71	return (VM_WIMG_COPYBACK);
	72	}
	73
	74	void pmap_set_cache_attributes(ppnum_t pn, unsigned int cacheattr) {
	75	unsigned int current, template = 0;
	76	int pai;
	77
	78	if (cacheattr & VM_MEM_NOT_CACHEABLE) {
	79	if(!(cacheattr & VM_MEM_GUARDED))
	80	template \|= PHYS_PTA;
	81	template \|= PHYS_NCACHE;
	82	}
	83
	84	pmap_intr_assert();
	85
	86	assert((pn != vm_page_fictitious_addr) && (pn != vm_page_guard_addr));
	87
	88	pai = ppn_to_pai(pn);
	89
	90	if (!IS_MANAGED_PAGE(pai)) {
	91	return;
	92	}
	93
	94	/* override cache attributes for this phys page
	95	* Does not walk through existing mappings to adjust,
	96	* assumes page is disconnected
	97	*/
	98
	99	LOCK_PVH(pai);
	100
	101	pmap_update_cache_attributes_locked(pn, template);
	102
	103	current = pmap_phys_attributes[pai] & PHYS_CACHEABILITY_MASK;
	104	pmap_phys_attributes[pai] &= ~PHYS_CACHEABILITY_MASK;
	105	pmap_phys_attributes[pai] \|= template;
	106
	107	UNLOCK_PVH(pai);
	108
	109	if ((template & PHYS_NCACHE) && !(current & PHYS_NCACHE)) {
	110	pmap_sync_page_attributes_phys(pn);
	111	}
	112	}
	113
3e170ce0	114	unsigned pmap_get_cache_attributes(ppnum_t pn, boolean_t is_ept) {
6d2010ae A	115	if (last_managed_page == 0)
	116	return 0;
	117
3e170ce0 A	118	if (!IS_MANAGED_PAGE(ppn_to_pai(pn)))
3e170ce0 A	119	return PTE_NCACHE(is_ept);
6d2010ae A	120
	121	/*
	122	* The cache attributes are read locklessly for efficiency.
	123	*/
	124	unsigned int attr = pmap_phys_attributes[ppn_to_pai(pn)];
	125	unsigned int template = 0;
3e170ce0 A	126
	127	/*
	128	* The PTA bit is currently unsupported for EPT PTEs.
	129	*/
	130	if ((attr & PHYS_PTA) && !is_ept)
6d2010ae	131	template \|= INTEL_PTE_PTA;
3e170ce0 A	132
	133	/*
	134	* If the page isn't marked as NCACHE, the default for EPT entries
	135	* is WB.
	136	*/
6d2010ae	137	if (attr & PHYS_NCACHE)
3e170ce0 A	138	template \|= PTE_NCACHE(is_ept);
	139	else if (is_ept)
	140	template \|= INTEL_EPT_WB;
	141
6d2010ae A	142	return template;
	143	}
	144
3e170ce0 A	145	boolean_t
	146	pmap_has_managed_page(ppnum_t first, ppnum_t last)
	147	{
	148	ppnum_t pn;
	149	boolean_t result;
	150
	151	assert(last_managed_page);
	152	assert(first <= last);
	153
	154	for (result = FALSE, pn = first;
	155	!result
	156	&& (pn <= last)
	157	&& (pn <= last_managed_page);
	158	pn++)
	159	{
	160	result = (0 != (pmap_phys_attributes[pn] & PHYS_MANAGED));
	161	}
	162
	163	return (result);
	164	}
6d2010ae A	165
	166	boolean_t
	167	pmap_is_noencrypt(ppnum_t pn)
	168	{
	169	int pai;
	170
	171	pai = ppn_to_pai(pn);
	172
	173	if (!IS_MANAGED_PAGE(pai))
7ddcb079	174	return (FALSE);
6d2010ae A	175
	176	if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT)
	177	return (TRUE);
	178
	179	return (FALSE);
	180	}
	181
	182
	183	void
	184	pmap_set_noencrypt(ppnum_t pn)
	185	{
	186	int pai;
	187
	188	pai = ppn_to_pai(pn);
	189
	190	if (IS_MANAGED_PAGE(pai)) {
	191	LOCK_PVH(pai);
	192
	193	pmap_phys_attributes[pai] \|= PHYS_NOENCRYPT;
	194
	195	UNLOCK_PVH(pai);
	196	}
	197	}
	198
	199
	200	void
	201	pmap_clear_noencrypt(ppnum_t pn)
	202	{
	203	int pai;
	204
	205	pai = ppn_to_pai(pn);
	206
	207	if (IS_MANAGED_PAGE(pai)) {
7ddcb079 A	208	/*
	209	* synchronization at VM layer prevents PHYS_NOENCRYPT
	210	* from changing state, so we don't need the lock to inspect
	211	*/
	212	if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT) {
	213	LOCK_PVH(pai);
6d2010ae	214
7ddcb079	215	pmap_phys_attributes[pai] &= ~PHYS_NOENCRYPT;
6d2010ae	216
7ddcb079 A	217	UNLOCK_PVH(pai);
7ddcb079 A	218	}
6d2010ae A	219	}
	220	}
	221
	222	void
	223	compute_pmap_gc_throttle(void *arg __unused)
	224	{
	225
	226	}
	227
	228
fe8ab488 A	229	void
	230	pmap_lock_phys_page(ppnum_t pn)
	231	{
	232	int pai;
	233
	234	pai = ppn_to_pai(pn);
	235
	236	if (IS_MANAGED_PAGE(pai)) {
	237	LOCK_PVH(pai);
	238	} else
	239	simple_lock(&phys_backup_lock);
	240	}
	241
	242
	243	void
	244	pmap_unlock_phys_page(ppnum_t pn)
	245	{
	246	int pai;
	247
	248	pai = ppn_to_pai(pn);
	249
	250	if (IS_MANAGED_PAGE(pai)) {
	251	UNLOCK_PVH(pai);
	252	} else
	253	simple_unlock(&phys_backup_lock);
	254	}
	255
	256
	257
6d2010ae A	258	__private_extern__ void
	259	pmap_pagetable_corruption_msg_log(int (log_func)(const char fmt, ...)__printflike(1,2)) {
	260	if (pmap_pagetable_corruption_incidents > 0) {
	261	int i, e = MIN(pmap_pagetable_corruption_incidents, PMAP_PAGETABLE_CORRUPTION_MAX_LOG);
	262	(*log_func)("%u pagetable corruption incident(s) detected, timeout: %u\n", pmap_pagetable_corruption_incidents, pmap_pagetable_corruption_timeout);
	263	for (i = 0; i < e; i++) {
	264	(*log_func)("Incident 0x%x, reason: 0x%x, action: 0x%x, time: 0x%llx\n", pmap_pagetable_corruption_records[i].incident, pmap_pagetable_corruption_records[i].reason, pmap_pagetable_corruption_records[i].action, pmap_pagetable_corruption_records[i].abstime);
	265	}
	266	}
	267	}
	268
	269	static inline void
	270	pmap_pagetable_corruption_log_setup(void) {
	271	if (pmap_pagetable_corruption_log_call == NULL) {
	272	nanotime_to_absolutetime(PMAP_PAGETABLE_CORRUPTION_INTERVAL, 0, &pmap_pagetable_corruption_interval_abstime);
	273	thread_call_setup(&pmap_pagetable_corruption_log_call_data,
	274	(thread_call_func_t) pmap_pagetable_corruption_msg_log,
	275	(thread_call_param_t) &printf);
	276	pmap_pagetable_corruption_log_call = &pmap_pagetable_corruption_log_call_data;
	277	}
	278	}
	279
	280	void
	281	mapping_free_prime(void)
	282	{
	283	unsigned i;
	284	pv_hashed_entry_t pvh_e;
	285	pv_hashed_entry_t pvh_eh;
	286	pv_hashed_entry_t pvh_et;
	287	int pv_cnt;
	288
	289	/* Scale based on DRAM size */
	290	pv_hashed_low_water_mark = MAX(PV_HASHED_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 2000);
	291	pv_hashed_low_water_mark = MIN(pv_hashed_low_water_mark, 16000);
	292	/* Alterable via sysctl */
	293	pv_hashed_kern_low_water_mark = MAX(PV_HASHED_KERN_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 1000);
	294	pv_hashed_kern_low_water_mark = MIN(pv_hashed_kern_low_water_mark, 16000);
	295	pv_hashed_kern_alloc_chunk = PV_HASHED_KERN_ALLOC_CHUNK_INITIAL;
	296	pv_hashed_alloc_chunk = PV_HASHED_ALLOC_CHUNK_INITIAL;
	297
	298	pv_cnt = 0;
	299	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
	300
	301	for (i = 0; i < (5 * PV_HASHED_ALLOC_CHUNK_INITIAL); i++) {
	302	pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
	303
	304	pvh_e->qlink.next = (queue_entry_t)pvh_eh;
	305	pvh_eh = pvh_e;
	306
	307	if (pvh_et == PV_HASHED_ENTRY_NULL)
	308	pvh_et = pvh_e;
	309	pv_cnt++;
	310	}
	311	PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
	312
	313	pv_cnt = 0;
	314	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
	315	for (i = 0; i < PV_HASHED_KERN_ALLOC_CHUNK_INITIAL; i++) {
	316	pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
	317
	318	pvh_e->qlink.next = (queue_entry_t)pvh_eh;
	319	pvh_eh = pvh_e;
	320
	321	if (pvh_et == PV_HASHED_ENTRY_NULL)
322	pvh_et = pvh_e;
323	pv_cnt++;
324	}
325	PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
326	}
327
328	void mapping_replenish(void);
329
330	void mapping_adjust(void) {
331	kern_return_t mres;
332
333	pmap_pagetable_corruption_log_setup();
334
335	mres = kernel_thread_start_priority((thread_continue_t)mapping_replenish, NULL, MAXPRI_KERNEL, &mapping_replenish_thread);
336	if (mres != KERN_SUCCESS) {
337	panic("pmap: mapping_replenish_thread creation failed");
338	}
339	thread_deallocate(mapping_replenish_thread);
340	}
341
342	unsigned pmap_mapping_thread_wakeups;
343	unsigned pmap_kernel_reserve_replenish_stat;
344	unsigned pmap_user_reserve_replenish_stat;
345	unsigned pmap_kern_reserve_alloc_stat;
346
39037602 A	347	__attribute__((noreturn))
	348	void
	349	mapping_replenish(void)
6d2010ae A	350	{
	351	pv_hashed_entry_t pvh_e;
	352	pv_hashed_entry_t pvh_eh;
	353	pv_hashed_entry_t pvh_et;
	354	int pv_cnt;
	355	unsigned i;
	356
	357	/* We qualify for VM privileges...*/
	358	current_thread()->options \|= TH_OPT_VMPRIV;
	359
	360	for (;;) {
	361
	362	while (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark) {
	363	pv_cnt = 0;
	364	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
	365
	366	for (i = 0; i < pv_hashed_kern_alloc_chunk; i++) {
	367	pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
	368	pvh_e->qlink.next = (queue_entry_t)pvh_eh;
	369	pvh_eh = pvh_e;
	370
	371	if (pvh_et == PV_HASHED_ENTRY_NULL)
	372	pvh_et = pvh_e;
	373	pv_cnt++;
	374	}
	375	pmap_kernel_reserve_replenish_stat += pv_cnt;
	376	PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
	377	}
	378
	379	pv_cnt = 0;
	380	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
	381
	382	if (pv_hashed_free_count < pv_hashed_low_water_mark) {
	383	for (i = 0; i < pv_hashed_alloc_chunk; i++) {
	384	pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
	385
	386	pvh_e->qlink.next = (queue_entry_t)pvh_eh;
	387	pvh_eh = pvh_e;
	388
	389	if (pvh_et == PV_HASHED_ENTRY_NULL)
	390	pvh_et = pvh_e;
	391	pv_cnt++;
	392	}
	393	pmap_user_reserve_replenish_stat += pv_cnt;
	394	PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
	395	}
	396	/* Wake threads throttled while the kernel reserve was being replenished.
	397	*/
	398	if (pmap_pv_throttled_waiters) {
	399	pmap_pv_throttled_waiters = 0;
	400	thread_wakeup(&pmap_user_pv_throttle_event);
	401	}
	402	/* Check if the kernel pool has been depleted since the
	403	* first pass, to reduce refill latency.
	404	*/
	405	if (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark)
	406	continue;
	407	/* Block sans continuation to avoid yielding kernel stack */
	408	assert_wait(&mapping_replenish_event, THREAD_UNINT);
	409	mappingrecurse = 0;
	410	thread_block(THREAD_CONTINUE_NULL);
	411	pmap_mapping_thread_wakeups++;
	412	}
	413	}
414
415	/*
416	* Set specified attribute bits.
417	*/
418
419	void
420	phys_attribute_set(
421	ppnum_t pn,
422	int bits)
423	{
424	int pai;
425
426	pmap_intr_assert();
427	assert(pn != vm_page_fictitious_addr);
428	if (pn == vm_page_guard_addr)
429	return;
430
431	pai = ppn_to_pai(pn);
432
433	if (!IS_MANAGED_PAGE(pai)) {
434	/* Not a managed page. */
435	return;
436	}
437
438	LOCK_PVH(pai);
439	pmap_phys_attributes[pai] \|= bits;
440	UNLOCK_PVH(pai);
441	}
442
443	/*
444	* Set the modify bit on the specified physical page.
445	*/
446
447	void
448	pmap_set_modify(ppnum_t pn)
449	{
450	phys_attribute_set(pn, PHYS_MODIFIED);
451	}
452
453	/*
454	* Clear the modify bits on the specified physical page.
455	*/
456
457	void
458	pmap_clear_modify(ppnum_t pn)
459	{
39236c6e	460	phys_attribute_clear(pn, PHYS_MODIFIED, 0, NULL);
6d2010ae A	461	}
	462
	463	/*
	464	* pmap_is_modified:
	465	*
	466	* Return whether or not the specified physical page is modified
	467	* by any physical maps.
	468	*/
	469
	470	boolean_t
	471	pmap_is_modified(ppnum_t pn)
	472	{
	473	if (phys_attribute_test(pn, PHYS_MODIFIED))
	474	return TRUE;
	475	return FALSE;
	476	}
	477
	478
	479	/*
	480	* pmap_clear_reference:
	481	*
	482	* Clear the reference bit on the specified physical page.
	483	*/
	484
	485	void
	486	pmap_clear_reference(ppnum_t pn)
	487	{
39236c6e	488	phys_attribute_clear(pn, PHYS_REFERENCED, 0, NULL);
6d2010ae A	489	}
	490
	491	void
	492	pmap_set_reference(ppnum_t pn)
	493	{
	494	phys_attribute_set(pn, PHYS_REFERENCED);
	495	}
	496
	497	/*
	498	* pmap_is_referenced:
	499	*
	500	* Return whether or not the specified physical page is referenced
	501	* by any physical maps.
	502	*/
	503
	504	boolean_t
	505	pmap_is_referenced(ppnum_t pn)
	506	{
	507	if (phys_attribute_test(pn, PHYS_REFERENCED))
	508	return TRUE;
	509	return FALSE;
	510	}
	511
	512
	513	/*
	514	* pmap_get_refmod(phys)
	515	* returns the referenced and modified bits of the specified
	516	* physical page.
	517	*/
	518	unsigned int
	519	pmap_get_refmod(ppnum_t pn)
	520	{
	521	int refmod;
	522	unsigned int retval = 0;
	523
	524	refmod = phys_attribute_test(pn, PHYS_MODIFIED \| PHYS_REFERENCED);
	525
	526	if (refmod & PHYS_MODIFIED)
	527	retval \|= VM_MEM_MODIFIED;
	528	if (refmod & PHYS_REFERENCED)
	529	retval \|= VM_MEM_REFERENCED;
	530
	531	return (retval);
	532	}
	533
39236c6e A	534
	535	void
	536	pmap_clear_refmod_options(ppnum_t pn, unsigned int mask, unsigned int options, void *arg)
	537	{
	538	unsigned int x86Mask;
	539
	540	x86Mask = ( ((mask & VM_MEM_MODIFIED)? PHYS_MODIFIED : 0)
	541	\| ((mask & VM_MEM_REFERENCED)? PHYS_REFERENCED : 0));
	542
	543	phys_attribute_clear(pn, x86Mask, options, arg);
	544	}
	545
6d2010ae A	546	/*
	547	* pmap_clear_refmod(phys, mask)
	548	* clears the referenced and modified bits as specified by the mask
	549	* of the specified physical page.
	550	*/
	551	void
	552	pmap_clear_refmod(ppnum_t pn, unsigned int mask)
	553	{
	554	unsigned int x86Mask;
	555
	556	x86Mask = ( ((mask & VM_MEM_MODIFIED)? PHYS_MODIFIED : 0)
	557	\| ((mask & VM_MEM_REFERENCED)? PHYS_REFERENCED : 0));
39236c6e A	558
	559	phys_attribute_clear(pn, x86Mask, 0, NULL);
	560	}
	561
	562	unsigned int
	563	pmap_disconnect(ppnum_t pa)
	564	{
	565	return (pmap_disconnect_options(pa, 0, NULL));
6d2010ae A	566	}
	567
	568	/*
	569	* Routine:
39236c6e	570	* pmap_disconnect_options
6d2010ae A	571	*
	572	* Function:
	573	* Disconnect all mappings for this page and return reference and change status
	574	* in generic format.
	575	*
	576	*/
	577	unsigned int
39236c6e	578	pmap_disconnect_options(ppnum_t pa, unsigned int options, void *arg)
6d2010ae A	579	{
	580	unsigned refmod, vmrefmod = 0;
	581
39236c6e	582	pmap_page_protect_options(pa, 0, options, arg); /* disconnect the page */
6d2010ae A	583
6d2010ae A	584	pmap_assert(pa != vm_page_fictitious_addr);
39236c6e	585	if ((pa == vm_page_guard_addr) \|\| !IS_MANAGED_PAGE(pa) \|\| (options & PMAP_OPTIONS_NOREFMOD))
6d2010ae A	586	return 0;
	587	refmod = pmap_phys_attributes[pa] & (PHYS_MODIFIED \| PHYS_REFERENCED);
	588
	589	if (refmod & PHYS_MODIFIED)
	590	vmrefmod \|= VM_MEM_MODIFIED;
	591	if (refmod & PHYS_REFERENCED)
	592	vmrefmod \|= VM_MEM_REFERENCED;
	593
	594	return vmrefmod;
	595	}