[apple/xnu.git] / osfmk / vm / vm_page.h

/*
 * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code 
 * as defined in and that are subject to the Apple Public Source License 
 * Version 2.0 (the 'License'). You may not use this file except in 
 * compliance with the License.  The rights granted to you under the 
 * License may not be used to create, or enable the creation or 
 * redistribution of, unlawful or unlicensed copies of an Apple operating 
 * system, or to circumvent, violate, or enable the circumvention or 
 * violation of, any terms of an Apple operating system software license 
 * agreement.
 *
 * Please obtain a copy of the License at 
 * http://www.opensource.apple.com/apsl/ and read it before using this 
 * file.
 *
 * The Original Code and all software distributed under the License are 
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 
 * Please see the License for the specific language governing rights and 
 * limitations under the License.
 *
 * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 */
/* 
 * Mach Operating System
 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
 * All Rights Reserved.
 * 
 * Permission to use, copy, modify and distribute this software and its
 * documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 * 
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 * 
 * Carnegie Mellon requests users of this software to return to
 * 
 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 * 
 * any improvements or extensions that they make and grant Carnegie Mellon
 * the rights to redistribute these changes.
 */
/*
 */
/*
 *	File:	vm/vm_page.h
 *	Author:	Avadis Tevanian, Jr., Michael Wayne Young
 *	Date:	1985
 *
 *	Resident memory system definitions.
 */

#ifndef	_VM_VM_PAGE_H_
#define _VM_VM_PAGE_H_

#include <debug.h>

#include <mach/boolean.h>
#include <mach/vm_prot.h>
#include <mach/vm_param.h>
#include <vm/vm_object.h>
#include <kern/queue.h>
#include <kern/lock.h>

#include <kern/macro_help.h>

/* 
 * Each page entered on the inactive queue obtains a ticket from a
 * particular ticket roll.  Pages granted tickets from a particular 
 * roll  generally flow through the queue as a group.  In this way when a
 * page with a ticket from a particular roll is pulled from the top of the
 * queue it is extremely likely that the pages near the top will have tickets
 * from the same or adjacent rolls.  In this way the proximity to the top
 * of the queue can be loosely ascertained by determining the identity of
 * the roll the pages ticket came from. 
 */


extern unsigned int	vm_page_ticket_roll;
extern unsigned int	vm_page_ticket;


#define VM_PAGE_TICKETS_IN_ROLL  512
#define VM_PAGE_TICKET_ROLL_IDS  16

/*
 *	Management of resident (logical) pages.
 *
 *	A small structure is kept for each resident
 *	page, indexed by page number.  Each structure
 *	is an element of several lists:
 *
 *		A hash table bucket used to quickly
 *		perform object/offset lookups
 *
 *		A list of all pages for a given object,
 *		so they can be quickly deactivated at
 *		time of deallocation.
 *
 *		An ordered list of pages due for pageout.
 *
 *	In addition, the structure contains the object
 *	and offset to which this page belongs (for pageout),
 *	and sundry status bits.
 *
 *	Fields in this structure are locked either by the lock on the
 *	object that the page belongs to (O) or by the lock on the page
 *	queues (P).  [Some fields require that both locks be held to
 *	change that field; holding either lock is sufficient to read.]
 */

struct vm_page {
	queue_chain_t	pageq;		/* queue info for FIFO
					 * queue or free list (P) */
	queue_chain_t	listq;		/* all pages in same object (O) */
	struct vm_page	*next;		/* VP bucket link (O) */

	vm_object_t	object;		/* which object am I in (O&P) */
	vm_object_offset_t offset;	/* offset into that object (O,P) */

	/*
	 * The following word of flags is protected
	 * by the "page queues" lock.
	 */
	unsigned int	wire_count:16,	/* how many wired down maps use me? (O&P) */
			page_ticket:4,	/* age of the page on the       */
					/* inactive queue.		    */
	/* boolean_t */	inactive:1,	/* page is in inactive list (P) */
			active:1,	/* page is in active list (P) */
			pageout_queue:1,/* page is on queue for pageout (P) */
			laundry:1,	/* page is being cleaned now (P)*/
			free:1,		/* page is on free list (P) */
			reference:1,	/* page has been used (P) */
			pageout:1,	/* page wired & busy for pageout (P) */
			gobbled:1,      /* page used internally (P) */
			private:1,	/* Page should not be returned to
					 *  the free list (P) */
			zero_fill:1,
			:0;

	/*
	 * The following word of flags is protected
	 * by the "VM object" lock.
	 */
	unsigned int
	                page_error:8,   /* error from I/O operations */
	/* boolean_t */	busy:1,		/* page is in transit (O) */
			wanted:1,	/* someone is waiting for page (O) */
			tabled:1,	/* page is in VP table (O) */
			fictitious:1,	/* Physical page doesn't exist (O) */
			no_isync:1,     /* page has not been instruction synced */
			absent:1,	/* Data has been requested, but is
					 *  not yet available (O) */
			error:1,	/* Data manager was unable to provide
					 *  data due to error (O) */
			dirty:1,	/* Page must be cleaned (O) */
			cleaning:1,	/* Page clean has begun (O) */
			precious:1,	/* Page is precious; data must be
					 *  returned even if clean (O) */
			clustered:1,	/* page is not the faulted page (O) */
			overwriting:1,  /* Request to unlock has been made
					 * without having data. (O)
					 * [See vm_fault_page_overwrite] */
			restart:1,	/* Page was pushed higher in shadow
					   chain by copy_call-related pagers;
					   start again at top of chain */
			lock_supplied:1,/* protection supplied by pager (O) */
	/* vm_prot_t */	page_lock:3,	/* Uses prohibited by pager (O) */
	/* vm_prot_t */	unlock_request:3,/* Outstanding unlock request (O) */
			unusual:1,	/* Page is absent, error, restart or
					   page locked */
			encrypted:1,	/* encrypted for secure swap (O) */
			list_req_pending:1, /* pagein/pageout alt mechanism */
					    /* allows creation of list      */
					    /* requests on pages that are   */
					    /* actively being paged.        */
			dump_cleaning:1;   /* set by the pageout daemon when */
					   /* a page being cleaned is       */
					   /* encountered and targeted as   */
					   /* a pageout candidate           */
        /* we've used up all 32 bits */

	ppnum_t		phys_page;	/* Physical address of page, passed
					 *  to pmap_enter (read-only) */
};

#define DEBUG_ENCRYPTED_SWAP	1
#if DEBUG_ENCRYPTED_SWAP
#define ASSERT_PAGE_DECRYPTED(page) 					\
	MACRO_BEGIN							\
	if ((page)->encrypted) {					\
		panic("VM page %p should not be encrypted here\n",	\
		      (page));						\
	}								\
	MACRO_END
#else	/* DEBUG_ENCRYPTED_SWAP */
#define ASSERT_PAGE_DECRYPTED(page) assert(!(page)->encrypted)
#endif	/* DEBUG_ENCRYPTED_SWAP */

typedef struct vm_page	*vm_page_t;

#define VM_PAGE_NULL		((vm_page_t) 0)
#define NEXT_PAGE(m)		((vm_page_t) (m)->pageq.next)
#define NEXT_PAGE_PTR(m)	((vm_page_t *) &(m)->pageq.next)

/*
 * XXX	The unusual bit should not be necessary.  Most of the bit
 * XXX	fields above really want to be masks.
 */

/*
 *	For debugging, this macro can be defined to perform
 *	some useful check on a page structure.
 */

#define VM_PAGE_CHECK(mem)

/*
 *	Each pageable resident page falls into one of three lists:
 *
 *	free	
 *		Available for allocation now.
 *	inactive
 *		Not referenced in any map, but still has an
 *		object/offset-page mapping, and may be dirty.
 *		This is the list of pages that should be
 *		paged out next.
 *	active
 *		A list of pages which have been placed in
 *		at least one physical map.  This list is
 *		ordered, in LRU-like fashion.
 */

extern
vm_page_t	vm_page_queue_free;	/* memory free queue */
extern
vm_page_t	vm_page_queue_fictitious;	/* fictitious free queue */
extern
queue_head_t	vm_page_queue_active;	/* active memory queue */
extern
queue_head_t	vm_page_queue_inactive;	/* inactive memory queue */
queue_head_t	vm_page_queue_zf;	/* inactive memory queue for zero fill */

extern
vm_offset_t	first_phys_addr;	/* physical address for first_page */
extern
vm_offset_t	last_phys_addr;		/* physical address for last_page */

extern
unsigned int	vm_page_free_count;	/* How many pages are free? */
extern
unsigned int	vm_page_fictitious_count;/* How many fictitious pages are free? */
extern
unsigned int	vm_page_active_count;	/* How many pages are active? */
extern
unsigned int	vm_page_inactive_count;	/* How many pages are inactive? */
extern
unsigned int	vm_page_wire_count;	/* How many pages are wired? */
extern
unsigned int	vm_page_free_target;	/* How many do we want free? */
extern
unsigned int	vm_page_free_min;	/* When to wakeup pageout */
extern
unsigned int	vm_page_inactive_target;/* How many do we want inactive? */
extern
unsigned int	vm_page_free_reserved;	/* How many pages reserved to do pageout */
extern
unsigned int	vm_page_throttled_count;/* Count of zero-fill allocations throttled */
extern
unsigned int	vm_page_gobble_count;

extern
unsigned int	vm_page_purgeable_count;/* How many pages are purgeable now ? */
extern
uint64_t	vm_page_purged_count;	/* How many pages got purged so far ? */

decl_mutex_data(,vm_page_queue_lock)
				/* lock on active and inactive page queues */
decl_mutex_data(,vm_page_queue_free_lock)
				/* lock on free page queue */

extern unsigned int	vm_page_free_wanted;
				/* how many threads are waiting for memory */

extern vm_offset_t	vm_page_fictitious_addr;
				/* (fake) phys_addr of fictitious pages */

extern boolean_t	vm_page_deactivate_hint;

// 0 = all pages avail, 1 = disable high mem, 2 = prefer himem
extern int		vm_himemory_mode;

extern ppnum_t		vm_lopage_poolend;
extern int		vm_lopage_poolsize;
extern uint64_t		max_valid_dma_address;


/*
 * Prototypes for functions exported by this module.
 */
extern void		vm_page_bootstrap(
					vm_offset_t	*startp,
					vm_offset_t	*endp);

extern void		vm_page_module_init(void);

extern void		vm_page_create(
					ppnum_t		start,
					ppnum_t		end);

extern vm_page_t	vm_page_lookup(
					vm_object_t		object,
					vm_object_offset_t	offset);

extern vm_page_t	vm_page_grab_fictitious(void);

extern void		vm_page_release_fictitious(
					vm_page_t page);

extern boolean_t	vm_page_convert(
					vm_page_t	page);

extern void		vm_page_more_fictitious(void);

extern int		vm_pool_low(void);

extern vm_page_t	vm_page_grab(void);

extern vm_page_t	vm_page_grablo(void);

extern void		vm_page_release(
					vm_page_t	page);

extern boolean_t	vm_page_wait(
					int		interruptible );

extern vm_page_t	vm_page_alloc(
					vm_object_t		object,
					vm_object_offset_t	offset);

extern vm_page_t	vm_page_alloclo(
					vm_object_t		object,
					vm_object_offset_t	offset);

extern void		vm_page_init(
					vm_page_t	page,
					ppnum_t		phys_page);

extern void		vm_page_free(
					vm_page_t	page);

extern void		vm_page_activate(
					vm_page_t	page);

extern void		vm_page_deactivate(
					vm_page_t	page);

extern void		vm_page_rename(
					vm_page_t		page,
					vm_object_t		new_object,
					vm_object_offset_t	new_offset);

extern void		vm_page_insert(
					vm_page_t		page,
					vm_object_t		object,
					vm_object_offset_t	offset);

extern void		vm_page_replace(
					vm_page_t		mem,
					vm_object_t		object,
					vm_object_offset_t	offset);

extern void		vm_page_remove(
					vm_page_t	page);

extern void		vm_page_zero_fill(
					vm_page_t	page);

extern void		vm_page_part_zero_fill(
					vm_page_t	m,
					vm_offset_t	m_pa,
					vm_size_t	len);

extern void		vm_page_copy(
					vm_page_t	src_page,
					vm_page_t	dest_page);

extern void		vm_page_part_copy(
					vm_page_t	src_m,
					vm_offset_t	src_pa,
					vm_page_t	dst_m,
					vm_offset_t	dst_pa,
					vm_size_t	len);

extern void		vm_page_wire(
					vm_page_t	page);

extern void		vm_page_unwire(
					vm_page_t	page);

extern void		vm_set_page_size(void);

extern void		vm_page_gobble(
				        vm_page_t      page);

/*
 *	Functions implemented as macros. m->wanted and m->busy are
 *	protected by the object lock.
 */

#define PAGE_ASSERT_WAIT(m, interruptible)			\
		(((m)->wanted = TRUE),				\
		 assert_wait((event_t) (m), (interruptible)))

#define PAGE_SLEEP(o, m, interruptible)				\
		(((m)->wanted = TRUE),				\
		 thread_sleep_vm_object((o), (m), (interruptible)))

#define PAGE_WAKEUP_DONE(m)					\
		MACRO_BEGIN					\
		(m)->busy = FALSE;				\
		if ((m)->wanted) {				\
			(m)->wanted = FALSE;			\
			thread_wakeup((event_t) (m));		\
		}						\
		MACRO_END

#define PAGE_WAKEUP(m)						\
		MACRO_BEGIN					\
		if ((m)->wanted) {				\
			(m)->wanted = FALSE;			\
			thread_wakeup((event_t) (m));		\
		}						\
		MACRO_END

#define VM_PAGE_FREE(p) 			\
		MACRO_BEGIN			\
		vm_page_lock_queues();		\
		vm_page_free(p);		\
		vm_page_unlock_queues();	\
		MACRO_END

#define VM_PAGE_GRAB_FICTITIOUS(M)					\
		MACRO_BEGIN						\
		while ((M = vm_page_grab_fictitious()) == VM_PAGE_NULL)	\
			vm_page_more_fictitious();			\
		MACRO_END

#define VM_PAGE_THROTTLED()						\
		(vm_page_free_count < vm_page_free_min &&		\
		 !(current_thread()->options & TH_OPT_VMPRIV) && 			\
		 ++vm_page_throttled_count)

#define	VM_PAGE_WAIT()		((void)vm_page_wait(THREAD_UNINT))

#define vm_page_lock_queues()	mutex_lock(&vm_page_queue_lock)
#define vm_page_unlock_queues()	mutex_unlock(&vm_page_queue_lock)

#define VM_PAGE_QUEUES_REMOVE(mem)				\
	MACRO_BEGIN						\
	assert(!mem->laundry);					\
	if (mem->active) {					\
		assert(mem->object != kernel_object);		\
		assert(!mem->inactive);				\
		queue_remove(&vm_page_queue_active,		\
			mem, vm_page_t, pageq);			\
		mem->pageq.next = NULL;				\
		mem->pageq.prev = NULL;			       	\
		mem->active = FALSE;				\
		if (!mem->fictitious)				\
			vm_page_active_count--;			\
	}							\
								\
	if (mem->inactive) {					\
		assert(mem->object != kernel_object);		\
		assert(!mem->active);				\
		if (mem->zero_fill) {				\
			queue_remove(&vm_page_queue_zf,		\
			mem, vm_page_t, pageq);			\
		} else {					\
			queue_remove(&vm_page_queue_inactive,	\
			mem, vm_page_t, pageq);			\
		}						\
		mem->pageq.next = NULL;				\
		mem->pageq.prev = NULL;			       	\
		mem->inactive = FALSE;				\
		if (!mem->fictitious)				\
			vm_page_inactive_count--;		\
	}							\
	MACRO_END

#endif	/* _VM_VM_PAGE_H_ */
Commit	Line	Data
1c79356b	1	/*
91447636	2	* Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
1c79356b	3	*
8ad349bb	4	* @APPLE_LICENSE_OSREFERENCE_HEADER_START@
1c79356b	5	*
8ad349bb A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the
	10	* License may not be used to create, or enable the creation or
	11	* redistribution of, unlawful or unlicensed copies of an Apple operating
	12	* system, or to circumvent, violate, or enable the circumvention or
	13	* violation of, any terms of an Apple operating system software license
	14	* agreement.
	15	*
	16	* Please obtain a copy of the License at
	17	* http://www.opensource.apple.com/apsl/ and read it before using this
	18	* file.
	19	*
	20	* The Original Code and all software distributed under the License are
	21	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	22	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	23	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	24	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	25	* Please see the License for the specific language governing rights and
	26	* limitations under the License.
	27	*
	28	* @APPLE_LICENSE_OSREFERENCE_HEADER_END@
1c79356b A	29	*/
	30	/*
	31	* @OSF_COPYRIGHT@
	32	*/
	33	/*
	34	* Mach Operating System
	35	* Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
	36	* All Rights Reserved.
	37	*
	38	* Permission to use, copy, modify and distribute this software and its
	39	* documentation is hereby granted, provided that both the copyright
	40	* notice and this permission notice appear in all copies of the
	41	* software, derivative works or modified versions, and any portions
	42	* thereof, and that both notices appear in supporting documentation.
	43	*
	44	* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
	45	* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
	46	* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
	47	*
	48	* Carnegie Mellon requests users of this software to return to
	49	*
	50	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
	51	* School of Computer Science
	52	* Carnegie Mellon University
	53	* Pittsburgh PA 15213-3890
	54	*
	55	* any improvements or extensions that they make and grant Carnegie Mellon
	56	* the rights to redistribute these changes.
	57	*/
	58	/*
	59	*/
	60	/*
	61	* File: vm/vm_page.h
	62	* Author: Avadis Tevanian, Jr., Michael Wayne Young
	63	* Date: 1985
	64	*
	65	* Resident memory system definitions.
	66	*/
	67
	68	#ifndef _VM_VM_PAGE_H_
	69	#define _VM_VM_PAGE_H_
	70
91447636 A	71	#include <debug.h>
91447636 A	72
1c79356b A	73	#include <mach/boolean.h>
	74	#include <mach/vm_prot.h>
	75	#include <mach/vm_param.h>
	76	#include <vm/vm_object.h>
	77	#include <kern/queue.h>
	78	#include <kern/lock.h>
	79
	80	#include <kern/macro_help.h>
	81
0b4e3aa0 A	82	/*
	83	* Each page entered on the inactive queue obtains a ticket from a
	84	* particular ticket roll. Pages granted tickets from a particular
	85	* roll generally flow through the queue as a group. In this way when a
	86	* page with a ticket from a particular roll is pulled from the top of the
	87	* queue it is extremely likely that the pages near the top will have tickets
	88	* from the same or adjacent rolls. In this way the proximity to the top
	89	* of the queue can be loosely ascertained by determining the identity of
	90	* the roll the pages ticket came from.
	91	*/
	92
	93
91447636 A	94	extern unsigned int vm_page_ticket_roll;
91447636 A	95	extern unsigned int vm_page_ticket;
0b4e3aa0	96
9bccf70c	97
0b4e3aa0 A	98	#define VM_PAGE_TICKETS_IN_ROLL 512
	99	#define VM_PAGE_TICKET_ROLL_IDS 16
	100
1c79356b A	101	/*
	102	* Management of resident (logical) pages.
	103	*
	104	* A small structure is kept for each resident
	105	* page, indexed by page number. Each structure
	106	* is an element of several lists:
	107	*
	108	* A hash table bucket used to quickly
	109	* perform object/offset lookups
	110	*
	111	* A list of all pages for a given object,
	112	* so they can be quickly deactivated at
	113	* time of deallocation.
	114	*
	115	* An ordered list of pages due for pageout.
	116	*
	117	* In addition, the structure contains the object
	118	* and offset to which this page belongs (for pageout),
	119	* and sundry status bits.
	120	*
	121	* Fields in this structure are locked either by the lock on the
	122	* object that the page belongs to (O) or by the lock on the page
	123	* queues (P). [Some fields require that both locks be held to
	124	* change that field; holding either lock is sufficient to read.]
	125	*/
	126
	127	struct vm_page {
	128	queue_chain_t pageq; /* queue info for FIFO
	129	* queue or free list (P) */
	130	queue_chain_t listq; /* all pages in same object (O) */
	131	struct vm_page next; / VP bucket link (O) */
	132
	133	vm_object_t object; /* which object am I in (O&P) */
	134	vm_object_offset_t offset; /* offset into that object (O,P) */
	135
91447636 A	136	/*
	137	* The following word of flags is protected
	138	* by the "page queues" lock.
	139	*/
1c79356b	140	unsigned int wire_count:16, /* how many wired down maps use me? (O&P) */
0b4e3aa0 A	141	page_ticket:4, /* age of the page on the */
0b4e3aa0 A	142	/* inactive queue. */
1c79356b A	143	/* boolean_t / inactive:1, / page is in inactive list (P) */
1c79356b A	144	active:1, /* page is in active list (P) */
91447636	145	pageout_queue:1,/* page is on queue for pageout (P) */
1c79356b A	146	laundry:1, /* page is being cleaned now (P)*/
	147	free:1, /* page is on free list (P) */
	148	reference:1, /* page has been used (P) */
1c79356b	149	pageout:1, /* page wired & busy for pageout (P) */
0b4e3aa0 A	150	gobbled:1, /* page used internally (P) */
0b4e3aa0 A	151	private:1, /* Page should not be returned to
91447636	152	* the free list (P) */
9bccf70c	153	zero_fill:1,
0b4e3aa0	154	:0;
1c79356b	155
91447636 A	156	/*
	157	* The following word of flags is protected
	158	* by the "VM object" lock.
	159	*/
1c79356b	160	unsigned int
0b4e3aa0	161	page_error:8, /* error from I/O operations */
1c79356b A	162	/* boolean_t / busy:1, / page is in transit (O) */
	163	wanted:1, /* someone is waiting for page (O) */
	164	tabled:1, /* page is in VP table (O) */
	165	fictitious:1, /* Physical page doesn't exist (O) */
0b4e3aa0	166	no_isync:1, /* page has not been instruction synced */
1c79356b A	167	absent:1, /* Data has been requested, but is
	168	* not yet available (O) */
	169	error:1, /* Data manager was unable to provide
	170	* data due to error (O) */
	171	dirty:1, /* Page must be cleaned (O) */
	172	cleaning:1, /* Page clean has begun (O) */
	173	precious:1, /* Page is precious; data must be
	174	* returned even if clean (O) */
	175	clustered:1, /* page is not the faulted page (O) */
	176	overwriting:1, /* Request to unlock has been made
	177	* without having data. (O)
	178	* [See vm_fault_page_overwrite] */
	179	restart:1, /* Page was pushed higher in shadow
	180	chain by copy_call-related pagers;
	181	start again at top of chain */
	182	lock_supplied:1,/* protection supplied by pager (O) */
	183	/* vm_prot_t / page_lock:3, / Uses prohibited by pager (O) */
	184	/* vm_prot_t / unlock_request:3,/ Outstanding unlock request (O) */
	185	unusual:1, /* Page is absent, error, restart or
	186	page locked */
91447636	187	encrypted:1, /* encrypted for secure swap (O) */
1c79356b A	188	list_req_pending:1, /* pagein/pageout alt mechanism */
	189	/* allows creation of list */
	190	/* requests on pages that are */
	191	/* actively being paged. */
0b4e3aa0 A	192	dump_cleaning:1; /* set by the pageout daemon when */
	193	/* a page being cleaned is */
	194	/* encountered and targeted as */
	195	/* a pageout candidate */
	196	/* we've used up all 32 bits */
1c79356b	197
91447636	198	ppnum_t phys_page; /* Physical address of page, passed
1c79356b	199	* to pmap_enter (read-only) */
1c79356b A	200	};
1c79356b A	201
91447636 A	202	#define DEBUG_ENCRYPTED_SWAP 1
	203	#if DEBUG_ENCRYPTED_SWAP
	204	#define ASSERT_PAGE_DECRYPTED(page) \
	205	MACRO_BEGIN \
	206	if ((page)->encrypted) { \
	207	panic("VM page %p should not be encrypted here\n", \
	208	(page)); \
	209	} \
	210	MACRO_END
	211	#else /* DEBUG_ENCRYPTED_SWAP */
	212	#define ASSERT_PAGE_DECRYPTED(page) assert(!(page)->encrypted)
	213	#endif /* DEBUG_ENCRYPTED_SWAP */
	214
1c79356b A	215	typedef struct vm_page *vm_page_t;
1c79356b A	216
1c79356b	217	#define VM_PAGE_NULL ((vm_page_t) 0)
91447636	218	#define NEXT_PAGE(m) ((vm_page_t) (m)->pageq.next)
e5568f75	219	#define NEXT_PAGE_PTR(m) ((vm_page_t *) &(m)->pageq.next)
1c79356b A	220
	221	/*
	222	* XXX The unusual bit should not be necessary. Most of the bit
	223	* XXX fields above really want to be masks.
	224	*/
	225
	226	/*
	227	* For debugging, this macro can be defined to perform
	228	* some useful check on a page structure.
	229	*/
	230
	231	#define VM_PAGE_CHECK(mem)
	232
	233	/*
	234	* Each pageable resident page falls into one of three lists:
	235	*
	236	* free
	237	* Available for allocation now.
	238	* inactive
	239	* Not referenced in any map, but still has an
	240	* object/offset-page mapping, and may be dirty.
	241	* This is the list of pages that should be
	242	* paged out next.
	243	* active
	244	* A list of pages which have been placed in
	245	* at least one physical map. This list is
	246	* ordered, in LRU-like fashion.
	247	*/
	248
	249	extern
	250	vm_page_t vm_page_queue_free; /* memory free queue */
	251	extern
	252	vm_page_t vm_page_queue_fictitious; /* fictitious free queue */
	253	extern
	254	queue_head_t vm_page_queue_active; /* active memory queue */
	255	extern
	256	queue_head_t vm_page_queue_inactive; /* inactive memory queue */
9bccf70c	257	queue_head_t vm_page_queue_zf; /* inactive memory queue for zero fill */
1c79356b A	258
	259	extern
	260	vm_offset_t first_phys_addr; /* physical address for first_page */
	261	extern
	262	vm_offset_t last_phys_addr; /* physical address for last_page */
	263
	264	extern
91447636	265	unsigned int vm_page_free_count; /* How many pages are free? */
1c79356b	266	extern
91447636	267	unsigned int vm_page_fictitious_count;/* How many fictitious pages are free? */
1c79356b	268	extern
91447636	269	unsigned int vm_page_active_count; /* How many pages are active? */
1c79356b	270	extern
91447636	271	unsigned int vm_page_inactive_count; /* How many pages are inactive? */
1c79356b	272	extern
91447636	273	unsigned int vm_page_wire_count; /* How many pages are wired? */
1c79356b	274	extern
91447636	275	unsigned int vm_page_free_target; /* How many do we want free? */
1c79356b	276	extern
91447636	277	unsigned int vm_page_free_min; /* When to wakeup pageout */
1c79356b	278	extern
91447636	279	unsigned int vm_page_inactive_target;/* How many do we want inactive? */
1c79356b	280	extern
91447636	281	unsigned int vm_page_free_reserved; /* How many pages reserved to do pageout */
1c79356b	282	extern
91447636 A	283	unsigned int vm_page_throttled_count;/* Count of zero-fill allocations throttled */
	284	extern
	285	unsigned int vm_page_gobble_count;
	286
55e303ae	287	extern
91447636	288	unsigned int vm_page_purgeable_count;/* How many pages are purgeable now ? */
55e303ae	289	extern
91447636	290	uint64_t vm_page_purged_count; /* How many pages got purged so far ? */
1c79356b A	291
	292	decl_mutex_data(,vm_page_queue_lock)
	293	/* lock on active and inactive page queues */
	294	decl_mutex_data(,vm_page_queue_free_lock)
	295	/* lock on free page queue */
1c79356b A	296
	297	extern unsigned int vm_page_free_wanted;
	298	/* how many threads are waiting for memory */
	299
	300	extern vm_offset_t vm_page_fictitious_addr;
	301	/* (fake) phys_addr of fictitious pages */
	302
91447636 A	303	extern boolean_t vm_page_deactivate_hint;
91447636 A	304
5d5c5d0d A	305	// 0 = all pages avail, 1 = disable high mem, 2 = prefer himem
	306	extern int vm_himemory_mode;
	307
	308	extern ppnum_t vm_lopage_poolend;
	309	extern int vm_lopage_poolsize;
	310	extern uint64_t max_valid_dma_address;
	311
	312
1c79356b A	313	/*
	314	* Prototypes for functions exported by this module.
	315	*/
	316	extern void vm_page_bootstrap(
	317	vm_offset_t *startp,
	318	vm_offset_t *endp);
	319
	320	extern void vm_page_module_init(void);
	321
	322	extern void vm_page_create(
55e303ae A	323	ppnum_t start,
55e303ae A	324	ppnum_t end);
1c79356b A	325
	326	extern vm_page_t vm_page_lookup(
	327	vm_object_t object,
	328	vm_object_offset_t offset);
	329
	330	extern vm_page_t vm_page_grab_fictitious(void);
	331
	332	extern void vm_page_release_fictitious(
	333	vm_page_t page);
	334
	335	extern boolean_t vm_page_convert(
	336	vm_page_t page);
	337
	338	extern void vm_page_more_fictitious(void);
	339
	340	extern int vm_pool_low(void);
	341
	342	extern vm_page_t vm_page_grab(void);
	343
5d5c5d0d A	344	extern vm_page_t vm_page_grablo(void);
5d5c5d0d A	345
1c79356b A	346	extern void vm_page_release(
	347	vm_page_t page);
	348
1c79356b A	349	extern boolean_t vm_page_wait(
	350	int interruptible );
	351
	352	extern vm_page_t vm_page_alloc(
	353	vm_object_t object,
	354	vm_object_offset_t offset);
	355
5d5c5d0d A	356	extern vm_page_t vm_page_alloclo(
	357	vm_object_t object,
	358	vm_object_offset_t offset);
	359
1c79356b A	360	extern void vm_page_init(
1c79356b A	361	vm_page_t page,
55e303ae	362	ppnum_t phys_page);
1c79356b A	363
	364	extern void vm_page_free(
	365	vm_page_t page);
	366
	367	extern void vm_page_activate(
	368	vm_page_t page);
	369
	370	extern void vm_page_deactivate(
	371	vm_page_t page);
	372
	373	extern void vm_page_rename(
	374	vm_page_t page,
	375	vm_object_t new_object,
	376	vm_object_offset_t new_offset);
	377
	378	extern void vm_page_insert(
	379	vm_page_t page,
	380	vm_object_t object,
	381	vm_object_offset_t offset);
	382
	383	extern void vm_page_replace(
	384	vm_page_t mem,
	385	vm_object_t object,
	386	vm_object_offset_t offset);
	387
	388	extern void vm_page_remove(
	389	vm_page_t page);
	390
	391	extern void vm_page_zero_fill(
	392	vm_page_t page);
	393
	394	extern void vm_page_part_zero_fill(
	395	vm_page_t m,
	396	vm_offset_t m_pa,
	397	vm_size_t len);
	398
	399	extern void vm_page_copy(
	400	vm_page_t src_page,
	401	vm_page_t dest_page);
	402
	403	extern void vm_page_part_copy(
	404	vm_page_t src_m,
	405	vm_offset_t src_pa,
	406	vm_page_t dst_m,
	407	vm_offset_t dst_pa,
	408	vm_size_t len);
	409
	410	extern void vm_page_wire(
	411	vm_page_t page);
	412
	413	extern void vm_page_unwire(
	414	vm_page_t page);
	415
	416	extern void vm_set_page_size(void);
	417
	418	extern void vm_page_gobble(
	419	vm_page_t page);
	420
1c79356b A	421	/*
	422	* Functions implemented as macros. m->wanted and m->busy are
	423	* protected by the object lock.
	424	*/
	425
	426	#define PAGE_ASSERT_WAIT(m, interruptible) \
9bccf70c A	427	(((m)->wanted = TRUE), \
	428	assert_wait((event_t) (m), (interruptible)))
	429
	430	#define PAGE_SLEEP(o, m, interruptible) \
	431	(((m)->wanted = TRUE), \
	432	thread_sleep_vm_object((o), (m), (interruptible)))
1c79356b A	433
	434	#define PAGE_WAKEUP_DONE(m) \
	435	MACRO_BEGIN \
	436	(m)->busy = FALSE; \
	437	if ((m)->wanted) { \
	438	(m)->wanted = FALSE; \
	439	thread_wakeup((event_t) (m)); \
	440	} \
	441	MACRO_END
	442
	443	#define PAGE_WAKEUP(m) \
	444	MACRO_BEGIN \
	445	if ((m)->wanted) { \
	446	(m)->wanted = FALSE; \
	447	thread_wakeup((event_t) (m)); \
	448	} \
	449	MACRO_END
	450
	451	#define VM_PAGE_FREE(p) \
	452	MACRO_BEGIN \
	453	vm_page_lock_queues(); \
	454	vm_page_free(p); \
	455	vm_page_unlock_queues(); \
	456	MACRO_END
	457
	458	#define VM_PAGE_GRAB_FICTITIOUS(M) \
	459	MACRO_BEGIN \
	460	while ((M = vm_page_grab_fictitious()) == VM_PAGE_NULL) \
	461	vm_page_more_fictitious(); \
	462	MACRO_END
	463
	464	#define VM_PAGE_THROTTLED() \
55e303ae	465	(vm_page_free_count < vm_page_free_min && \
91447636	466	!(current_thread()->options & TH_OPT_VMPRIV) && \
55e303ae	467	++vm_page_throttled_count)
1c79356b A	468
	469	#define VM_PAGE_WAIT() ((void)vm_page_wait(THREAD_UNINT))
	470
	471	#define vm_page_lock_queues() mutex_lock(&vm_page_queue_lock)
	472	#define vm_page_unlock_queues() mutex_unlock(&vm_page_queue_lock)
1c79356b A	473
	474	#define VM_PAGE_QUEUES_REMOVE(mem) \
	475	MACRO_BEGIN \
91447636	476	assert(!mem->laundry); \
1c79356b	477	if (mem->active) { \
91447636	478	assert(mem->object != kernel_object); \
1c79356b A	479	assert(!mem->inactive); \
	480	queue_remove(&vm_page_queue_active, \
	481	mem, vm_page_t, pageq); \
91447636 A	482	mem->pageq.next = NULL; \
91447636 A	483	mem->pageq.prev = NULL; \
1c79356b A	484	mem->active = FALSE; \
	485	if (!mem->fictitious) \
	486	vm_page_active_count--; \
	487	} \
	488	\
	489	if (mem->inactive) { \
91447636	490	assert(mem->object != kernel_object); \
1c79356b	491	assert(!mem->active); \
9bccf70c A	492	if (mem->zero_fill) { \
	493	queue_remove(&vm_page_queue_zf, \
	494	mem, vm_page_t, pageq); \
	495	} else { \
	496	queue_remove(&vm_page_queue_inactive, \
1c79356b	497	mem, vm_page_t, pageq); \
9bccf70c	498	} \
91447636 A	499	mem->pageq.next = NULL; \
91447636 A	500	mem->pageq.prev = NULL; \
1c79356b A	501	mem->inactive = FALSE; \
	502	if (!mem->fictitious) \
	503	vm_page_inactive_count--; \
	504	} \
	505	MACRO_END
	506
	507	#endif /* _VM_VM_PAGE_H_ */