[apple/xnu.git] / osfmk / vm / vm_object.h

/*
 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * The contents of this file constitute Original Code as defined in and
 * are subject to the Apple Public Source License Version 1.1 (the
 * "License").  You may not use this file except in compliance with the
 * License.  Please obtain a copy of the License at
 * http://www.apple.com/publicsource and read it before using this file.
 * 
 * This Original Code and all software distributed under the License are
 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 * License for the specific language governing rights and limitations
 * under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 */
/* 
 * Mach Operating System
 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
 * All Rights Reserved.
 * 
 * Permission to use, copy, modify and distribute this software and its
 * documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 * 
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 * 
 * Carnegie Mellon requests users of this software to return to
 * 
 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 * 
 * any improvements or extensions that they make and grant Carnegie Mellon
 * the rights to redistribute these changes.
 */
/*
 */
/*
 *	File:	vm_object.h
 *	Author:	Avadis Tevanian, Jr., Michael Wayne Young
 *	Date:	1985
 *
 *	Virtual memory object module definitions.
 */

#ifndef	_VM_VM_OBJECT_H_
#define _VM_VM_OBJECT_H_

#include <mach_pagemap.h>
#include <task_swapper.h>

#include <mach/kern_return.h>
#include <mach/boolean.h>
#include <mach/memory_object_types.h>
#include <mach/port.h>
#include <mach/vm_prot.h>
#include <mach/machine/vm_types.h>
#include <kern/queue.h>
#include <kern/lock.h>
#include <kern/assert.h>
#include <kern/ipc_mig.h>
#include <kern/misc_protos.h>
#include <kern/macro_help.h>
#include <ipc/ipc_types.h>
#include <vm/pmap.h>

#if	MACH_PAGEMAP
#include <vm/vm_external.h>
#endif	/* MACH_PAGEMAP */

typedef memory_object_control_t	pager_request_t;
#define	PAGER_REQUEST_NULL	((pager_request_t) 0)

/*
 *	Types defined:
 *
 *	vm_object_t		Virtual memory object.
 */

typedef unsigned long long vm_object_size_t;


struct vm_object {
	queue_head_t		memq;		/* Resident memory */
	decl_mutex_data(,	Lock)		/* Synchronization */

	vm_object_size_t	size;		/* Object size (only valid
						 * if internal)
						 */
	vm_object_size_t	frozen_size;	/* How much has been marked
						 * copy-on-write (only
						 * valid if copy_symmetric)
						 */
	int			ref_count;	/* Number of references */
#if	TASK_SWAPPER
	int			res_count;	/* Residency references (swap)*/
#endif	/* TASK_SWAPPER */
	unsigned int		resident_page_count;
						/* number of resident pages */

	struct vm_object	*copy;		/* Object that should receive
						 * a copy of my changed pages,
						 * for copy_delay, or just the
						 * temporary object that
						 * shadows this object, for
						 * copy_call.
						 */
	struct vm_object	*shadow;	/* My shadow */
	vm_object_offset_t	shadow_offset;	/* Offset into shadow */

	memory_object_t		pager;		/* Where to get data */
	vm_object_offset_t	paging_offset;	/* Offset into memory object */
	pager_request_t		pager_request;	/* Where data comes back */

	memory_object_copy_strategy_t
				copy_strategy;	/* How to handle data copy */

	unsigned int		absent_count;	/* The number of pages that
						 * have been requested but
						 * not filled.  That is, the
						 * number of pages for which
						 * the "absent" attribute is
						 * asserted.
						 */

	unsigned int		paging_in_progress;
						/* The memory object ports are
						 * being used (e.g., for pagein
						 * or pageout) -- don't change
						 * any of these fields (i.e.,
						 * don't collapse, destroy or
						 * terminate)
						 */
	unsigned int
	/* boolean_t array */	all_wanted:11,	/* Bit array of "want to be
						 * awakened" notations.  See
						 * VM_OBJECT_EVENT_* items
						 * below */
	/* boolean_t */	pager_created:1,	/* Has pager been created? */
	/* boolean_t */	pager_initialized:1,	/* Are fields ready to use? */
	/* boolean_t */	pager_ready:1,		/* Will pager take requests? */

	/* boolean_t */		pager_trusted:1,/* The pager for this object
						 * is trusted. This is true for
						 * all internal objects (backed
						 * by the default pager)
						 */
	/* boolean_t */		can_persist:1,	/* The kernel may keep the data
						 * for this object (and rights
						 * to the memory object) after
						 * all address map references 
						 * are deallocated?
						 */
	/* boolean_t */		internal:1,	/* Created by the kernel (and
						 * therefore, managed by the
						 * default memory manger)
						 */
	/* boolean_t */		temporary:1,	/* Permanent objects may be
						 * changed externally by the 
						 * memory manager, and changes
						 * made in memory must be
						 * reflected back to the memory
						 * manager.  Temporary objects
						 * lack both of these
						 * characteristics.
						 */
	/* boolean_t */		private:1,	/* magic device_pager object,
						 * holds private pages only */
	/* boolean_t */		pageout:1,	/* pageout object. contains
						 * private pages that refer to
						 * a real memory object. */
	/* boolean_t */		alive:1,	/* Not yet terminated */

	/* boolean_t */		lock_in_progress:1,
						/* Is a multi-page lock
						 * request in progress?
						 */
	/* boolean_t */		lock_restart:1,
						/* Should lock request in
						 * progress restart search?
						 */
	/* boolean_t */		shadowed:1,	/* Shadow may exist */
	/* boolean_t */		silent_overwrite:1,
						/* Allow full page overwrite
						 * without data_request if
						 * page is absent */
	/* boolean_t */		advisory_pageout:1,
						/* Instead of sending page
						 * via OOL, just notify
						 * pager that the kernel
						 * wants to discard it, page
						 * remains in object */
	/* boolean_t */		true_share:1,
						/* This object is mapped
						 * in more than one place
						 * and hence cannot be 
						 * coalesced */
	/* boolean_t */		terminating:1,
						/* Allows vm_object_lookup
						 * and vm_object_deallocate
						 * to special case their
						 * behavior when they are
						 * called as a result of
						 * page cleaning during
						 * object termination
						 */
	/* boolean_t */		named:1,	/* An enforces an internal
						 * naming convention, by
						 * calling the right routines
						 * for allocation and 
						 * destruction, UBC references
						 * against the vm_object are
						 * checked.
						 */
	/* boolean_t */		shadow_severed:1,
						/* When a permanent object 
						 * backing a COW goes away
					  	 * unexpectedly.  This bit
						 * allows vm_fault to return
						 * an error rather than a
						 * zero filled page.
						 */
	/* boolean_t */		phys_contiguous:1,
						/* Memory is wired and
						 * guaranteed physically 
						 * contiguous.  However
						 * it is not device memory
						 * and obeys normal virtual
						 * memory rules w.r.t pmap
						 * access bits.
						 */
	/* boolean_t */		nophyscache:1;
						/* When mapped at the 
						 * pmap level, don't allow
						 * primary caching. (for
						 * I/O)
						 */
						

	queue_chain_t		cached_list;	/* Attachment point for the
						 * list of objects cached as a
						 * result of their can_persist
						 * value
						 */

	queue_head_t		msr_q;		/* memory object synchronise
						   request queue */

	vm_object_offset_t	last_alloc;	/* last allocation offset */
	vm_size_t		cluster_size;	/* size of paging cluster */
#if	MACH_PAGEMAP
	vm_external_map_t	existence_map;	/* bitmap of pages written to
						 * backing storage */
#endif	/* MACH_PAGEMAP */
	int			cow_hint;	/* last page present in     */
						/* shadow but not in object */
#if	MACH_ASSERT
	struct vm_object	*paging_object;	/* object which pages to be
						 * swapped out are temporary
						 * put in current object
						 */
#endif
					/* hold object lock when altering */
	unsigned	int			/* cache WIMG bits         */		
			wimg_bits:8,		/* wimg plus some expansion*/
			not_in_use:24;
#ifdef	UBC_DEBUG
	queue_head_t		uplq;		/* List of outstanding upls */
#endif /* UBC_DEBUG */
};

__private_extern__
vm_object_t	kernel_object;		/* the single kernel object */

__private_extern__
int		vm_object_absent_max;	/* maximum number of absent pages
					   at a time for each object */

# define	VM_MSYNC_INITIALIZED			0
# define	VM_MSYNC_SYNCHRONIZING			1
# define	VM_MSYNC_DONE				2

struct msync_req {
	queue_chain_t		msr_q;		/* object request queue */
	queue_chain_t		req_q;		/* vm_msync request queue */
	unsigned int		flag;
	vm_object_offset_t	offset;
	vm_object_size_t	length;
	vm_object_t		object;		/* back pointer */
	decl_mutex_data(,	msync_req_lock)	/* Lock for this structure */
};

typedef struct msync_req	*msync_req_t;
#define MSYNC_REQ_NULL		((msync_req_t) 0)

/*
 * Macros to allocate and free msync_reqs
 */
#define msync_req_alloc(msr)						\
	MACRO_BEGIN							\
        (msr) = (msync_req_t)kalloc(sizeof(struct msync_req));		\
        mutex_init(&(msr)->msync_req_lock, ETAP_VM_MSYNC);		\
	msr->flag = VM_MSYNC_INITIALIZED;				\
        MACRO_END

#define msync_req_free(msr)						\
	(kfree((vm_offset_t)(msr), sizeof(struct msync_req)))

#define msr_lock(msr)   mutex_lock(&(msr)->msync_req_lock)
#define msr_unlock(msr) mutex_unlock(&(msr)->msync_req_lock)

/*
 *	Declare procedures that operate on VM objects.
 */

__private_extern__ void		vm_object_bootstrap(void);

__private_extern__ void		vm_object_init(void);

__private_extern__ vm_object_t	vm_object_allocate(
					vm_object_size_t	size);

#if	TASK_SWAPPER

__private_extern__ void	vm_object_res_reference(
				vm_object_t 		object);
__private_extern__ void	vm_object_res_deallocate(
				vm_object_t		object);
#define	VM_OBJ_RES_INCR(object)	(object)->res_count++
#define	VM_OBJ_RES_DECR(object)	(object)->res_count--

#else	/* TASK_SWAPPER */

#define	VM_OBJ_RES_INCR(object)
#define	VM_OBJ_RES_DECR(object)
#define vm_object_res_reference(object)
#define vm_object_res_deallocate(object)

#endif	/* TASK_SWAPPER */

#define vm_object_reference_locked(object)		\
MACRO_BEGIN						\
		vm_object_t RLObject = (object);	\
		assert((RLObject)->ref_count > 0);	\
		(RLObject)->ref_count++;		\
		vm_object_res_reference(RLObject);	\
MACRO_END


#if	MACH_ASSERT

__private_extern__ void		vm_object_reference(
					vm_object_t	object);

#else	/* MACH_ASSERT */

#define	vm_object_reference(object)			\
MACRO_BEGIN						\
	vm_object_t RObject = (object);			\
	if (RObject) {					\
		vm_object_lock(RObject);		\
		vm_object_reference_locked(RObject);	\
		vm_object_unlock(RObject);		\
	}						\
MACRO_END

#endif	/* MACH_ASSERT */

__private_extern__ void		vm_object_deallocate(
					vm_object_t	object);

__private_extern__ kern_return_t vm_object_release_name(
					vm_object_t	object,
					int		flags);
							
__private_extern__ void		vm_object_pmap_protect(
					vm_object_t		object,
					vm_object_offset_t	offset,
					vm_size_t		size,
					pmap_t			pmap,
					vm_offset_t		pmap_start,
					vm_prot_t		prot);

__private_extern__ void		vm_object_page_remove(
					vm_object_t		object,
					vm_object_offset_t	start,
					vm_object_offset_t	end);

__private_extern__ void		vm_object_deactivate_pages(
					vm_object_t		object,
					vm_object_offset_t	offset,
					vm_object_size_t	size,
					boolean_t               kill_page);

__private_extern__ boolean_t	vm_object_coalesce(
					vm_object_t		prev_object,
					vm_object_t		next_object,
					vm_object_offset_t	prev_offset,
					vm_object_offset_t	next_offset,
					vm_object_size_t	prev_size,
					vm_object_size_t	next_size);

__private_extern__ boolean_t	vm_object_shadow(
					vm_object_t		*object,
					vm_object_offset_t	*offset,
					vm_object_size_t	length);

__private_extern__ void		vm_object_collapse(
					vm_object_t	object);

__private_extern__ boolean_t	vm_object_copy_quickly(
				vm_object_t		*_object,
				vm_object_offset_t	src_offset,
				vm_object_size_t	size,
				boolean_t		*_src_needs_copy,
				boolean_t		*_dst_needs_copy);

__private_extern__ kern_return_t	vm_object_copy_strategically(
				vm_object_t		src_object,
				vm_object_offset_t	src_offset,
				vm_object_size_t	size,
				vm_object_t		*dst_object,
				vm_object_offset_t	*dst_offset,
				boolean_t		*dst_needs_copy);

__private_extern__ kern_return_t	vm_object_copy_slowly(
				vm_object_t		src_object,
				vm_object_offset_t	src_offset,
				vm_object_size_t	size,
				int			interruptible,
				vm_object_t		*_result_object);

__private_extern__ vm_object_t	vm_object_copy_delayed(
				vm_object_t		src_object,
				vm_object_offset_t	src_offset,
				vm_object_size_t	size);


__private_extern__ kern_return_t	vm_object_destroy(
					vm_object_t	object,
					kern_return_t	reason);

__private_extern__ void		vm_object_pager_create(
					vm_object_t	object);

__private_extern__ void		vm_object_page_map(
				vm_object_t	object,
				vm_object_offset_t	offset,
				vm_object_size_t	size,
				vm_object_offset_t	(*map_fn)
					(void *, vm_object_offset_t),
					void 		*map_fn_data);

__private_extern__ kern_return_t vm_object_upl_request(
				vm_object_t		object, 
				vm_object_offset_t	offset,
				vm_size_t		size,
				upl_t			*upl,
				upl_page_info_t		*page_info,
				unsigned int		*count,
				int			flags);

__private_extern__ boolean_t vm_object_sync(
				vm_object_t		object,
				vm_object_offset_t	offset,
				vm_size_t		size,
				boolean_t		should_flush,
				boolean_t		should_return);

__private_extern__ kern_return_t vm_object_update(
				vm_object_t		object,
				vm_object_offset_t	offset,
				vm_size_t		size, /* should be 64 */
				memory_object_return_t	should_return,
				int			flags,
				vm_prot_t		prot);

__private_extern__ kern_return_t vm_object_lock_request(
				vm_object_t		object,
				vm_object_offset_t	offset,
				vm_object_size_t	size,
				memory_object_return_t	should_return,
				int			flags,
				vm_prot_t		prot);


__private_extern__ vm_object_t	vm_object_enter(
					memory_object_t		pager,
					vm_object_size_t	size,
					boolean_t		internal,
					boolean_t		init,
					boolean_t		check_named);


/*
 *	Event waiting handling
 */

#define	VM_OBJECT_EVENT_INITIALIZED		0
#define	VM_OBJECT_EVENT_PAGER_READY		1
#define	VM_OBJECT_EVENT_PAGING_IN_PROGRESS	2
#define	VM_OBJECT_EVENT_ABSENT_COUNT		3
#define	VM_OBJECT_EVENT_LOCK_IN_PROGRESS	4
#define	VM_OBJECT_EVENT_UNCACHING		5
#define	VM_OBJECT_EVENT_COPY_CALL		6
#define	VM_OBJECT_EVENT_CACHING			7

#define	vm_object_assert_wait(object, event, interruptible)		\
	(((object)->all_wanted |= 1 << (event)),			\
	 assert_wait((event_t)((vm_offset_t)(object)+(event)),(interruptible)))

#define	vm_object_wait(object, event, interruptible)			\
	(vm_object_assert_wait((object),(event),(interruptible)),	\
	vm_object_unlock(object),					\
	thread_block(THREAD_CONTINUE_NULL))				\

#define thread_sleep_vm_object(object, event, interruptible)		\
	thread_sleep_mutex((event_t)(event), &(object)->Lock, (interruptible))

#define vm_object_sleep(object, event, interruptible)			\
	(((object)->all_wanted |= 1 << (event)),			\
	 thread_sleep_vm_object((object), 				\
		((vm_offset_t)(object)+(event)), (interruptible)))

#define	vm_object_wakeup(object, event)					\
	MACRO_BEGIN							\
	if ((object)->all_wanted & (1 << (event)))			\
		thread_wakeup((event_t)((vm_offset_t)(object) + (event))); \
	(object)->all_wanted &= ~(1 << (event));			\
	MACRO_END

#define	vm_object_set_wanted(object, event)				\
	MACRO_BEGIN							\
	((object)->all_wanted |= (1 << (event)));			\
	MACRO_END

#define	vm_object_wanted(object, event)					\
	((object)->all_wanted & (1 << (event)))

/*
 *	Routines implemented as macros
 */

#define		vm_object_paging_begin(object) 				\
	MACRO_BEGIN							\
	(object)->paging_in_progress++;					\
	MACRO_END

#define		vm_object_paging_end(object) 				\
	MACRO_BEGIN							\
	assert((object)->paging_in_progress != 0);			\
	if (--(object)->paging_in_progress == 0) {			\
		vm_object_wakeup(object,				\
			VM_OBJECT_EVENT_PAGING_IN_PROGRESS);		\
	}								\
	MACRO_END

#define		vm_object_paging_wait(object, interruptible)		\
	MACRO_BEGIN							\
	while ((object)->paging_in_progress != 0) {			\
		wait_result_t  _wr;					\
									\
		_wr = vm_object_sleep((object),				\
				VM_OBJECT_EVENT_PAGING_IN_PROGRESS,	\
				(interruptible));			\
									\
		/*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
			/*XXX break; */					\
	}								\
	MACRO_END

#define	vm_object_absent_assert_wait(object, interruptible)		\
	MACRO_BEGIN							\
	vm_object_assert_wait(	(object),				\
			VM_OBJECT_EVENT_ABSENT_COUNT,			\
			(interruptible));				\
	MACRO_END


#define	vm_object_absent_release(object)				\
	MACRO_BEGIN							\
	(object)->absent_count--;					\
	vm_object_wakeup((object),					\
			 VM_OBJECT_EVENT_ABSENT_COUNT);			\
	MACRO_END

/*
 *	Object locking macros
 */

#define vm_object_lock_init(object)	mutex_init(&(object)->Lock, ETAP_VM_OBJ)
#define vm_object_lock(object)		mutex_lock(&(object)->Lock)
#define vm_object_unlock(object)	mutex_unlock(&(object)->Lock)
#define vm_object_lock_try(object)	mutex_try(&(object)->Lock)

#endif	/* _VM_VM_OBJECT_H_ */
Commit	Line	Data
1c79356b A	1	/*
	2	* Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
	6	* The contents of this file constitute Original Code as defined in and
	7	* are subject to the Apple Public Source License Version 1.1 (the
	8	* "License"). You may not use this file except in compliance with the
	9	* License. Please obtain a copy of the License at
	10	* http://www.apple.com/publicsource and read it before using this file.
	11	*
	12	* This Original Code and all software distributed under the License are
	13	* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	14	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	15	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
	17	* License for the specific language governing rights and limitations
	18	* under the License.
	19	*
	20	* @APPLE_LICENSE_HEADER_END@
	21	*/
	22	/*
	23	* @OSF_COPYRIGHT@
	24	*/
	25	/*
	26	* Mach Operating System
	27	* Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
	28	* All Rights Reserved.
	29	*
	30	* Permission to use, copy, modify and distribute this software and its
	31	* documentation is hereby granted, provided that both the copyright
	32	* notice and this permission notice appear in all copies of the
	33	* software, derivative works or modified versions, and any portions
	34	* thereof, and that both notices appear in supporting documentation.
	35	*
	36	* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
	37	* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
	38	* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
	39	*
	40	* Carnegie Mellon requests users of this software to return to
	41	*
	42	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
	43	* School of Computer Science
	44	* Carnegie Mellon University
	45	* Pittsburgh PA 15213-3890
	46	*
	47	* any improvements or extensions that they make and grant Carnegie Mellon
	48	* the rights to redistribute these changes.
	49	*/
	50	/*
	51	*/
	52	/*
	53	* File: vm_object.h
	54	* Author: Avadis Tevanian, Jr., Michael Wayne Young
	55	* Date: 1985
	56	*
	57	* Virtual memory object module definitions.
	58	*/
	59
	60	#ifndef _VM_VM_OBJECT_H_
	61	#define _VM_VM_OBJECT_H_
	62
	63	#include <mach_pagemap.h>
	64	#include <task_swapper.h>
65
66	#include <mach/kern_return.h>
67	#include <mach/boolean.h>
68	#include <mach/memory_object_types.h>
69	#include <mach/port.h>
70	#include <mach/vm_prot.h>
71	#include <mach/machine/vm_types.h>
72	#include <kern/queue.h>
73	#include <kern/lock.h>
74	#include <kern/assert.h>
0b4e3aa0 A	75	#include <kern/ipc_mig.h>
0b4e3aa0 A	76	#include <kern/misc_protos.h>
1c79356b A	77	#include <kern/macro_help.h>
	78	#include <ipc/ipc_types.h>
	79	#include <vm/pmap.h>
1c79356b A	80
	81	#if MACH_PAGEMAP
	82	#include <vm/vm_external.h>
	83	#endif /* MACH_PAGEMAP */
	84
0b4e3aa0	85	typedef memory_object_control_t pager_request_t;
1c79356b A	86	#define PAGER_REQUEST_NULL ((pager_request_t) 0)
	87
	88	/*
	89	* Types defined:
	90	*
	91	* vm_object_t Virtual memory object.
1c79356b A	92	*/
	93
	94	typedef unsigned long long vm_object_size_t;
	95
	96
	97	struct vm_object {
	98	queue_head_t memq; /* Resident memory */
	99	decl_mutex_data(, Lock) /* Synchronization */
	100
	101	vm_object_size_t size; /* Object size (only valid
	102	* if internal)
	103	*/
	104	vm_object_size_t frozen_size; /* How much has been marked
	105	* copy-on-write (only
	106	* valid if copy_symmetric)
	107	*/
	108	int ref_count; /* Number of references */
	109	#if TASK_SWAPPER
	110	int res_count; /* Residency references (swap)*/
	111	#endif /* TASK_SWAPPER */
	112	unsigned int resident_page_count;
	113	/* number of resident pages */
	114
	115	struct vm_object copy; / Object that should receive
	116	* a copy of my changed pages,
	117	* for copy_delay, or just the
	118	* temporary object that
	119	* shadows this object, for
	120	* copy_call.
	121	*/
	122	struct vm_object shadow; / My shadow */
	123	vm_object_offset_t shadow_offset; /* Offset into shadow */
	124
0b4e3aa0	125	memory_object_t pager; /* Where to get data */
1c79356b A	126	vm_object_offset_t paging_offset; /* Offset into memory object */
	127	pager_request_t pager_request; /* Where data comes back */
	128
	129	memory_object_copy_strategy_t
	130	copy_strategy; /* How to handle data copy */
	131
	132	unsigned int absent_count; /* The number of pages that
	133	* have been requested but
	134	* not filled. That is, the
	135	* number of pages for which
	136	* the "absent" attribute is
	137	* asserted.
	138	*/
	139
	140	unsigned int paging_in_progress;
	141	/* The memory object ports are
	142	* being used (e.g., for pagein
	143	* or pageout) -- don't change
	144	* any of these fields (i.e.,
	145	* don't collapse, destroy or
	146	* terminate)
	147	*/
	148	unsigned int
	149	/* boolean_t array / all_wanted:11, / Bit array of "want to be
	150	* awakened" notations. See
	151	* VM_OBJECT_EVENT_* items
	152	* below */
	153	/* boolean_t / pager_created:1, / Has pager been created? */
	154	/* boolean_t / pager_initialized:1, / Are fields ready to use? */
	155	/* boolean_t / pager_ready:1, / Will pager take requests? */
	156
	157	/* boolean_t / pager_trusted:1,/ The pager for this object
	158	* is trusted. This is true for
	159	* all internal objects (backed
	160	* by the default pager)
	161	*/
	162	/* boolean_t / can_persist:1, / The kernel may keep the data
	163	* for this object (and rights
	164	* to the memory object) after
	165	* all address map references
	166	* are deallocated?
	167	*/
	168	/* boolean_t / internal:1, / Created by the kernel (and
	169	* therefore, managed by the
	170	* default memory manger)
	171	*/
	172	/* boolean_t / temporary:1, / Permanent objects may be
	173	* changed externally by the
	174	* memory manager, and changes
	175	* made in memory must be
	176	* reflected back to the memory
	177	* manager. Temporary objects
	178	* lack both of these
	179	* characteristics.
	180	*/
	181	/* boolean_t / private:1, / magic device_pager object,
	182	* holds private pages only */
	183	/* boolean_t / pageout:1, / pageout object. contains
	184	* private pages that refer to
	185	* a real memory object. */
	186	/* boolean_t / alive:1, / Not yet terminated */
	187
	188	/* boolean_t */ lock_in_progress:1,
	189	/* Is a multi-page lock
190	* request in progress?
191	*/
192	/* boolean_t */ lock_restart:1,
193	/* Should lock request in
194	* progress restart search?
195	*/
196	/* boolean_t / shadowed:1, / Shadow may exist */
197	/* boolean_t */ silent_overwrite:1,
198	/* Allow full page overwrite
199	* without data_request if
200	* page is absent */
201	/* boolean_t */ advisory_pageout:1,
202	/* Instead of sending page
203	* via OOL, just notify
204	* pager that the kernel
205	* wants to discard it, page
206	* remains in object */
207	/* boolean_t */ true_share:1,
208	/* This object is mapped
209	* in more than one place
210	* and hence cannot be
211	* coalesced */
212	/* boolean_t */ terminating:1,
213	/* Allows vm_object_lookup
214	* and vm_object_deallocate
215	* to special case their
216	* behavior when they are
217	* called as a result of
218	* page cleaning during
219	* object termination
220	*/
221	/* boolean_t / named:1, / An enforces an internal
222	* naming convention, by
223	* calling the right routines
224	* for allocation and
225	* destruction, UBC references
226	* against the vm_object are
227	* checked.
228	*/
229	/* boolean_t */ shadow_severed:1,
230	/* When a permanent object
231	* backing a COW goes away
232	* unexpectedly. This bit
233	* allows vm_fault to return
234	* an error rather than a
235	* zero filled page.
236	*/
0b4e3aa0	237	/* boolean_t */ phys_contiguous:1,
1c79356b A	238	/* Memory is wired and
	239	* guaranteed physically
	240	* contiguous. However
	241	* it is not device memory
	242	* and obeys normal virtual
	243	* memory rules w.r.t pmap
	244	* access bits.
	245	*/
0b4e3aa0 A	246	/* boolean_t */ nophyscache:1;
	247	/* When mapped at the
	248	* pmap level, don't allow
	249	* primary caching. (for
	250	* I/O)
	251	*/
1c79356b A	252
	253
	254
	255	queue_chain_t cached_list; /* Attachment point for the
	256	* list of objects cached as a
	257	* result of their can_persist
	258	* value
	259	*/
	260
	261	queue_head_t msr_q; /* memory object synchronise
	262	request queue */
	263
	264	vm_object_offset_t last_alloc; /* last allocation offset */
	265	vm_size_t cluster_size; /* size of paging cluster */
	266	#if MACH_PAGEMAP
	267	vm_external_map_t existence_map; /* bitmap of pages written to
	268	* backing storage */
	269	#endif /* MACH_PAGEMAP */
0b4e3aa0 A	270	int cow_hint; /* last page present in */
0b4e3aa0 A	271	/* shadow but not in object */
1c79356b A	272	#if MACH_ASSERT
	273	struct vm_object paging_object; / object which pages to be
	274	* swapped out are temporary
	275	* put in current object
	276	*/
	277	#endif
9bccf70c A	278	/* hold object lock when altering */
	279	unsigned int /* cache WIMG bits */
	280	wimg_bits:8, /* wimg plus some expansion*/
	281	not_in_use:24;
1c79356b A	282	#ifdef UBC_DEBUG
	283	queue_head_t uplq; /* List of outstanding upls */
	284	#endif /* UBC_DEBUG */
	285	};
	286
0b4e3aa0	287	__private_extern__
1c79356b A	288	vm_object_t kernel_object; /* the single kernel object */
1c79356b A	289
0b4e3aa0	290	__private_extern__
1c79356b A	291	int vm_object_absent_max; /* maximum number of absent pages
	292	at a time for each object */
	293
	294	# define VM_MSYNC_INITIALIZED 0
	295	# define VM_MSYNC_SYNCHRONIZING 1
	296	# define VM_MSYNC_DONE 2
	297
	298	struct msync_req {
	299	queue_chain_t msr_q; /* object request queue */
	300	queue_chain_t req_q; /* vm_msync request queue */
	301	unsigned int flag;
	302	vm_object_offset_t offset;
	303	vm_object_size_t length;
	304	vm_object_t object; /* back pointer */
	305	decl_mutex_data(, msync_req_lock) /* Lock for this structure */
	306	};
	307
	308	typedef struct msync_req *msync_req_t;
	309	#define MSYNC_REQ_NULL ((msync_req_t) 0)
	310
	311	/*
	312	* Macros to allocate and free msync_reqs
	313	*/
	314	#define msync_req_alloc(msr) \
	315	MACRO_BEGIN \
	316	(msr) = (msync_req_t)kalloc(sizeof(struct msync_req)); \
	317	mutex_init(&(msr)->msync_req_lock, ETAP_VM_MSYNC); \
	318	msr->flag = VM_MSYNC_INITIALIZED; \
	319	MACRO_END
	320
	321	#define msync_req_free(msr) \
	322	(kfree((vm_offset_t)(msr), sizeof(struct msync_req)))
	323
	324	#define msr_lock(msr) mutex_lock(&(msr)->msync_req_lock)
	325	#define msr_unlock(msr) mutex_unlock(&(msr)->msync_req_lock)
	326
	327	/*
	328	* Declare procedures that operate on VM objects.
	329	*/
	330
0b4e3aa0	331	__private_extern__ void vm_object_bootstrap(void);
1c79356b	332
0b4e3aa0	333	__private_extern__ void vm_object_init(void);
1c79356b	334
0b4e3aa0	335	__private_extern__ vm_object_t vm_object_allocate(
1c79356b A	336	vm_object_size_t size);
1c79356b A	337
0b4e3aa0 A	338	#if TASK_SWAPPER
	339
	340	__private_extern__ void vm_object_res_reference(
	341	vm_object_t object);
	342	__private_extern__ void vm_object_res_deallocate(
	343	vm_object_t object);
	344	#define VM_OBJ_RES_INCR(object) (object)->res_count++
	345	#define VM_OBJ_RES_DECR(object) (object)->res_count--
	346
	347	#else /* TASK_SWAPPER */
	348
	349	#define VM_OBJ_RES_INCR(object)
	350	#define VM_OBJ_RES_DECR(object)
	351	#define vm_object_res_reference(object)
	352	#define vm_object_res_deallocate(object)
	353
	354	#endif /* TASK_SWAPPER */
	355
	356	#define vm_object_reference_locked(object) \
	357	MACRO_BEGIN \
	358	vm_object_t RLObject = (object); \
	359	assert((RLObject)->ref_count > 0); \
	360	(RLObject)->ref_count++; \
	361	vm_object_res_reference(RLObject); \
	362	MACRO_END
	363
	364
1c79356b	365	#if MACH_ASSERT
0b4e3aa0 A	366
0b4e3aa0 A	367	__private_extern__ void vm_object_reference(
1c79356b	368	vm_object_t object);
0b4e3aa0	369
1c79356b	370	#else /* MACH_ASSERT */
0b4e3aa0	371
1c79356b A	372	#define vm_object_reference(object) \
1c79356b A	373	MACRO_BEGIN \
0b4e3aa0 A	374	vm_object_t RObject = (object); \
	375	if (RObject) { \
	376	vm_object_lock(RObject); \
	377	vm_object_reference_locked(RObject); \
	378	vm_object_unlock(RObject); \
1c79356b A	379	} \
1c79356b A	380	MACRO_END
0b4e3aa0	381
1c79356b A	382	#endif /* MACH_ASSERT */
1c79356b A	383
0b4e3aa0	384	__private_extern__ void vm_object_deallocate(
1c79356b A	385	vm_object_t object);
1c79356b A	386
0b4e3aa0 A	387	__private_extern__ kern_return_t vm_object_release_name(
	388	vm_object_t object,
	389	int flags);
	390
	391	__private_extern__ void vm_object_pmap_protect(
1c79356b A	392	vm_object_t object,
	393	vm_object_offset_t offset,
	394	vm_size_t size,
	395	pmap_t pmap,
	396	vm_offset_t pmap_start,
	397	vm_prot_t prot);
	398
0b4e3aa0	399	__private_extern__ void vm_object_page_remove(
1c79356b A	400	vm_object_t object,
	401	vm_object_offset_t start,
	402	vm_object_offset_t end);
	403
0b4e3aa0 A	404	__private_extern__ void vm_object_deactivate_pages(
	405	vm_object_t object,
	406	vm_object_offset_t offset,
	407	vm_object_size_t size,
	408	boolean_t kill_page);
	409
	410	__private_extern__ boolean_t vm_object_coalesce(
1c79356b A	411	vm_object_t prev_object,
	412	vm_object_t next_object,
	413	vm_object_offset_t prev_offset,
	414	vm_object_offset_t next_offset,
	415	vm_object_size_t prev_size,
	416	vm_object_size_t next_size);
	417
0b4e3aa0	418	__private_extern__ boolean_t vm_object_shadow(
1c79356b A	419	vm_object_t *object,
	420	vm_object_offset_t *offset,
	421	vm_object_size_t length);
	422
0b4e3aa0	423	__private_extern__ void vm_object_collapse(
1c79356b A	424	vm_object_t object);
1c79356b A	425
0b4e3aa0	426	__private_extern__ boolean_t vm_object_copy_quickly(
1c79356b A	427	vm_object_t *_object,
	428	vm_object_offset_t src_offset,
	429	vm_object_size_t size,
	430	boolean_t *_src_needs_copy,
	431	boolean_t *_dst_needs_copy);
	432
0b4e3aa0	433	__private_extern__ kern_return_t vm_object_copy_strategically(
1c79356b A	434	vm_object_t src_object,
	435	vm_object_offset_t src_offset,
	436	vm_object_size_t size,
	437	vm_object_t *dst_object,
	438	vm_object_offset_t *dst_offset,
	439	boolean_t *dst_needs_copy);
	440
0b4e3aa0	441	__private_extern__ kern_return_t vm_object_copy_slowly(
1c79356b A	442	vm_object_t src_object,
	443	vm_object_offset_t src_offset,
	444	vm_object_size_t size,
	445	int interruptible,
	446	vm_object_t *_result_object);
	447
0b4e3aa0 A	448	__private_extern__ vm_object_t vm_object_copy_delayed(
	449	vm_object_t src_object,
	450	vm_object_offset_t src_offset,
	451	vm_object_size_t size);
	452
1c79356b	453
1c79356b	454
0b4e3aa0 A	455	__private_extern__ kern_return_t vm_object_destroy(
	456	vm_object_t object,
	457	kern_return_t reason);
1c79356b	458
0b4e3aa0 A	459	__private_extern__ void vm_object_pager_create(
	460	vm_object_t object);
	461
	462	__private_extern__ void vm_object_page_map(
1c79356b A	463	vm_object_t object,
	464	vm_object_offset_t offset,
	465	vm_object_size_t size,
	466	vm_object_offset_t (*map_fn)
	467	(void *, vm_object_offset_t),
	468	void *map_fn_data);
	469
0b4e3aa0 A	470	__private_extern__ kern_return_t vm_object_upl_request(
	471	vm_object_t object,
	472	vm_object_offset_t offset,
	473	vm_size_t size,
	474	upl_t *upl,
	475	upl_page_info_t *page_info,
	476	unsigned int *count,
	477	int flags);
	478
	479	__private_extern__ boolean_t vm_object_sync(
	480	vm_object_t object,
	481	vm_object_offset_t offset,
	482	vm_size_t size,
	483	boolean_t should_flush,
	484	boolean_t should_return);
1c79356b	485
0b4e3aa0 A	486	__private_extern__ kern_return_t vm_object_update(
	487	vm_object_t object,
	488	vm_object_offset_t offset,
	489	vm_size_t size, /* should be 64 */
	490	memory_object_return_t should_return,
	491	int flags,
	492	vm_prot_t prot);
1c79356b	493
0b4e3aa0 A	494	__private_extern__ kern_return_t vm_object_lock_request(
	495	vm_object_t object,
	496	vm_object_offset_t offset,
	497	vm_object_size_t size,
	498	memory_object_return_t should_return,
	499	int flags,
	500	vm_prot_t prot);
1c79356b	501
1c79356b	502
1c79356b	503
0b4e3aa0 A	504	__private_extern__ vm_object_t vm_object_enter(
0b4e3aa0 A	505	memory_object_t pager,
1c79356b A	506	vm_object_size_t size,
	507	boolean_t internal,
	508	boolean_t init,
	509	boolean_t check_named);
	510
	511
1c79356b A	512	/*
	513	* Event waiting handling
	514	*/
	515
	516	#define VM_OBJECT_EVENT_INITIALIZED 0
	517	#define VM_OBJECT_EVENT_PAGER_READY 1
	518	#define VM_OBJECT_EVENT_PAGING_IN_PROGRESS 2
	519	#define VM_OBJECT_EVENT_ABSENT_COUNT 3
	520	#define VM_OBJECT_EVENT_LOCK_IN_PROGRESS 4
	521	#define VM_OBJECT_EVENT_UNCACHING 5
	522	#define VM_OBJECT_EVENT_COPY_CALL 6
	523	#define VM_OBJECT_EVENT_CACHING 7
	524
	525	#define vm_object_assert_wait(object, event, interruptible) \
9bccf70c A	526	(((object)->all_wanted \|= 1 << (event)), \
9bccf70c A	527	assert_wait((event_t)((vm_offset_t)(object)+(event)),(interruptible)))
1c79356b A	528
1c79356b A	529	#define vm_object_wait(object, event, interruptible) \
9bccf70c A	530	(vm_object_assert_wait((object),(event),(interruptible)), \
	531	vm_object_unlock(object), \
	532	thread_block(THREAD_CONTINUE_NULL)) \
	533
	534	#define thread_sleep_vm_object(object, event, interruptible) \
	535	thread_sleep_mutex((event_t)(event), &(object)->Lock, (interruptible))
	536
	537	#define vm_object_sleep(object, event, interruptible) \
	538	(((object)->all_wanted \|= 1 << (event)), \
	539	thread_sleep_vm_object((object), \
	540	((vm_offset_t)(object)+(event)), (interruptible)))
1c79356b A	541
	542	#define vm_object_wakeup(object, event) \
	543	MACRO_BEGIN \
	544	if ((object)->all_wanted & (1 << (event))) \
	545	thread_wakeup((event_t)((vm_offset_t)(object) + (event))); \
	546	(object)->all_wanted &= ~(1 << (event)); \
	547	MACRO_END
	548
	549	#define vm_object_set_wanted(object, event) \
	550	MACRO_BEGIN \
	551	((object)->all_wanted \|= (1 << (event))); \
	552	MACRO_END
	553
	554	#define vm_object_wanted(object, event) \
	555	((object)->all_wanted & (1 << (event)))
	556
	557	/*
	558	* Routines implemented as macros
	559	*/
	560
	561	#define vm_object_paging_begin(object) \
	562	MACRO_BEGIN \
	563	(object)->paging_in_progress++; \
	564	MACRO_END
	565
	566	#define vm_object_paging_end(object) \
	567	MACRO_BEGIN \
	568	assert((object)->paging_in_progress != 0); \
	569	if (--(object)->paging_in_progress == 0) { \
	570	vm_object_wakeup(object, \
	571	VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
	572	} \
	573	MACRO_END
	574
	575	#define vm_object_paging_wait(object, interruptible) \
	576	MACRO_BEGIN \
	577	while ((object)->paging_in_progress != 0) { \
9bccf70c A	578	wait_result_t _wr; \
	579	\
	580	_wr = vm_object_sleep((object), \
1c79356b A	581	VM_OBJECT_EVENT_PAGING_IN_PROGRESS, \
1c79356b A	582	(interruptible)); \
1c79356b	583	\
9bccf70c	584	/XXX if ((interruptible) && (_wr != THREAD_AWAKENED))/\
1c79356b A	585	/XXX break; / \
	586	} \
	587	MACRO_END
	588
	589	#define vm_object_absent_assert_wait(object, interruptible) \
	590	MACRO_BEGIN \
	591	vm_object_assert_wait( (object), \
	592	VM_OBJECT_EVENT_ABSENT_COUNT, \
	593	(interruptible)); \
	594	MACRO_END
	595
	596
	597	#define vm_object_absent_release(object) \
	598	MACRO_BEGIN \
	599	(object)->absent_count--; \
	600	vm_object_wakeup((object), \
	601	VM_OBJECT_EVENT_ABSENT_COUNT); \
	602	MACRO_END
	603
	604	/*
	605	* Object locking macros
	606	*/
	607
	608	#define vm_object_lock_init(object) mutex_init(&(object)->Lock, ETAP_VM_OBJ)
	609	#define vm_object_lock(object) mutex_lock(&(object)->Lock)
	610	#define vm_object_unlock(object) mutex_unlock(&(object)->Lock)
	611	#define vm_object_lock_try(object) mutex_try(&(object)->Lock)
	612
	613	#endif /* _VM_VM_OBJECT_H_ */