[apple/xnu.git] / osfmk / vm / vm_object.h

/*
 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 */
/* 
 * Mach Operating System
 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
 * All Rights Reserved.
 * 
 * Permission to use, copy, modify and distribute this software and its
 * documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 * 
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 * 
 * Carnegie Mellon requests users of this software to return to
 * 
 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 * 
 * any improvements or extensions that they make and grant Carnegie Mellon
 * the rights to redistribute these changes.
 */
/*
 */
/*
 *	File:	vm_object.h
 *	Author:	Avadis Tevanian, Jr., Michael Wayne Young
 *	Date:	1985
 *
 *	Virtual memory object module definitions.
 */

#ifndef	_VM_VM_OBJECT_H_
#define _VM_VM_OBJECT_H_

#include <mach_pagemap.h>
#include <task_swapper.h>

#include <mach/kern_return.h>
#include <mach/boolean.h>
#include <mach/memory_object_types.h>
#include <mach/port.h>
#include <mach/vm_prot.h>
#include <mach/machine/vm_types.h>
#include <kern/queue.h>
#include <kern/lock.h>
#include <kern/assert.h>
#include <kern/ipc_mig.h>
#include <kern/misc_protos.h>
#include <kern/macro_help.h>
#include <ipc/ipc_types.h>
#include <vm/pmap.h>

#if	MACH_PAGEMAP
#include <vm/vm_external.h>
#endif	/* MACH_PAGEMAP */

typedef memory_object_control_t	pager_request_t;
#define	PAGER_REQUEST_NULL	((pager_request_t) 0)

/*
 *	Types defined:
 *
 *	vm_object_t		Virtual memory object.
 */

typedef unsigned long long vm_object_size_t;


struct vm_object {
	queue_head_t		memq;		/* Resident memory */
	decl_mutex_data(,	Lock)		/* Synchronization */

	vm_object_size_t	size;		/* Object size (only valid
						 * if internal)
						 */
	vm_object_size_t	frozen_size;	/* How much has been marked
						 * copy-on-write (only
						 * valid if copy_symmetric)
						 */
	int			ref_count;	/* Number of references */
#if	TASK_SWAPPER
	int			res_count;	/* Residency references (swap)*/
#endif	/* TASK_SWAPPER */
	unsigned int		resident_page_count;
						/* number of resident pages */

	struct vm_object	*copy;		/* Object that should receive
						 * a copy of my changed pages,
						 * for copy_delay, or just the
						 * temporary object that
						 * shadows this object, for
						 * copy_call.
						 */
	struct vm_object	*shadow;	/* My shadow */
	vm_object_offset_t	shadow_offset;	/* Offset into shadow */

	memory_object_t		pager;		/* Where to get data */
	vm_object_offset_t	paging_offset;	/* Offset into memory object */
	pager_request_t		pager_request;	/* Where data comes back */

	memory_object_copy_strategy_t
				copy_strategy;	/* How to handle data copy */

	unsigned int		absent_count;	/* The number of pages that
						 * have been requested but
						 * not filled.  That is, the
						 * number of pages for which
						 * the "absent" attribute is
						 * asserted.
						 */

	unsigned int		paging_in_progress;
						/* The memory object ports are
						 * being used (e.g., for pagein
						 * or pageout) -- don't change
						 * any of these fields (i.e.,
						 * don't collapse, destroy or
						 * terminate)
						 */
	unsigned int
	/* boolean_t array */	all_wanted:11,	/* Bit array of "want to be
						 * awakened" notations.  See
						 * VM_OBJECT_EVENT_* items
						 * below */
	/* boolean_t */	pager_created:1,	/* Has pager been created? */
	/* boolean_t */	pager_initialized:1,	/* Are fields ready to use? */
	/* boolean_t */	pager_ready:1,		/* Will pager take requests? */

	/* boolean_t */		pager_trusted:1,/* The pager for this object
						 * is trusted. This is true for
						 * all internal objects (backed
						 * by the default pager)
						 */
	/* boolean_t */		can_persist:1,	/* The kernel may keep the data
						 * for this object (and rights
						 * to the memory object) after
						 * all address map references 
						 * are deallocated?
						 */
	/* boolean_t */		internal:1,	/* Created by the kernel (and
						 * therefore, managed by the
						 * default memory manger)
						 */
	/* boolean_t */		temporary:1,	/* Permanent objects may be
						 * changed externally by the 
						 * memory manager, and changes
						 * made in memory must be
						 * reflected back to the memory
						 * manager.  Temporary objects
						 * lack both of these
						 * characteristics.
						 */
	/* boolean_t */		private:1,	/* magic device_pager object,
						 * holds private pages only */
	/* boolean_t */		pageout:1,	/* pageout object. contains
						 * private pages that refer to
						 * a real memory object. */
	/* boolean_t */		alive:1,	/* Not yet terminated */

	/* boolean_t */		lock_in_progress:1,
						/* Is a multi-page lock
						 * request in progress?
						 */
	/* boolean_t */		lock_restart:1,
						/* Should lock request in
						 * progress restart search?
						 */
	/* boolean_t */		shadowed:1,	/* Shadow may exist */
	/* boolean_t */		silent_overwrite:1,
						/* Allow full page overwrite
						 * without data_request if
						 * page is absent */
	/* boolean_t */		advisory_pageout:1,
						/* Instead of sending page
						 * via OOL, just notify
						 * pager that the kernel
						 * wants to discard it, page
						 * remains in object */
	/* boolean_t */		true_share:1,
						/* This object is mapped
						 * in more than one place
						 * and hence cannot be 
						 * coalesced */
	/* boolean_t */		terminating:1,
						/* Allows vm_object_lookup
						 * and vm_object_deallocate
						 * to special case their
						 * behavior when they are
						 * called as a result of
						 * page cleaning during
						 * object termination
						 */
	/* boolean_t */		named:1,	/* An enforces an internal
						 * naming convention, by
						 * calling the right routines
						 * for allocation and 
						 * destruction, UBC references
						 * against the vm_object are
						 * checked.
						 */
	/* boolean_t */		shadow_severed:1,
						/* When a permanent object 
						 * backing a COW goes away
					  	 * unexpectedly.  This bit
						 * allows vm_fault to return
						 * an error rather than a
						 * zero filled page.
						 */
	/* boolean_t */		phys_contiguous:1,
						/* Memory is wired and
						 * guaranteed physically 
						 * contiguous.  However
						 * it is not device memory
						 * and obeys normal virtual
						 * memory rules w.r.t pmap
						 * access bits.
						 */
	/* boolean_t */		nophyscache:1;
						/* When mapped at the 
						 * pmap level, don't allow
						 * primary caching. (for
						 * I/O)
						 */
						

	queue_chain_t		cached_list;	/* Attachment point for the
						 * list of objects cached as a
						 * result of their can_persist
						 * value
						 */

	queue_head_t		msr_q;		/* memory object synchronise
						   request queue */

	vm_object_offset_t	last_alloc;	/* last allocation offset */
	vm_size_t		cluster_size;	/* size of paging cluster */
#if	MACH_PAGEMAP
	vm_external_map_t	existence_map;	/* bitmap of pages written to
						 * backing storage */
#endif	/* MACH_PAGEMAP */
	int			cow_hint;	/* last page present in     */
						/* shadow but not in object */
#if	MACH_ASSERT
	struct vm_object	*paging_object;	/* object which pages to be
						 * swapped out are temporary
						 * put in current object
						 */
#endif
					/* hold object lock when altering */
	unsigned	int			/* cache WIMG bits         */		
			wimg_bits:8,		/* wimg plus some expansion*/
			not_in_use:24;
#ifdef	UBC_DEBUG
	queue_head_t		uplq;		/* List of outstanding upls */
#endif /* UBC_DEBUG */
};

__private_extern__
vm_object_t	kernel_object;		/* the single kernel object */

__private_extern__
int		vm_object_absent_max;	/* maximum number of absent pages
					   at a time for each object */

# define	VM_MSYNC_INITIALIZED			0
# define	VM_MSYNC_SYNCHRONIZING			1
# define	VM_MSYNC_DONE				2

struct msync_req {
	queue_chain_t		msr_q;		/* object request queue */
	queue_chain_t		req_q;		/* vm_msync request queue */
	unsigned int		flag;
	vm_object_offset_t	offset;
	vm_object_size_t	length;
	vm_object_t		object;		/* back pointer */
	decl_mutex_data(,	msync_req_lock)	/* Lock for this structure */
};

typedef struct msync_req	*msync_req_t;
#define MSYNC_REQ_NULL		((msync_req_t) 0)

/*
 * Macros to allocate and free msync_reqs
 */
#define msync_req_alloc(msr)						\
	MACRO_BEGIN							\
        (msr) = (msync_req_t)kalloc(sizeof(struct msync_req));		\
        mutex_init(&(msr)->msync_req_lock, ETAP_VM_MSYNC);		\
	msr->flag = VM_MSYNC_INITIALIZED;				\
        MACRO_END

#define msync_req_free(msr)						\
	(kfree((vm_offset_t)(msr), sizeof(struct msync_req)))

#define msr_lock(msr)   mutex_lock(&(msr)->msync_req_lock)
#define msr_unlock(msr) mutex_unlock(&(msr)->msync_req_lock)

/*
 *	Declare procedures that operate on VM objects.
 */

__private_extern__ void		vm_object_bootstrap(void);

__private_extern__ void		vm_object_init(void);

__private_extern__ vm_object_t	vm_object_allocate(
					vm_object_size_t	size);

#if	TASK_SWAPPER

__private_extern__ void	vm_object_res_reference(
				vm_object_t 		object);
__private_extern__ void	vm_object_res_deallocate(
				vm_object_t		object);
#define	VM_OBJ_RES_INCR(object)	(object)->res_count++
#define	VM_OBJ_RES_DECR(object)	(object)->res_count--

#else	/* TASK_SWAPPER */

#define	VM_OBJ_RES_INCR(object)
#define	VM_OBJ_RES_DECR(object)
#define vm_object_res_reference(object)
#define vm_object_res_deallocate(object)

#endif	/* TASK_SWAPPER */

#define vm_object_reference_locked(object)		\
MACRO_BEGIN						\
		vm_object_t RLObject = (object);	\
		assert((RLObject)->ref_count > 0);	\
		(RLObject)->ref_count++;		\
		vm_object_res_reference(RLObject);	\
MACRO_END


#if	MACH_ASSERT

__private_extern__ void		vm_object_reference(
					vm_object_t	object);

#else	/* MACH_ASSERT */

#define	vm_object_reference(object)			\
MACRO_BEGIN						\
	vm_object_t RObject = (object);			\
	if (RObject) {					\
		vm_object_lock(RObject);		\
		vm_object_reference_locked(RObject);	\
		vm_object_unlock(RObject);		\
	}						\
MACRO_END

#endif	/* MACH_ASSERT */

__private_extern__ void		vm_object_deallocate(
					vm_object_t	object);

__private_extern__ kern_return_t vm_object_release_name(
					vm_object_t	object,
					int		flags);
							
__private_extern__ void		vm_object_pmap_protect(
					vm_object_t		object,
					vm_object_offset_t	offset,
					vm_size_t		size,
					pmap_t			pmap,
					vm_offset_t		pmap_start,
					vm_prot_t		prot);

__private_extern__ void		vm_object_page_remove(
					vm_object_t		object,
					vm_object_offset_t	start,
					vm_object_offset_t	end);

__private_extern__ void		vm_object_deactivate_pages(
					vm_object_t		object,
					vm_object_offset_t	offset,
					vm_object_size_t	size,
					boolean_t               kill_page);

__private_extern__ boolean_t	vm_object_coalesce(
					vm_object_t		prev_object,
					vm_object_t		next_object,
					vm_object_offset_t	prev_offset,
					vm_object_offset_t	next_offset,
					vm_object_size_t	prev_size,
					vm_object_size_t	next_size);

__private_extern__ boolean_t	vm_object_shadow(
					vm_object_t		*object,
					vm_object_offset_t	*offset,
					vm_object_size_t	length);

__private_extern__ void		vm_object_collapse(
					vm_object_t	object);

__private_extern__ boolean_t	vm_object_copy_quickly(
				vm_object_t		*_object,
				vm_object_offset_t	src_offset,
				vm_object_size_t	size,
				boolean_t		*_src_needs_copy,
				boolean_t		*_dst_needs_copy);

__private_extern__ kern_return_t	vm_object_copy_strategically(
				vm_object_t		src_object,
				vm_object_offset_t	src_offset,
				vm_object_size_t	size,
				vm_object_t		*dst_object,
				vm_object_offset_t	*dst_offset,
				boolean_t		*dst_needs_copy);

__private_extern__ kern_return_t	vm_object_copy_slowly(
				vm_object_t		src_object,
				vm_object_offset_t	src_offset,
				vm_object_size_t	size,
				int			interruptible,
				vm_object_t		*_result_object);

__private_extern__ vm_object_t	vm_object_copy_delayed(
				vm_object_t		src_object,
				vm_object_offset_t	src_offset,
				vm_object_size_t	size);


__private_extern__ kern_return_t	vm_object_destroy(
					vm_object_t	object,
					kern_return_t	reason);

__private_extern__ void		vm_object_pager_create(
					vm_object_t	object);

__private_extern__ void		vm_object_page_map(
				vm_object_t	object,
				vm_object_offset_t	offset,
				vm_object_size_t	size,
				vm_object_offset_t	(*map_fn)
					(void *, vm_object_offset_t),
					void 		*map_fn_data);

__private_extern__ kern_return_t vm_object_upl_request(
				vm_object_t		object, 
				vm_object_offset_t	offset,
				vm_size_t		size,
				upl_t			*upl,
				upl_page_info_t		*page_info,
				unsigned int		*count,
				int			flags);

__private_extern__ boolean_t vm_object_sync(
				vm_object_t		object,
				vm_object_offset_t	offset,
				vm_size_t		size,
				boolean_t		should_flush,
				boolean_t		should_return);

__private_extern__ kern_return_t vm_object_update(
				vm_object_t		object,
				vm_object_offset_t	offset,
				vm_size_t		size, /* should be 64 */
				memory_object_return_t	should_return,
				int			flags,
				vm_prot_t		prot);

__private_extern__ kern_return_t vm_object_lock_request(
				vm_object_t		object,
				vm_object_offset_t	offset,
				vm_object_size_t	size,
				memory_object_return_t	should_return,
				int			flags,
				vm_prot_t		prot);


__private_extern__ vm_object_t	vm_object_enter(
					memory_object_t		pager,
					vm_object_size_t	size,
					boolean_t		internal,
					boolean_t		init,
					boolean_t		check_named);


/*
 *	Event waiting handling
 */

#define	VM_OBJECT_EVENT_INITIALIZED		0
#define	VM_OBJECT_EVENT_PAGER_READY		1
#define	VM_OBJECT_EVENT_PAGING_IN_PROGRESS	2
#define	VM_OBJECT_EVENT_ABSENT_COUNT		3
#define	VM_OBJECT_EVENT_LOCK_IN_PROGRESS	4
#define	VM_OBJECT_EVENT_UNCACHING		5
#define	VM_OBJECT_EVENT_COPY_CALL		6
#define	VM_OBJECT_EVENT_CACHING			7

#define	vm_object_assert_wait(object, event, interruptible)		\
	(((object)->all_wanted |= 1 << (event)),			\
	 assert_wait((event_t)((vm_offset_t)(object)+(event)),(interruptible)))

#define	vm_object_wait(object, event, interruptible)			\
	(vm_object_assert_wait((object),(event),(interruptible)),	\
	vm_object_unlock(object),					\
	thread_block(THREAD_CONTINUE_NULL))				\

#define thread_sleep_vm_object(object, event, interruptible)		\
	thread_sleep_mutex((event_t)(event), &(object)->Lock, (interruptible))

#define vm_object_sleep(object, event, interruptible)			\
	(((object)->all_wanted |= 1 << (event)),			\
	 thread_sleep_vm_object((object), 				\
		((vm_offset_t)(object)+(event)), (interruptible)))

#define	vm_object_wakeup(object, event)					\
	MACRO_BEGIN							\
	if ((object)->all_wanted & (1 << (event)))			\
		thread_wakeup((event_t)((vm_offset_t)(object) + (event))); \
	(object)->all_wanted &= ~(1 << (event));			\
	MACRO_END

#define	vm_object_set_wanted(object, event)				\
	MACRO_BEGIN							\
	((object)->all_wanted |= (1 << (event)));			\
	MACRO_END

#define	vm_object_wanted(object, event)					\
	((object)->all_wanted & (1 << (event)))

/*
 *	Routines implemented as macros
 */

#define		vm_object_paging_begin(object) 				\
	MACRO_BEGIN							\
	(object)->paging_in_progress++;					\
	MACRO_END

#define		vm_object_paging_end(object) 				\
	MACRO_BEGIN							\
	assert((object)->paging_in_progress != 0);			\
	if (--(object)->paging_in_progress == 0) {			\
		vm_object_wakeup(object,				\
			VM_OBJECT_EVENT_PAGING_IN_PROGRESS);		\
	}								\
	MACRO_END

#define		vm_object_paging_wait(object, interruptible)		\
	MACRO_BEGIN							\
	while ((object)->paging_in_progress != 0) {			\
		wait_result_t  _wr;					\
									\
		_wr = vm_object_sleep((object),				\
				VM_OBJECT_EVENT_PAGING_IN_PROGRESS,	\
				(interruptible));			\
									\
		/*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
			/*XXX break; */					\
	}								\
	MACRO_END

#define	vm_object_absent_assert_wait(object, interruptible)		\
	MACRO_BEGIN							\
	vm_object_assert_wait(	(object),				\
			VM_OBJECT_EVENT_ABSENT_COUNT,			\
			(interruptible));				\
	MACRO_END


#define	vm_object_absent_release(object)				\
	MACRO_BEGIN							\
	(object)->absent_count--;					\
	vm_object_wakeup((object),					\
			 VM_OBJECT_EVENT_ABSENT_COUNT);			\
	MACRO_END

/*
 *	Object locking macros
 */

#define vm_object_lock_init(object)	mutex_init(&(object)->Lock, ETAP_VM_OBJ)
#define vm_object_lock(object)		mutex_lock(&(object)->Lock)
#define vm_object_unlock(object)	mutex_unlock(&(object)->Lock)
#define vm_object_lock_try(object)	mutex_try(&(object)->Lock)

#endif	/* _VM_VM_OBJECT_H_ */
Commit	Line	Data
1c79356b A	1	/*
	2	* Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
43866e37	6	* Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
1c79356b	7	*
43866e37 A	8	* This file contains Original Code and/or Modifications of Original Code
	9	* as defined in and that are subject to the Apple Public Source License
	10	* Version 2.0 (the 'License'). You may not use this file except in
	11	* compliance with the License. Please obtain a copy of the License at
	12	* http://www.opensource.apple.com/apsl/ and read it before using this
	13	* file.
	14	*
	15	* The Original Code and all software distributed under the License are
	16	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
1c79356b A	17	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
1c79356b A	18	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
43866e37 A	19	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	20	* Please see the License for the specific language governing rights and
	21	* limitations under the License.
1c79356b A	22	*
	23	* @APPLE_LICENSE_HEADER_END@
	24	*/
	25	/*
	26	* @OSF_COPYRIGHT@
	27	*/
	28	/*
	29	* Mach Operating System
	30	* Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
	31	* All Rights Reserved.
	32	*
	33	* Permission to use, copy, modify and distribute this software and its
	34	* documentation is hereby granted, provided that both the copyright
	35	* notice and this permission notice appear in all copies of the
	36	* software, derivative works or modified versions, and any portions
	37	* thereof, and that both notices appear in supporting documentation.
	38	*
	39	* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
	40	* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
	41	* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
	42	*
	43	* Carnegie Mellon requests users of this software to return to
	44	*
	45	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
	46	* School of Computer Science
	47	* Carnegie Mellon University
	48	* Pittsburgh PA 15213-3890
	49	*
	50	* any improvements or extensions that they make and grant Carnegie Mellon
	51	* the rights to redistribute these changes.
	52	*/
	53	/*
	54	*/
	55	/*
	56	* File: vm_object.h
	57	* Author: Avadis Tevanian, Jr., Michael Wayne Young
	58	* Date: 1985
	59	*
	60	* Virtual memory object module definitions.
	61	*/
	62
	63	#ifndef _VM_VM_OBJECT_H_
	64	#define _VM_VM_OBJECT_H_
	65
	66	#include <mach_pagemap.h>
	67	#include <task_swapper.h>
	68
	69	#include <mach/kern_return.h>
	70	#include <mach/boolean.h>
	71	#include <mach/memory_object_types.h>
	72	#include <mach/port.h>
	73	#include <mach/vm_prot.h>
	74	#include <mach/machine/vm_types.h>
	75	#include <kern/queue.h>
	76	#include <kern/lock.h>
	77	#include <kern/assert.h>
0b4e3aa0 A	78	#include <kern/ipc_mig.h>
0b4e3aa0 A	79	#include <kern/misc_protos.h>
1c79356b A	80	#include <kern/macro_help.h>
	81	#include <ipc/ipc_types.h>
	82	#include <vm/pmap.h>
1c79356b A	83
	84	#if MACH_PAGEMAP
	85	#include <vm/vm_external.h>
	86	#endif /* MACH_PAGEMAP */
	87
0b4e3aa0	88	typedef memory_object_control_t pager_request_t;
1c79356b A	89	#define PAGER_REQUEST_NULL ((pager_request_t) 0)
	90
	91	/*
	92	* Types defined:
	93	*
	94	* vm_object_t Virtual memory object.
1c79356b A	95	*/
	96
	97	typedef unsigned long long vm_object_size_t;
	98
	99
	100	struct vm_object {
	101	queue_head_t memq; /* Resident memory */
	102	decl_mutex_data(, Lock) /* Synchronization */
	103
	104	vm_object_size_t size; /* Object size (only valid
	105	* if internal)
	106	*/
	107	vm_object_size_t frozen_size; /* How much has been marked
	108	* copy-on-write (only
	109	* valid if copy_symmetric)
	110	*/
	111	int ref_count; /* Number of references */
	112	#if TASK_SWAPPER
	113	int res_count; /* Residency references (swap)*/
	114	#endif /* TASK_SWAPPER */
	115	unsigned int resident_page_count;
	116	/* number of resident pages */
	117
	118	struct vm_object copy; / Object that should receive
	119	* a copy of my changed pages,
	120	* for copy_delay, or just the
	121	* temporary object that
	122	* shadows this object, for
	123	* copy_call.
	124	*/
	125	struct vm_object shadow; / My shadow */
	126	vm_object_offset_t shadow_offset; /* Offset into shadow */
	127
0b4e3aa0	128	memory_object_t pager; /* Where to get data */
1c79356b A	129	vm_object_offset_t paging_offset; /* Offset into memory object */
	130	pager_request_t pager_request; /* Where data comes back */
	131
	132	memory_object_copy_strategy_t
	133	copy_strategy; /* How to handle data copy */
	134
	135	unsigned int absent_count; /* The number of pages that
	136	* have been requested but
	137	* not filled. That is, the
	138	* number of pages for which
	139	* the "absent" attribute is
	140	* asserted.
	141	*/
	142
	143	unsigned int paging_in_progress;
	144	/* The memory object ports are
	145	* being used (e.g., for pagein
	146	* or pageout) -- don't change
	147	* any of these fields (i.e.,
	148	* don't collapse, destroy or
	149	* terminate)
	150	*/
	151	unsigned int
	152	/* boolean_t array / all_wanted:11, / Bit array of "want to be
	153	* awakened" notations. See
	154	* VM_OBJECT_EVENT_* items
	155	* below */
	156	/* boolean_t / pager_created:1, / Has pager been created? */
	157	/* boolean_t / pager_initialized:1, / Are fields ready to use? */
	158	/* boolean_t / pager_ready:1, / Will pager take requests? */
	159
	160	/* boolean_t / pager_trusted:1,/ The pager for this object
	161	* is trusted. This is true for
	162	* all internal objects (backed
	163	* by the default pager)
	164	*/
	165	/* boolean_t / can_persist:1, / The kernel may keep the data
	166	* for this object (and rights
	167	* to the memory object) after
	168	* all address map references
	169	* are deallocated?
	170	*/
	171	/* boolean_t / internal:1, / Created by the kernel (and
	172	* therefore, managed by the
	173	* default memory manger)
	174	*/
	175	/* boolean_t / temporary:1, / Permanent objects may be
	176	* changed externally by the
	177	* memory manager, and changes
	178	* made in memory must be
	179	* reflected back to the memory
	180	* manager. Temporary objects
	181	* lack both of these
	182	* characteristics.
	183	*/
	184	/* boolean_t / private:1, / magic device_pager object,
	185	* holds private pages only */
	186	/* boolean_t / pageout:1, / pageout object. contains
	187	* private pages that refer to
	188	* a real memory object. */
	189	/* boolean_t / alive:1, / Not yet terminated */
	190
	191	/* boolean_t */ lock_in_progress:1,
	192	/* Is a multi-page lock
193	* request in progress?
194	*/
195	/* boolean_t */ lock_restart:1,
196	/* Should lock request in
197	* progress restart search?
198	*/
199	/* boolean_t / shadowed:1, / Shadow may exist */
200	/* boolean_t */ silent_overwrite:1,
201	/* Allow full page overwrite
202	* without data_request if
203	* page is absent */
204	/* boolean_t */ advisory_pageout:1,
205	/* Instead of sending page
206	* via OOL, just notify
207	* pager that the kernel
208	* wants to discard it, page
209	* remains in object */
210	/* boolean_t */ true_share:1,
211	/* This object is mapped
212	* in more than one place
213	* and hence cannot be
214	* coalesced */
215	/* boolean_t */ terminating:1,
216	/* Allows vm_object_lookup
217	* and vm_object_deallocate
218	* to special case their
219	* behavior when they are
220	* called as a result of
221	* page cleaning during
222	* object termination
223	*/
224	/* boolean_t / named:1, / An enforces an internal
225	* naming convention, by
226	* calling the right routines
227	* for allocation and
228	* destruction, UBC references
229	* against the vm_object are
230	* checked.
231	*/
232	/* boolean_t */ shadow_severed:1,
233	/* When a permanent object
234	* backing a COW goes away
235	* unexpectedly. This bit
236	* allows vm_fault to return
237	* an error rather than a
238	* zero filled page.
239	*/
0b4e3aa0	240	/* boolean_t */ phys_contiguous:1,
1c79356b A	241	/* Memory is wired and
	242	* guaranteed physically
	243	* contiguous. However
	244	* it is not device memory
	245	* and obeys normal virtual
	246	* memory rules w.r.t pmap
	247	* access bits.
	248	*/
0b4e3aa0 A	249	/* boolean_t */ nophyscache:1;
	250	/* When mapped at the
	251	* pmap level, don't allow
	252	* primary caching. (for
	253	* I/O)
	254	*/
1c79356b A	255
	256
	257
	258	queue_chain_t cached_list; /* Attachment point for the
	259	* list of objects cached as a
	260	* result of their can_persist
	261	* value
	262	*/
	263
	264	queue_head_t msr_q; /* memory object synchronise
	265	request queue */
	266
	267	vm_object_offset_t last_alloc; /* last allocation offset */
	268	vm_size_t cluster_size; /* size of paging cluster */
	269	#if MACH_PAGEMAP
	270	vm_external_map_t existence_map; /* bitmap of pages written to
	271	* backing storage */
	272	#endif /* MACH_PAGEMAP */
0b4e3aa0 A	273	int cow_hint; /* last page present in */
0b4e3aa0 A	274	/* shadow but not in object */
1c79356b A	275	#if MACH_ASSERT
	276	struct vm_object paging_object; / object which pages to be
	277	* swapped out are temporary
	278	* put in current object
	279	*/
	280	#endif
9bccf70c A	281	/* hold object lock when altering */
	282	unsigned int /* cache WIMG bits */
	283	wimg_bits:8, /* wimg plus some expansion*/
	284	not_in_use:24;
1c79356b A	285	#ifdef UBC_DEBUG
	286	queue_head_t uplq; /* List of outstanding upls */
	287	#endif /* UBC_DEBUG */
	288	};
	289
0b4e3aa0	290	__private_extern__
1c79356b A	291	vm_object_t kernel_object; /* the single kernel object */
1c79356b A	292
0b4e3aa0	293	__private_extern__
1c79356b A	294	int vm_object_absent_max; /* maximum number of absent pages
	295	at a time for each object */
	296
	297	# define VM_MSYNC_INITIALIZED 0
	298	# define VM_MSYNC_SYNCHRONIZING 1
	299	# define VM_MSYNC_DONE 2
	300
	301	struct msync_req {
	302	queue_chain_t msr_q; /* object request queue */
	303	queue_chain_t req_q; /* vm_msync request queue */
	304	unsigned int flag;
	305	vm_object_offset_t offset;
	306	vm_object_size_t length;
	307	vm_object_t object; /* back pointer */
	308	decl_mutex_data(, msync_req_lock) /* Lock for this structure */
	309	};
	310
	311	typedef struct msync_req *msync_req_t;
	312	#define MSYNC_REQ_NULL ((msync_req_t) 0)
	313
	314	/*
	315	* Macros to allocate and free msync_reqs
	316	*/
	317	#define msync_req_alloc(msr) \
	318	MACRO_BEGIN \
	319	(msr) = (msync_req_t)kalloc(sizeof(struct msync_req)); \
	320	mutex_init(&(msr)->msync_req_lock, ETAP_VM_MSYNC); \
	321	msr->flag = VM_MSYNC_INITIALIZED; \
	322	MACRO_END
	323
	324	#define msync_req_free(msr) \
	325	(kfree((vm_offset_t)(msr), sizeof(struct msync_req)))
	326
	327	#define msr_lock(msr) mutex_lock(&(msr)->msync_req_lock)
	328	#define msr_unlock(msr) mutex_unlock(&(msr)->msync_req_lock)
	329
	330	/*
	331	* Declare procedures that operate on VM objects.
	332	*/
	333
0b4e3aa0	334	__private_extern__ void vm_object_bootstrap(void);
1c79356b	335
0b4e3aa0	336	__private_extern__ void vm_object_init(void);
1c79356b	337
0b4e3aa0	338	__private_extern__ vm_object_t vm_object_allocate(
1c79356b A	339	vm_object_size_t size);
1c79356b A	340
0b4e3aa0 A	341	#if TASK_SWAPPER
	342
	343	__private_extern__ void vm_object_res_reference(
	344	vm_object_t object);
	345	__private_extern__ void vm_object_res_deallocate(
	346	vm_object_t object);
	347	#define VM_OBJ_RES_INCR(object) (object)->res_count++
	348	#define VM_OBJ_RES_DECR(object) (object)->res_count--
	349
	350	#else /* TASK_SWAPPER */
	351
	352	#define VM_OBJ_RES_INCR(object)
	353	#define VM_OBJ_RES_DECR(object)
	354	#define vm_object_res_reference(object)
	355	#define vm_object_res_deallocate(object)
	356
	357	#endif /* TASK_SWAPPER */
	358
	359	#define vm_object_reference_locked(object) \
	360	MACRO_BEGIN \
	361	vm_object_t RLObject = (object); \
	362	assert((RLObject)->ref_count > 0); \
	363	(RLObject)->ref_count++; \
	364	vm_object_res_reference(RLObject); \
	365	MACRO_END
	366
	367
1c79356b	368	#if MACH_ASSERT
0b4e3aa0 A	369
0b4e3aa0 A	370	__private_extern__ void vm_object_reference(
1c79356b	371	vm_object_t object);
0b4e3aa0	372
1c79356b	373	#else /* MACH_ASSERT */
0b4e3aa0	374
1c79356b A	375	#define vm_object_reference(object) \
1c79356b A	376	MACRO_BEGIN \
0b4e3aa0 A	377	vm_object_t RObject = (object); \
	378	if (RObject) { \
	379	vm_object_lock(RObject); \
	380	vm_object_reference_locked(RObject); \
	381	vm_object_unlock(RObject); \
1c79356b A	382	} \
1c79356b A	383	MACRO_END
0b4e3aa0	384
1c79356b A	385	#endif /* MACH_ASSERT */
1c79356b A	386
0b4e3aa0	387	__private_extern__ void vm_object_deallocate(
1c79356b A	388	vm_object_t object);
1c79356b A	389
0b4e3aa0 A	390	__private_extern__ kern_return_t vm_object_release_name(
	391	vm_object_t object,
	392	int flags);
	393
	394	__private_extern__ void vm_object_pmap_protect(
1c79356b A	395	vm_object_t object,
	396	vm_object_offset_t offset,
	397	vm_size_t size,
	398	pmap_t pmap,
	399	vm_offset_t pmap_start,
	400	vm_prot_t prot);
	401
0b4e3aa0	402	__private_extern__ void vm_object_page_remove(
1c79356b A	403	vm_object_t object,
	404	vm_object_offset_t start,
	405	vm_object_offset_t end);
	406
0b4e3aa0 A	407	__private_extern__ void vm_object_deactivate_pages(
	408	vm_object_t object,
	409	vm_object_offset_t offset,
	410	vm_object_size_t size,
	411	boolean_t kill_page);
	412
	413	__private_extern__ boolean_t vm_object_coalesce(
1c79356b A	414	vm_object_t prev_object,
	415	vm_object_t next_object,
	416	vm_object_offset_t prev_offset,
	417	vm_object_offset_t next_offset,
	418	vm_object_size_t prev_size,
	419	vm_object_size_t next_size);
	420
0b4e3aa0	421	__private_extern__ boolean_t vm_object_shadow(
1c79356b A	422	vm_object_t *object,
	423	vm_object_offset_t *offset,
	424	vm_object_size_t length);
	425
0b4e3aa0	426	__private_extern__ void vm_object_collapse(
1c79356b A	427	vm_object_t object);
1c79356b A	428
0b4e3aa0	429	__private_extern__ boolean_t vm_object_copy_quickly(
1c79356b A	430	vm_object_t *_object,
	431	vm_object_offset_t src_offset,
	432	vm_object_size_t size,
	433	boolean_t *_src_needs_copy,
	434	boolean_t *_dst_needs_copy);
	435
0b4e3aa0	436	__private_extern__ kern_return_t vm_object_copy_strategically(
1c79356b A	437	vm_object_t src_object,
	438	vm_object_offset_t src_offset,
	439	vm_object_size_t size,
	440	vm_object_t *dst_object,
	441	vm_object_offset_t *dst_offset,
	442	boolean_t *dst_needs_copy);
	443
0b4e3aa0	444	__private_extern__ kern_return_t vm_object_copy_slowly(
1c79356b A	445	vm_object_t src_object,
	446	vm_object_offset_t src_offset,
	447	vm_object_size_t size,
	448	int interruptible,
	449	vm_object_t *_result_object);
	450
0b4e3aa0 A	451	__private_extern__ vm_object_t vm_object_copy_delayed(
	452	vm_object_t src_object,
	453	vm_object_offset_t src_offset,
	454	vm_object_size_t size);
	455
1c79356b	456
1c79356b	457
0b4e3aa0 A	458	__private_extern__ kern_return_t vm_object_destroy(
	459	vm_object_t object,
	460	kern_return_t reason);
1c79356b	461
0b4e3aa0 A	462	__private_extern__ void vm_object_pager_create(
	463	vm_object_t object);
	464
	465	__private_extern__ void vm_object_page_map(
1c79356b A	466	vm_object_t object,
	467	vm_object_offset_t offset,
	468	vm_object_size_t size,
	469	vm_object_offset_t (*map_fn)
	470	(void *, vm_object_offset_t),
	471	void *map_fn_data);
	472
0b4e3aa0 A	473	__private_extern__ kern_return_t vm_object_upl_request(
	474	vm_object_t object,
	475	vm_object_offset_t offset,
	476	vm_size_t size,
	477	upl_t *upl,
	478	upl_page_info_t *page_info,
	479	unsigned int *count,
	480	int flags);
	481
	482	__private_extern__ boolean_t vm_object_sync(
	483	vm_object_t object,
	484	vm_object_offset_t offset,
	485	vm_size_t size,
	486	boolean_t should_flush,
	487	boolean_t should_return);
1c79356b	488
0b4e3aa0 A	489	__private_extern__ kern_return_t vm_object_update(
	490	vm_object_t object,
	491	vm_object_offset_t offset,
	492	vm_size_t size, /* should be 64 */
	493	memory_object_return_t should_return,
	494	int flags,
	495	vm_prot_t prot);
1c79356b	496
0b4e3aa0 A	497	__private_extern__ kern_return_t vm_object_lock_request(
	498	vm_object_t object,
	499	vm_object_offset_t offset,
	500	vm_object_size_t size,
	501	memory_object_return_t should_return,
	502	int flags,
	503	vm_prot_t prot);
1c79356b	504
1c79356b	505
1c79356b	506
0b4e3aa0 A	507	__private_extern__ vm_object_t vm_object_enter(
0b4e3aa0 A	508	memory_object_t pager,
1c79356b A	509	vm_object_size_t size,
	510	boolean_t internal,
	511	boolean_t init,
	512	boolean_t check_named);
	513
	514
1c79356b A	515	/*
	516	* Event waiting handling
	517	*/
	518
	519	#define VM_OBJECT_EVENT_INITIALIZED 0
	520	#define VM_OBJECT_EVENT_PAGER_READY 1
	521	#define VM_OBJECT_EVENT_PAGING_IN_PROGRESS 2
	522	#define VM_OBJECT_EVENT_ABSENT_COUNT 3
	523	#define VM_OBJECT_EVENT_LOCK_IN_PROGRESS 4
	524	#define VM_OBJECT_EVENT_UNCACHING 5
	525	#define VM_OBJECT_EVENT_COPY_CALL 6
	526	#define VM_OBJECT_EVENT_CACHING 7
	527
	528	#define vm_object_assert_wait(object, event, interruptible) \
9bccf70c A	529	(((object)->all_wanted \|= 1 << (event)), \
9bccf70c A	530	assert_wait((event_t)((vm_offset_t)(object)+(event)),(interruptible)))
1c79356b A	531
1c79356b A	532	#define vm_object_wait(object, event, interruptible) \
9bccf70c A	533	(vm_object_assert_wait((object),(event),(interruptible)), \
	534	vm_object_unlock(object), \
	535	thread_block(THREAD_CONTINUE_NULL)) \
	536
	537	#define thread_sleep_vm_object(object, event, interruptible) \
	538	thread_sleep_mutex((event_t)(event), &(object)->Lock, (interruptible))
	539
	540	#define vm_object_sleep(object, event, interruptible) \
	541	(((object)->all_wanted \|= 1 << (event)), \
	542	thread_sleep_vm_object((object), \
	543	((vm_offset_t)(object)+(event)), (interruptible)))
1c79356b A	544
	545	#define vm_object_wakeup(object, event) \
	546	MACRO_BEGIN \
	547	if ((object)->all_wanted & (1 << (event))) \
	548	thread_wakeup((event_t)((vm_offset_t)(object) + (event))); \
	549	(object)->all_wanted &= ~(1 << (event)); \
	550	MACRO_END
	551
	552	#define vm_object_set_wanted(object, event) \
	553	MACRO_BEGIN \
	554	((object)->all_wanted \|= (1 << (event))); \
	555	MACRO_END
	556
	557	#define vm_object_wanted(object, event) \
	558	((object)->all_wanted & (1 << (event)))
	559
	560	/*
	561	* Routines implemented as macros
	562	*/
	563
	564	#define vm_object_paging_begin(object) \
	565	MACRO_BEGIN \
	566	(object)->paging_in_progress++; \
	567	MACRO_END
	568
	569	#define vm_object_paging_end(object) \
	570	MACRO_BEGIN \
	571	assert((object)->paging_in_progress != 0); \
	572	if (--(object)->paging_in_progress == 0) { \
	573	vm_object_wakeup(object, \
	574	VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
	575	} \
	576	MACRO_END
	577
	578	#define vm_object_paging_wait(object, interruptible) \
	579	MACRO_BEGIN \
	580	while ((object)->paging_in_progress != 0) { \
9bccf70c A	581	wait_result_t _wr; \
	582	\
	583	_wr = vm_object_sleep((object), \
1c79356b A	584	VM_OBJECT_EVENT_PAGING_IN_PROGRESS, \
1c79356b A	585	(interruptible)); \
1c79356b	586	\
9bccf70c	587	/XXX if ((interruptible) && (_wr != THREAD_AWAKENED))/\
1c79356b A	588	/XXX break; / \
	589	} \
	590	MACRO_END
	591
	592	#define vm_object_absent_assert_wait(object, interruptible) \
	593	MACRO_BEGIN \
	594	vm_object_assert_wait( (object), \
	595	VM_OBJECT_EVENT_ABSENT_COUNT, \
	596	(interruptible)); \
	597	MACRO_END
	598
	599
	600	#define vm_object_absent_release(object) \
	601	MACRO_BEGIN \
	602	(object)->absent_count--; \
	603	vm_object_wakeup((object), \
	604	VM_OBJECT_EVENT_ABSENT_COUNT); \
	605	MACRO_END
	606
	607	/*
	608	* Object locking macros
	609	*/
	610
	611	#define vm_object_lock_init(object) mutex_init(&(object)->Lock, ETAP_VM_OBJ)
	612	#define vm_object_lock(object) mutex_lock(&(object)->Lock)
	613	#define vm_object_unlock(object) mutex_unlock(&(object)->Lock)
	614	#define vm_object_lock_try(object) mutex_try(&(object)->Lock)
	615
	616	#endif /* _VM_VM_OBJECT_H_ */