[apple/xnu.git] / osfmk / vm / bsd_vm.c

/*
 * Copyright (c) 2000-2001 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <sys/errno.h>
#include <kern/host.h>
#include <mach/mach_types.h>
#include <vm/vm_map.h>
#include <vm/vm_kern.h>
#include <vm/vm_pageout.h>
#include <mach/kern_return.h>
#include <mach/memory_object_types.h>
#include <mach/port.h>
#include <mach/policy.h>
#include <ipc/ipc_port.h>
#include <ipc/ipc_space.h>
#include <kern/thread.h>
#include <vm/memory_object.h>
#include <vm/vm_pageout.h>

#include <default_pager/default_pager_types.h>

/* BSD VM COMPONENT INTERFACES */
int
get_map_nentries(
	vm_map_t);

vm_offset_t
get_map_start(
	vm_map_t);

vm_offset_t
get_map_end(
	vm_map_t);

/*
 * 
 */
int
get_map_nentries(
	vm_map_t map)
{
	return(map->hdr.nentries);
}

/*
 * 
 */
vm_offset_t
get_map_start(
	vm_map_t map)
{
	return(vm_map_first_entry(map)->vme_start);
}

/*
 * 
 */
vm_offset_t
get_map_end(
	vm_map_t map)
{
	return(vm_map_last_entry(map)->vme_end);
}

/* 
 * BSD VNODE PAGER 
 */

/* until component support available */
int	vnode_pager_workaround;

typedef int vnode_port_t;

typedef struct vnode_pager {
	int 			*pager;		/* pager workaround pointer  */
	unsigned int		pager_ikot;	/* JMM: fake ip_kotype()     */
	unsigned int		ref_count;	/* reference count	     */
	memory_object_control_t control_handle;	/* mem object control handle */
	vnode_port_t 		vnode_handle;	/* vnode handle 	     */
} *vnode_pager_t;


ipc_port_t
trigger_name_to_port(
	mach_port_t);

void 
vnode_pager_bootstrap(
	void);

void
vnode_pager_alloc_map(
	void);

memory_object_t
vnode_pager_setup(
	vnode_port_t,
	memory_object_t);


kern_return_t
vnode_pager_init(
	memory_object_t, 
	memory_object_control_t, 
	vm_size_t);

kern_return_t
vnode_pager_get_object_size(
	memory_object_t,
	memory_object_offset_t *);

kern_return_t
vnode_pager_data_request( 
	memory_object_t, 
	memory_object_offset_t, 
	vm_size_t, 
	vm_prot_t);

kern_return_t
vnode_pager_data_return(
	memory_object_t,
	memory_object_offset_t,
	vm_size_t,
	boolean_t,
	boolean_t);

kern_return_t
vnode_pager_data_initialize(
	memory_object_t,
	memory_object_offset_t,
	vm_size_t);

void
vnode_pager_deallocate(
	memory_object_t);

kern_return_t
vnode_pager_terminate(
	memory_object_t);

kern_return_t
vnode_pager_cluster_read(
	vnode_pager_t, 
	vm_object_offset_t, 
	vm_size_t);

void
vnode_pager_cluster_write(
	vnode_pager_t,
	vm_object_offset_t,
	vm_size_t);


int    
vnode_pagein(
	vnode_port_t,
	upl_t,
	vm_offset_t,
	vm_object_offset_t, 
	int, 
	int,
	int *);
int    
vnode_pageout(
	vnode_port_t,
	upl_t,
	vm_offset_t,
	vm_object_offset_t, 
	int, 
	int,
	int *);

vm_object_offset_t
vnode_pager_get_filesize(
	vnode_port_t);

vnode_pager_t
vnode_object_create(
	vnode_port_t	vp);

vnode_pager_t
vnode_pager_lookup(
	memory_object_t);

void
vnode_pager_release_from_cache(
	int	*cnt);

zone_t	vnode_pager_zone;


#define	VNODE_PAGER_NULL	((vnode_pager_t) 0)

/* TODO: Should be set dynamically by vnode_pager_init() */
#define CLUSTER_SHIFT 	1

/* TODO: Should be set dynamically by vnode_pager_bootstrap() */
#define	MAX_VNODE		10000


#if DEBUG
int pagerdebug=0;

#define PAGER_ALL		0xffffffff
#define	PAGER_INIT		0x00000001
#define	PAGER_PAGEIN	0x00000002

#define PAGER_DEBUG(LEVEL, A) {if ((pagerdebug & LEVEL)==LEVEL){printf A;}}
#else
#define PAGER_DEBUG(LEVEL, A)
#endif

/*
 *	Routine:	macx_triggers
 *	Function:
 *		Syscall interface to set the call backs for low and
 *		high water marks.
 */
int
macx_triggers(
	int	hi_water,
	int	low_water,
	int	flags,
	mach_port_t	trigger_name)
{
	kern_return_t kr;
	memory_object_default_t	default_pager;
	ipc_port_t		trigger_port;

	default_pager = MEMORY_OBJECT_DEFAULT_NULL;
	kr = host_default_memory_manager(host_priv_self(), 
					&default_pager, 0);
	if(kr != KERN_SUCCESS) {
		return EINVAL;
	}
	if (flags & HI_WAT_ALERT) {
		trigger_port = trigger_name_to_port(trigger_name);
		if(trigger_port == NULL) {
			return EINVAL;
		}
		/* trigger_port is locked and active */
		ipc_port_make_send_locked(trigger_port); 
		/* now unlocked */
		default_pager_triggers(default_pager, 
				       hi_water, low_water,
				       HI_WAT_ALERT, trigger_port);
	}

	if (flags & LO_WAT_ALERT) {
		trigger_port = trigger_name_to_port(trigger_name);
		if(trigger_port == NULL) {
			return EINVAL;
		}
		/* trigger_port is locked and active */
		ipc_port_make_send_locked(trigger_port);
		/* and now its unlocked */
		default_pager_triggers(default_pager, 
				       hi_water, low_water,
				       LO_WAT_ALERT, trigger_port);
	}

	/*
	 * Set thread scheduling priority and policy for the current thread
	 * it is assumed for the time being that the thread setting the alert
	 * is the same one which will be servicing it.
	 *
	 * XXX This does not belong in the kernel XXX
	 */
	{
		thread_precedence_policy_data_t		pre;
		thread_extended_policy_data_t		ext;

		ext.timeshare = FALSE;
		pre.importance = INT32_MAX;

		thread_policy_set(current_act(),
							THREAD_EXTENDED_POLICY, (thread_policy_t)&ext,
									THREAD_EXTENDED_POLICY_COUNT);

		thread_policy_set(current_act(),
							THREAD_PRECEDENCE_POLICY, (thread_policy_t)&pre,
									THREAD_PRECEDENCE_POLICY_COUNT);
	}
 
	current_thread()->vm_privilege = TRUE;
}

/*
 *
 */
ipc_port_t
trigger_name_to_port(
	mach_port_t	trigger_name)
{
	ipc_port_t	trigger_port;
	ipc_space_t	space;

	if (trigger_name == 0)
		return (NULL);

	space  = current_space();
	if(ipc_port_translate_receive(space, (mach_port_name_t)trigger_name, 
						&trigger_port) != KERN_SUCCESS)
		return (NULL);
	return trigger_port;
}

/*
 *
 */
void
vnode_pager_bootstrap(void)
{
	register vm_size_t      size;

	size = (vm_size_t) sizeof(struct vnode_pager);
	vnode_pager_zone = zinit(size, (vm_size_t) MAX_VNODE*size,
				PAGE_SIZE, "vnode pager structures");
	return;
}

/*
 *
 */
memory_object_t
vnode_pager_setup(
	vnode_port_t	vp,
	memory_object_t	pager)
{
	vnode_pager_t	vnode_object;

	vnode_object = vnode_object_create(vp);
	if (vnode_object == VNODE_PAGER_NULL)
		panic("vnode_pager_setup: vnode_object_create() failed");
	return((memory_object_t)vnode_object);
}

/*
 *
 */
kern_return_t
vnode_pager_init(memory_object_t mem_obj, 
		memory_object_control_t control, 
		vm_size_t pg_size)
{
	vnode_pager_t   vnode_object;
	kern_return_t   kr;
	memory_object_attr_info_data_t  attributes;


	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %x, %x, %x\n", pager, pager_request, pg_size));

	if (control == MEMORY_OBJECT_CONTROL_NULL)
		return KERN_INVALID_ARGUMENT;

	vnode_object = vnode_pager_lookup(mem_obj);

	memory_object_control_reference(control);
	vnode_object->control_handle = control;

	attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
	/* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
	attributes.cluster_size = (1 << (PAGE_SHIFT));
	attributes.may_cache_object = TRUE;
	attributes.temporary = TRUE;

	kr = memory_object_change_attributes(
					control,
					MEMORY_OBJECT_ATTRIBUTE_INFO,
					(memory_object_info_t) &attributes,
					MEMORY_OBJECT_ATTR_INFO_COUNT);
	if (kr != KERN_SUCCESS)
		panic("vnode_pager_init: memory_object_change_attributes() failed");

	return(KERN_SUCCESS);
}

/*
 *
 */
kern_return_t
vnode_pager_data_return(
        memory_object_t		mem_obj,
        memory_object_offset_t	offset,
        vm_size_t		data_cnt,
        boolean_t		dirty,
        boolean_t		kernel_copy)  
{
	register vnode_pager_t	vnode_object;

	vnode_object = vnode_pager_lookup(mem_obj);

	vnode_pager_cluster_write(vnode_object, offset, data_cnt);

	return KERN_SUCCESS;
}

kern_return_t
vnode_pager_data_initialize(
        memory_object_t		mem_obj,
        memory_object_offset_t	offset,
        vm_size_t		data_cnt)
{
	return KERN_FAILURE;
}

kern_return_t
vnode_pager_data_unlock(
	memory_object_t		mem_obj,
	memory_object_offset_t	offset,
	vm_size_t		size,
	vm_prot_t		desired_access)
{
	return KERN_FAILURE;
}

kern_return_t
vnode_pager_get_object_size(
	memory_object_t		mem_obj,
	memory_object_offset_t	*length)
{
	vnode_pager_t	vnode_object;

	vnode_object = vnode_pager_lookup(mem_obj);

	*length = vnode_pager_get_filesize(vnode_object->vnode_handle);
	return KERN_SUCCESS;
}

/*
 *
 */
kern_return_t	
vnode_pager_data_request(
	memory_object_t		mem_obj,
	memory_object_offset_t	offset,
	vm_size_t		length,
	vm_prot_t		protection_required)
{
	register vnode_pager_t	vnode_object;

	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x\n", mem_obj, offset, length, protection_required));

	vnode_object = vnode_pager_lookup(mem_obj);

	PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, vnode_object %x\n", mem_obj, offset, length, protection_required, vnode_object));
		
	vnode_pager_cluster_read(vnode_object, offset, length);

	return KERN_SUCCESS;
}

/*
 *
 */
void
vnode_pager_reference(
	memory_object_t		mem_obj)
{	
	register vnode_pager_t	vnode_object;
	unsigned int		new_ref_count;

	vnode_object = vnode_pager_lookup(mem_obj);
	new_ref_count = hw_atomic_add(&vnode_object->ref_count, 1);
	assert(new_ref_count > 1);
}

/*
 *
 */
void
vnode_pager_deallocate(
	memory_object_t		mem_obj)
{
	register vnode_pager_t	vnode_object;

	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %x\n", mem_obj));

	vnode_object = vnode_pager_lookup(mem_obj);

	if (hw_atomic_sub(&vnode_object->ref_count, 1) == 0) {
		if (vnode_object->vnode_handle != (vnode_port_t) NULL) {
			vnode_pager_vrele(vnode_object->vnode_handle);
		}
		zfree(vnode_pager_zone, (vm_offset_t) vnode_object);
	}
	return;
}

/*
 *
 */
kern_return_t
vnode_pager_terminate(
	memory_object_t	mem_obj)
{
	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x\n", mem_obj));

	return(KERN_SUCCESS);
}

/*
 *
 */
kern_return_t
vnode_pager_synchronize(
	memory_object_t		mem_obj,
	memory_object_offset_t	offset,
	vm_size_t		length,
	vm_sync_t		sync_flags)
{
	register vnode_pager_t	vnode_object;

	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_synchronize: %x\n", mem_obj));

	vnode_object = vnode_pager_lookup(mem_obj);

	memory_object_synchronize_completed(vnode_object->control_handle, offset, length);

	return (KERN_SUCCESS);
}

/*
 *
 */
kern_return_t
vnode_pager_unmap(
	memory_object_t		mem_obj)
{
	register vnode_pager_t	vnode_object;

	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_unmap: %x\n", mem_obj));

	vnode_object = vnode_pager_lookup(mem_obj);

	ubc_unmap(vnode_object->vnode_handle);
	return KERN_SUCCESS;
}


/*
 *
 */
void
vnode_pager_cluster_write(
	vnode_pager_t		vnode_object,
	vm_object_offset_t	offset,
	vm_size_t		cnt)
{
	int		error = 0;
	int		local_error = 0;
	int		kret;
	int		size;

	if (cnt & PAGE_MASK) {
		panic("vs_cluster_write: cnt not a multiple of PAGE_SIZE");
	}
	size = (cnt < (PAGE_SIZE*32)) ? cnt : (PAGE_SIZE*32); /* effective min */
	
	while (cnt) {

		kret = vnode_pageout(vnode_object->vnode_handle, 
			(upl_t )NULL, (vm_offset_t)NULL, 
			offset, size, 0, &local_error);
/*
		if(kret == PAGER_ABSENT) {
			Need to work out the defs here, 1 corresponds to 
			PAGER_ABSENT defined in bsd/vm/vm_pager.h  However, 
			we should not be including that file here it is a 
			layering violation.
*/
		if(kret == 1) {
			int	uplflags;
			upl_t	upl = NULL;
			int	count = 0;
			kern_return_t	kr;

        		uplflags = (UPL_NO_SYNC | UPL_CLEAN_IN_PLACE |
					UPL_SET_INTERNAL | UPL_COPYOUT_FROM);
		        count = 0;
        		kr = memory_object_upl_request(
					vnode_object->control_handle, 
					offset, size, &upl, NULL, &count, uplflags);
			if(kr != KERN_SUCCESS) {
				panic("vnode_pager_cluster_write: upl request failed\n");
			}
			upl_abort(upl, 0);
			upl_deallocate(upl);

			error = 0;
			local_error = 0;
		}

		if (local_error != 0) {
			error = local_error;
			local_error = 0;
		}
		cnt -= size;
		offset += size;
		size = (cnt < (PAGE_SIZE*32)) ? cnt : (PAGE_SIZE*32); /* effective min */
	}
#if 0
	if (error != 0)
		return(KERN_FAILURE);
	
	return(KERN_SUCCESS);
#endif /* 0 */
}


/*
 *
 */
kern_return_t
vnode_pager_cluster_read(
	vnode_pager_t		vnode_object,
	vm_object_offset_t	offset,
	vm_size_t		cnt)
{
	int		error = 0;
	int		local_error = 0;
	int		kret;

	if(cnt & PAGE_MASK) {
		panic("vs_cluster_read: cnt not a multiple of PAGE_SIZE");
	}

	kret = vnode_pagein(vnode_object->vnode_handle, (upl_t)NULL, (vm_offset_t)NULL, offset, cnt, 0, &local_error);
/*
	if(kret == PAGER_ABSENT) {
	Need to work out the defs here, 1 corresponds to PAGER_ABSENT 
	defined in bsd/vm/vm_pager.h  However, we should not be including 
	that file here it is a layering violation.
*/
	if(kret == 1) {
			int	uplflags;
			upl_t	upl = NULL;
			int	count = 0;
			kern_return_t	kr;

        		uplflags = (UPL_NO_SYNC | 
				UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL);
		        count = 0;
        		kr = memory_object_upl_request(
				vnode_object->control_handle, offset, cnt,
				&upl, NULL, &count, uplflags);
			if(kr != KERN_SUCCESS) {
				panic("vnode_pager_cluster_read: upl request failed\n");
			}
			upl_abort(upl, 0);
			upl_deallocate(upl);

		error = 1;
	}

	if (error != 0)
		return(KERN_FAILURE);

	return(KERN_SUCCESS);

}


/*
 *
 */
void
vnode_pager_release_from_cache(
		int	*cnt)
{
	memory_object_free_from_cache(
			&realhost, &vnode_pager_workaround, cnt);
}

/*
 *
 */
vnode_pager_t
vnode_object_create(
        vnode_port_t   vp)
{
	register vnode_pager_t  vnode_object;

	vnode_object = (struct vnode_pager *) zalloc(vnode_pager_zone);
	if (vnode_object == VNODE_PAGER_NULL)
		return(VNODE_PAGER_NULL);

	/*
	 * The vm_map call takes both named entry ports and raw memory
	 * objects in the same parameter.  We need to make sure that
	 * vm_map does not see this object as a named entry port.  So,
	 * we reserve the second word in the object for a fake ip_kotype
	 * setting - that will tell vm_map to use it as a memory object.
	 */
	vnode_object->pager = &vnode_pager_workaround;
	vnode_object->pager_ikot = IKOT_MEMORY_OBJECT;
	vnode_object->ref_count = 1;
	vnode_object->control_handle = MEMORY_OBJECT_CONTROL_NULL;
	vnode_object->vnode_handle = vp;

	return(vnode_object);
}

/*
 *
 */
vnode_pager_t
vnode_pager_lookup(
	memory_object_t	 name)
{
	vnode_pager_t	vnode_object;

	vnode_object = (vnode_pager_t)name;
	assert(vnode_object->pager == &vnode_pager_workaround);
	return (vnode_object);
}
Commit	Line	Data
1c79356b	1	/*
0b4e3aa0	2	* Copyright (c) 2000-2001 Apple Computer, Inc. All rights reserved.
1c79356b A	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
43866e37	6	* Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
1c79356b	7	*
43866e37 A	8	* This file contains Original Code and/or Modifications of Original Code
	9	* as defined in and that are subject to the Apple Public Source License
	10	* Version 2.0 (the 'License'). You may not use this file except in
	11	* compliance with the License. Please obtain a copy of the License at
	12	* http://www.opensource.apple.com/apsl/ and read it before using this
	13	* file.
	14	*
	15	* The Original Code and all software distributed under the License are
	16	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
1c79356b A	17	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
1c79356b A	18	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
43866e37 A	19	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	20	* Please see the License for the specific language governing rights and
	21	* limitations under the License.
1c79356b A	22	*
	23	* @APPLE_LICENSE_HEADER_END@
	24	*/
	25
	26	#include <sys/errno.h>
	27	#include <kern/host.h>
	28	#include <mach/mach_types.h>
	29	#include <vm/vm_map.h>
	30	#include <vm/vm_kern.h>
	31	#include <vm/vm_pageout.h>
	32	#include <mach/kern_return.h>
	33	#include <mach/memory_object_types.h>
	34	#include <mach/port.h>
	35	#include <mach/policy.h>
	36	#include <ipc/ipc_port.h>
	37	#include <ipc/ipc_space.h>
	38	#include <kern/thread.h>
0b4e3aa0	39	#include <vm/memory_object.h>
1c79356b A	40	#include <vm/vm_pageout.h>
1c79356b A	41
0b4e3aa0	42	#include <default_pager/default_pager_types.h>
1c79356b A	43
	44	/* BSD VM COMPONENT INTERFACES */
	45	int
	46	get_map_nentries(
	47	vm_map_t);
	48
	49	vm_offset_t
	50	get_map_start(
	51	vm_map_t);
	52
	53	vm_offset_t
	54	get_map_end(
	55	vm_map_t);
	56
	57	/*
	58	*
	59	*/
	60	int
	61	get_map_nentries(
	62	vm_map_t map)
	63	{
	64	return(map->hdr.nentries);
	65	}
	66
	67	/*
	68	*
	69	*/
	70	vm_offset_t
	71	get_map_start(
	72	vm_map_t map)
	73	{
	74	return(vm_map_first_entry(map)->vme_start);
	75	}
	76
	77	/*
	78	*
	79	*/
	80	vm_offset_t
	81	get_map_end(
	82	vm_map_t map)
	83	{
	84	return(vm_map_last_entry(map)->vme_end);
	85	}
	86
	87	/*
	88	* BSD VNODE PAGER
	89	*/
	90
	91	/* until component support available */
	92	int vnode_pager_workaround;
	93
	94	typedef int vnode_port_t;
	95
	96	typedef struct vnode_pager {
0b4e3aa0 A	97	int pager; / pager workaround pointer */
	98	unsigned int pager_ikot; /* JMM: fake ip_kotype() */
	99	unsigned int ref_count; /* reference count */
	100	memory_object_control_t control_handle; /* mem object control handle */
	101	vnode_port_t vnode_handle; /* vnode handle */
1c79356b A	102	} *vnode_pager_t;
1c79356b A	103
1c79356b A	104
	105	ipc_port_t
	106	trigger_name_to_port(
	107	mach_port_t);
	108
	109	void
	110	vnode_pager_bootstrap(
	111	void);
	112
	113	void
	114	vnode_pager_alloc_map(
	115	void);
	116
0b4e3aa0	117	memory_object_t
1c79356b A	118	vnode_pager_setup(
1c79356b A	119	vnode_port_t,
0b4e3aa0	120	memory_object_t);
1c79356b	121
1c79356b A	122
	123	kern_return_t
	124	vnode_pager_init(
0b4e3aa0 A	125	memory_object_t,
0b4e3aa0 A	126	memory_object_control_t,
1c79356b A	127	vm_size_t);
1c79356b A	128
0b4e3aa0 A	129	kern_return_t
	130	vnode_pager_get_object_size(
	131	memory_object_t,
	132	memory_object_offset_t *);
	133
1c79356b A	134	kern_return_t
1c79356b A	135	vnode_pager_data_request(
0b4e3aa0 A	136	memory_object_t,
0b4e3aa0 A	137	memory_object_offset_t,
1c79356b A	138	vm_size_t,
	139	vm_prot_t);
	140
	141	kern_return_t
	142	vnode_pager_data_return(
0b4e3aa0 A	143	memory_object_t,
0b4e3aa0 A	144	memory_object_offset_t,
1c79356b A	145	vm_size_t,
	146	boolean_t,
	147	boolean_t);
	148
0b4e3aa0 A	149	kern_return_t
	150	vnode_pager_data_initialize(
	151	memory_object_t,
	152	memory_object_offset_t,
	153	vm_size_t);
	154
1c79356b	155	void
0b4e3aa0 A	156	vnode_pager_deallocate(
0b4e3aa0 A	157	memory_object_t);
1c79356b A	158
	159	kern_return_t
	160	vnode_pager_terminate(
0b4e3aa0	161	memory_object_t);
1c79356b A	162
	163	kern_return_t
	164	vnode_pager_cluster_read(
	165	vnode_pager_t,
	166	vm_object_offset_t,
	167	vm_size_t);
	168
	169	void
	170	vnode_pager_cluster_write(
	171	vnode_pager_t,
	172	vm_object_offset_t,
	173	vm_size_t);
	174
1c79356b A	175
	176	int
	177	vnode_pagein(
	178	vnode_port_t,
	179	upl_t,
	180	vm_offset_t,
	181	vm_object_offset_t,
	182	int,
	183	int,
	184	int *);
	185	int
	186	vnode_pageout(
	187	vnode_port_t,
	188	upl_t,
	189	vm_offset_t,
	190	vm_object_offset_t,
	191	int,
	192	int,
	193	int *);
	194
0b4e3aa0 A	195	vm_object_offset_t
	196	vnode_pager_get_filesize(
	197	vnode_port_t);
	198
1c79356b A	199	vnode_pager_t
	200	vnode_object_create(
	201	vnode_port_t vp);
	202
1c79356b	203	vnode_pager_t
0b4e3aa0 A	204	vnode_pager_lookup(
0b4e3aa0 A	205	memory_object_t);
1c79356b A	206
	207	void
	208	vnode_pager_release_from_cache(
	209	int *cnt);
	210
	211	zone_t vnode_pager_zone;
	212
	213
	214	#define VNODE_PAGER_NULL ((vnode_pager_t) 0)
	215
	216	/* TODO: Should be set dynamically by vnode_pager_init() */
	217	#define CLUSTER_SHIFT 1
	218
	219	/* TODO: Should be set dynamically by vnode_pager_bootstrap() */
	220	#define MAX_VNODE 10000
	221
	222
	223	#if DEBUG
	224	int pagerdebug=0;
	225
	226	#define PAGER_ALL 0xffffffff
	227	#define PAGER_INIT 0x00000001
	228	#define PAGER_PAGEIN 0x00000002
	229
	230	#define PAGER_DEBUG(LEVEL, A) {if ((pagerdebug & LEVEL)==LEVEL){printf A;}}
	231	#else
	232	#define PAGER_DEBUG(LEVEL, A)
	233	#endif
	234
	235	/*
	236	* Routine: macx_triggers
	237	* Function:
	238	* Syscall interface to set the call backs for low and
	239	* high water marks.
	240	*/
	241	int
	242	macx_triggers(
	243	int hi_water,
	244	int low_water,
	245	int flags,
	246	mach_port_t trigger_name)
	247	{
	248	kern_return_t kr;
0b4e3aa0	249	memory_object_default_t default_pager;
1c79356b A	250	ipc_port_t trigger_port;
1c79356b A	251
0b4e3aa0	252	default_pager = MEMORY_OBJECT_DEFAULT_NULL;
1c79356b	253	kr = host_default_memory_manager(host_priv_self(),
0b4e3aa0	254	&default_pager, 0);
1c79356b A	255	if(kr != KERN_SUCCESS) {
	256	return EINVAL;
	257	}
0b4e3aa0 A	258	if (flags & HI_WAT_ALERT) {
	259	trigger_port = trigger_name_to_port(trigger_name);
	260	if(trigger_port == NULL) {
	261	return EINVAL;
	262	}
	263	/* trigger_port is locked and active */
	264	ipc_port_make_send_locked(trigger_port);
	265	/* now unlocked */
	266	default_pager_triggers(default_pager,
	267	hi_water, low_water,
	268	HI_WAT_ALERT, trigger_port);
	269	}
	270
	271	if (flags & LO_WAT_ALERT) {
	272	trigger_port = trigger_name_to_port(trigger_name);
	273	if(trigger_port == NULL) {
	274	return EINVAL;
	275	}
	276	/* trigger_port is locked and active */
	277	ipc_port_make_send_locked(trigger_port);
	278	/* and now its unlocked */
	279	default_pager_triggers(default_pager,
	280	hi_water, low_water,
	281	LO_WAT_ALERT, trigger_port);
1c79356b	282	}
1c79356b A	283
	284	/*
	285	* Set thread scheduling priority and policy for the current thread
	286	* it is assumed for the time being that the thread setting the alert
55e303ae A	287	* is the same one which will be servicing it.
	288	*
	289	* XXX This does not belong in the kernel XXX
1c79356b A	290	*/
1c79356b A	291	{
55e303ae A	292	thread_precedence_policy_data_t pre;
	293	thread_extended_policy_data_t ext;
	294
	295	ext.timeshare = FALSE;
	296	pre.importance = INT32_MAX;
	297
	298	thread_policy_set(current_act(),
	299	THREAD_EXTENDED_POLICY, (thread_policy_t)&ext,
	300	THREAD_EXTENDED_POLICY_COUNT);
	301
	302	thread_policy_set(current_act(),
	303	THREAD_PRECEDENCE_POLICY, (thread_policy_t)&pre,
	304	THREAD_PRECEDENCE_POLICY_COUNT);
1c79356b A	305	}
	306
	307	current_thread()->vm_privilege = TRUE;
	308	}
	309
	310	/*
	311	*
	312	*/
	313	ipc_port_t
	314	trigger_name_to_port(
	315	mach_port_t trigger_name)
	316	{
	317	ipc_port_t trigger_port;
	318	ipc_space_t space;
	319
	320	if (trigger_name == 0)
	321	return (NULL);
	322
	323	space = current_space();
	324	if(ipc_port_translate_receive(space, (mach_port_name_t)trigger_name,
	325	&trigger_port) != KERN_SUCCESS)
	326	return (NULL);
	327	return trigger_port;
	328	}
	329
	330	/*
	331	*
	332	*/
	333	void
	334	vnode_pager_bootstrap(void)
	335	{
	336	register vm_size_t size;
	337
	338	size = (vm_size_t) sizeof(struct vnode_pager);
	339	vnode_pager_zone = zinit(size, (vm_size_t) MAX_VNODE*size,
	340	PAGE_SIZE, "vnode pager structures");
1c79356b A	341	return;
	342	}
	343
	344	/*
	345	*
	346	*/
0b4e3aa0	347	memory_object_t
1c79356b A	348	vnode_pager_setup(
1c79356b A	349	vnode_port_t vp,
0b4e3aa0	350	memory_object_t pager)
1c79356b A	351	{
1c79356b A	352	vnode_pager_t vnode_object;
1c79356b A	353
	354	vnode_object = vnode_object_create(vp);
	355	if (vnode_object == VNODE_PAGER_NULL)
	356	panic("vnode_pager_setup: vnode_object_create() failed");
0b4e3aa0	357	return((memory_object_t)vnode_object);
1c79356b A	358	}
	359
	360	/*
	361	*
	362	*/
	363	kern_return_t
0b4e3aa0 A	364	vnode_pager_init(memory_object_t mem_obj,
0b4e3aa0 A	365	memory_object_control_t control,
1c79356b A	366	vm_size_t pg_size)
	367	{
	368	vnode_pager_t vnode_object;
	369	kern_return_t kr;
	370	memory_object_attr_info_data_t attributes;
1c79356b A	371
	372
	373	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %x, %x, %x\n", pager, pager_request, pg_size));
	374
0b4e3aa0 A	375	if (control == MEMORY_OBJECT_CONTROL_NULL)
0b4e3aa0 A	376	return KERN_INVALID_ARGUMENT;
1c79356b	377
0b4e3aa0	378	vnode_object = vnode_pager_lookup(mem_obj);
1c79356b	379
0b4e3aa0 A	380	memory_object_control_reference(control);
0b4e3aa0 A	381	vnode_object->control_handle = control;
1c79356b A	382
	383	attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
	384	/* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
	385	attributes.cluster_size = (1 << (PAGE_SHIFT));
	386	attributes.may_cache_object = TRUE;
	387	attributes.temporary = TRUE;
	388
	389	kr = memory_object_change_attributes(
0b4e3aa0	390	control,
1c79356b A	391	MEMORY_OBJECT_ATTRIBUTE_INFO,
1c79356b A	392	(memory_object_info_t) &attributes,
0b4e3aa0	393	MEMORY_OBJECT_ATTR_INFO_COUNT);
1c79356b A	394	if (kr != KERN_SUCCESS)
	395	panic("vnode_pager_init: memory_object_change_attributes() failed");
	396
	397	return(KERN_SUCCESS);
	398	}
	399
	400	/*
	401	*
	402	*/
	403	kern_return_t
	404	vnode_pager_data_return(
0b4e3aa0 A	405	memory_object_t mem_obj,
0b4e3aa0 A	406	memory_object_offset_t offset,
1c79356b A	407	vm_size_t data_cnt,
	408	boolean_t dirty,
	409	boolean_t kernel_copy)
	410	{
	411	register vnode_pager_t vnode_object;
	412
0b4e3aa0	413	vnode_object = vnode_pager_lookup(mem_obj);
1c79356b A	414
	415	vnode_pager_cluster_write(vnode_object, offset, data_cnt);
	416
	417	return KERN_SUCCESS;
	418	}
	419
0b4e3aa0 A	420	kern_return_t
	421	vnode_pager_data_initialize(
	422	memory_object_t mem_obj,
	423	memory_object_offset_t offset,
	424	vm_size_t data_cnt)
	425	{
	426	return KERN_FAILURE;
	427	}
	428
	429	kern_return_t
	430	vnode_pager_data_unlock(
	431	memory_object_t mem_obj,
	432	memory_object_offset_t offset,
	433	vm_size_t size,
	434	vm_prot_t desired_access)
	435	{
	436	return KERN_FAILURE;
	437	}
	438
	439	kern_return_t
	440	vnode_pager_get_object_size(
	441	memory_object_t mem_obj,
	442	memory_object_offset_t *length)
	443	{
	444	vnode_pager_t vnode_object;
	445
	446	vnode_object = vnode_pager_lookup(mem_obj);
	447
	448	*length = vnode_pager_get_filesize(vnode_object->vnode_handle);
	449	return KERN_SUCCESS;
	450	}
	451
1c79356b A	452	/*
	453	*
	454	*/
	455	kern_return_t
	456	vnode_pager_data_request(
0b4e3aa0 A	457	memory_object_t mem_obj,
0b4e3aa0 A	458	memory_object_offset_t offset,
1c79356b A	459	vm_size_t length,
	460	vm_prot_t protection_required)
	461	{
	462	register vnode_pager_t vnode_object;
	463
0b4e3aa0	464	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x\n", mem_obj, offset, length, protection_required));
1c79356b	465
0b4e3aa0	466	vnode_object = vnode_pager_lookup(mem_obj);
1c79356b	467
0b4e3aa0	468	PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, vnode_object %x\n", mem_obj, offset, length, protection_required, vnode_object));
1c79356b	469
1c79356b A	470	vnode_pager_cluster_read(vnode_object, offset, length);
	471
	472	return KERN_SUCCESS;
	473	}
	474
	475	/*
	476	*
	477	*/
	478	void
0b4e3aa0 A	479	vnode_pager_reference(
	480	memory_object_t mem_obj)
	481	{
1c79356b	482	register vnode_pager_t vnode_object;
9bccf70c	483	unsigned int new_ref_count;
1c79356b	484
0b4e3aa0	485	vnode_object = vnode_pager_lookup(mem_obj);
9bccf70c A	486	new_ref_count = hw_atomic_add(&vnode_object->ref_count, 1);
9bccf70c A	487	assert(new_ref_count > 1);
0b4e3aa0	488	}
1c79356b	489
0b4e3aa0 A	490	/*
	491	*
	492	*/
	493	void
	494	vnode_pager_deallocate(
	495	memory_object_t mem_obj)
	496	{
	497	register vnode_pager_t vnode_object;
1c79356b	498
0b4e3aa0	499	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %x\n", mem_obj));
1c79356b	500
0b4e3aa0	501	vnode_object = vnode_pager_lookup(mem_obj);
1c79356b	502
9bccf70c	503	if (hw_atomic_sub(&vnode_object->ref_count, 1) == 0) {
0b4e3aa0 A	504	if (vnode_object->vnode_handle != (vnode_port_t) NULL) {
	505	vnode_pager_vrele(vnode_object->vnode_handle);
	506	}
	507	zfree(vnode_pager_zone, (vm_offset_t) vnode_object);
	508	}
1c79356b A	509	return;
	510	}
	511
	512	/*
	513	*
	514	*/
	515	kern_return_t
	516	vnode_pager_terminate(
0b4e3aa0	517	memory_object_t mem_obj)
1c79356b	518	{
0b4e3aa0	519	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x\n", mem_obj));
1c79356b	520
0b4e3aa0 A	521	return(KERN_SUCCESS);
0b4e3aa0 A	522	}
1c79356b	523
0b4e3aa0 A	524	/*
	525	*
	526	*/
	527	kern_return_t
	528	vnode_pager_synchronize(
	529	memory_object_t mem_obj,
	530	memory_object_offset_t offset,
	531	vm_size_t length,
	532	vm_sync_t sync_flags)
	533	{
	534	register vnode_pager_t vnode_object;
1c79356b	535
0b4e3aa0	536	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_synchronize: %x\n", mem_obj));
1c79356b	537
0b4e3aa0	538	vnode_object = vnode_pager_lookup(mem_obj);
1c79356b	539
0b4e3aa0	540	memory_object_synchronize_completed(vnode_object->control_handle, offset, length);
1c79356b	541
0b4e3aa0	542	return (KERN_SUCCESS);
1c79356b A	543	}
	544
	545	/*
	546	*
	547	*/
	548	kern_return_t
0b4e3aa0 A	549	vnode_pager_unmap(
0b4e3aa0 A	550	memory_object_t mem_obj)
1c79356b	551	{
0b4e3aa0	552	register vnode_pager_t vnode_object;
1c79356b	553
0b4e3aa0 A	554	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_unmap: %x\n", mem_obj));
	555
	556	vnode_object = vnode_pager_lookup(mem_obj);
	557
	558	ubc_unmap(vnode_object->vnode_handle);
	559	return KERN_SUCCESS;
1c79356b A	560	}
1c79356b A	561
0b4e3aa0	562
1c79356b A	563	/*
	564	*
	565	*/
	566	void
	567	vnode_pager_cluster_write(
	568	vnode_pager_t vnode_object,
	569	vm_object_offset_t offset,
	570	vm_size_t cnt)
	571	{
	572	int error = 0;
	573	int local_error = 0;
	574	int kret;
	575	int size;
	576
	577	if (cnt & PAGE_MASK) {
	578	panic("vs_cluster_write: cnt not a multiple of PAGE_SIZE");
	579	}
	580	size = (cnt < (PAGE_SIZE32)) ? cnt : (PAGE_SIZE32); /* effective min */
	581
	582	while (cnt) {
	583
0b4e3aa0 A	584	kret = vnode_pageout(vnode_object->vnode_handle,
	585	(upl_t )NULL, (vm_offset_t)NULL,
	586	offset, size, 0, &local_error);
	587	/*
	588	if(kret == PAGER_ABSENT) {
	589	Need to work out the defs here, 1 corresponds to
	590	PAGER_ABSENT defined in bsd/vm/vm_pager.h However,
	591	we should not be including that file here it is a
	592	layering violation.
	593	*/
	594	if(kret == 1) {
	595	int uplflags;
	596	upl_t upl = NULL;
	597	int count = 0;
	598	kern_return_t kr;
	599
	600	uplflags = (UPL_NO_SYNC \| UPL_CLEAN_IN_PLACE \|
	601	UPL_SET_INTERNAL \| UPL_COPYOUT_FROM);
	602	count = 0;
	603	kr = memory_object_upl_request(
	604	vnode_object->control_handle,
	605	offset, size, &upl, NULL, &count, uplflags);
	606	if(kr != KERN_SUCCESS) {
	607	panic("vnode_pager_cluster_write: upl request failed\n");
	608	}
	609	upl_abort(upl, 0);
	610	upl_deallocate(upl);
	611
	612	error = 0;
	613	local_error = 0;
	614	}
1c79356b A	615
	616	if (local_error != 0) {
	617	error = local_error;
	618	local_error = 0;
	619	}
	620	cnt -= size;
	621	offset += size;
0b4e3aa0	622	size = (cnt < (PAGE_SIZE32)) ? cnt : (PAGE_SIZE32); /* effective min */
1c79356b A	623	}
	624	#if 0
	625	if (error != 0)
	626	return(KERN_FAILURE);
	627
	628	return(KERN_SUCCESS);
	629	#endif /* 0 */
	630	}
	631
	632
	633	/*
	634	*
	635	*/
	636	kern_return_t
	637	vnode_pager_cluster_read(
	638	vnode_pager_t vnode_object,
	639	vm_object_offset_t offset,
	640	vm_size_t cnt)
	641	{
	642	int error = 0;
	643	int local_error = 0;
	644	int kret;
1c79356b A	645
	646	if(cnt & PAGE_MASK) {
	647	panic("vs_cluster_read: cnt not a multiple of PAGE_SIZE");
	648	}
	649
9bccf70c	650	kret = vnode_pagein(vnode_object->vnode_handle, (upl_t)NULL, (vm_offset_t)NULL, offset, cnt, 0, &local_error);
0b4e3aa0 A	651	/*
	652	if(kret == PAGER_ABSENT) {
	653	Need to work out the defs here, 1 corresponds to PAGER_ABSENT
	654	defined in bsd/vm/vm_pager.h However, we should not be including
	655	that file here it is a layering violation.
	656	*/
	657	if(kret == 1) {
	658	int uplflags;
	659	upl_t upl = NULL;
	660	int count = 0;
	661	kern_return_t kr;
	662
	663	uplflags = (UPL_NO_SYNC \|
	664	UPL_CLEAN_IN_PLACE \| UPL_SET_INTERNAL);
	665	count = 0;
	666	kr = memory_object_upl_request(
	667	vnode_object->control_handle, offset, cnt,
	668	&upl, NULL, &count, uplflags);
	669	if(kr != KERN_SUCCESS) {
	670	panic("vnode_pager_cluster_read: upl request failed\n");
	671	}
	672	upl_abort(upl, 0);
	673	upl_deallocate(upl);
	674
	675	error = 1;
1c79356b	676	}
0b4e3aa0	677
1c79356b A	678	if (error != 0)
	679	return(KERN_FAILURE);
	680
	681	return(KERN_SUCCESS);
	682
	683	}
	684
	685
	686	/*
	687	*
	688	*/
	689	void
	690	vnode_pager_release_from_cache(
	691	int *cnt)
	692	{
	693	memory_object_free_from_cache(
0b4e3aa0	694	&realhost, &vnode_pager_workaround, cnt);
1c79356b A	695	}
	696
	697	/*
	698	*
	699	*/
	700	vnode_pager_t
	701	vnode_object_create(
	702	vnode_port_t vp)
	703	{
	704	register vnode_pager_t vnode_object;
	705
	706	vnode_object = (struct vnode_pager *) zalloc(vnode_pager_zone);
	707	if (vnode_object == VNODE_PAGER_NULL)
	708	return(VNODE_PAGER_NULL);
1c79356b	709
1c79356b	710	/*
0b4e3aa0 A	711	* The vm_map call takes both named entry ports and raw memory
	712	* objects in the same parameter. We need to make sure that
	713	* vm_map does not see this object as a named entry port. So,
	714	* we reserve the second word in the object for a fake ip_kotype
	715	* setting - that will tell vm_map to use it as a memory object.
1c79356b	716	*/
0b4e3aa0 A	717	vnode_object->pager = &vnode_pager_workaround;
	718	vnode_object->pager_ikot = IKOT_MEMORY_OBJECT;
	719	vnode_object->ref_count = 1;
	720	vnode_object->control_handle = MEMORY_OBJECT_CONTROL_NULL;
	721	vnode_object->vnode_handle = vp;
	722
	723	return(vnode_object);
1c79356b A	724	}
	725
	726	/*
	727	*
	728	*/
	729	vnode_pager_t
0b4e3aa0 A	730	vnode_pager_lookup(
0b4e3aa0 A	731	memory_object_t name)
1c79356b	732	{
0b4e3aa0	733	vnode_pager_t vnode_object;
1c79356b	734
0b4e3aa0 A	735	vnode_object = (vnode_pager_t)name;
	736	assert(vnode_object->pager == &vnode_pager_workaround);
	737	return (vnode_object);
1c79356b	738	}
0b4e3aa0	739