[apple/xnu.git] / osfmk / vm / bsd_vm.c

/*
 * Copyright (c) 2000-2001 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <sys/errno.h>
#include <kern/host.h>
#include <mach/mach_types.h>
#include <vm/vm_map.h>
#include <vm/vm_kern.h>
#include <vm/vm_pageout.h>
#include <mach/kern_return.h>
#include <mach/memory_object_types.h>
#include <mach/port.h>
#include <mach/policy.h>
#include <ipc/ipc_port.h>
#include <ipc/ipc_space.h>
#include <kern/thread.h>
#include <vm/memory_object.h>
#include <vm/vm_pageout.h>

#include <default_pager/default_pager_types.h>

/* BSD VM COMPONENT INTERFACES */
int
get_map_nentries(
	vm_map_t);

vm_offset_t
get_map_start(
	vm_map_t);

vm_offset_t
get_map_end(
	vm_map_t);

/*
 * 
 */
int
get_map_nentries(
	vm_map_t map)
{
	return(map->hdr.nentries);
}

/*
 * 
 */
vm_offset_t
get_map_start(
	vm_map_t map)
{
	return(vm_map_first_entry(map)->vme_start);
}

/*
 * 
 */
vm_offset_t
get_map_end(
	vm_map_t map)
{
	return(vm_map_last_entry(map)->vme_end);
}

/* 
 * BSD VNODE PAGER 
 */

/* until component support available */
int	vnode_pager_workaround;

typedef int vnode_port_t;

typedef struct vnode_pager {
	int 			*pager;		/* pager workaround pointer  */
	unsigned int		pager_ikot;	/* JMM: fake ip_kotype()     */
	unsigned int		ref_count;	/* reference count	     */
	memory_object_control_t control_handle;	/* mem object control handle */
	vnode_port_t 		vnode_handle;	/* vnode handle 	     */
} *vnode_pager_t;


ipc_port_t
trigger_name_to_port(
	mach_port_t);

void 
vnode_pager_bootstrap(
	void);

void
vnode_pager_alloc_map(
	void);

memory_object_t
vnode_pager_setup(
	vnode_port_t,
	memory_object_t);


kern_return_t
vnode_pager_init(
	memory_object_t, 
	memory_object_control_t, 
	vm_size_t);

kern_return_t
vnode_pager_get_object_size(
	memory_object_t,
	memory_object_offset_t *);

kern_return_t
vnode_pager_data_request( 
	memory_object_t, 
	memory_object_offset_t, 
	vm_size_t, 
	vm_prot_t);

kern_return_t
vnode_pager_data_return(
	memory_object_t,
	memory_object_offset_t,
	vm_size_t,
	boolean_t,
	boolean_t);

kern_return_t
vnode_pager_data_initialize(
	memory_object_t,
	memory_object_offset_t,
	vm_size_t);

void
vnode_pager_deallocate(
	memory_object_t);

kern_return_t
vnode_pager_terminate(
	memory_object_t);

kern_return_t
vnode_pager_cluster_read(
	vnode_pager_t, 
	vm_object_offset_t, 
	vm_size_t);

void
vnode_pager_cluster_write(
	vnode_pager_t,
	vm_object_offset_t,
	vm_size_t);


int    
vnode_pagein(
	vnode_port_t,
	upl_t,
	vm_offset_t,
	vm_object_offset_t, 
	int, 
	int,
	int *);
int    
vnode_pageout(
	vnode_port_t,
	upl_t,
	vm_offset_t,
	vm_object_offset_t, 
	int, 
	int,
	int *);

vm_object_offset_t
vnode_pager_get_filesize(
	vnode_port_t);

vnode_pager_t
vnode_object_create(
	vnode_port_t	vp);

vnode_pager_t
vnode_pager_lookup(
	memory_object_t);

void
vnode_pager_release_from_cache(
	int	*cnt);

zone_t	vnode_pager_zone;


#define	VNODE_PAGER_NULL	((vnode_pager_t) 0)

/* TODO: Should be set dynamically by vnode_pager_init() */
#define CLUSTER_SHIFT 	1

/* TODO: Should be set dynamically by vnode_pager_bootstrap() */
#define	MAX_VNODE		10000


#if DEBUG
int pagerdebug=0;

#define PAGER_ALL		0xffffffff
#define	PAGER_INIT		0x00000001
#define	PAGER_PAGEIN	0x00000002

#define PAGER_DEBUG(LEVEL, A) {if ((pagerdebug & LEVEL)==LEVEL){printf A;}}
#else
#define PAGER_DEBUG(LEVEL, A)
#endif

/*
 *	Routine:	macx_triggers
 *	Function:
 *		Syscall interface to set the call backs for low and
 *		high water marks.
 */
int
macx_triggers(
	int	hi_water,
	int	low_water,
	int	flags,
	mach_port_t	trigger_name)
{
	kern_return_t kr;
	memory_object_default_t	default_pager;
	ipc_port_t		trigger_port;

	default_pager = MEMORY_OBJECT_DEFAULT_NULL;
	kr = host_default_memory_manager(host_priv_self(), 
					&default_pager, 0);
	if(kr != KERN_SUCCESS) {
		return EINVAL;
	}
	if (flags & HI_WAT_ALERT) {
		trigger_port = trigger_name_to_port(trigger_name);
		if(trigger_port == NULL) {
			return EINVAL;
		}
		/* trigger_port is locked and active */
		ipc_port_make_send_locked(trigger_port); 
		/* now unlocked */
		default_pager_triggers(default_pager, 
				       hi_water, low_water,
				       HI_WAT_ALERT, trigger_port);
	}

	if (flags & LO_WAT_ALERT) {
		trigger_port = trigger_name_to_port(trigger_name);
		if(trigger_port == NULL) {
			return EINVAL;
		}
		/* trigger_port is locked and active */
		ipc_port_make_send_locked(trigger_port);
		/* and now its unlocked */
		default_pager_triggers(default_pager, 
				       hi_water, low_water,
				       LO_WAT_ALERT, trigger_port);
	}

	/*
	 * Set thread scheduling priority and policy for the current thread
	 * it is assumed for the time being that the thread setting the alert
	 * is the same one which will be servicing it. 
	 */
	{
		struct policy_timeshare_base	 fifo_base;
		struct policy_timeshare_limit fifo_limit;
		policy_base_t 	base;
		processor_set_t	pset;
		policy_limit_t	limit;

		pset = (current_thread())->processor_set;  
		base = (policy_base_t) &fifo_base;
		limit = (policy_limit_t) &fifo_limit;
		fifo_limit.max_priority = fifo_base.base_priority = MAXPRI_STANDARD;
		thread_set_policy((current_thread())->top_act, pset, POLICY_FIFO, base, POLICY_TIMESHARE_BASE_COUNT, limit, POLICY_TIMESHARE_LIMIT_COUNT);
	}
 
	current_thread()->vm_privilege = TRUE;
}

/*
 *
 */
ipc_port_t
trigger_name_to_port(
	mach_port_t	trigger_name)
{
	ipc_port_t	trigger_port;
	ipc_space_t	space;

	if (trigger_name == 0)
		return (NULL);

	space  = current_space();
	if(ipc_port_translate_receive(space, (mach_port_name_t)trigger_name, 
						&trigger_port) != KERN_SUCCESS)
		return (NULL);
	return trigger_port;
}

/*
 *
 */
void
vnode_pager_bootstrap(void)
{
	register vm_size_t      size;

	size = (vm_size_t) sizeof(struct vnode_pager);
	vnode_pager_zone = zinit(size, (vm_size_t) MAX_VNODE*size,
				PAGE_SIZE, "vnode pager structures");
	return;
}

/*
 *
 */
memory_object_t
vnode_pager_setup(
	vnode_port_t	vp,
	memory_object_t	pager)
{
	vnode_pager_t	vnode_object;

	vnode_object = vnode_object_create(vp);
	if (vnode_object == VNODE_PAGER_NULL)
		panic("vnode_pager_setup: vnode_object_create() failed");
	return((memory_object_t)vnode_object);
}

/*
 *
 */
kern_return_t
vnode_pager_init(memory_object_t mem_obj, 
		memory_object_control_t control, 
		vm_size_t pg_size)
{
	vnode_pager_t   vnode_object;
	kern_return_t   kr;
	memory_object_attr_info_data_t  attributes;


	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %x, %x, %x\n", pager, pager_request, pg_size));

	if (control == MEMORY_OBJECT_CONTROL_NULL)
		return KERN_INVALID_ARGUMENT;

	vnode_object = vnode_pager_lookup(mem_obj);

	memory_object_control_reference(control);
	vnode_object->control_handle = control;

	attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
	/* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
	attributes.cluster_size = (1 << (PAGE_SHIFT));
	attributes.may_cache_object = TRUE;
	attributes.temporary = TRUE;

	kr = memory_object_change_attributes(
					control,
					MEMORY_OBJECT_ATTRIBUTE_INFO,
					(memory_object_info_t) &attributes,
					MEMORY_OBJECT_ATTR_INFO_COUNT);
	if (kr != KERN_SUCCESS)
		panic("vnode_pager_init: memory_object_change_attributes() failed");

	return(KERN_SUCCESS);
}

/*
 *
 */
kern_return_t
vnode_pager_data_return(
        memory_object_t		mem_obj,
        memory_object_offset_t	offset,
        vm_size_t		data_cnt,
        boolean_t		dirty,
        boolean_t		kernel_copy)  
{
	register vnode_pager_t	vnode_object;

	vnode_object = vnode_pager_lookup(mem_obj);

	vnode_pager_cluster_write(vnode_object, offset, data_cnt);

	return KERN_SUCCESS;
}

kern_return_t
vnode_pager_data_initialize(
        memory_object_t		mem_obj,
        memory_object_offset_t	offset,
        vm_size_t		data_cnt)
{
	return KERN_FAILURE;
}

kern_return_t
vnode_pager_data_unlock(
	memory_object_t		mem_obj,
	memory_object_offset_t	offset,
	vm_size_t		size,
	vm_prot_t		desired_access)
{
	return KERN_FAILURE;
}

kern_return_t
vnode_pager_get_object_size(
	memory_object_t		mem_obj,
	memory_object_offset_t	*length)
{
	vnode_pager_t	vnode_object;

	vnode_object = vnode_pager_lookup(mem_obj);

	*length = vnode_pager_get_filesize(vnode_object->vnode_handle);
	return KERN_SUCCESS;
}

/*
 *
 */
kern_return_t	
vnode_pager_data_request(
	memory_object_t		mem_obj,
	memory_object_offset_t	offset,
	vm_size_t		length,
	vm_prot_t		protection_required)
{
	register vnode_pager_t	vnode_object;

	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x\n", mem_obj, offset, length, protection_required));

	vnode_object = vnode_pager_lookup(mem_obj);

	PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, vnode_object %x\n", mem_obj, offset, length, protection_required, vnode_object));
		
	vnode_pager_cluster_read(vnode_object, offset, length);

	return KERN_SUCCESS;
}

/*
 *
 */
void
vnode_pager_reference(
	memory_object_t		mem_obj)
{	
	register vnode_pager_t	vnode_object;
	unsigned int		new_ref_count;

	vnode_object = vnode_pager_lookup(mem_obj);
	new_ref_count = hw_atomic_add(&vnode_object->ref_count, 1);
	assert(new_ref_count > 1);
}

/*
 *
 */
void
vnode_pager_deallocate(
	memory_object_t		mem_obj)
{
	register vnode_pager_t	vnode_object;

	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %x\n", mem_obj));

	vnode_object = vnode_pager_lookup(mem_obj);

	if (hw_atomic_sub(&vnode_object->ref_count, 1) == 0) {
		if (vnode_object->vnode_handle != (vnode_port_t) NULL) {
			vnode_pager_vrele(vnode_object->vnode_handle);
		}
		zfree(vnode_pager_zone, (vm_offset_t) vnode_object);
	}
	return;
}

/*
 *
 */
kern_return_t
vnode_pager_terminate(
	memory_object_t	mem_obj)
{
	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x\n", mem_obj));

	return(KERN_SUCCESS);
}

/*
 *
 */
kern_return_t
vnode_pager_synchronize(
	memory_object_t		mem_obj,
	memory_object_offset_t	offset,
	vm_size_t		length,
	vm_sync_t		sync_flags)
{
	register vnode_pager_t	vnode_object;

	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_synchronize: %x\n", mem_obj));

	vnode_object = vnode_pager_lookup(mem_obj);

	memory_object_synchronize_completed(vnode_object->control_handle, offset, length);

	return (KERN_SUCCESS);
}

/*
 *
 */
kern_return_t
vnode_pager_unmap(
	memory_object_t		mem_obj)
{
	register vnode_pager_t	vnode_object;

	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_unmap: %x\n", mem_obj));

	vnode_object = vnode_pager_lookup(mem_obj);

	ubc_unmap(vnode_object->vnode_handle);
	return KERN_SUCCESS;
}


/*
 *
 */
void
vnode_pager_cluster_write(
	vnode_pager_t		vnode_object,
	vm_object_offset_t	offset,
	vm_size_t		cnt)
{
	int		error = 0;
	int		local_error = 0;
	int		kret;
	int		size;

	if (cnt & PAGE_MASK) {
		panic("vs_cluster_write: cnt not a multiple of PAGE_SIZE");
	}
	size = (cnt < (PAGE_SIZE*32)) ? cnt : (PAGE_SIZE*32); /* effective min */
	
	while (cnt) {

		kret = vnode_pageout(vnode_object->vnode_handle, 
			(upl_t )NULL, (vm_offset_t)NULL, 
			offset, size, 0, &local_error);
/*
		if(kret == PAGER_ABSENT) {
			Need to work out the defs here, 1 corresponds to 
			PAGER_ABSENT defined in bsd/vm/vm_pager.h  However, 
			we should not be including that file here it is a 
			layering violation.
*/
		if(kret == 1) {
			int	uplflags;
			upl_t	upl = NULL;
			int	count = 0;
			kern_return_t	kr;

        		uplflags = (UPL_NO_SYNC | UPL_CLEAN_IN_PLACE |
					UPL_SET_INTERNAL | UPL_COPYOUT_FROM);
		        count = 0;
        		kr = memory_object_upl_request(
					vnode_object->control_handle, 
					offset, size, &upl, NULL, &count, uplflags);
			if(kr != KERN_SUCCESS) {
				panic("vnode_pager_cluster_write: upl request failed\n");
			}
			upl_abort(upl, 0);
			upl_deallocate(upl);

			error = 0;
			local_error = 0;
		}

		if (local_error != 0) {
			error = local_error;
			local_error = 0;
		}
		cnt -= size;
		offset += size;
		size = (cnt < (PAGE_SIZE*32)) ? cnt : (PAGE_SIZE*32); /* effective min */
	}
#if 0
	if (error != 0)
		return(KERN_FAILURE);
	
	return(KERN_SUCCESS);
#endif /* 0 */
}


/*
 *
 */
kern_return_t
vnode_pager_cluster_read(
	vnode_pager_t		vnode_object,
	vm_object_offset_t	offset,
	vm_size_t		cnt)
{
	int		error = 0;
	int		local_error = 0;
	int		kret;

	if(cnt & PAGE_MASK) {
		panic("vs_cluster_read: cnt not a multiple of PAGE_SIZE");
	}

	kret = vnode_pagein(vnode_object->vnode_handle, (upl_t)NULL, (vm_offset_t)NULL, offset, cnt, 0, &local_error);
/*
	if(kret == PAGER_ABSENT) {
	Need to work out the defs here, 1 corresponds to PAGER_ABSENT 
	defined in bsd/vm/vm_pager.h  However, we should not be including 
	that file here it is a layering violation.
*/
	if(kret == 1) {
			int	uplflags;
			upl_t	upl = NULL;
			int	count = 0;
			kern_return_t	kr;

        		uplflags = (UPL_NO_SYNC | 
				UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL);
		        count = 0;
        		kr = memory_object_upl_request(
				vnode_object->control_handle, offset, cnt,
				&upl, NULL, &count, uplflags);
			if(kr != KERN_SUCCESS) {
				panic("vnode_pager_cluster_read: upl request failed\n");
			}
			upl_abort(upl, 0);
			upl_deallocate(upl);

		error = 1;
	}

	if (error != 0)
		return(KERN_FAILURE);

	return(KERN_SUCCESS);

}


/*
 *
 */
void
vnode_pager_release_from_cache(
		int	*cnt)
{
	memory_object_free_from_cache(
			&realhost, &vnode_pager_workaround, cnt);
}

/*
 *
 */
vnode_pager_t
vnode_object_create(
        vnode_port_t   vp)
{
	register vnode_pager_t  vnode_object;

	vnode_object = (struct vnode_pager *) zalloc(vnode_pager_zone);
	if (vnode_object == VNODE_PAGER_NULL)
		return(VNODE_PAGER_NULL);

	/*
	 * The vm_map call takes both named entry ports and raw memory
	 * objects in the same parameter.  We need to make sure that
	 * vm_map does not see this object as a named entry port.  So,
	 * we reserve the second word in the object for a fake ip_kotype
	 * setting - that will tell vm_map to use it as a memory object.
	 */
	vnode_object->pager = &vnode_pager_workaround;
	vnode_object->pager_ikot = IKOT_MEMORY_OBJECT;
	vnode_object->ref_count = 1;
	vnode_object->control_handle = MEMORY_OBJECT_CONTROL_NULL;
	vnode_object->vnode_handle = vp;

	return(vnode_object);
}

/*
 *
 */
vnode_pager_t
vnode_pager_lookup(
	memory_object_t	 name)
{
	vnode_pager_t	vnode_object;

	vnode_object = (vnode_pager_t)name;
	assert(vnode_object->pager == &vnode_pager_workaround);
	return (vnode_object);
}
Commit	Line	Data
1c79356b	1	/*
0b4e3aa0	2	* Copyright (c) 2000-2001 Apple Computer, Inc. All rights reserved.
1c79356b A	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
43866e37	6	* Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
1c79356b	7	*
43866e37 A	8	* This file contains Original Code and/or Modifications of Original Code
	9	* as defined in and that are subject to the Apple Public Source License
	10	* Version 2.0 (the 'License'). You may not use this file except in
	11	* compliance with the License. Please obtain a copy of the License at
	12	* http://www.opensource.apple.com/apsl/ and read it before using this
	13	* file.
	14	*
	15	* The Original Code and all software distributed under the License are
	16	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
1c79356b A	17	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
1c79356b A	18	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
43866e37 A	19	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	20	* Please see the License for the specific language governing rights and
	21	* limitations under the License.
1c79356b A	22	*
	23	* @APPLE_LICENSE_HEADER_END@
	24	*/
	25
	26	#include <sys/errno.h>
	27	#include <kern/host.h>
	28	#include <mach/mach_types.h>
	29	#include <vm/vm_map.h>
	30	#include <vm/vm_kern.h>
	31	#include <vm/vm_pageout.h>
	32	#include <mach/kern_return.h>
	33	#include <mach/memory_object_types.h>
	34	#include <mach/port.h>
	35	#include <mach/policy.h>
	36	#include <ipc/ipc_port.h>
	37	#include <ipc/ipc_space.h>
	38	#include <kern/thread.h>
0b4e3aa0	39	#include <vm/memory_object.h>
1c79356b A	40	#include <vm/vm_pageout.h>
1c79356b A	41
0b4e3aa0	42	#include <default_pager/default_pager_types.h>
1c79356b A	43
	44	/* BSD VM COMPONENT INTERFACES */
	45	int
	46	get_map_nentries(
	47	vm_map_t);
	48
	49	vm_offset_t
	50	get_map_start(
	51	vm_map_t);
	52
	53	vm_offset_t
	54	get_map_end(
	55	vm_map_t);
	56
	57	/*
	58	*
	59	*/
	60	int
	61	get_map_nentries(
	62	vm_map_t map)
	63	{
	64	return(map->hdr.nentries);
	65	}
	66
	67	/*
	68	*
	69	*/
	70	vm_offset_t
	71	get_map_start(
	72	vm_map_t map)
	73	{
	74	return(vm_map_first_entry(map)->vme_start);
	75	}
	76
	77	/*
	78	*
	79	*/
	80	vm_offset_t
	81	get_map_end(
	82	vm_map_t map)
	83	{
	84	return(vm_map_last_entry(map)->vme_end);
	85	}
	86
	87	/*
	88	* BSD VNODE PAGER
	89	*/
	90
	91	/* until component support available */
	92	int vnode_pager_workaround;
	93
	94	typedef int vnode_port_t;
	95
	96	typedef struct vnode_pager {
0b4e3aa0 A	97	int pager; / pager workaround pointer */
	98	unsigned int pager_ikot; /* JMM: fake ip_kotype() */
	99	unsigned int ref_count; /* reference count */
	100	memory_object_control_t control_handle; /* mem object control handle */
	101	vnode_port_t vnode_handle; /* vnode handle */
1c79356b A	102	} *vnode_pager_t;
1c79356b A	103
1c79356b A	104
	105	ipc_port_t
	106	trigger_name_to_port(
	107	mach_port_t);
	108
	109	void
	110	vnode_pager_bootstrap(
	111	void);
	112
	113	void
	114	vnode_pager_alloc_map(
	115	void);
	116
0b4e3aa0	117	memory_object_t
1c79356b A	118	vnode_pager_setup(
1c79356b A	119	vnode_port_t,
0b4e3aa0	120	memory_object_t);
1c79356b	121
1c79356b A	122
	123	kern_return_t
	124	vnode_pager_init(
0b4e3aa0 A	125	memory_object_t,
0b4e3aa0 A	126	memory_object_control_t,
1c79356b A	127	vm_size_t);
1c79356b A	128
0b4e3aa0 A	129	kern_return_t
	130	vnode_pager_get_object_size(
	131	memory_object_t,
	132	memory_object_offset_t *);
	133
1c79356b A	134	kern_return_t
1c79356b A	135	vnode_pager_data_request(
0b4e3aa0 A	136	memory_object_t,
0b4e3aa0 A	137	memory_object_offset_t,
1c79356b A	138	vm_size_t,
	139	vm_prot_t);
	140
	141	kern_return_t
	142	vnode_pager_data_return(
0b4e3aa0 A	143	memory_object_t,
0b4e3aa0 A	144	memory_object_offset_t,
1c79356b A	145	vm_size_t,
	146	boolean_t,
	147	boolean_t);
	148
0b4e3aa0 A	149	kern_return_t
	150	vnode_pager_data_initialize(
	151	memory_object_t,
	152	memory_object_offset_t,
	153	vm_size_t);
	154
1c79356b	155	void
0b4e3aa0 A	156	vnode_pager_deallocate(
0b4e3aa0 A	157	memory_object_t);
1c79356b A	158
	159	kern_return_t
	160	vnode_pager_terminate(
0b4e3aa0	161	memory_object_t);
1c79356b A	162
	163	kern_return_t
	164	vnode_pager_cluster_read(
	165	vnode_pager_t,
	166	vm_object_offset_t,
	167	vm_size_t);
	168
	169	void
	170	vnode_pager_cluster_write(
	171	vnode_pager_t,
	172	vm_object_offset_t,
	173	vm_size_t);
	174
1c79356b A	175
	176	int
	177	vnode_pagein(
	178	vnode_port_t,
	179	upl_t,
	180	vm_offset_t,
	181	vm_object_offset_t,
	182	int,
	183	int,
	184	int *);
	185	int
	186	vnode_pageout(
	187	vnode_port_t,
	188	upl_t,
	189	vm_offset_t,
	190	vm_object_offset_t,
	191	int,
	192	int,
	193	int *);
	194
0b4e3aa0 A	195	vm_object_offset_t
	196	vnode_pager_get_filesize(
	197	vnode_port_t);
	198
1c79356b A	199	vnode_pager_t
	200	vnode_object_create(
	201	vnode_port_t vp);
	202
1c79356b	203	vnode_pager_t
0b4e3aa0 A	204	vnode_pager_lookup(
0b4e3aa0 A	205	memory_object_t);
1c79356b A	206
	207	void
	208	vnode_pager_release_from_cache(
	209	int *cnt);
	210
	211	zone_t vnode_pager_zone;
	212
	213
	214	#define VNODE_PAGER_NULL ((vnode_pager_t) 0)
	215
	216	/* TODO: Should be set dynamically by vnode_pager_init() */
	217	#define CLUSTER_SHIFT 1
	218
	219	/* TODO: Should be set dynamically by vnode_pager_bootstrap() */
	220	#define MAX_VNODE 10000
	221
	222
	223	#if DEBUG
	224	int pagerdebug=0;
	225
	226	#define PAGER_ALL 0xffffffff
	227	#define PAGER_INIT 0x00000001
	228	#define PAGER_PAGEIN 0x00000002
	229
	230	#define PAGER_DEBUG(LEVEL, A) {if ((pagerdebug & LEVEL)==LEVEL){printf A;}}
	231	#else
	232	#define PAGER_DEBUG(LEVEL, A)
	233	#endif
	234
	235	/*
	236	* Routine: macx_triggers
	237	* Function:
	238	* Syscall interface to set the call backs for low and
	239	* high water marks.
	240	*/
	241	int
	242	macx_triggers(
	243	int hi_water,
	244	int low_water,
	245	int flags,
	246	mach_port_t trigger_name)
	247	{
	248	kern_return_t kr;
0b4e3aa0	249	memory_object_default_t default_pager;
1c79356b A	250	ipc_port_t trigger_port;
1c79356b A	251
0b4e3aa0	252	default_pager = MEMORY_OBJECT_DEFAULT_NULL;
1c79356b	253	kr = host_default_memory_manager(host_priv_self(),
0b4e3aa0	254	&default_pager, 0);
1c79356b A	255	if(kr != KERN_SUCCESS) {
	256	return EINVAL;
	257	}
0b4e3aa0 A	258	if (flags & HI_WAT_ALERT) {
	259	trigger_port = trigger_name_to_port(trigger_name);
	260	if(trigger_port == NULL) {
	261	return EINVAL;
	262	}
	263	/* trigger_port is locked and active */
	264	ipc_port_make_send_locked(trigger_port);
	265	/* now unlocked */
	266	default_pager_triggers(default_pager,
	267	hi_water, low_water,
	268	HI_WAT_ALERT, trigger_port);
	269	}
	270
	271	if (flags & LO_WAT_ALERT) {
	272	trigger_port = trigger_name_to_port(trigger_name);
	273	if(trigger_port == NULL) {
	274	return EINVAL;
	275	}
	276	/* trigger_port is locked and active */
	277	ipc_port_make_send_locked(trigger_port);
	278	/* and now its unlocked */
	279	default_pager_triggers(default_pager,
	280	hi_water, low_water,
	281	LO_WAT_ALERT, trigger_port);
1c79356b	282	}
1c79356b A	283
	284	/*
	285	* Set thread scheduling priority and policy for the current thread
	286	* it is assumed for the time being that the thread setting the alert
	287	* is the same one which will be servicing it.
	288	*/
	289	{
	290	struct policy_timeshare_base fifo_base;
	291	struct policy_timeshare_limit fifo_limit;
	292	policy_base_t base;
	293	processor_set_t pset;
	294	policy_limit_t limit;
	295
	296	pset = (current_thread())->processor_set;
	297	base = (policy_base_t) &fifo_base;
	298	limit = (policy_limit_t) &fifo_limit;
	299	fifo_limit.max_priority = fifo_base.base_priority = MAXPRI_STANDARD;
	300	thread_set_policy((current_thread())->top_act, pset, POLICY_FIFO, base, POLICY_TIMESHARE_BASE_COUNT, limit, POLICY_TIMESHARE_LIMIT_COUNT);
	301	}
	302
	303	current_thread()->vm_privilege = TRUE;
	304	}
	305
	306	/*
	307	*
	308	*/
	309	ipc_port_t
	310	trigger_name_to_port(
	311	mach_port_t trigger_name)
	312	{
	313	ipc_port_t trigger_port;
	314	ipc_space_t space;
	315
	316	if (trigger_name == 0)
	317	return (NULL);
	318
	319	space = current_space();
	320	if(ipc_port_translate_receive(space, (mach_port_name_t)trigger_name,
	321	&trigger_port) != KERN_SUCCESS)
	322	return (NULL);
	323	return trigger_port;
	324	}
	325
	326	/*
	327	*
	328	*/
	329	void
	330	vnode_pager_bootstrap(void)
	331	{
	332	register vm_size_t size;
	333
	334	size = (vm_size_t) sizeof(struct vnode_pager);
	335	vnode_pager_zone = zinit(size, (vm_size_t) MAX_VNODE*size,
	336	PAGE_SIZE, "vnode pager structures");
1c79356b A	337	return;
	338	}
	339
	340	/*
	341	*
	342	*/
0b4e3aa0	343	memory_object_t
1c79356b A	344	vnode_pager_setup(
1c79356b A	345	vnode_port_t vp,
0b4e3aa0	346	memory_object_t pager)
1c79356b A	347	{
1c79356b A	348	vnode_pager_t vnode_object;
1c79356b A	349
	350	vnode_object = vnode_object_create(vp);
	351	if (vnode_object == VNODE_PAGER_NULL)
	352	panic("vnode_pager_setup: vnode_object_create() failed");
0b4e3aa0	353	return((memory_object_t)vnode_object);
1c79356b A	354	}
	355
	356	/*
	357	*
	358	*/
	359	kern_return_t
0b4e3aa0 A	360	vnode_pager_init(memory_object_t mem_obj,
0b4e3aa0 A	361	memory_object_control_t control,
1c79356b A	362	vm_size_t pg_size)
	363	{
	364	vnode_pager_t vnode_object;
	365	kern_return_t kr;
	366	memory_object_attr_info_data_t attributes;
1c79356b A	367
	368
	369	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %x, %x, %x\n", pager, pager_request, pg_size));
	370
0b4e3aa0 A	371	if (control == MEMORY_OBJECT_CONTROL_NULL)
0b4e3aa0 A	372	return KERN_INVALID_ARGUMENT;
1c79356b	373
0b4e3aa0	374	vnode_object = vnode_pager_lookup(mem_obj);
1c79356b	375
0b4e3aa0 A	376	memory_object_control_reference(control);
0b4e3aa0 A	377	vnode_object->control_handle = control;
1c79356b A	378
	379	attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
	380	/* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
	381	attributes.cluster_size = (1 << (PAGE_SHIFT));
	382	attributes.may_cache_object = TRUE;
	383	attributes.temporary = TRUE;
	384
	385	kr = memory_object_change_attributes(
0b4e3aa0	386	control,
1c79356b A	387	MEMORY_OBJECT_ATTRIBUTE_INFO,
1c79356b A	388	(memory_object_info_t) &attributes,
0b4e3aa0	389	MEMORY_OBJECT_ATTR_INFO_COUNT);
1c79356b A	390	if (kr != KERN_SUCCESS)
	391	panic("vnode_pager_init: memory_object_change_attributes() failed");
	392
	393	return(KERN_SUCCESS);
	394	}
	395
	396	/*
	397	*
	398	*/
	399	kern_return_t
	400	vnode_pager_data_return(
0b4e3aa0 A	401	memory_object_t mem_obj,
0b4e3aa0 A	402	memory_object_offset_t offset,
1c79356b A	403	vm_size_t data_cnt,
	404	boolean_t dirty,
	405	boolean_t kernel_copy)
	406	{
	407	register vnode_pager_t vnode_object;
	408
0b4e3aa0	409	vnode_object = vnode_pager_lookup(mem_obj);
1c79356b A	410
	411	vnode_pager_cluster_write(vnode_object, offset, data_cnt);
	412
	413	return KERN_SUCCESS;
	414	}
	415
0b4e3aa0 A	416	kern_return_t
	417	vnode_pager_data_initialize(
	418	memory_object_t mem_obj,
	419	memory_object_offset_t offset,
	420	vm_size_t data_cnt)
	421	{
	422	return KERN_FAILURE;
	423	}
	424
	425	kern_return_t
	426	vnode_pager_data_unlock(
	427	memory_object_t mem_obj,
	428	memory_object_offset_t offset,
	429	vm_size_t size,
	430	vm_prot_t desired_access)
	431	{
	432	return KERN_FAILURE;
	433	}
	434
	435	kern_return_t
	436	vnode_pager_get_object_size(
	437	memory_object_t mem_obj,
	438	memory_object_offset_t *length)
	439	{
	440	vnode_pager_t vnode_object;
	441
	442	vnode_object = vnode_pager_lookup(mem_obj);
	443
	444	*length = vnode_pager_get_filesize(vnode_object->vnode_handle);
	445	return KERN_SUCCESS;
	446	}
	447
1c79356b A	448	/*
	449	*
	450	*/
	451	kern_return_t
	452	vnode_pager_data_request(
0b4e3aa0 A	453	memory_object_t mem_obj,
0b4e3aa0 A	454	memory_object_offset_t offset,
1c79356b A	455	vm_size_t length,
	456	vm_prot_t protection_required)
	457	{
	458	register vnode_pager_t vnode_object;
	459
0b4e3aa0	460	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x\n", mem_obj, offset, length, protection_required));
1c79356b	461
0b4e3aa0	462	vnode_object = vnode_pager_lookup(mem_obj);
1c79356b	463
0b4e3aa0	464	PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, vnode_object %x\n", mem_obj, offset, length, protection_required, vnode_object));
1c79356b	465
1c79356b A	466	vnode_pager_cluster_read(vnode_object, offset, length);
	467
	468	return KERN_SUCCESS;
	469	}
	470
	471	/*
	472	*
	473	*/
	474	void
0b4e3aa0 A	475	vnode_pager_reference(
	476	memory_object_t mem_obj)
	477	{
1c79356b	478	register vnode_pager_t vnode_object;
9bccf70c	479	unsigned int new_ref_count;
1c79356b	480
0b4e3aa0	481	vnode_object = vnode_pager_lookup(mem_obj);
9bccf70c A	482	new_ref_count = hw_atomic_add(&vnode_object->ref_count, 1);
9bccf70c A	483	assert(new_ref_count > 1);
0b4e3aa0	484	}
1c79356b	485
0b4e3aa0 A	486	/*
	487	*
	488	*/
	489	void
	490	vnode_pager_deallocate(
	491	memory_object_t mem_obj)
	492	{
	493	register vnode_pager_t vnode_object;
1c79356b	494
0b4e3aa0	495	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %x\n", mem_obj));
1c79356b	496
0b4e3aa0	497	vnode_object = vnode_pager_lookup(mem_obj);
1c79356b	498
9bccf70c	499	if (hw_atomic_sub(&vnode_object->ref_count, 1) == 0) {
0b4e3aa0 A	500	if (vnode_object->vnode_handle != (vnode_port_t) NULL) {
	501	vnode_pager_vrele(vnode_object->vnode_handle);
	502	}
	503	zfree(vnode_pager_zone, (vm_offset_t) vnode_object);
	504	}
1c79356b A	505	return;
	506	}
	507
	508	/*
	509	*
	510	*/
	511	kern_return_t
	512	vnode_pager_terminate(
0b4e3aa0	513	memory_object_t mem_obj)
1c79356b	514	{
0b4e3aa0	515	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x\n", mem_obj));
1c79356b	516
0b4e3aa0 A	517	return(KERN_SUCCESS);
0b4e3aa0 A	518	}
1c79356b	519
0b4e3aa0 A	520	/*
	521	*
	522	*/
	523	kern_return_t
	524	vnode_pager_synchronize(
	525	memory_object_t mem_obj,
	526	memory_object_offset_t offset,
	527	vm_size_t length,
	528	vm_sync_t sync_flags)
	529	{
	530	register vnode_pager_t vnode_object;
1c79356b	531
0b4e3aa0	532	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_synchronize: %x\n", mem_obj));
1c79356b	533
0b4e3aa0	534	vnode_object = vnode_pager_lookup(mem_obj);
1c79356b	535
0b4e3aa0	536	memory_object_synchronize_completed(vnode_object->control_handle, offset, length);
1c79356b	537
0b4e3aa0	538	return (KERN_SUCCESS);
1c79356b A	539	}
	540
	541	/*
	542	*
	543	*/
	544	kern_return_t
0b4e3aa0 A	545	vnode_pager_unmap(
0b4e3aa0 A	546	memory_object_t mem_obj)
1c79356b	547	{
0b4e3aa0	548	register vnode_pager_t vnode_object;
1c79356b	549
0b4e3aa0 A	550	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_unmap: %x\n", mem_obj));
	551
	552	vnode_object = vnode_pager_lookup(mem_obj);
	553
	554	ubc_unmap(vnode_object->vnode_handle);
	555	return KERN_SUCCESS;
1c79356b A	556	}
1c79356b A	557
0b4e3aa0	558
1c79356b A	559	/*
	560	*
	561	*/
	562	void
	563	vnode_pager_cluster_write(
	564	vnode_pager_t vnode_object,
	565	vm_object_offset_t offset,
	566	vm_size_t cnt)
	567	{
	568	int error = 0;
	569	int local_error = 0;
	570	int kret;
	571	int size;
	572
	573	if (cnt & PAGE_MASK) {
	574	panic("vs_cluster_write: cnt not a multiple of PAGE_SIZE");
	575	}
	576	size = (cnt < (PAGE_SIZE32)) ? cnt : (PAGE_SIZE32); /* effective min */
	577
	578	while (cnt) {
	579
0b4e3aa0 A	580	kret = vnode_pageout(vnode_object->vnode_handle,
	581	(upl_t )NULL, (vm_offset_t)NULL,
	582	offset, size, 0, &local_error);
	583	/*
	584	if(kret == PAGER_ABSENT) {
	585	Need to work out the defs here, 1 corresponds to
	586	PAGER_ABSENT defined in bsd/vm/vm_pager.h However,
	587	we should not be including that file here it is a
	588	layering violation.
	589	*/
	590	if(kret == 1) {
	591	int uplflags;
	592	upl_t upl = NULL;
	593	int count = 0;
	594	kern_return_t kr;
	595
	596	uplflags = (UPL_NO_SYNC \| UPL_CLEAN_IN_PLACE \|
	597	UPL_SET_INTERNAL \| UPL_COPYOUT_FROM);
	598	count = 0;
	599	kr = memory_object_upl_request(
	600	vnode_object->control_handle,
	601	offset, size, &upl, NULL, &count, uplflags);
	602	if(kr != KERN_SUCCESS) {
	603	panic("vnode_pager_cluster_write: upl request failed\n");
	604	}
	605	upl_abort(upl, 0);
	606	upl_deallocate(upl);
	607
	608	error = 0;
	609	local_error = 0;
	610	}
1c79356b A	611
	612	if (local_error != 0) {
	613	error = local_error;
	614	local_error = 0;
	615	}
	616	cnt -= size;
	617	offset += size;
0b4e3aa0	618	size = (cnt < (PAGE_SIZE32)) ? cnt : (PAGE_SIZE32); /* effective min */
1c79356b A	619	}
	620	#if 0
	621	if (error != 0)
	622	return(KERN_FAILURE);
	623
	624	return(KERN_SUCCESS);
	625	#endif /* 0 */
	626	}
	627
	628
	629	/*
	630	*
	631	*/
	632	kern_return_t
	633	vnode_pager_cluster_read(
	634	vnode_pager_t vnode_object,
	635	vm_object_offset_t offset,
	636	vm_size_t cnt)
	637	{
	638	int error = 0;
	639	int local_error = 0;
	640	int kret;
1c79356b A	641
	642	if(cnt & PAGE_MASK) {
	643	panic("vs_cluster_read: cnt not a multiple of PAGE_SIZE");
	644	}
	645
9bccf70c	646	kret = vnode_pagein(vnode_object->vnode_handle, (upl_t)NULL, (vm_offset_t)NULL, offset, cnt, 0, &local_error);
0b4e3aa0 A	647	/*
	648	if(kret == PAGER_ABSENT) {
	649	Need to work out the defs here, 1 corresponds to PAGER_ABSENT
	650	defined in bsd/vm/vm_pager.h However, we should not be including
	651	that file here it is a layering violation.
	652	*/
	653	if(kret == 1) {
	654	int uplflags;
	655	upl_t upl = NULL;
	656	int count = 0;
	657	kern_return_t kr;
	658
	659	uplflags = (UPL_NO_SYNC \|
	660	UPL_CLEAN_IN_PLACE \| UPL_SET_INTERNAL);
	661	count = 0;
	662	kr = memory_object_upl_request(
	663	vnode_object->control_handle, offset, cnt,
	664	&upl, NULL, &count, uplflags);
	665	if(kr != KERN_SUCCESS) {
	666	panic("vnode_pager_cluster_read: upl request failed\n");
	667	}
	668	upl_abort(upl, 0);
	669	upl_deallocate(upl);
	670
	671	error = 1;
1c79356b	672	}
0b4e3aa0	673
1c79356b A	674	if (error != 0)
	675	return(KERN_FAILURE);
	676
	677	return(KERN_SUCCESS);
	678
	679	}
	680
	681
	682	/*
	683	*
	684	*/
	685	void
	686	vnode_pager_release_from_cache(
	687	int *cnt)
	688	{
	689	memory_object_free_from_cache(
0b4e3aa0	690	&realhost, &vnode_pager_workaround, cnt);
1c79356b A	691	}
	692
	693	/*
	694	*
	695	*/
	696	vnode_pager_t
	697	vnode_object_create(
	698	vnode_port_t vp)
	699	{
	700	register vnode_pager_t vnode_object;
	701
	702	vnode_object = (struct vnode_pager *) zalloc(vnode_pager_zone);
	703	if (vnode_object == VNODE_PAGER_NULL)
	704	return(VNODE_PAGER_NULL);
1c79356b	705
1c79356b	706	/*
0b4e3aa0 A	707	* The vm_map call takes both named entry ports and raw memory
	708	* objects in the same parameter. We need to make sure that
	709	* vm_map does not see this object as a named entry port. So,
	710	* we reserve the second word in the object for a fake ip_kotype
	711	* setting - that will tell vm_map to use it as a memory object.
1c79356b	712	*/
0b4e3aa0 A	713	vnode_object->pager = &vnode_pager_workaround;
	714	vnode_object->pager_ikot = IKOT_MEMORY_OBJECT;
	715	vnode_object->ref_count = 1;
	716	vnode_object->control_handle = MEMORY_OBJECT_CONTROL_NULL;
	717	vnode_object->vnode_handle = vp;
	718
	719	return(vnode_object);
1c79356b A	720	}
	721
	722	/*
	723	*
	724	*/
	725	vnode_pager_t
0b4e3aa0 A	726	vnode_pager_lookup(
0b4e3aa0 A	727	memory_object_t name)
1c79356b	728	{
0b4e3aa0	729	vnode_pager_t vnode_object;
1c79356b	730
0b4e3aa0 A	731	vnode_object = (vnode_pager_t)name;
	732	assert(vnode_object->pager == &vnode_pager_workaround);
	733	return (vnode_object);
1c79356b	734	}
0b4e3aa0	735