[apple/xnu.git] / osfmk / vm / bsd_vm.c

/*
 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * The contents of this file constitute Original Code as defined in and
 * are subject to the Apple Public Source License Version 1.1 (the
 * "License").  You may not use this file except in compliance with the
 * License.  Please obtain a copy of the License at
 * http://www.apple.com/publicsource and read it before using this file.
 * 
 * This Original Code and all software distributed under the License are
 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 * License for the specific language governing rights and limitations
 * under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <sys/errno.h>
#include <kern/host.h>
#include <mach/mach_types.h>
#include <vm/vm_map.h>
#include <vm/vm_kern.h>
#include <vm/vm_pageout.h>
#include <mach/kern_return.h>
#include <mach/memory_object_types.h>
#include <mach/port.h>
#include <mach/policy.h>
#include <ipc/ipc_port.h>
#include <ipc/ipc_space.h>
#include <kern/thread.h>
#include <vm/vm_pageout.h>


/* BSD VM COMPONENT INTERFACES */
int
get_map_nentries(
	vm_map_t);

vm_offset_t
get_map_start(
	vm_map_t);

vm_offset_t
get_map_end(
	vm_map_t);

/*
 * 
 */
int
get_map_nentries(
	vm_map_t map)
{
	return(map->hdr.nentries);
}

/*
 * 
 */
vm_offset_t
get_map_start(
	vm_map_t map)
{
	return(vm_map_first_entry(map)->vme_start);
}

/*
 * 
 */
vm_offset_t
get_map_end(
	vm_map_t map)
{
	return(vm_map_last_entry(map)->vme_end);
}

/* 
 * BSD VNODE PAGER 
 */

/* until component support available */
int	vnode_pager_workaround;

typedef int vnode_port_t;

typedef struct vnode_pager {
	ipc_port_t pager;			/* pager */
	ipc_port_t pager_handle;	/* pager handle */
	ipc_port_t vm_obj_handle;	/* memory object's control handle */
	vnode_port_t vnode_handle;	/* vnode handle */
} *vnode_pager_t;

typedef struct vnode_port_entry {
	queue_chain_t	links; 		/* queue links */
	ipc_port_t	name;			/* port name */
	vnode_pager_t	pager_rec;	/* pager record */
} *vnode_port_entry_t;


#define         VNODE_PORT_HASH_COUNT   127
#define         vnode_port_hash(name_port) \
                        (((int)(name_port) & 0xffffff) % VNODE_PORT_HASH_COUNT)

queue_head_t	vnode_port_hashtable[VNODE_PORT_HASH_COUNT];
zone_t			vnode_port_hash_zone;
decl_simple_lock_data(,vnode_port_hash_lock)


ipc_port_t
trigger_name_to_port(
	mach_port_t);

void 
vnode_pager_bootstrap(
	void);

void
vnode_pager_alloc_map(
	void);

ipc_port_t
vnode_pager_setup(
	vnode_port_t,
	ipc_port_t);

ipc_port_t
vnode_pager_lookup(
	vnode_port_t,
	ipc_port_t);

kern_return_t
vnode_pager_init(
	ipc_port_t, 
	ipc_port_t, 
	vm_size_t);

kern_return_t
vnode_pager_data_request( 
	ipc_port_t, 
	ipc_port_t,
	vm_object_offset_t, 
	vm_size_t, 
	vm_prot_t);

kern_return_t
vnode_pager_data_return(
	ipc_port_t,
	ipc_port_t,
	vm_object_offset_t,
	pointer_t,
	vm_size_t,
	boolean_t,
	boolean_t);

void
vnode_pager_no_senders(
	ipc_port_t,
	mach_port_mscount_t);

kern_return_t
vnode_pager_terminate(
	ipc_port_t,
	ipc_port_t);

kern_return_t
vnode_pager_cluster_read(
	vnode_pager_t, 
	vm_object_offset_t, 
	vm_size_t);

void
vnode_pager_cluster_write(
	vnode_pager_t,
	vm_object_offset_t,
	vm_size_t);

kern_return_t
memory_object_change_attributes(
        vm_object_t,
        memory_object_flavor_t,
        memory_object_info_t,
        mach_msg_type_number_t,
        ipc_port_t,
        mach_msg_type_name_t);

int    
vnode_pagein(
	vnode_port_t,
	upl_t,
	vm_offset_t,
	vm_object_offset_t, 
	int, 
	int,
	int *);
int    
vnode_pageout(
	vnode_port_t,
	upl_t,
	vm_offset_t,
	vm_object_offset_t, 
	int, 
	int,
	int *);

vnode_pager_t
vnode_object_create(
	vnode_port_t	vp);

void
vnode_port_hash_init(void);

void
vnode_port_hash_insert(
	ipc_port_t,   
	vnode_pager_t);

vnode_pager_t
vnode_port_hash_lookup(
	ipc_port_t);

void
vnode_port_hash_delete(
	ipc_port_t);

void
vnode_pager_release_from_cache(
	int	*cnt);

zone_t	vnode_pager_zone;


#define	VNODE_PAGER_NULL	((vnode_pager_t) 0)

/* TODO: Should be set dynamically by vnode_pager_init() */
#define CLUSTER_SHIFT 	1

/* TODO: Should be set dynamically by vnode_pager_bootstrap() */
#define	MAX_VNODE		10000


#if DEBUG
int pagerdebug=0;

#define PAGER_ALL		0xffffffff
#define	PAGER_INIT		0x00000001
#define	PAGER_PAGEIN	0x00000002

#define PAGER_DEBUG(LEVEL, A) {if ((pagerdebug & LEVEL)==LEVEL){printf A;}}
#else
#define PAGER_DEBUG(LEVEL, A)
#endif

/*
 *	Routine:	macx_triggers
 *	Function:
 *		Syscall interface to set the call backs for low and
 *		high water marks.
 */
int
macx_triggers(
	int	hi_water,
	int	low_water,
	int	flags,
	mach_port_t	trigger_name)
{
	kern_return_t kr;
	ipc_port_t		default_pager_port = MACH_PORT_NULL;
	ipc_port_t		trigger_port;

	kr = host_default_memory_manager(host_priv_self(), 
					&default_pager_port, 0);
	if(kr != KERN_SUCCESS) {
		return EINVAL;
	}
	trigger_port = trigger_name_to_port(trigger_name);
	if(trigger_port == NULL) {
		return EINVAL;
	}
        /* trigger_port is locked and active */
        ip_unlock(trigger_port);
	default_pager_triggers(default_pager_port, 
			hi_water, low_water, flags, trigger_port);
	ipc_port_make_send(trigger_port); 

	/*
	 * Set thread scheduling priority and policy for the current thread
	 * it is assumed for the time being that the thread setting the alert
	 * is the same one which will be servicing it. 
	 */
	{
		struct policy_timeshare_base	 fifo_base;
		struct policy_timeshare_limit fifo_limit;
		policy_base_t 	base;
		processor_set_t	pset;
		policy_limit_t	limit;

		pset = (current_thread())->processor_set;  
		base = (policy_base_t) &fifo_base;
		limit = (policy_limit_t) &fifo_limit;
		fifo_limit.max_priority = fifo_base.base_priority = MAXPRI_STANDARD;
		thread_set_policy((current_thread())->top_act, pset, POLICY_FIFO, base, POLICY_TIMESHARE_BASE_COUNT, limit, POLICY_TIMESHARE_LIMIT_COUNT);
	}
 
	current_thread()->vm_privilege = TRUE;
}

/*
 *
 */
ipc_port_t
trigger_name_to_port(
	mach_port_t	trigger_name)
{
	ipc_port_t	trigger_port;
	ipc_space_t	space;

	if (trigger_name == 0)
		return (NULL);

	space  = current_space();
	if(ipc_port_translate_receive(space, (mach_port_name_t)trigger_name, 
						&trigger_port) != KERN_SUCCESS)
		return (NULL);
	return trigger_port;
}

/*
 *
 */
void
vnode_pager_bootstrap(void)
{
	register vm_size_t      size;

	size = (vm_size_t) sizeof(struct vnode_pager);
	vnode_pager_zone = zinit(size, (vm_size_t) MAX_VNODE*size,
				PAGE_SIZE, "vnode pager structures");
	vnode_port_hash_init();

	return;
}

/*
 *
 */
ipc_port_t
vnode_pager_setup(
	vnode_port_t	vp,
	ipc_port_t	pager)
{
	vnode_pager_t	vnode_object;
	kern_return_t	kr;
	ipc_port_t	previous;

	if (pager &&
	    (vnode_object = vnode_port_hash_lookup(pager))) {
		if (vnode_object->vnode_handle == vp) 
			return(pager);
	}

	vnode_object = vnode_object_create(vp);
	if (vnode_object == VNODE_PAGER_NULL)
		panic("vnode_pager_setup: vnode_object_create() failed");

	vnode_object->pager = ipc_port_alloc_kernel();
	assert (vnode_object->pager != IP_NULL);
	pager_mux_hash_insert(vnode_object->pager, 
			(rpc_subsystem_t)&vnode_pager_workaround);

	vnode_object->pager_handle = ipc_port_make_send(vnode_object->pager);

	vnode_port_hash_insert(vnode_object->pager_handle, vnode_object);

	ipc_port_make_sonce(vnode_object->pager);
	ip_lock(vnode_object->pager);	/* unlocked in nsrequest below */
	ipc_port_nsrequest(vnode_object->pager, 1, vnode_object->pager, &previous);

	PAGER_DEBUG(PAGER_INIT, ("vnode_pager_setup: vp %x pager %x vnode_pager %x\n", vp, vnode_object->pager_handle, vnode_object));

	ubc_setpager( vp, vnode_object->pager_handle);
	return(vnode_object->pager_handle);
}

/*
 *
 */
ipc_port_t
vnode_pager_lookup(
	vnode_port_t    vp,
	ipc_port_t	pager)
{
	vnode_pager_t	vnode_object;
	kern_return_t	kr;

	if (pager &&
	    (vnode_object = vnode_port_hash_lookup(pager))) {
		if (vnode_object->vnode_handle == vp) 
			return(vnode_object->vm_obj_handle);
		else 
			return NULL;
	}
	else 
		return NULL;
}

/*
 *
 */
kern_return_t
vnode_pager_init(ipc_port_t pager, 
		ipc_port_t pager_request, 
		vm_size_t pg_size)
{
	vnode_pager_t   vnode_object;
	kern_return_t   kr;
	memory_object_attr_info_data_t  attributes;
	vm_object_t	vm_object;


	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %x, %x, %x\n", pager, pager_request, pg_size));

	vnode_object = vnode_port_hash_lookup(pager);
	if (vnode_object == VNODE_PAGER_NULL)
		panic("vnode_pager_init: lookup failed");

	vnode_object->vm_obj_handle = pager_request;

	vm_object = vm_object_lookup(pager_request);

	if (vm_object == VM_OBJECT_NULL)
		panic("vnode_pager_init: vm_object_lookup() failed");

	attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
	/* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
	attributes.cluster_size = (1 << (PAGE_SHIFT));
	attributes.may_cache_object = TRUE;
	attributes.temporary = TRUE;

	kr = memory_object_change_attributes(
					vm_object,
					MEMORY_OBJECT_ATTRIBUTE_INFO,
					(memory_object_info_t) &attributes,
					MEMORY_OBJECT_ATTR_INFO_COUNT,
					MACH_PORT_NULL, 0);
	if (kr != KERN_SUCCESS)
		panic("vnode_pager_init: memory_object_change_attributes() failed");

	return(KERN_SUCCESS);
}

/*
 *
 */
kern_return_t
vnode_pager_data_return(
        ipc_port_t		mem_obj,
        ipc_port_t		control_port,
        vm_object_offset_t	offset,
        pointer_t		addr,
        vm_size_t		data_cnt,
        boolean_t		dirty,
        boolean_t		kernel_copy)  
{
	register vnode_pager_t	vnode_object;

	vnode_object = vnode_port_hash_lookup(mem_obj);
	if (vnode_object == VNODE_PAGER_NULL)
		panic("vnode_pager_data_return: lookup failed");

	vnode_pager_cluster_write(vnode_object, offset, data_cnt);

	return KERN_SUCCESS;
}

/*
 *
 */
kern_return_t	
vnode_pager_data_request(
	ipc_port_t		mem_obj,
	ipc_port_t		mem_obj_control,
	vm_object_offset_t	offset,
	vm_size_t		length,
	vm_prot_t		protection_required)
{
	register vnode_pager_t	vnode_object;

	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x, %x\n", mem_obj, mem_obj_control, offset, length, protection_required));

	vnode_object = vnode_port_hash_lookup(mem_obj);

	PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, %x, vnode_object %x\n", mem_obj, mem_obj_control, offset, length, protection_required, vnode_object));
		
	if (vnode_object == VNODE_PAGER_NULL)
		panic("vnode_pager_data_request: lookup failed");

	vnode_pager_cluster_read(vnode_object, offset, length);

	return KERN_SUCCESS;
}

/*
 *
 */
void
vnode_pager_no_senders(
	ipc_port_t	mem_obj,
	mach_port_mscount_t mscount)
{
	register vnode_pager_t	vnode_object;

	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_nosenders: %x, %x\n", mem_obj, mscount));

	vnode_object = vnode_port_hash_lookup(mem_obj);
	if (vnode_object == VNODE_PAGER_NULL)
		panic("vnode_pager_no_senders: lookup failed");

	assert(vnode_object->pager_handle == mem_obj);

	pager_mux_hash_delete((ipc_port_t) vnode_object->pager_handle);
	ipc_port_dealloc_kernel(vnode_object->pager);
	vnode_port_hash_delete(vnode_object->pager_handle);
	if (vnode_object->vnode_handle != (vnode_port_t) NULL) {
		vnode_pager_vrele(vnode_object->vnode_handle);
	}
	zfree(vnode_pager_zone, (vm_offset_t) vnode_object);

	return;
}

/*
 *
 */
kern_return_t
vnode_pager_terminate(
	ipc_port_t	mem_obj,
	ipc_port_t	mem_obj_control)
{
	register vnode_pager_t	vnode_object;

	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x, %x\n", mem_obj, mem_obj_control));

	vnode_object = vnode_port_hash_lookup(mem_obj);
	if (vnode_object == VNODE_PAGER_NULL)
		panic("vnode_pager_terminate: lookup failed");

	assert(vnode_object->pager_handle == mem_obj);

	/* release extra send right created by the fact that the caller */
	/* of vnode_pager_setup does not establish a mapping between a  */
	/* cache object and the mem_obj (AMO).  When a subsequent vm_map */
	/* is done, vm_map will bump the send right count */
	ipc_port_release_send(mem_obj);

	/* release a send right because terminate is called directly and */
	/* not through IPC, the right won't disappear quietly */
	ipc_port_release_send(mem_obj);

	ipc_port_dealloc_kernel(mem_obj_control);

	return(KERN_SUCCESS);
}

/*
 *
 */
kern_return_t
vnode_pager_synchronize(
	ipc_port_t		pager,
	ipc_port_t		pager_request,
	vm_object_offset_t	offset,
	vm_offset_t		length,
	vm_sync_t		sync_flags)
{
	memory_object_synchronize_completed(vm_object_lookup(pager_request), offset, length);

	return (KERN_SUCCESS);
}

/*
 *
 */
void
vnode_pager_cluster_write(
	vnode_pager_t		vnode_object,
	vm_object_offset_t	offset,
	vm_size_t		cnt)
{
	int		error = 0;
	int		local_error = 0;
	int		kret;
	int		size;

	if (cnt & PAGE_MASK) {
		panic("vs_cluster_write: cnt not a multiple of PAGE_SIZE");
	}
	size = (cnt < (PAGE_SIZE*32)) ? cnt : (PAGE_SIZE*32); /* effective min */
	
	while (cnt) {

		kret = vnode_pageout(vnode_object->vnode_handle, (upl_t )NULL, (vm_offset_t)NULL, offset, size, 0, &local_error);

		if (local_error != 0) {
			error = local_error;
			local_error = 0;
		}
		cnt -= size;
		offset += size;
	}
#if 0
	if (error != 0)
		return(KERN_FAILURE);
	
	return(KERN_SUCCESS);
#endif /* 0 */
}


/*
 *
 */
kern_return_t
vnode_pager_cluster_read(
	vnode_pager_t		vnode_object,
	vm_object_offset_t	offset,
	vm_size_t		cnt)
{
	int		error = 0;
	int		local_error = 0;
	int		kret;
	int		size;

	if(cnt & PAGE_MASK) {
		panic("vs_cluster_read: cnt not a multiple of PAGE_SIZE");
	}

	size = PAGE_SIZE;

	while (cnt) {

		kret = vnode_pagein(vnode_object->vnode_handle, (upl_t)NULL, (vm_offset_t)NULL, offset, size, 0, &local_error);

		if (local_error != 0) {
			error = local_error;
			local_error = 0;
		}
		cnt -= size;
		offset += size;
	}
	if (error != 0)
		return(KERN_FAILURE);

	return(KERN_SUCCESS);

}


/*
 *
 */
void
vnode_pager_release_from_cache(
		int	*cnt)
{
	memory_object_free_from_cache(
			&realhost, (int)&vnode_pager_workaround, cnt);
}

/*
 *
 */
vnode_pager_t
vnode_object_create(
        vnode_port_t   vp)
{
	register vnode_pager_t  vnode_object;

	vnode_object = (struct vnode_pager *) zalloc(vnode_pager_zone);
	if (vnode_object == VNODE_PAGER_NULL)
		return(VNODE_PAGER_NULL);
	vnode_object->pager_handle = IP_NULL;
	vnode_object->vm_obj_handle = IP_NULL;
	vnode_object->vnode_handle = vp;

	return(vnode_object);
}

/*
 *
 */
void
vnode_port_hash_init(void)
{
	register vm_size_t	size;
	register int		i;


	size = (vm_size_t) sizeof(struct vnode_port_entry);

	vnode_port_hash_zone = zinit(size,
				     (vm_size_t) MAX_VNODE * size,
				     PAGE_SIZE, "vnode_pager port hash");

	for (i = 0; i < VNODE_PORT_HASH_COUNT; i++) 
		queue_init(&vnode_port_hashtable[i]);

	simple_lock_init(&vnode_port_hash_lock,ETAP_NO_TRACE);
}

/*
 *
 */
void
vnode_port_hash_insert(
	ipc_port_t		name_port,
	vnode_pager_t	rec)
{
	register vnode_port_entry_t	new_entry;

	new_entry = (vnode_port_entry_t) zalloc(vnode_port_hash_zone);
	/*
	 * TODO: Delete the following check once MAX_VNODE is removed 
	 */
	if (!new_entry)
		panic("vnode_port_hash_insert: no space");
	new_entry->name = name_port;
	new_entry->pager_rec = rec;

	simple_lock(&vnode_port_hash_lock);
	queue_enter(&vnode_port_hashtable[vnode_port_hash(name_port)],
			new_entry, vnode_port_entry_t, links);
	simple_unlock(&vnode_port_hash_lock);
}

/*
 *
 */
vnode_pager_t
vnode_port_hash_lookup(
	ipc_port_t	 name_port)
{
	register queue_t 		bucket;
	register vnode_port_entry_t	entry;
	vnode_pager_t			rec;

	bucket = (queue_t) &vnode_port_hashtable[vnode_port_hash(name_port)];

	simple_lock(&vnode_port_hash_lock);
	entry = (vnode_port_entry_t) queue_first(bucket);
	while (!queue_end(bucket,&entry->links)) {
		if (entry->name == name_port) {
			rec = entry->pager_rec;
			simple_unlock(&vnode_port_hash_lock);
			return(rec);
		}
		entry = (vnode_port_entry_t)queue_next(&entry->links);
	}
	simple_unlock(&vnode_port_hash_lock);
	return(VNODE_PAGER_NULL);
}

/*
 *
 */
void
vnode_port_hash_delete(
	ipc_port_t name_port)
{
	register queue_t bucket;
	register vnode_port_entry_t entry;

	bucket = (queue_t) &vnode_port_hashtable[vnode_port_hash(name_port)];

	simple_lock(&vnode_port_hash_lock);
	entry = (vnode_port_entry_t) queue_first(bucket);
	while (!queue_end(bucket,&entry->links)) {
		if (entry->name == name_port) {
			queue_remove(bucket, entry, vnode_port_entry_t,links);
			simple_unlock(&vnode_port_hash_lock);
			zfree(vnode_port_hash_zone, (vm_offset_t) entry);
			return;
		}
		entry = (vnode_port_entry_t)queue_next(&entry->links);
	}
	simple_unlock(&vnode_port_hash_lock);
}
Commit	Line	Data
1c79356b A	1	/*
	2	* Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
	6	* The contents of this file constitute Original Code as defined in and
	7	* are subject to the Apple Public Source License Version 1.1 (the
	8	* "License"). You may not use this file except in compliance with the
	9	* License. Please obtain a copy of the License at
	10	* http://www.apple.com/publicsource and read it before using this file.
	11	*
	12	* This Original Code and all software distributed under the License are
	13	* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	14	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	15	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
	17	* License for the specific language governing rights and limitations
	18	* under the License.
	19	*
	20	* @APPLE_LICENSE_HEADER_END@
	21	*/
	22
	23	#include <sys/errno.h>
	24	#include <kern/host.h>
	25	#include <mach/mach_types.h>
	26	#include <vm/vm_map.h>
	27	#include <vm/vm_kern.h>
	28	#include <vm/vm_pageout.h>
	29	#include <mach/kern_return.h>
	30	#include <mach/memory_object_types.h>
	31	#include <mach/port.h>
	32	#include <mach/policy.h>
	33	#include <ipc/ipc_port.h>
	34	#include <ipc/ipc_space.h>
	35	#include <kern/thread.h>
	36	#include <vm/vm_pageout.h>
	37
	38
	39	/* BSD VM COMPONENT INTERFACES */
	40	int
	41	get_map_nentries(
	42	vm_map_t);
	43
	44	vm_offset_t
	45	get_map_start(
	46	vm_map_t);
	47
	48	vm_offset_t
	49	get_map_end(
	50	vm_map_t);
	51
	52	/*
	53	*
	54	*/
	55	int
	56	get_map_nentries(
	57	vm_map_t map)
	58	{
	59	return(map->hdr.nentries);
	60	}
	61
	62	/*
	63	*
	64	*/
65	vm_offset_t
66	get_map_start(
67	vm_map_t map)
68	{
69	return(vm_map_first_entry(map)->vme_start);
70	}
71
72	/*
73	*
74	*/
75	vm_offset_t
76	get_map_end(
77	vm_map_t map)
78	{
79	return(vm_map_last_entry(map)->vme_end);
80	}
81
82	/*
83	* BSD VNODE PAGER
84	*/
85
86	/* until component support available */
87	int vnode_pager_workaround;
88
89	typedef int vnode_port_t;
90
91	typedef struct vnode_pager {
92	ipc_port_t pager; /* pager */
93	ipc_port_t pager_handle; /* pager handle */
94	ipc_port_t vm_obj_handle; /* memory object's control handle */
95	vnode_port_t vnode_handle; /* vnode handle */
96	} *vnode_pager_t;
97
98	typedef struct vnode_port_entry {
99	queue_chain_t links; /* queue links */
100	ipc_port_t name; /* port name */
101	vnode_pager_t pager_rec; /* pager record */
102	} *vnode_port_entry_t;
103
104
105	#define VNODE_PORT_HASH_COUNT 127
106	#define vnode_port_hash(name_port) \
107	(((int)(name_port) & 0xffffff) % VNODE_PORT_HASH_COUNT)
108
109	queue_head_t vnode_port_hashtable[VNODE_PORT_HASH_COUNT];
110	zone_t vnode_port_hash_zone;
111	decl_simple_lock_data(,vnode_port_hash_lock)
112
113
114	ipc_port_t
115	trigger_name_to_port(
116	mach_port_t);
117
118	void
119	vnode_pager_bootstrap(
120	void);
121
122	void
123	vnode_pager_alloc_map(
124	void);
125
126	ipc_port_t
127	vnode_pager_setup(
128	vnode_port_t,
129	ipc_port_t);
130
131	ipc_port_t
132	vnode_pager_lookup(
133	vnode_port_t,
134	ipc_port_t);
135
136	kern_return_t
137	vnode_pager_init(
138	ipc_port_t,
139	ipc_port_t,
140	vm_size_t);
141
142	kern_return_t
143	vnode_pager_data_request(
144	ipc_port_t,
145	ipc_port_t,
146	vm_object_offset_t,
147	vm_size_t,
148	vm_prot_t);
149
150	kern_return_t
151	vnode_pager_data_return(
152	ipc_port_t,
153	ipc_port_t,
154	vm_object_offset_t,
155	pointer_t,
156	vm_size_t,
157	boolean_t,
158	boolean_t);
159
160	void
161	vnode_pager_no_senders(
162	ipc_port_t,
163	mach_port_mscount_t);
164
165	kern_return_t
166	vnode_pager_terminate(
167	ipc_port_t,
168	ipc_port_t);
169
170	kern_return_t
171	vnode_pager_cluster_read(
172	vnode_pager_t,
173	vm_object_offset_t,
174	vm_size_t);
175
176	void
177	vnode_pager_cluster_write(
178	vnode_pager_t,
179	vm_object_offset_t,
180	vm_size_t);
181
182	kern_return_t
183	memory_object_change_attributes(
184	vm_object_t,
185	memory_object_flavor_t,
186	memory_object_info_t,
187	mach_msg_type_number_t,
188	ipc_port_t,
189	mach_msg_type_name_t);
190
191	int
192	vnode_pagein(
193	vnode_port_t,
194	upl_t,
195	vm_offset_t,
196	vm_object_offset_t,
197	int,
198	int,
199	int *);
200	int
201	vnode_pageout(
202	vnode_port_t,
203	upl_t,
204	vm_offset_t,
205	vm_object_offset_t,
206	int,
207	int,
208	int *);
209
210	vnode_pager_t
211	vnode_object_create(
212	vnode_port_t vp);
213
214	void
215	vnode_port_hash_init(void);
216
217	void
218	vnode_port_hash_insert(
219	ipc_port_t,
220	vnode_pager_t);
221
222	vnode_pager_t
223	vnode_port_hash_lookup(
224	ipc_port_t);
225
226	void
227	vnode_port_hash_delete(
228	ipc_port_t);
229
230	void
231	vnode_pager_release_from_cache(
232	int *cnt);
233
234	zone_t vnode_pager_zone;
235
236
237	#define VNODE_PAGER_NULL ((vnode_pager_t) 0)
238
239	/* TODO: Should be set dynamically by vnode_pager_init() */
240	#define CLUSTER_SHIFT 1
241
242	/* TODO: Should be set dynamically by vnode_pager_bootstrap() */
243	#define MAX_VNODE 10000
244
245
246	#if DEBUG
247	int pagerdebug=0;
248
249	#define PAGER_ALL 0xffffffff
250	#define PAGER_INIT 0x00000001
251	#define PAGER_PAGEIN 0x00000002
252
253	#define PAGER_DEBUG(LEVEL, A) {if ((pagerdebug & LEVEL)==LEVEL){printf A;}}
254	#else
255	#define PAGER_DEBUG(LEVEL, A)
256	#endif
257
258	/*
259	* Routine: macx_triggers
260	* Function:
261	* Syscall interface to set the call backs for low and
262	* high water marks.
263	*/
264	int
265	macx_triggers(
266	int hi_water,
267	int low_water,
268	int flags,
269	mach_port_t trigger_name)
270	{
271	kern_return_t kr;
272	ipc_port_t default_pager_port = MACH_PORT_NULL;
273	ipc_port_t trigger_port;
274
275	kr = host_default_memory_manager(host_priv_self(),
276	&default_pager_port, 0);
277	if(kr != KERN_SUCCESS) {
278	return EINVAL;
279	}
280	trigger_port = trigger_name_to_port(trigger_name);
281	if(trigger_port == NULL) {
282	return EINVAL;
283	}
284	/* trigger_port is locked and active */
285	ip_unlock(trigger_port);
286	default_pager_triggers(default_pager_port,
287	hi_water, low_water, flags, trigger_port);
288	ipc_port_make_send(trigger_port);
289
290	/*
291	* Set thread scheduling priority and policy for the current thread
292	* it is assumed for the time being that the thread setting the alert
293	* is the same one which will be servicing it.
294	*/
295	{
296	struct policy_timeshare_base fifo_base;
297	struct policy_timeshare_limit fifo_limit;
298	policy_base_t base;
299	processor_set_t pset;
300	policy_limit_t limit;
301
302	pset = (current_thread())->processor_set;
303	base = (policy_base_t) &fifo_base;
304	limit = (policy_limit_t) &fifo_limit;
305	fifo_limit.max_priority = fifo_base.base_priority = MAXPRI_STANDARD;
306	thread_set_policy((current_thread())->top_act, pset, POLICY_FIFO, base, POLICY_TIMESHARE_BASE_COUNT, limit, POLICY_TIMESHARE_LIMIT_COUNT);
307	}
308
309	current_thread()->vm_privilege = TRUE;
310	}
311
312	/*
313	*
314	*/
315	ipc_port_t
316	trigger_name_to_port(
317	mach_port_t trigger_name)
318	{
319	ipc_port_t trigger_port;
320	ipc_space_t space;
321
322	if (trigger_name == 0)
323	return (NULL);
324
325	space = current_space();
326	if(ipc_port_translate_receive(space, (mach_port_name_t)trigger_name,
327	&trigger_port) != KERN_SUCCESS)
328	return (NULL);
329	return trigger_port;
330	}
331
332	/*
333	*
334	*/
335	void
336	vnode_pager_bootstrap(void)
337	{
338	register vm_size_t size;
339
340	size = (vm_size_t) sizeof(struct vnode_pager);
341	vnode_pager_zone = zinit(size, (vm_size_t) MAX_VNODE*size,
342	PAGE_SIZE, "vnode pager structures");
343	vnode_port_hash_init();
344
345	return;
346	}
347
348	/*
349	*
350	*/
351	ipc_port_t
352	vnode_pager_setup(
353	vnode_port_t vp,
354	ipc_port_t pager)
355	{
356	vnode_pager_t vnode_object;
357	kern_return_t kr;
358	ipc_port_t previous;
359
360	if (pager &&
361	(vnode_object = vnode_port_hash_lookup(pager))) {
362	if (vnode_object->vnode_handle == vp)
363	return(pager);
364	}
365
366	vnode_object = vnode_object_create(vp);
367	if (vnode_object == VNODE_PAGER_NULL)
368	panic("vnode_pager_setup: vnode_object_create() failed");
369
370	vnode_object->pager = ipc_port_alloc_kernel();
371	assert (vnode_object->pager != IP_NULL);
372	pager_mux_hash_insert(vnode_object->pager,
373	(rpc_subsystem_t)&vnode_pager_workaround);
374
375	vnode_object->pager_handle = ipc_port_make_send(vnode_object->pager);
376
377	vnode_port_hash_insert(vnode_object->pager_handle, vnode_object);
378
379	ipc_port_make_sonce(vnode_object->pager);
380	ip_lock(vnode_object->pager); /* unlocked in nsrequest below */
381	ipc_port_nsrequest(vnode_object->pager, 1, vnode_object->pager, &previous);
382
383	PAGER_DEBUG(PAGER_INIT, ("vnode_pager_setup: vp %x pager %x vnode_pager %x\n", vp, vnode_object->pager_handle, vnode_object));
384
385	ubc_setpager( vp, vnode_object->pager_handle);
386	return(vnode_object->pager_handle);
387	}
388
389	/*
390	*
391	*/
392	ipc_port_t
393	vnode_pager_lookup(
394	vnode_port_t vp,
395	ipc_port_t pager)
396	{
397	vnode_pager_t vnode_object;
398	kern_return_t kr;
399
400	if (pager &&
401	(vnode_object = vnode_port_hash_lookup(pager))) {
402	if (vnode_object->vnode_handle == vp)
403	return(vnode_object->vm_obj_handle);
404	else
405	return NULL;
406	}
407	else
408	return NULL;
409	}
410
411	/*
412	*
413	*/
414	kern_return_t
415	vnode_pager_init(ipc_port_t pager,
416	ipc_port_t pager_request,
417	vm_size_t pg_size)
418	{
419	vnode_pager_t vnode_object;
420	kern_return_t kr;
421	memory_object_attr_info_data_t attributes;
422	vm_object_t vm_object;
423
424
425	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %x, %x, %x\n", pager, pager_request, pg_size));
426
427	vnode_object = vnode_port_hash_lookup(pager);
428	if (vnode_object == VNODE_PAGER_NULL)
429	panic("vnode_pager_init: lookup failed");
430
431	vnode_object->vm_obj_handle = pager_request;
432
433	vm_object = vm_object_lookup(pager_request);
434
435	if (vm_object == VM_OBJECT_NULL)
436	panic("vnode_pager_init: vm_object_lookup() failed");
437
438	attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
439	/* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
440	attributes.cluster_size = (1 << (PAGE_SHIFT));
441	attributes.may_cache_object = TRUE;
442	attributes.temporary = TRUE;
443
444	kr = memory_object_change_attributes(
445	vm_object,
446	MEMORY_OBJECT_ATTRIBUTE_INFO,
447	(memory_object_info_t) &attributes,
448	MEMORY_OBJECT_ATTR_INFO_COUNT,
449	MACH_PORT_NULL, 0);
450	if (kr != KERN_SUCCESS)
451	panic("vnode_pager_init: memory_object_change_attributes() failed");
452
453	return(KERN_SUCCESS);
454	}
455
456	/*
457	*
458	*/
459	kern_return_t
460	vnode_pager_data_return(
461	ipc_port_t mem_obj,
462	ipc_port_t control_port,
463	vm_object_offset_t offset,
464	pointer_t addr,
465	vm_size_t data_cnt,
466	boolean_t dirty,
467	boolean_t kernel_copy)
468	{
469	register vnode_pager_t vnode_object;
470
471	vnode_object = vnode_port_hash_lookup(mem_obj);
472	if (vnode_object == VNODE_PAGER_NULL)
473	panic("vnode_pager_data_return: lookup failed");
474
475	vnode_pager_cluster_write(vnode_object, offset, data_cnt);
476
477	return KERN_SUCCESS;
478	}
479
480	/*
481	*
482	*/
483	kern_return_t
484	vnode_pager_data_request(
485	ipc_port_t mem_obj,
486	ipc_port_t mem_obj_control,
487	vm_object_offset_t offset,
488	vm_size_t length,
489	vm_prot_t protection_required)
490	{
491	register vnode_pager_t vnode_object;
492
493	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x, %x\n", mem_obj, mem_obj_control, offset, length, protection_required));
494
495	vnode_object = vnode_port_hash_lookup(mem_obj);
496
497	PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, %x, vnode_object %x\n", mem_obj, mem_obj_control, offset, length, protection_required, vnode_object));
498
499	if (vnode_object == VNODE_PAGER_NULL)
500	panic("vnode_pager_data_request: lookup failed");
501
502	vnode_pager_cluster_read(vnode_object, offset, length);
503
504	return KERN_SUCCESS;
505	}
506
507	/*
508	*
509	*/
510	void
511	vnode_pager_no_senders(
512	ipc_port_t mem_obj,
513	mach_port_mscount_t mscount)
514	{
515	register vnode_pager_t vnode_object;
516
517	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_nosenders: %x, %x\n", mem_obj, mscount));
518
519	vnode_object = vnode_port_hash_lookup(mem_obj);
520	if (vnode_object == VNODE_PAGER_NULL)
521	panic("vnode_pager_no_senders: lookup failed");
522
523	assert(vnode_object->pager_handle == mem_obj);
524
525	pager_mux_hash_delete((ipc_port_t) vnode_object->pager_handle);
526	ipc_port_dealloc_kernel(vnode_object->pager);
527	vnode_port_hash_delete(vnode_object->pager_handle);
528	if (vnode_object->vnode_handle != (vnode_port_t) NULL) {
529	vnode_pager_vrele(vnode_object->vnode_handle);
530	}
531	zfree(vnode_pager_zone, (vm_offset_t) vnode_object);
532
533	return;
534	}
535
536	/*
537	*
538	*/
539	kern_return_t
540	vnode_pager_terminate(
541	ipc_port_t mem_obj,
542	ipc_port_t mem_obj_control)
543	{
544	register vnode_pager_t vnode_object;
545
546	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x, %x\n", mem_obj, mem_obj_control));
547
548	vnode_object = vnode_port_hash_lookup(mem_obj);
549	if (vnode_object == VNODE_PAGER_NULL)
550	panic("vnode_pager_terminate: lookup failed");
551
552	assert(vnode_object->pager_handle == mem_obj);
553
554	/* release extra send right created by the fact that the caller */
555	/* of vnode_pager_setup does not establish a mapping between a */
556	/* cache object and the mem_obj (AMO). When a subsequent vm_map */
557	/* is done, vm_map will bump the send right count */
558	ipc_port_release_send(mem_obj);
559
560	/* release a send right because terminate is called directly and */
561	/* not through IPC, the right won't disappear quietly */
562	ipc_port_release_send(mem_obj);
563
564	ipc_port_dealloc_kernel(mem_obj_control);
565
566	return(KERN_SUCCESS);
567	}
568
569	/*
570	*
571	*/
572	kern_return_t
573	vnode_pager_synchronize(
574	ipc_port_t pager,
575	ipc_port_t pager_request,
576	vm_object_offset_t offset,
577	vm_offset_t length,
578	vm_sync_t sync_flags)
579	{
580	memory_object_synchronize_completed(vm_object_lookup(pager_request), offset, length);
581
582	return (KERN_SUCCESS);
583	}
584
585	/*
586	*
587	*/
588	void
589	vnode_pager_cluster_write(
590	vnode_pager_t vnode_object,
591	vm_object_offset_t offset,
592	vm_size_t cnt)
593	{
594	int error = 0;
595	int local_error = 0;
596	int kret;
597	int size;
598
599	if (cnt & PAGE_MASK) {
600	panic("vs_cluster_write: cnt not a multiple of PAGE_SIZE");
601	}
602	size = (cnt < (PAGE_SIZE32)) ? cnt : (PAGE_SIZE32); /* effective min */
603
604	while (cnt) {
605
606	kret = vnode_pageout(vnode_object->vnode_handle, (upl_t )NULL, (vm_offset_t)NULL, offset, size, 0, &local_error);
607
608	if (local_error != 0) {
609	error = local_error;
610	local_error = 0;
611	}
612	cnt -= size;
613	offset += size;
614	}
615	#if 0
616	if (error != 0)
617	return(KERN_FAILURE);
618
619	return(KERN_SUCCESS);
620	#endif /* 0 */
621	}
622
623
624	/*
625	*
626	*/
627	kern_return_t
628	vnode_pager_cluster_read(
629	vnode_pager_t vnode_object,
630	vm_object_offset_t offset,
631	vm_size_t cnt)
632	{
633	int error = 0;
634	int local_error = 0;
635	int kret;
636	int size;
637
638	if(cnt & PAGE_MASK) {
639	panic("vs_cluster_read: cnt not a multiple of PAGE_SIZE");
640	}
641
642	size = PAGE_SIZE;
643
644	while (cnt) {
645
646	kret = vnode_pagein(vnode_object->vnode_handle, (upl_t)NULL, (vm_offset_t)NULL, offset, size, 0, &local_error);
647
648	if (local_error != 0) {
649	error = local_error;
650	local_error = 0;
651	}
652	cnt -= size;
653	offset += size;
654	}
655	if (error != 0)
656	return(KERN_FAILURE);
657
658	return(KERN_SUCCESS);
659
660	}
661
662
663	/*
664	*
665	*/
666	void
667	vnode_pager_release_from_cache(
668	int *cnt)
669	{
670	memory_object_free_from_cache(
671	&realhost, (int)&vnode_pager_workaround, cnt);
672	}
673
674	/*
675	*
676	*/
677	vnode_pager_t
678	vnode_object_create(
679	vnode_port_t vp)
680	{
681	register vnode_pager_t vnode_object;
682
683	vnode_object = (struct vnode_pager *) zalloc(vnode_pager_zone);
684	if (vnode_object == VNODE_PAGER_NULL)
685	return(VNODE_PAGER_NULL);
686	vnode_object->pager_handle = IP_NULL;
687	vnode_object->vm_obj_handle = IP_NULL;
688	vnode_object->vnode_handle = vp;
689
690	return(vnode_object);
691	}
692
693	/*
694	*
695	*/
696	void
697	vnode_port_hash_init(void)
698	{
699	register vm_size_t size;
700	register int i;
701
702
703	size = (vm_size_t) sizeof(struct vnode_port_entry);
704
705	vnode_port_hash_zone = zinit(size,
706	(vm_size_t) MAX_VNODE * size,
707	PAGE_SIZE, "vnode_pager port hash");
708
709	for (i = 0; i < VNODE_PORT_HASH_COUNT; i++)
710	queue_init(&vnode_port_hashtable[i]);
711
712	simple_lock_init(&vnode_port_hash_lock,ETAP_NO_TRACE);
713	}
714
715	/*
716	*
717	*/
718	void
719	vnode_port_hash_insert(
720	ipc_port_t name_port,
721	vnode_pager_t rec)
722	{
723	register vnode_port_entry_t new_entry;
724
725	new_entry = (vnode_port_entry_t) zalloc(vnode_port_hash_zone);
726	/*
727	* TODO: Delete the following check once MAX_VNODE is removed
728	*/
729	if (!new_entry)
730	panic("vnode_port_hash_insert: no space");
731	new_entry->name = name_port;
732	new_entry->pager_rec = rec;
733
734	simple_lock(&vnode_port_hash_lock);
735	queue_enter(&vnode_port_hashtable[vnode_port_hash(name_port)],
736	new_entry, vnode_port_entry_t, links);
737	simple_unlock(&vnode_port_hash_lock);
738	}
739
740	/*
741	*
742	*/
743	vnode_pager_t
744	vnode_port_hash_lookup(
745	ipc_port_t name_port)
746	{
747	register queue_t bucket;
748	register vnode_port_entry_t entry;
749	vnode_pager_t rec;
750
751	bucket = (queue_t) &vnode_port_hashtable[vnode_port_hash(name_port)];
752
753	simple_lock(&vnode_port_hash_lock);
754	entry = (vnode_port_entry_t) queue_first(bucket);
755	while (!queue_end(bucket,&entry->links)) {
756	if (entry->name == name_port) {
757	rec = entry->pager_rec;
758	simple_unlock(&vnode_port_hash_lock);
759	return(rec);
760	}
761	entry = (vnode_port_entry_t)queue_next(&entry->links);
762	}
763	simple_unlock(&vnode_port_hash_lock);
764	return(VNODE_PAGER_NULL);
765	}
766
767	/*
768	*
769	*/
770	void
771	vnode_port_hash_delete(
772	ipc_port_t name_port)
773	{
774	register queue_t bucket;
775	register vnode_port_entry_t entry;
776
777	bucket = (queue_t) &vnode_port_hashtable[vnode_port_hash(name_port)];
778
779	simple_lock(&vnode_port_hash_lock);
780	entry = (vnode_port_entry_t) queue_first(bucket);
781	while (!queue_end(bucket,&entry->links)) {
782	if (entry->name == name_port) {
783	queue_remove(bucket, entry, vnode_port_entry_t,links);
784	simple_unlock(&vnode_port_hash_lock);
785	zfree(vnode_port_hash_zone, (vm_offset_t) entry);
786	return;
787	}
788	entry = (vnode_port_entry_t)queue_next(&entry->links);
789	}
790	simple_unlock(&vnode_port_hash_lock);
791	}