[apple/xnu.git] / osfmk / vm / device_vm.c

/*
 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <sys/errno.h>

#include <mach/mach_types.h>
#include <mach/kern_return.h>
#include <mach/memory_object_control.h>
#include <mach/memory_object_types.h>
#include <mach/port.h>
#include <mach/policy.h>
#include <mach/upl.h>
#include <kern/kern_types.h>
#include <kern/ipc_kobject.h>
#include <kern/host.h>
#include <kern/thread.h>
#include <ipc/ipc_port.h>
#include <ipc/ipc_space.h>
#include <device/device_port.h>
#include <vm/memory_object.h>
#include <vm/vm_pageout.h>
#include <vm/vm_map.h>
#include <vm/vm_kern.h>
#include <vm/vm_pageout.h>
#include <vm/vm_protos.h>
#include <mach/sdt.h>


/* Device VM COMPONENT INTERFACES */


/* 
 * Device PAGER 
 */


/* until component support available */


/* until component support available */
const struct memory_object_pager_ops device_pager_ops = {
	device_pager_reference,
	device_pager_deallocate,
	device_pager_init,
	device_pager_terminate,
	device_pager_data_request,
	device_pager_data_return,
	device_pager_data_initialize,
	device_pager_data_unlock,
	device_pager_synchronize,
	device_pager_map,
	device_pager_last_unmap,
	NULL, /* data_reclaim */
	"device pager"
};

typedef uintptr_t device_port_t;

/*
 * The start of "struct device_pager" MUST match a "struct memory_object".
 */
typedef struct device_pager {
	/* mandatory generic header */
	struct memory_object dev_pgr_hdr;

	/* pager-specific data */
	lck_mtx_t	lock;
	unsigned int	ref_count;	/* reference count		*/
	device_port_t   device_handle;  /* device_handle */
	vm_size_t	size;
	int		flags;
	boolean_t	is_mapped;
} *device_pager_t;

lck_grp_t	device_pager_lck_grp;
lck_grp_attr_t	device_pager_lck_grp_attr;
lck_attr_t	device_pager_lck_attr;

#define device_pager_lock_init(pager)				\
	lck_mtx_init(&(pager)->lock,				\
		     &device_pager_lck_grp,			\
		     &device_pager_lck_attr)
#define device_pager_lock_destroy(pager)			\
	lck_mtx_destroy(&(pager)->lock, &device_pager_lck_grp)
#define device_pager_lock(pager) lck_mtx_lock(&(pager)->lock)
#define device_pager_unlock(pager) lck_mtx_unlock(&(pager)->lock)

device_pager_t
device_pager_lookup(		/* forward */
	memory_object_t);

device_pager_t
device_object_create(void);	/* forward */

zone_t	device_pager_zone;


#define	DEVICE_PAGER_NULL	((device_pager_t) 0)


#define	MAX_DNODE		10000


/*
 *
 */
void
device_pager_bootstrap(void)
{
	vm_size_t      size;

	size = (vm_size_t) sizeof(struct device_pager);
	device_pager_zone = zinit(size, (vm_size_t) MAX_DNODE*size,
				PAGE_SIZE, "device node pager structures");
	zone_change(device_pager_zone, Z_CALLERACCT, FALSE);

	lck_grp_attr_setdefault(&device_pager_lck_grp_attr);
	lck_grp_init(&device_pager_lck_grp, "device_pager", &device_pager_lck_grp_attr);
	lck_attr_setdefault(&device_pager_lck_attr);

	return;
}

/*
 *
 */
memory_object_t
device_pager_setup(
	__unused memory_object_t device,
	uintptr_t		device_handle,
	vm_size_t	size,
	int		flags)
{
	device_pager_t	device_object;
	memory_object_control_t control;
	vm_object_t	object;

	device_object = device_object_create();
	if (device_object == DEVICE_PAGER_NULL)
		panic("device_pager_setup: device_object_create() failed");

	device_object->device_handle = device_handle;
	device_object->size = size;
	device_object->flags = flags;

	memory_object_create_named((memory_object_t) device_object,
				   size,
				   &control);
	object = memory_object_control_to_vm_object(control);

	assert(object != VM_OBJECT_NULL);
	vm_object_lock(object);
	object->true_share = TRUE;
	if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
		object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
	}
	vm_object_unlock(object);

	return (memory_object_t)device_object;
}

/*
 *
 */
kern_return_t
device_pager_populate_object(
	memory_object_t		device,
	memory_object_offset_t	offset,
	ppnum_t			page_num,
	vm_size_t		size)
{
	device_pager_t	device_object;
	vm_object_t	vm_object;
	kern_return_t	kr;
	upl_t		upl;

	device_object = device_pager_lookup(device);
	if(device_object == DEVICE_PAGER_NULL)
		return KERN_FAILURE;

	vm_object = (vm_object_t)memory_object_control_to_vm_object(
					device_object->dev_pgr_hdr.mo_control);
	if(vm_object == NULL) 
		return KERN_FAILURE;

	kr =  vm_object_populate_with_private(
				vm_object, offset, page_num, size);
	if(kr != KERN_SUCCESS)
		return kr;

	if(!vm_object->phys_contiguous) {
		unsigned int null_size = 0;
		assert((upl_size_t) size == size);
        	kr = vm_object_upl_request(vm_object,
					   (vm_object_offset_t)offset,
					   (upl_size_t) size, &upl,  NULL,
					   &null_size,
					   (UPL_NO_SYNC | UPL_CLEAN_IN_PLACE),
					   VM_KERN_MEMORY_NONE);
		if(kr != KERN_SUCCESS)
			panic("device_pager_populate_object: list_req failed");

		upl_commit(upl, NULL, 0);
		upl_deallocate(upl);
	}


	return kr;
}

/*
 *
 */
device_pager_t
device_pager_lookup(
	memory_object_t	mem_obj)
{
	device_pager_t	device_object;

	assert(mem_obj->mo_pager_ops == &device_pager_ops);
	device_object = (device_pager_t)mem_obj;
	assert(device_object->ref_count > 0);
	return device_object;
}

/*
 *
 */
kern_return_t
device_pager_init(
	memory_object_t mem_obj, 
	memory_object_control_t control, 
	__unused memory_object_cluster_size_t pg_size)
{
	device_pager_t   device_object;
	kern_return_t   kr;
	memory_object_attr_info_data_t  attributes;

	vm_object_t	vm_object;


	if (control == MEMORY_OBJECT_CONTROL_NULL)
		return KERN_INVALID_ARGUMENT;

	device_object = device_pager_lookup(mem_obj);

	memory_object_control_reference(control);
	device_object->dev_pgr_hdr.mo_control = control;


/* The following settings should be done through an expanded change */
/* attributes call */

	vm_object = (vm_object_t)memory_object_control_to_vm_object(control);
	vm_object_lock(vm_object);
	vm_object->private = TRUE;
	if(device_object->flags & DEVICE_PAGER_CONTIGUOUS)
		vm_object->phys_contiguous = TRUE;
	if(device_object->flags & DEVICE_PAGER_NOPHYSCACHE)
		vm_object->nophyscache = TRUE;

	vm_object->wimg_bits = device_object->flags & VM_WIMG_MASK;
	vm_object_unlock(vm_object);


	attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
	/* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
	attributes.cluster_size = (1 << (PAGE_SHIFT));
	attributes.may_cache_object = FALSE;
	attributes.temporary = TRUE;

	kr = memory_object_change_attributes(
					control,
					MEMORY_OBJECT_ATTRIBUTE_INFO,
					(memory_object_info_t) &attributes,
					MEMORY_OBJECT_ATTR_INFO_COUNT);
	if (kr != KERN_SUCCESS)
		panic("device_pager_init: memory_object_change_attributes() failed");

	return(KERN_SUCCESS);
}

/*
 *
 */
/*ARGSUSED6*/
kern_return_t
device_pager_data_return(
	memory_object_t			mem_obj,
	memory_object_offset_t		offset,
	memory_object_cluster_size_t			data_cnt,
	__unused memory_object_offset_t	*resid_offset,
	__unused int			*io_error,
	__unused boolean_t		dirty,
	__unused boolean_t		kernel_copy,
	__unused int			upl_flags)  
{
	device_pager_t	device_object;

	device_object = device_pager_lookup(mem_obj);
	if (device_object == DEVICE_PAGER_NULL)
		panic("device_pager_data_return: lookup failed");

	__IGNORE_WCASTALIGN(return device_data_action(device_object->device_handle,
			  (ipc_port_t) device_object,
			  VM_PROT_READ | VM_PROT_WRITE,
			  offset, data_cnt));
}

/*
 *
 */
kern_return_t	
device_pager_data_request(
	memory_object_t		mem_obj,
	memory_object_offset_t	offset,
	memory_object_cluster_size_t		length,
	__unused vm_prot_t	protection_required,
        __unused memory_object_fault_info_t 	fault_info)
{
	device_pager_t	device_object;

	device_object = device_pager_lookup(mem_obj);

	if (device_object == DEVICE_PAGER_NULL)
		panic("device_pager_data_request: lookup failed");

	__IGNORE_WCASTALIGN(device_data_action(device_object->device_handle,
			   (ipc_port_t) device_object,
			   VM_PROT_READ, offset, length));
	return KERN_SUCCESS;
}

/*
 *
 */
void
device_pager_reference(
	memory_object_t		mem_obj)
{	
	device_pager_t		device_object;
	unsigned int		new_ref_count;

	device_object = device_pager_lookup(mem_obj);
	new_ref_count = hw_atomic_add(&device_object->ref_count, 1);
	assert(new_ref_count > 1);
	DTRACE_VM2(device_pager_reference,
		   device_pager_t, device_object,
		   unsigned int, device_object->ref_count);

}

/*
 *
 */
void
device_pager_deallocate(
	memory_object_t		mem_obj)
{
	device_pager_t		device_object;
	memory_object_control_t	device_control;
	unsigned int		ref_count;

	device_object = device_pager_lookup(mem_obj);
	assert(device_object->ref_count > 0);

	DTRACE_VM2(device_pager_deallocate,
		   device_pager_t, device_object,
		   unsigned int, device_object->ref_count);

	ref_count = hw_atomic_sub(&device_object->ref_count, 1);

	if (ref_count == 1) {
		/*
		 * The last reference is our "named" reference.
		 * Close the device and "destroy" the VM object.
		 */

		DTRACE_VM2(device_pager_destroy,
			   device_pager_t, device_object,
			   unsigned int, device_object->ref_count);

		assert(device_object->is_mapped == FALSE);
		if (device_object->device_handle != (device_port_t) NULL) {
			device_close(device_object->device_handle);
			device_object->device_handle = (device_port_t) NULL;
		}
		device_control = device_object->dev_pgr_hdr.mo_control;
		memory_object_destroy(device_control, 0);
	} else if (ref_count == 0) {
		/*
		 * No more references: free the pager.
		 */
		DTRACE_VM2(device_pager_free,
			   device_pager_t, device_object,
			   unsigned int, device_object->ref_count);

		device_pager_lock_destroy(device_object);

		zfree(device_pager_zone, device_object);
	}
	return;
}

kern_return_t
device_pager_data_initialize(
        __unused memory_object_t		mem_obj,
        __unused memory_object_offset_t	offset,
        __unused memory_object_cluster_size_t		data_cnt)
{
	panic("device_pager_data_initialize");
	return KERN_FAILURE;
}

kern_return_t
device_pager_data_unlock(
	__unused memory_object_t		mem_obj,
	__unused memory_object_offset_t	offset,
	__unused memory_object_size_t		size,
	__unused vm_prot_t		desired_access)
{
	return KERN_FAILURE;
}

kern_return_t
device_pager_terminate(
	__unused memory_object_t	mem_obj)
{
	return KERN_SUCCESS;
}


/*
 *
 */
kern_return_t
device_pager_synchronize(
	__unused memory_object_t	mem_obj,
	__unused memory_object_offset_t	offset,
	__unused memory_object_size_t	length,
	__unused vm_sync_t		sync_flags)
{
	panic("device_pager_synchronize: memory_object_synchronize no longer supported\n");
	return KERN_FAILURE;
}

/*
 *
 */
kern_return_t
device_pager_map(
	memory_object_t	mem_obj,
	__unused vm_prot_t		prot)
{
	device_pager_t		device_object;

	device_object = device_pager_lookup(mem_obj);

	device_pager_lock(device_object);
	assert(device_object->ref_count > 0);
	if (device_object->is_mapped == FALSE) {
		/*
		 * First mapping of this pager: take an extra reference
		 * that will remain until all the mappings of this pager
		 * are removed.
		 */
		device_object->is_mapped = TRUE;
		device_pager_reference(mem_obj);
	}
	device_pager_unlock(device_object);

	return KERN_SUCCESS;
}

kern_return_t
device_pager_last_unmap(
	memory_object_t	mem_obj)
{
	device_pager_t	device_object;
	boolean_t	drop_ref;

	device_object = device_pager_lookup(mem_obj);

	device_pager_lock(device_object);
	assert(device_object->ref_count > 0);
	if (device_object->is_mapped) {
		device_object->is_mapped = FALSE;
		drop_ref = TRUE;
	} else {
		drop_ref = FALSE;
	}
	device_pager_unlock(device_object);

	if (drop_ref) {
		device_pager_deallocate(mem_obj);
	}

	return KERN_SUCCESS;
}


/*
 *
 */
device_pager_t
device_object_create(void)
{
	device_pager_t  device_object;

	device_object = (struct device_pager *) zalloc(device_pager_zone);
	if (device_object == DEVICE_PAGER_NULL)
		return(DEVICE_PAGER_NULL);

	bzero(device_object, sizeof (*device_object));

	device_object->dev_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT;
	device_object->dev_pgr_hdr.mo_pager_ops = &device_pager_ops;
	device_object->dev_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;

	device_pager_lock_init(device_object);
	device_object->ref_count = 1;
	device_object->is_mapped = FALSE;

	DTRACE_VM2(device_pager_create,
		   device_pager_t, device_object,
		   unsigned int, device_object->ref_count);

	return(device_object);
}

boolean_t
is_device_pager_ops(const struct memory_object_pager_ops *pager_ops)
{
	if (pager_ops == &device_pager_ops) {
		return TRUE;
	}
	return FALSE;
}
Commit	Line	Data
0b4e3aa0	1	/*
2d21ac55	2	* Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
0b4e3aa0	3	*
2d21ac55	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0b4e3aa0	5	*
2d21ac55 A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
8f6c56a5	14	*
2d21ac55 A	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5 A	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
8f6c56a5 A	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55 A	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
8f6c56a5	25	*
2d21ac55	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
0b4e3aa0 A	27	*/
	28
	29	#include <sys/errno.h>
91447636	30
0b4e3aa0	31	#include <mach/mach_types.h>
0b4e3aa0	32	#include <mach/kern_return.h>
91447636	33	#include <mach/memory_object_control.h>
0b4e3aa0 A	34	#include <mach/memory_object_types.h>
	35	#include <mach/port.h>
	36	#include <mach/policy.h>
91447636 A	37	#include <mach/upl.h>
	38	#include <kern/kern_types.h>
	39	#include <kern/ipc_kobject.h>
	40	#include <kern/host.h>
	41	#include <kern/thread.h>
0b4e3aa0 A	42	#include <ipc/ipc_port.h>
0b4e3aa0 A	43	#include <ipc/ipc_space.h>
0b4e3aa0	44	#include <device/device_port.h>
91447636	45	#include <vm/memory_object.h>
0b4e3aa0	46	#include <vm/vm_pageout.h>
91447636 A	47	#include <vm/vm_map.h>
	48	#include <vm/vm_kern.h>
	49	#include <vm/vm_pageout.h>
	50	#include <vm/vm_protos.h>
5ba3f43e	51	#include <mach/sdt.h>
91447636	52
0b4e3aa0	53
0b4e3aa0 A	54	/* Device VM COMPONENT INTERFACES */
	55
	56
	57	/*
	58	* Device PAGER
	59	*/
	60
	61
	62	/* until component support available */
	63
	64
	65
	66	/* until component support available */
0c530ab8 A	67	const struct memory_object_pager_ops device_pager_ops = {
	68	device_pager_reference,
	69	device_pager_deallocate,
	70	device_pager_init,
	71	device_pager_terminate,
	72	device_pager_data_request,
	73	device_pager_data_return,
	74	device_pager_data_initialize,
	75	device_pager_data_unlock,
	76	device_pager_synchronize,
593a1d5f A	77	device_pager_map,
593a1d5f A	78	device_pager_last_unmap,
6d2010ae	79	NULL, /* data_reclaim */
0c530ab8 A	80	"device pager"
0c530ab8 A	81	};
0b4e3aa0	82
b0d623f7	83	typedef uintptr_t device_port_t;
0b4e3aa0	84
0c530ab8 A	85	/*
	86	* The start of "struct device_pager" MUST match a "struct memory_object".
	87	*/
0b4e3aa0	88	typedef struct device_pager {
5ba3f43e A	89	/* mandatory generic header */
	90	struct memory_object dev_pgr_hdr;
	91
	92	/* pager-specific data */
	93	lck_mtx_t lock;
0b4e3aa0	94	unsigned int ref_count; /* reference count */
0b4e3aa0 A	95	device_port_t device_handle; /* device_handle */
	96	vm_size_t size;
	97	int flags;
5ba3f43e	98	boolean_t is_mapped;
0b4e3aa0 A	99	} *device_pager_t;
0b4e3aa0 A	100
5ba3f43e A	101	lck_grp_t device_pager_lck_grp;
	102	lck_grp_attr_t device_pager_lck_grp_attr;
	103	lck_attr_t device_pager_lck_attr;
0b4e3aa0	104
5ba3f43e A	105	#define device_pager_lock_init(pager) \
	106	lck_mtx_init(&(pager)->lock, \
	107	&device_pager_lck_grp, \
	108	&device_pager_lck_attr)
	109	#define device_pager_lock_destroy(pager) \
	110	lck_mtx_destroy(&(pager)->lock, &device_pager_lck_grp)
	111	#define device_pager_lock(pager) lck_mtx_lock(&(pager)->lock)
	112	#define device_pager_unlock(pager) lck_mtx_unlock(&(pager)->lock)
0b4e3aa0 A	113
0b4e3aa0 A	114	device_pager_t
91447636	115	device_pager_lookup( /* forward */
0b4e3aa0 A	116	memory_object_t);
	117
	118	device_pager_t
91447636	119	device_object_create(void); /* forward */
0b4e3aa0 A	120
	121	zone_t device_pager_zone;
	122
	123
	124	#define DEVICE_PAGER_NULL ((device_pager_t) 0)
	125
	126
	127	#define MAX_DNODE 10000
	128
	129
	130
	131
	132
	133	/*
	134	*
	135	*/
	136	void
	137	device_pager_bootstrap(void)
	138	{
39037602	139	vm_size_t size;
0b4e3aa0 A	140
	141	size = (vm_size_t) sizeof(struct device_pager);
	142	device_pager_zone = zinit(size, (vm_size_t) MAX_DNODE*size,
	143	PAGE_SIZE, "device node pager structures");
6d2010ae	144	zone_change(device_pager_zone, Z_CALLERACCT, FALSE);
5ba3f43e A	145
	146	lck_grp_attr_setdefault(&device_pager_lck_grp_attr);
	147	lck_grp_init(&device_pager_lck_grp, "device_pager", &device_pager_lck_grp_attr);
	148	lck_attr_setdefault(&device_pager_lck_attr);
	149
0b4e3aa0 A	150	return;
	151	}
	152
	153	/*
	154	*
	155	*/
	156	memory_object_t
	157	device_pager_setup(
91447636	158	__unused memory_object_t device,
b0d623f7	159	uintptr_t device_handle,
0b4e3aa0 A	160	vm_size_t size,
	161	int flags)
	162	{
	163	device_pager_t device_object;
5ba3f43e A	164	memory_object_control_t control;
5ba3f43e A	165	vm_object_t object;
0b4e3aa0 A	166
	167	device_object = device_object_create();
	168	if (device_object == DEVICE_PAGER_NULL)
	169	panic("device_pager_setup: device_object_create() failed");
	170
	171	device_object->device_handle = device_handle;
	172	device_object->size = size;
9bccf70c	173	device_object->flags = flags;
0b4e3aa0	174
5ba3f43e A	175	memory_object_create_named((memory_object_t) device_object,
	176	size,
	177	&control);
	178	object = memory_object_control_to_vm_object(control);
	179
	180	assert(object != VM_OBJECT_NULL);
	181	vm_object_lock(object);
	182	object->true_share = TRUE;
	183	if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
	184	object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
	185	}
	186	vm_object_unlock(object);
	187
	188	return (memory_object_t)device_object;
0b4e3aa0 A	189	}
	190
	191	/*
	192	*
	193	*/
	194	kern_return_t
	195	device_pager_populate_object(
	196	memory_object_t device,
	197	memory_object_offset_t offset,
55e303ae	198	ppnum_t page_num,
0b4e3aa0 A	199	vm_size_t size)
	200	{
	201	device_pager_t device_object;
	202	vm_object_t vm_object;
	203	kern_return_t kr;
	204	upl_t upl;
0b4e3aa0 A	205
	206	device_object = device_pager_lookup(device);
	207	if(device_object == DEVICE_PAGER_NULL)
	208	return KERN_FAILURE;
	209
	210	vm_object = (vm_object_t)memory_object_control_to_vm_object(
5ba3f43e	211	device_object->dev_pgr_hdr.mo_control);
0b4e3aa0 A	212	if(vm_object == NULL)
	213	return KERN_FAILURE;
	214
	215	kr = vm_object_populate_with_private(
55e303ae	216	vm_object, offset, page_num, size);
0b4e3aa0 A	217	if(kr != KERN_SUCCESS)
	218	return kr;
	219
	220	if(!vm_object->phys_contiguous) {
0c530ab8	221	unsigned int null_size = 0;
b0d623f7	222	assert((upl_size_t) size == size);
0b4e3aa0	223	kr = vm_object_upl_request(vm_object,
b0d623f7 A	224	(vm_object_offset_t)offset,
	225	(upl_size_t) size, &upl, NULL,
	226	&null_size,
5ba3f43e A	227	(UPL_NO_SYNC \| UPL_CLEAN_IN_PLACE),
5ba3f43e A	228	VM_KERN_MEMORY_NONE);
0b4e3aa0 A	229	if(kr != KERN_SUCCESS)
	230	panic("device_pager_populate_object: list_req failed");
	231
91447636	232	upl_commit(upl, NULL, 0);
0b4e3aa0 A	233	upl_deallocate(upl);
	234	}
	235
	236
	237	return kr;
	238	}
	239
	240	/*
	241	*
	242	*/
	243	device_pager_t
	244	device_pager_lookup(
5ba3f43e	245	memory_object_t mem_obj)
0b4e3aa0 A	246	{
	247	device_pager_t device_object;
	248
5ba3f43e A	249	assert(mem_obj->mo_pager_ops == &device_pager_ops);
	250	device_object = (device_pager_t)mem_obj;
	251	assert(device_object->ref_count > 0);
	252	return device_object;
0b4e3aa0 A	253	}
	254
	255	/*
	256	*
	257	*/
	258	kern_return_t
91447636 A	259	device_pager_init(
	260	memory_object_t mem_obj,
	261	memory_object_control_t control,
b0d623f7	262	__unused memory_object_cluster_size_t pg_size)
0b4e3aa0 A	263	{
	264	device_pager_t device_object;
	265	kern_return_t kr;
	266	memory_object_attr_info_data_t attributes;
	267
	268	vm_object_t vm_object;
	269
	270
	271	if (control == MEMORY_OBJECT_CONTROL_NULL)
	272	return KERN_INVALID_ARGUMENT;
	273
	274	device_object = device_pager_lookup(mem_obj);
	275
	276	memory_object_control_reference(control);
5ba3f43e	277	device_object->dev_pgr_hdr.mo_control = control;
0b4e3aa0 A	278
	279
	280	/* The following settings should be done through an expanded change */
	281	/* attributes call */
	282
	283	vm_object = (vm_object_t)memory_object_control_to_vm_object(control);
	284	vm_object_lock(vm_object);
	285	vm_object->private = TRUE;
	286	if(device_object->flags & DEVICE_PAGER_CONTIGUOUS)
	287	vm_object->phys_contiguous = TRUE;
	288	if(device_object->flags & DEVICE_PAGER_NOPHYSCACHE)
	289	vm_object->nophyscache = TRUE;
9bccf70c A	290
9bccf70c A	291	vm_object->wimg_bits = device_object->flags & VM_WIMG_MASK;
0b4e3aa0 A	292	vm_object_unlock(vm_object);
	293
	294
	295	attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
	296	/* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
	297	attributes.cluster_size = (1 << (PAGE_SHIFT));
	298	attributes.may_cache_object = FALSE;
	299	attributes.temporary = TRUE;
	300
	301	kr = memory_object_change_attributes(
	302	control,
	303	MEMORY_OBJECT_ATTRIBUTE_INFO,
	304	(memory_object_info_t) &attributes,
	305	MEMORY_OBJECT_ATTR_INFO_COUNT);
	306	if (kr != KERN_SUCCESS)
	307	panic("device_pager_init: memory_object_change_attributes() failed");
	308
	309	return(KERN_SUCCESS);
	310	}
	311
	312	/*
	313	*
	314	*/
91447636	315	/ARGSUSED6/
0b4e3aa0 A	316	kern_return_t
0b4e3aa0 A	317	device_pager_data_return(
0c530ab8 A	318	memory_object_t mem_obj,
0c530ab8 A	319	memory_object_offset_t offset,
b0d623f7	320	memory_object_cluster_size_t data_cnt,
0c530ab8 A	321	__unused memory_object_offset_t *resid_offset,
0c530ab8 A	322	__unused int *io_error,
91447636 A	323	__unused boolean_t dirty,
	324	__unused boolean_t kernel_copy,
	325	__unused int upl_flags)
0b4e3aa0 A	326	{
	327	device_pager_t device_object;
	328
	329	device_object = device_pager_lookup(mem_obj);
	330	if (device_object == DEVICE_PAGER_NULL)
	331	panic("device_pager_data_return: lookup failed");
	332
3e170ce0 A	333	__IGNORE_WCASTALIGN(return device_data_action(device_object->device_handle,
	334	(ipc_port_t) device_object,
	335	VM_PROT_READ \| VM_PROT_WRITE,
	336	offset, data_cnt));
0b4e3aa0 A	337	}
	338
	339	/*
	340	*
	341	*/
	342	kern_return_t
	343	device_pager_data_request(
	344	memory_object_t mem_obj,
	345	memory_object_offset_t offset,
b0d623f7	346	memory_object_cluster_size_t length,
2d21ac55 A	347	__unused vm_prot_t protection_required,
2d21ac55 A	348	__unused memory_object_fault_info_t fault_info)
0b4e3aa0 A	349	{
	350	device_pager_t device_object;
	351
	352	device_object = device_pager_lookup(mem_obj);
	353
	354	if (device_object == DEVICE_PAGER_NULL)
	355	panic("device_pager_data_request: lookup failed");
	356
3e170ce0	357	__IGNORE_WCASTALIGN(device_data_action(device_object->device_handle,
91447636	358	(ipc_port_t) device_object,
3e170ce0	359	VM_PROT_READ, offset, length));
0b4e3aa0 A	360	return KERN_SUCCESS;
	361	}
	362
	363	/*
	364	*
	365	*/
	366	void
	367	device_pager_reference(
	368	memory_object_t mem_obj)
	369	{
	370	device_pager_t device_object;
9bccf70c	371	unsigned int new_ref_count;
0b4e3aa0 A	372
0b4e3aa0 A	373	device_object = device_pager_lookup(mem_obj);
9bccf70c A	374	new_ref_count = hw_atomic_add(&device_object->ref_count, 1);
9bccf70c A	375	assert(new_ref_count > 1);
5ba3f43e A	376	DTRACE_VM2(device_pager_reference,
	377	device_pager_t, device_object,
	378	unsigned int, device_object->ref_count);
	379
0b4e3aa0 A	380	}
	381
	382	/*
	383	*
	384	*/
	385	void
	386	device_pager_deallocate(
	387	memory_object_t mem_obj)
	388	{
91447636 A	389	device_pager_t device_object;
91447636 A	390	memory_object_control_t device_control;
5ba3f43e	391	unsigned int ref_count;
0b4e3aa0 A	392
0b4e3aa0 A	393	device_object = device_pager_lookup(mem_obj);
5ba3f43e A	394	assert(device_object->ref_count > 0);
	395
	396	DTRACE_VM2(device_pager_deallocate,
	397	device_pager_t, device_object,
	398	unsigned int, device_object->ref_count);
	399
	400	ref_count = hw_atomic_sub(&device_object->ref_count, 1);
	401
	402	if (ref_count == 1) {
	403	/*
	404	* The last reference is our "named" reference.
	405	* Close the device and "destroy" the VM object.
	406	*/
	407
	408	DTRACE_VM2(device_pager_destroy,
	409	device_pager_t, device_object,
	410	unsigned int, device_object->ref_count);
0b4e3aa0	411
5ba3f43e	412	assert(device_object->is_mapped == FALSE);
0b4e3aa0 A	413	if (device_object->device_handle != (device_port_t) NULL) {
0b4e3aa0 A	414	device_close(device_object->device_handle);
91447636	415	device_object->device_handle = (device_port_t) NULL;
0b4e3aa0	416	}
5ba3f43e A	417	device_control = device_object->dev_pgr_hdr.mo_control;
	418	memory_object_destroy(device_control, 0);
	419	} else if (ref_count == 0) {
	420	/*
	421	* No more references: free the pager.
	422	*/
	423	DTRACE_VM2(device_pager_free,
	424	device_pager_t, device_object,
	425	unsigned int, device_object->ref_count);
	426
	427	device_pager_lock_destroy(device_object);
91447636 A	428
91447636 A	429	zfree(device_pager_zone, device_object);
0b4e3aa0 A	430	}
	431	return;
	432	}
	433
	434	kern_return_t
	435	device_pager_data_initialize(
91447636 A	436	__unused memory_object_t mem_obj,
91447636 A	437	__unused memory_object_offset_t offset,
b0d623f7	438	__unused memory_object_cluster_size_t data_cnt)
0b4e3aa0	439	{
91447636	440	panic("device_pager_data_initialize");
0b4e3aa0 A	441	return KERN_FAILURE;
	442	}
	443
	444	kern_return_t
	445	device_pager_data_unlock(
91447636 A	446	__unused memory_object_t mem_obj,
91447636 A	447	__unused memory_object_offset_t offset,
b0d623f7	448	__unused memory_object_size_t size,
91447636	449	__unused vm_prot_t desired_access)
0b4e3aa0 A	450	{
	451	return KERN_FAILURE;
	452	}
	453
91447636	454	kern_return_t
0b4e3aa0	455	device_pager_terminate(
91447636	456	__unused memory_object_t mem_obj)
0b4e3aa0 A	457	{
	458	return KERN_SUCCESS;
	459	}
	460
	461
	462
	463	/*
	464	*
	465	*/
	466	kern_return_t
	467	device_pager_synchronize(
5ba3f43e A	468	__unused memory_object_t mem_obj,
	469	__unused memory_object_offset_t offset,
	470	__unused memory_object_size_t length,
91447636	471	__unused vm_sync_t sync_flags)
0b4e3aa0	472	{
5ba3f43e A	473	panic("device_pager_synchronize: memory_object_synchronize no longer supported\n");
5ba3f43e A	474	return KERN_FAILURE;
0b4e3aa0 A	475	}
	476
	477	/*
	478	*
	479	*/
	480	kern_return_t
593a1d5f	481	device_pager_map(
5ba3f43e	482	memory_object_t mem_obj,
593a1d5f A	483	__unused vm_prot_t prot)
593a1d5f A	484	{
5ba3f43e A	485	device_pager_t device_object;
	486
	487	device_object = device_pager_lookup(mem_obj);
	488
	489	device_pager_lock(device_object);
	490	assert(device_object->ref_count > 0);
	491	if (device_object->is_mapped == FALSE) {
	492	/*
	493	* First mapping of this pager: take an extra reference
	494	* that will remain until all the mappings of this pager
	495	* are removed.
	496	*/
	497	device_object->is_mapped = TRUE;
	498	device_pager_reference(mem_obj);
	499	}
	500	device_pager_unlock(device_object);
	501
593a1d5f A	502	return KERN_SUCCESS;
	503	}
	504
	505	kern_return_t
	506	device_pager_last_unmap(
5ba3f43e	507	memory_object_t mem_obj)
0b4e3aa0	508	{
5ba3f43e A	509	device_pager_t device_object;
	510	boolean_t drop_ref;
	511
	512	device_object = device_pager_lookup(mem_obj);
	513
	514	device_pager_lock(device_object);
	515	assert(device_object->ref_count > 0);
	516	if (device_object->is_mapped) {
	517	device_object->is_mapped = FALSE;
	518	drop_ref = TRUE;
	519	} else {
	520	drop_ref = FALSE;
	521	}
	522	device_pager_unlock(device_object);
	523
	524	if (drop_ref) {
	525	device_pager_deallocate(mem_obj);
	526	}
	527
0b4e3aa0 A	528	return KERN_SUCCESS;
	529	}
	530
	531
	532
	533	/*
	534	*
	535	*/
	536	device_pager_t
2d21ac55	537	device_object_create(void)
0b4e3aa0	538	{
39037602	539	device_pager_t device_object;
0b4e3aa0 A	540
	541	device_object = (struct device_pager *) zalloc(device_pager_zone);
	542	if (device_object == DEVICE_PAGER_NULL)
	543	return(DEVICE_PAGER_NULL);
5ba3f43e A	544
	545	bzero(device_object, sizeof (*device_object));
	546
	547	device_object->dev_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT;
	548	device_object->dev_pgr_hdr.mo_pager_ops = &device_pager_ops;
	549	device_object->dev_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
	550
	551	device_pager_lock_init(device_object);
0b4e3aa0	552	device_object->ref_count = 1;
5ba3f43e	553	device_object->is_mapped = FALSE;
0b4e3aa0	554
5ba3f43e A	555	DTRACE_VM2(device_pager_create,
	556	device_pager_t, device_object,
	557	unsigned int, device_object->ref_count);
0b4e3aa0 A	558
	559	return(device_object);
	560	}
	561
39037602 A	562	boolean_t
	563	is_device_pager_ops(const struct memory_object_pager_ops *pager_ops)
	564	{
	565	if (pager_ops == &device_pager_ops) {
	566	return TRUE;
	567	}
	568	return FALSE;
	569	}