X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/2d21ac55c334faf3a56e5634905ed6987fc787d4..5ba3f43ea354af8ad55bea84372a2bc834d8757c:/osfmk/vm/device_vm.c diff --git a/osfmk/vm/device_vm.c b/osfmk/vm/device_vm.c index 015200a6a..c4f953e1c 100644 --- a/osfmk/vm/device_vm.c +++ b/osfmk/vm/device_vm.c @@ -48,6 +48,7 @@ #include #include #include +#include /* Device VM COMPONENT INTERFACES */ @@ -73,27 +74,42 @@ const struct memory_object_pager_ops device_pager_ops = { device_pager_data_initialize, device_pager_data_unlock, device_pager_synchronize, - device_pager_unmap, + device_pager_map, + device_pager_last_unmap, + NULL, /* data_reclaim */ "device pager" }; -typedef int device_port_t; +typedef uintptr_t device_port_t; /* * The start of "struct device_pager" MUST match a "struct memory_object". */ typedef struct device_pager { - memory_object_pager_ops_t pager_ops; /* == &device_pager_ops */ - unsigned int pager_ikot; /* fake ip_kotype() */ + /* mandatory generic header */ + struct memory_object dev_pgr_hdr; + + /* pager-specific data */ + lck_mtx_t lock; unsigned int ref_count; /* reference count */ - memory_object_control_t control_handle; /* mem object's cntrl handle */ device_port_t device_handle; /* device_handle */ vm_size_t size; int flags; + boolean_t is_mapped; } *device_pager_t; +lck_grp_t device_pager_lck_grp; +lck_grp_attr_t device_pager_lck_grp_attr; +lck_attr_t device_pager_lck_attr; - +#define device_pager_lock_init(pager) \ + lck_mtx_init(&(pager)->lock, \ + &device_pager_lck_grp, \ + &device_pager_lck_attr) +#define device_pager_lock_destroy(pager) \ + lck_mtx_destroy(&(pager)->lock, &device_pager_lck_grp) +#define device_pager_lock(pager) lck_mtx_lock(&(pager)->lock) +#define device_pager_unlock(pager) lck_mtx_unlock(&(pager)->lock) device_pager_t device_pager_lookup( /* forward */ @@ -120,11 +136,16 @@ zone_t device_pager_zone; void device_pager_bootstrap(void) { - register vm_size_t size; + vm_size_t size; size = (vm_size_t) sizeof(struct device_pager); device_pager_zone = zinit(size, (vm_size_t) MAX_DNODE*size, PAGE_SIZE, "device node pager structures"); + zone_change(device_pager_zone, Z_CALLERACCT, FALSE); + + lck_grp_attr_setdefault(&device_pager_lck_grp_attr); + lck_grp_init(&device_pager_lck_grp, "device_pager", &device_pager_lck_grp_attr); + lck_attr_setdefault(&device_pager_lck_attr); return; } @@ -135,11 +156,13 @@ device_pager_bootstrap(void) memory_object_t device_pager_setup( __unused memory_object_t device, - int device_handle, + uintptr_t device_handle, vm_size_t size, int flags) { device_pager_t device_object; + memory_object_control_t control; + vm_object_t object; device_object = device_object_create(); if (device_object == DEVICE_PAGER_NULL) @@ -149,7 +172,20 @@ device_pager_setup( device_object->size = size; device_object->flags = flags; - return((memory_object_t)device_object); + memory_object_create_named((memory_object_t) device_object, + size, + &control); + object = memory_object_control_to_vm_object(control); + + assert(object != VM_OBJECT_NULL); + vm_object_lock(object); + object->true_share = TRUE; + if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) { + object->copy_strategy = MEMORY_OBJECT_COPY_DELAY; + } + vm_object_unlock(object); + + return (memory_object_t)device_object; } /* @@ -172,7 +208,7 @@ device_pager_populate_object( return KERN_FAILURE; vm_object = (vm_object_t)memory_object_control_to_vm_object( - device_object->control_handle); + device_object->dev_pgr_hdr.mo_control); if(vm_object == NULL) return KERN_FAILURE; @@ -183,10 +219,13 @@ device_pager_populate_object( if(!vm_object->phys_contiguous) { unsigned int null_size = 0; + assert((upl_size_t) size == size); kr = vm_object_upl_request(vm_object, - (vm_object_offset_t)offset, size, &upl, NULL, - &null_size, (UPL_NO_SYNC | UPL_CLEAN_IN_PLACE)); - + (vm_object_offset_t)offset, + (upl_size_t) size, &upl, NULL, + &null_size, + (UPL_NO_SYNC | UPL_CLEAN_IN_PLACE), + VM_KERN_MEMORY_NONE); if(kr != KERN_SUCCESS) panic("device_pager_populate_object: list_req failed"); @@ -203,13 +242,14 @@ device_pager_populate_object( */ device_pager_t device_pager_lookup( - memory_object_t name) + memory_object_t mem_obj) { device_pager_t device_object; - device_object = (device_pager_t)name; - assert(device_object->pager_ops == &device_pager_ops); - return (device_object); + assert(mem_obj->mo_pager_ops == &device_pager_ops); + device_object = (device_pager_t)mem_obj; + assert(device_object->ref_count > 0); + return device_object; } /* @@ -219,7 +259,7 @@ kern_return_t device_pager_init( memory_object_t mem_obj, memory_object_control_t control, - __unused vm_size_t pg_size) + __unused memory_object_cluster_size_t pg_size) { device_pager_t device_object; kern_return_t kr; @@ -234,7 +274,7 @@ device_pager_init( device_object = device_pager_lookup(mem_obj); memory_object_control_reference(control); - device_object->control_handle = control; + device_object->dev_pgr_hdr.mo_control = control; /* The following settings should be done through an expanded change */ @@ -277,7 +317,7 @@ kern_return_t device_pager_data_return( memory_object_t mem_obj, memory_object_offset_t offset, - vm_size_t data_cnt, + memory_object_cluster_size_t data_cnt, __unused memory_object_offset_t *resid_offset, __unused int *io_error, __unused boolean_t dirty, @@ -290,10 +330,10 @@ device_pager_data_return( if (device_object == DEVICE_PAGER_NULL) panic("device_pager_data_return: lookup failed"); - return device_data_action(device_object->device_handle, - (ipc_port_t) device_object, - VM_PROT_READ | VM_PROT_WRITE, - offset, data_cnt); + __IGNORE_WCASTALIGN(return device_data_action(device_object->device_handle, + (ipc_port_t) device_object, + VM_PROT_READ | VM_PROT_WRITE, + offset, data_cnt)); } /* @@ -303,7 +343,7 @@ kern_return_t device_pager_data_request( memory_object_t mem_obj, memory_object_offset_t offset, - vm_size_t length, + memory_object_cluster_size_t length, __unused vm_prot_t protection_required, __unused memory_object_fault_info_t fault_info) { @@ -314,9 +354,9 @@ device_pager_data_request( if (device_object == DEVICE_PAGER_NULL) panic("device_pager_data_request: lookup failed"); - device_data_action(device_object->device_handle, + __IGNORE_WCASTALIGN(device_data_action(device_object->device_handle, (ipc_port_t) device_object, - VM_PROT_READ, offset, length); + VM_PROT_READ, offset, length)); return KERN_SUCCESS; } @@ -333,6 +373,10 @@ device_pager_reference( device_object = device_pager_lookup(mem_obj); new_ref_count = hw_atomic_add(&device_object->ref_count, 1); assert(new_ref_count > 1); + DTRACE_VM2(device_pager_reference, + device_pager_t, device_object, + unsigned int, device_object->ref_count); + } /* @@ -344,27 +388,43 @@ device_pager_deallocate( { device_pager_t device_object; memory_object_control_t device_control; + unsigned int ref_count; device_object = device_pager_lookup(mem_obj); + assert(device_object->ref_count > 0); + + DTRACE_VM2(device_pager_deallocate, + device_pager_t, device_object, + unsigned int, device_object->ref_count); + + ref_count = hw_atomic_sub(&device_object->ref_count, 1); + + if (ref_count == 1) { + /* + * The last reference is our "named" reference. + * Close the device and "destroy" the VM object. + */ + + DTRACE_VM2(device_pager_destroy, + device_pager_t, device_object, + unsigned int, device_object->ref_count); - if (hw_atomic_sub(&device_object->ref_count, 1) == 0) { + assert(device_object->is_mapped == FALSE); if (device_object->device_handle != (device_port_t) NULL) { device_close(device_object->device_handle); device_object->device_handle = (device_port_t) NULL; } - device_control = device_object->control_handle; - if (device_control != MEMORY_OBJECT_CONTROL_NULL) { - /* - * The VM object should already have been disconnected - * from the pager at this point. - * We still have to release the "memory object control" - * handle. - */ - assert(device_control->moc_object == VM_OBJECT_NULL); - memory_object_control_deallocate(device_control); - device_object->control_handle = - MEMORY_OBJECT_CONTROL_NULL; - } + device_control = device_object->dev_pgr_hdr.mo_control; + memory_object_destroy(device_control, 0); + } else if (ref_count == 0) { + /* + * No more references: free the pager. + */ + DTRACE_VM2(device_pager_free, + device_pager_t, device_object, + unsigned int, device_object->ref_count); + + device_pager_lock_destroy(device_object); zfree(device_pager_zone, device_object); } @@ -375,7 +435,7 @@ kern_return_t device_pager_data_initialize( __unused memory_object_t mem_obj, __unused memory_object_offset_t offset, - __unused vm_size_t data_cnt) + __unused memory_object_cluster_size_t data_cnt) { panic("device_pager_data_initialize"); return KERN_FAILURE; @@ -385,7 +445,7 @@ kern_return_t device_pager_data_unlock( __unused memory_object_t mem_obj, __unused memory_object_offset_t offset, - __unused vm_size_t size, + __unused memory_object_size_t size, __unused vm_prot_t desired_access) { return KERN_FAILURE; @@ -405,28 +465,66 @@ device_pager_terminate( */ kern_return_t device_pager_synchronize( - memory_object_t mem_obj, - memory_object_offset_t offset, - vm_offset_t length, + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_size_t length, __unused vm_sync_t sync_flags) { - device_pager_t device_object; + panic("device_pager_synchronize: memory_object_synchronize no longer supported\n"); + return KERN_FAILURE; +} + +/* + * + */ +kern_return_t +device_pager_map( + memory_object_t mem_obj, + __unused vm_prot_t prot) +{ + device_pager_t device_object; device_object = device_pager_lookup(mem_obj); - memory_object_synchronize_completed( - device_object->control_handle, offset, length); + device_pager_lock(device_object); + assert(device_object->ref_count > 0); + if (device_object->is_mapped == FALSE) { + /* + * First mapping of this pager: take an extra reference + * that will remain until all the mappings of this pager + * are removed. + */ + device_object->is_mapped = TRUE; + device_pager_reference(mem_obj); + } + device_pager_unlock(device_object); return KERN_SUCCESS; } -/* - * - */ kern_return_t -device_pager_unmap( - __unused memory_object_t mem_obj) +device_pager_last_unmap( + memory_object_t mem_obj) { + device_pager_t device_object; + boolean_t drop_ref; + + device_object = device_pager_lookup(mem_obj); + + device_pager_lock(device_object); + assert(device_object->ref_count > 0); + if (device_object->is_mapped) { + device_object->is_mapped = FALSE; + drop_ref = TRUE; + } else { + drop_ref = FALSE; + } + device_pager_unlock(device_object); + + if (drop_ref) { + device_pager_deallocate(mem_obj); + } + return KERN_SUCCESS; } @@ -438,17 +536,34 @@ device_pager_unmap( device_pager_t device_object_create(void) { - register device_pager_t device_object; + device_pager_t device_object; device_object = (struct device_pager *) zalloc(device_pager_zone); if (device_object == DEVICE_PAGER_NULL) return(DEVICE_PAGER_NULL); - device_object->pager_ops = &device_pager_ops; - device_object->pager_ikot = IKOT_MEMORY_OBJECT; + + bzero(device_object, sizeof (*device_object)); + + device_object->dev_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT; + device_object->dev_pgr_hdr.mo_pager_ops = &device_pager_ops; + device_object->dev_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; + + device_pager_lock_init(device_object); device_object->ref_count = 1; - device_object->control_handle = MEMORY_OBJECT_CONTROL_NULL; + device_object->is_mapped = FALSE; + DTRACE_VM2(device_pager_create, + device_pager_t, device_object, + unsigned int, device_object->ref_count); return(device_object); } +boolean_t +is_device_pager_ops(const struct memory_object_pager_ops *pager_ops) +{ + if (pager_ops == &device_pager_ops) { + return TRUE; + } + return FALSE; +}