X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/1c79356b52d46aa6b508fb032f5ae709b1f2897b..91447636331957f3d9b5ca5b508f07c526b0074d:/osfmk/vm/bsd_vm.c diff --git a/osfmk/vm/bsd_vm.c b/osfmk/vm/bsd_vm.c index 6c4a68dc7..fa1cfa94b 100644 --- a/osfmk/vm/bsd_vm.c +++ b/osfmk/vm/bsd_vm.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * @@ -21,20 +21,33 @@ */ #include -#include + #include -#include -#include -#include +#include +#include #include +#include #include #include #include +#include +#include + +#include +#include + #include #include -#include -#include +#include +#include + +#include +#include +#include +#include +#include +#include /* BSD VM COMPONENT INTERFACES */ int @@ -59,24 +72,36 @@ get_map_nentries( return(map->hdr.nentries); } +mach_vm_offset_t +mach_get_vm_start(vm_map_t map) +{ + return( vm_map_first_entry(map)->vme_start); +} + +mach_vm_offset_t +mach_get_vm_end(vm_map_t map) +{ + return( vm_map_last_entry(map)->vme_end); +} + /* - * + * Legacy routines to get the start and end for a vm_map_t. They + * return them in the vm_offset_t format. So, they should only be + * called on maps that are the same size as the kernel map for + * accurate results. */ vm_offset_t -get_map_start( +get_vm_start( vm_map_t map) { - return(vm_map_first_entry(map)->vme_start); + return(CAST_DOWN(vm_offset_t, vm_map_first_entry(map)->vme_start)); } -/* - * - */ vm_offset_t -get_map_end( +get_vm_end( vm_map_t map) { - return(vm_map_last_entry(map)->vme_end); + return(CAST_DOWN(vm_offset_t, vm_map_last_entry(map)->vme_end)); } /* @@ -86,150 +111,42 @@ get_map_end( /* until component support available */ int vnode_pager_workaround; -typedef int vnode_port_t; - typedef struct vnode_pager { - ipc_port_t pager; /* pager */ - ipc_port_t pager_handle; /* pager handle */ - ipc_port_t vm_obj_handle; /* memory object's control handle */ - vnode_port_t vnode_handle; /* vnode handle */ + int *pager; /* pager workaround pointer */ + unsigned int pager_ikot; /* JMM: fake ip_kotype() */ + unsigned int ref_count; /* reference count */ + memory_object_control_t control_handle; /* mem object control handle */ + struct vnode *vnode_handle; /* vnode handle */ } *vnode_pager_t; -typedef struct vnode_port_entry { - queue_chain_t links; /* queue links */ - ipc_port_t name; /* port name */ - vnode_pager_t pager_rec; /* pager record */ -} *vnode_port_entry_t; - - -#define VNODE_PORT_HASH_COUNT 127 -#define vnode_port_hash(name_port) \ - (((int)(name_port) & 0xffffff) % VNODE_PORT_HASH_COUNT) - -queue_head_t vnode_port_hashtable[VNODE_PORT_HASH_COUNT]; -zone_t vnode_port_hash_zone; -decl_simple_lock_data(,vnode_port_hash_lock) - ipc_port_t -trigger_name_to_port( +trigger_name_to_port( /* forward */ mach_port_t); -void -vnode_pager_bootstrap( - void); - -void -vnode_pager_alloc_map( - void); - -ipc_port_t -vnode_pager_setup( - vnode_port_t, - ipc_port_t); - -ipc_port_t -vnode_pager_lookup( - vnode_port_t, - ipc_port_t); - -kern_return_t -vnode_pager_init( - ipc_port_t, - ipc_port_t, - vm_size_t); - kern_return_t -vnode_pager_data_request( - ipc_port_t, - ipc_port_t, - vm_object_offset_t, - vm_size_t, - vm_prot_t); - -kern_return_t -vnode_pager_data_return( - ipc_port_t, - ipc_port_t, - vm_object_offset_t, - pointer_t, - vm_size_t, - boolean_t, - boolean_t); - -void -vnode_pager_no_senders( - ipc_port_t, - mach_port_mscount_t); - -kern_return_t -vnode_pager_terminate( - ipc_port_t, - ipc_port_t); - -kern_return_t -vnode_pager_cluster_read( +vnode_pager_cluster_read( /* forward */ vnode_pager_t, vm_object_offset_t, vm_size_t); void -vnode_pager_cluster_write( +vnode_pager_cluster_write( /* forward */ vnode_pager_t, vm_object_offset_t, - vm_size_t); + vm_size_t, + vm_object_offset_t *, + int *, + int); -kern_return_t -memory_object_change_attributes( - vm_object_t, - memory_object_flavor_t, - memory_object_info_t, - mach_msg_type_number_t, - ipc_port_t, - mach_msg_type_name_t); - -int -vnode_pagein( - vnode_port_t, - upl_t, - vm_offset_t, - vm_object_offset_t, - int, - int, - int *); -int -vnode_pageout( - vnode_port_t, - upl_t, - vm_offset_t, - vm_object_offset_t, - int, - int, - int *); vnode_pager_t -vnode_object_create( - vnode_port_t vp); - -void -vnode_port_hash_init(void); - -void -vnode_port_hash_insert( - ipc_port_t, - vnode_pager_t); +vnode_object_create( /* forward */ + struct vnode *); vnode_pager_t -vnode_port_hash_lookup( - ipc_port_t); - -void -vnode_port_hash_delete( - ipc_port_t); - -void -vnode_pager_release_from_cache( - int *cnt); +vnode_pager_lookup( /* forward */ + memory_object_t); zone_t vnode_pager_zone; @@ -263,50 +180,97 @@ int pagerdebug=0; */ int macx_triggers( - int hi_water, - int low_water, - int flags, - mach_port_t trigger_name) + struct macx_triggers_args *args) { + int hi_water = args->hi_water; + int low_water = args->low_water; + int flags = args->flags; + mach_port_t trigger_name = args->alert_port; kern_return_t kr; - ipc_port_t default_pager_port = MACH_PORT_NULL; + memory_object_default_t default_pager; ipc_port_t trigger_port; + default_pager = MEMORY_OBJECT_DEFAULT_NULL; kr = host_default_memory_manager(host_priv_self(), - &default_pager_port, 0); + &default_pager, 0); if(kr != KERN_SUCCESS) { return EINVAL; } - trigger_port = trigger_name_to_port(trigger_name); - if(trigger_port == NULL) { + + if ((flags & SWAP_ENCRYPT_ON) && + (flags & SWAP_ENCRYPT_OFF)) { + /* can't have it both ways */ return EINVAL; } - /* trigger_port is locked and active */ - ip_unlock(trigger_port); - default_pager_triggers(default_pager_port, - hi_water, low_water, flags, trigger_port); - ipc_port_make_send(trigger_port); + + if (flags & SWAP_ENCRYPT_ON) { + /* ENCRYPTED SWAP: tell default_pager to encrypt */ + default_pager_triggers(default_pager, + 0, 0, + SWAP_ENCRYPT_ON, + IP_NULL); + } else if (flags & SWAP_ENCRYPT_OFF) { + /* ENCRYPTED SWAP: tell default_pager not to encrypt */ + default_pager_triggers(default_pager, + 0, 0, + SWAP_ENCRYPT_OFF, + IP_NULL); + } + + if (flags & HI_WAT_ALERT) { + trigger_port = trigger_name_to_port(trigger_name); + if(trigger_port == NULL) { + return EINVAL; + } + /* trigger_port is locked and active */ + ipc_port_make_send_locked(trigger_port); + /* now unlocked */ + default_pager_triggers(default_pager, + hi_water, low_water, + HI_WAT_ALERT, trigger_port); + } + + if (flags & LO_WAT_ALERT) { + trigger_port = trigger_name_to_port(trigger_name); + if(trigger_port == NULL) { + return EINVAL; + } + /* trigger_port is locked and active */ + ipc_port_make_send_locked(trigger_port); + /* and now its unlocked */ + default_pager_triggers(default_pager, + hi_water, low_water, + LO_WAT_ALERT, trigger_port); + } /* * Set thread scheduling priority and policy for the current thread * it is assumed for the time being that the thread setting the alert - * is the same one which will be servicing it. + * is the same one which will be servicing it. + * + * XXX This does not belong in the kernel XXX */ { - struct policy_timeshare_base fifo_base; - struct policy_timeshare_limit fifo_limit; - policy_base_t base; - processor_set_t pset; - policy_limit_t limit; - - pset = (current_thread())->processor_set; - base = (policy_base_t) &fifo_base; - limit = (policy_limit_t) &fifo_limit; - fifo_limit.max_priority = fifo_base.base_priority = MAXPRI_STANDARD; - thread_set_policy((current_thread())->top_act, pset, POLICY_FIFO, base, POLICY_TIMESHARE_BASE_COUNT, limit, POLICY_TIMESHARE_LIMIT_COUNT); + thread_precedence_policy_data_t pre; + thread_extended_policy_data_t ext; + + ext.timeshare = FALSE; + pre.importance = INT32_MAX; + + thread_policy_set(current_thread(), + THREAD_EXTENDED_POLICY, + (thread_policy_t)&ext, + THREAD_EXTENDED_POLICY_COUNT); + + thread_policy_set(current_thread(), + THREAD_PRECEDENCE_POLICY, + (thread_policy_t)&pre, + THREAD_PRECEDENCE_POLICY_COUNT); } - current_thread()->vm_privilege = TRUE; + current_thread()->options |= TH_OPT_VMPRIV; + + return 0; } /* @@ -329,6 +293,126 @@ trigger_name_to_port( return trigger_port; } + +extern int uiomove64(addr64_t, int, void *); +#define MAX_RUN 32 + +int +memory_object_control_uiomove( + memory_object_control_t control, + memory_object_offset_t offset, + void * uio, + int start_offset, + int io_requested, + int mark_dirty) +{ + vm_object_t object; + vm_page_t dst_page; + int xsize; + int retval = 0; + int cur_run; + int cur_needed; + int i; + vm_page_t page_run[MAX_RUN]; + + + object = memory_object_control_to_vm_object(control); + if (object == VM_OBJECT_NULL) { + return (0); + } + assert(!object->internal); + + vm_object_lock(object); + + if (mark_dirty && object->copy != VM_OBJECT_NULL) { + /* + * We can't modify the pages without honoring + * copy-on-write obligations first, so fall off + * this optimized path and fall back to the regular + * path. + */ + vm_object_unlock(object); + return 0; + } + + while (io_requested && retval == 0) { + + cur_needed = (start_offset + io_requested + (PAGE_SIZE - 1)) / PAGE_SIZE; + + if (cur_needed > MAX_RUN) + cur_needed = MAX_RUN; + + for (cur_run = 0; cur_run < cur_needed; ) { + + if ((dst_page = vm_page_lookup(object, offset)) == VM_PAGE_NULL) + break; + /* + * Sync up on getting the busy bit + */ + if ((dst_page->busy || dst_page->cleaning)) { + /* + * someone else is playing with the page... if we've + * already collected pages into this run, go ahead + * and process now, we can't block on this + * page while holding other pages in the BUSY state + * otherwise we will wait + */ + if (cur_run) + break; + PAGE_SLEEP(object, dst_page, THREAD_UNINT); + continue; + } + /* + * this routine is only called when copying + * to/from real files... no need to consider + * encrypted swap pages + */ + assert(!dst_page->encrypted); + + if (mark_dirty) + dst_page->dirty = TRUE; + dst_page->busy = TRUE; + + page_run[cur_run++] = dst_page; + + offset += PAGE_SIZE_64; + } + if (cur_run == 0) + /* + * we hit a 'hole' in the cache + * we bail at this point + * we'll unlock the object below + */ + break; + vm_object_unlock(object); + + for (i = 0; i < cur_run; i++) { + + dst_page = page_run[i]; + + if ((xsize = PAGE_SIZE - start_offset) > io_requested) + xsize = io_requested; + + if ( (retval = uiomove64((addr64_t)(((addr64_t)(dst_page->phys_page) << 12) + start_offset), xsize, uio)) ) + break; + + io_requested -= xsize; + start_offset = 0; + } + vm_object_lock(object); + + for (i = 0; i < cur_run; i++) { + dst_page = page_run[i]; + + PAGE_WAKEUP_DONE(dst_page); + } + } + vm_object_unlock(object); + + return (retval); +} + + /* * */ @@ -340,100 +424,51 @@ vnode_pager_bootstrap(void) size = (vm_size_t) sizeof(struct vnode_pager); vnode_pager_zone = zinit(size, (vm_size_t) MAX_VNODE*size, PAGE_SIZE, "vnode pager structures"); - vnode_port_hash_init(); - return; } /* * */ -ipc_port_t +memory_object_t vnode_pager_setup( - vnode_port_t vp, - ipc_port_t pager) + struct vnode *vp, + __unused memory_object_t pager) { vnode_pager_t vnode_object; - kern_return_t kr; - ipc_port_t previous; - - if (pager && - (vnode_object = vnode_port_hash_lookup(pager))) { - if (vnode_object->vnode_handle == vp) - return(pager); - } vnode_object = vnode_object_create(vp); if (vnode_object == VNODE_PAGER_NULL) panic("vnode_pager_setup: vnode_object_create() failed"); - - vnode_object->pager = ipc_port_alloc_kernel(); - assert (vnode_object->pager != IP_NULL); - pager_mux_hash_insert(vnode_object->pager, - (rpc_subsystem_t)&vnode_pager_workaround); - - vnode_object->pager_handle = ipc_port_make_send(vnode_object->pager); - - vnode_port_hash_insert(vnode_object->pager_handle, vnode_object); - - ipc_port_make_sonce(vnode_object->pager); - ip_lock(vnode_object->pager); /* unlocked in nsrequest below */ - ipc_port_nsrequest(vnode_object->pager, 1, vnode_object->pager, &previous); - - PAGER_DEBUG(PAGER_INIT, ("vnode_pager_setup: vp %x pager %x vnode_pager %x\n", vp, vnode_object->pager_handle, vnode_object)); - - ubc_setpager( vp, vnode_object->pager_handle); - return(vnode_object->pager_handle); -} - -/* - * - */ -ipc_port_t -vnode_pager_lookup( - vnode_port_t vp, - ipc_port_t pager) -{ - vnode_pager_t vnode_object; - kern_return_t kr; - - if (pager && - (vnode_object = vnode_port_hash_lookup(pager))) { - if (vnode_object->vnode_handle == vp) - return(vnode_object->vm_obj_handle); - else - return NULL; - } - else - return NULL; + return((memory_object_t)vnode_object); } /* * */ kern_return_t -vnode_pager_init(ipc_port_t pager, - ipc_port_t pager_request, - vm_size_t pg_size) +vnode_pager_init(memory_object_t mem_obj, + memory_object_control_t control, +#if !DEBUG + __unused +#endif + vm_size_t pg_size) { vnode_pager_t vnode_object; kern_return_t kr; memory_object_attr_info_data_t attributes; - vm_object_t vm_object; - PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %x, %x, %x\n", pager, pager_request, pg_size)); + PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %p, %p, %x\n", mem_obj, control, pg_size)); - vnode_object = vnode_port_hash_lookup(pager); - if (vnode_object == VNODE_PAGER_NULL) - panic("vnode_pager_init: lookup failed"); + if (control == MEMORY_OBJECT_CONTROL_NULL) + return KERN_INVALID_ARGUMENT; - vnode_object->vm_obj_handle = pager_request; + vnode_object = vnode_pager_lookup(mem_obj); - vm_object = vm_object_lookup(pager_request); + memory_object_control_reference(control); - if (vm_object == VM_OBJECT_NULL) - panic("vnode_pager_init: vm_object_lookup() failed"); + vnode_object->control_handle = control; attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/ @@ -442,11 +477,10 @@ vnode_pager_init(ipc_port_t pager, attributes.temporary = TRUE; kr = memory_object_change_attributes( - vm_object, + control, MEMORY_OBJECT_ATTRIBUTE_INFO, (memory_object_info_t) &attributes, - MEMORY_OBJECT_ATTR_INFO_COUNT, - MACH_PORT_NULL, 0); + MEMORY_OBJECT_ATTR_INFO_COUNT); if (kr != KERN_SUCCESS) panic("vnode_pager_init: memory_object_change_attributes() failed"); @@ -458,78 +492,115 @@ vnode_pager_init(ipc_port_t pager, */ kern_return_t vnode_pager_data_return( - ipc_port_t mem_obj, - ipc_port_t control_port, - vm_object_offset_t offset, - pointer_t addr, + memory_object_t mem_obj, + memory_object_offset_t offset, vm_size_t data_cnt, - boolean_t dirty, - boolean_t kernel_copy) + memory_object_offset_t *resid_offset, + int *io_error, + __unused boolean_t dirty, + __unused boolean_t kernel_copy, + int upl_flags) { register vnode_pager_t vnode_object; - vnode_object = vnode_port_hash_lookup(mem_obj); - if (vnode_object == VNODE_PAGER_NULL) - panic("vnode_pager_data_return: lookup failed"); + vnode_object = vnode_pager_lookup(mem_obj); - vnode_pager_cluster_write(vnode_object, offset, data_cnt); + vnode_pager_cluster_write(vnode_object, offset, data_cnt, resid_offset, io_error, upl_flags); return KERN_SUCCESS; } +kern_return_t +vnode_pager_data_initialize( + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused vm_size_t data_cnt) +{ + panic("vnode_pager_data_initialize"); + return KERN_FAILURE; +} + +kern_return_t +vnode_pager_data_unlock( + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused vm_size_t size, + __unused vm_prot_t desired_access) +{ + return KERN_FAILURE; +} + +kern_return_t +vnode_pager_get_object_size( + memory_object_t mem_obj, + memory_object_offset_t *length) +{ + vnode_pager_t vnode_object; + + vnode_object = vnode_pager_lookup(mem_obj); + + *length = vnode_pager_get_filesize(vnode_object->vnode_handle); + return KERN_SUCCESS; +} + /* * */ kern_return_t vnode_pager_data_request( - ipc_port_t mem_obj, - ipc_port_t mem_obj_control, - vm_object_offset_t offset, + memory_object_t mem_obj, + memory_object_offset_t offset, vm_size_t length, - vm_prot_t protection_required) +#if !DEBUG + __unused +#endif +vm_prot_t protection_required) { register vnode_pager_t vnode_object; - PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x, %x\n", mem_obj, mem_obj_control, offset, length, protection_required)); + PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x\n", mem_obj, offset, length, protection_required)); - vnode_object = vnode_port_hash_lookup(mem_obj); + vnode_object = vnode_pager_lookup(mem_obj); - PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, %x, vnode_object %x\n", mem_obj, mem_obj_control, offset, length, protection_required, vnode_object)); + PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, vnode_object %x\n", mem_obj, offset, length, protection_required, vnode_object)); - if (vnode_object == VNODE_PAGER_NULL) - panic("vnode_pager_data_request: lookup failed"); + return vnode_pager_cluster_read(vnode_object, offset, length); +} - vnode_pager_cluster_read(vnode_object, offset, length); +/* + * + */ +void +vnode_pager_reference( + memory_object_t mem_obj) +{ + register vnode_pager_t vnode_object; + unsigned int new_ref_count; - return KERN_SUCCESS; + vnode_object = vnode_pager_lookup(mem_obj); + new_ref_count = hw_atomic_add(&vnode_object->ref_count, 1); + assert(new_ref_count > 1); } /* * */ void -vnode_pager_no_senders( - ipc_port_t mem_obj, - mach_port_mscount_t mscount) +vnode_pager_deallocate( + memory_object_t mem_obj) { register vnode_pager_t vnode_object; - PAGER_DEBUG(PAGER_ALL, ("vnode_pager_nosenders: %x, %x\n", mem_obj, mscount)); - - vnode_object = vnode_port_hash_lookup(mem_obj); - if (vnode_object == VNODE_PAGER_NULL) - panic("vnode_pager_no_senders: lookup failed"); + PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %x\n", mem_obj)); - assert(vnode_object->pager_handle == mem_obj); + vnode_object = vnode_pager_lookup(mem_obj); - pager_mux_hash_delete((ipc_port_t) vnode_object->pager_handle); - ipc_port_dealloc_kernel(vnode_object->pager); - vnode_port_hash_delete(vnode_object->pager_handle); - if (vnode_object->vnode_handle != (vnode_port_t) NULL) { - vnode_pager_vrele(vnode_object->vnode_handle); + if (hw_atomic_sub(&vnode_object->ref_count, 1) == 0) { + if (vnode_object->vnode_handle != NULL) { + vnode_pager_vrele(vnode_object->vnode_handle); + } + zfree(vnode_pager_zone, vnode_object); } - zfree(vnode_pager_zone, (vm_offset_t) vnode_object); - return; } @@ -538,50 +609,55 @@ vnode_pager_no_senders( */ kern_return_t vnode_pager_terminate( - ipc_port_t mem_obj, - ipc_port_t mem_obj_control) +#if !DEBUG + __unused +#endif + memory_object_t mem_obj) { - register vnode_pager_t vnode_object; - - PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x, %x\n", mem_obj, mem_obj_control)); + PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x\n", mem_obj)); - vnode_object = vnode_port_hash_lookup(mem_obj); - if (vnode_object == VNODE_PAGER_NULL) - panic("vnode_pager_terminate: lookup failed"); + return(KERN_SUCCESS); +} - assert(vnode_object->pager_handle == mem_obj); +/* + * + */ +kern_return_t +vnode_pager_synchronize( + memory_object_t mem_obj, + memory_object_offset_t offset, + vm_size_t length, + __unused vm_sync_t sync_flags) +{ + register vnode_pager_t vnode_object; - /* release extra send right created by the fact that the caller */ - /* of vnode_pager_setup does not establish a mapping between a */ - /* cache object and the mem_obj (AMO). When a subsequent vm_map */ - /* is done, vm_map will bump the send right count */ - ipc_port_release_send(mem_obj); + PAGER_DEBUG(PAGER_ALL, ("vnode_pager_synchronize: %x\n", mem_obj)); - /* release a send right because terminate is called directly and */ - /* not through IPC, the right won't disappear quietly */ - ipc_port_release_send(mem_obj); + vnode_object = vnode_pager_lookup(mem_obj); - ipc_port_dealloc_kernel(mem_obj_control); + memory_object_synchronize_completed(vnode_object->control_handle, offset, length); - return(KERN_SUCCESS); + return (KERN_SUCCESS); } /* * */ kern_return_t -vnode_pager_synchronize( - ipc_port_t pager, - ipc_port_t pager_request, - vm_object_offset_t offset, - vm_offset_t length, - vm_sync_t sync_flags) +vnode_pager_unmap( + memory_object_t mem_obj) { - memory_object_synchronize_completed(vm_object_lookup(pager_request), offset, length); + register vnode_pager_t vnode_object; - return (KERN_SUCCESS); + PAGER_DEBUG(PAGER_ALL, ("vnode_pager_unmap: %x\n", mem_obj)); + + vnode_object = vnode_pager_lookup(mem_obj); + + ubc_unmap(vnode_object->vnode_handle); + return KERN_SUCCESS; } + /* * */ @@ -589,35 +665,115 @@ void vnode_pager_cluster_write( vnode_pager_t vnode_object, vm_object_offset_t offset, - vm_size_t cnt) + vm_size_t cnt, + vm_object_offset_t * resid_offset, + int * io_error, + int upl_flags) { - int error = 0; - int local_error = 0; - int kret; - int size; + vm_size_t size; + upl_t upl = NULL; + int request_flags; + int errno; - if (cnt & PAGE_MASK) { - panic("vs_cluster_write: cnt not a multiple of PAGE_SIZE"); - } - size = (cnt < (PAGE_SIZE*32)) ? cnt : (PAGE_SIZE*32); /* effective min */ - - while (cnt) { + if (upl_flags & UPL_MSYNC) { - kret = vnode_pageout(vnode_object->vnode_handle, (upl_t )NULL, (vm_offset_t)NULL, offset, size, 0, &local_error); + upl_flags |= UPL_VNODE_PAGER; - if (local_error != 0) { - error = local_error; - local_error = 0; + if ( (upl_flags & UPL_IOSYNC) && io_error) + upl_flags |= UPL_KEEPCACHED; + + while (cnt) { + kern_return_t kr; + + size = (cnt < (PAGE_SIZE * MAX_UPL_TRANSFER)) ? cnt : (PAGE_SIZE * MAX_UPL_TRANSFER); /* effective max */ + + request_flags = UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM | UPL_CLEAN_IN_PLACE | + UPL_SET_INTERNAL | UPL_SET_LITE; + + kr = memory_object_upl_request(vnode_object->control_handle, + offset, size, &upl, NULL, NULL, request_flags); + if (kr != KERN_SUCCESS) + panic("vnode_pager_cluster_write: upl request failed\n"); + + vnode_pageout(vnode_object->vnode_handle, + upl, (vm_offset_t)0, offset, size, upl_flags, &errno); + + if ( (upl_flags & UPL_KEEPCACHED) ) { + if ( (*io_error = errno) ) + break; + } + cnt -= size; + offset += size; + } + if (resid_offset) + *resid_offset = offset; + + } else { + vm_object_offset_t vnode_size; + vm_object_offset_t base_offset; + vm_object_t object; + vm_page_t target_page; + int ticket; + + /* + * this is the pageout path + */ + vnode_size = vnode_pager_get_filesize(vnode_object->vnode_handle); + + if (vnode_size > (offset + PAGE_SIZE)) { + /* + * preset the maximum size of the cluster + * and put us on a nice cluster boundary... + * and then clip the size to insure we + * don't request past the end of the underlying file + */ + size = PAGE_SIZE * MAX_UPL_TRANSFER; + base_offset = offset & ~((signed)(size - 1)); + + if ((base_offset + size) > vnode_size) + size = round_page_32(((vm_size_t)(vnode_size - base_offset))); + } else { + /* + * we've been requested to page out a page beyond the current + * end of the 'file'... don't try to cluster in this case... + * we still need to send this page through because it might + * be marked precious and the underlying filesystem may need + * to do something with it (besides page it out)... + */ + base_offset = offset; + size = PAGE_SIZE; + } + object = memory_object_control_to_vm_object(vnode_object->control_handle); + + if (object == VM_OBJECT_NULL) + panic("vnode_pager_cluster_write: NULL vm_object in control handle\n"); + + request_flags = UPL_NOBLOCK | UPL_FOR_PAGEOUT | UPL_CLEAN_IN_PLACE | + UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM | + UPL_SET_INTERNAL | UPL_SET_LITE; + + vm_object_lock(object); + + if ((target_page = vm_page_lookup(object, offset)) != VM_PAGE_NULL) { + /* + * only pick up pages whose ticket number matches + * the ticket number of the page orginally targeted + * for pageout + */ + ticket = target_page->page_ticket; + + request_flags |= ((ticket << UPL_PAGE_TICKET_SHIFT) & UPL_PAGE_TICKET_MASK); } - cnt -= size; - offset += size; + vm_object_unlock(object); + + vm_object_upl_request(object, base_offset, size, + &upl, NULL, NULL, request_flags); + if (upl == NULL) + panic("vnode_pager_cluster_write: upl request failed\n"); + + vnode_pageout(vnode_object->vnode_handle, + upl, (vm_offset_t)0, upl->offset, upl->size, UPL_VNODE_PAGER, NULL); } -#if 0 - if (error != 0) - return(KERN_FAILURE); - - return(KERN_SUCCESS); -#endif /* 0 */ } @@ -630,32 +786,54 @@ vnode_pager_cluster_read( vm_object_offset_t offset, vm_size_t cnt) { - int error = 0; int local_error = 0; int kret; - int size; - if(cnt & PAGE_MASK) { - panic("vs_cluster_read: cnt not a multiple of PAGE_SIZE"); - } - - size = PAGE_SIZE; - - while (cnt) { - - kret = vnode_pagein(vnode_object->vnode_handle, (upl_t)NULL, (vm_offset_t)NULL, offset, size, 0, &local_error); + assert(! (cnt & PAGE_MASK)); - if (local_error != 0) { - error = local_error; - local_error = 0; + kret = vnode_pagein(vnode_object->vnode_handle, + (upl_t) NULL, + (vm_offset_t) NULL, + offset, + cnt, + 0, + &local_error); +/* + if(kret == PAGER_ABSENT) { + Need to work out the defs here, 1 corresponds to PAGER_ABSENT + defined in bsd/vm/vm_pager.h However, we should not be including + that file here it is a layering violation. +*/ + if (kret == 1) { + int uplflags; + upl_t upl = NULL; + int count = 0; + kern_return_t kr; + + uplflags = (UPL_NO_SYNC | + UPL_CLEAN_IN_PLACE | + UPL_SET_INTERNAL); + count = 0; + kr = memory_object_upl_request(vnode_object->control_handle, + offset, cnt, + &upl, NULL, &count, uplflags); + if (kr == KERN_SUCCESS) { + upl_abort(upl, 0); + upl_deallocate(upl); + } else { + /* + * We couldn't gather the page list, probably + * because the memory object doesn't have a link + * to a VM object anymore (forced unmount, for + * example). Just return an error to the vm_fault() + * path and let it handle it. + */ } - cnt -= size; - offset += size; + + return KERN_FAILURE; } - if (error != 0) - return(KERN_FAILURE); - return(KERN_SUCCESS); + return KERN_SUCCESS; } @@ -668,7 +846,7 @@ vnode_pager_release_from_cache( int *cnt) { memory_object_free_from_cache( - &realhost, (int)&vnode_pager_workaround, cnt); + &realhost, &vnode_pager_workaround, cnt); } /* @@ -676,116 +854,41 @@ vnode_pager_release_from_cache( */ vnode_pager_t vnode_object_create( - vnode_port_t vp) + struct vnode *vp) { register vnode_pager_t vnode_object; vnode_object = (struct vnode_pager *) zalloc(vnode_pager_zone); if (vnode_object == VNODE_PAGER_NULL) return(VNODE_PAGER_NULL); - vnode_object->pager_handle = IP_NULL; - vnode_object->vm_obj_handle = IP_NULL; - vnode_object->vnode_handle = vp; - - return(vnode_object); -} - -/* - * - */ -void -vnode_port_hash_init(void) -{ - register vm_size_t size; - register int i; - - - size = (vm_size_t) sizeof(struct vnode_port_entry); - - vnode_port_hash_zone = zinit(size, - (vm_size_t) MAX_VNODE * size, - PAGE_SIZE, "vnode_pager port hash"); - - for (i = 0; i < VNODE_PORT_HASH_COUNT; i++) - queue_init(&vnode_port_hashtable[i]); - - simple_lock_init(&vnode_port_hash_lock,ETAP_NO_TRACE); -} - -/* - * - */ -void -vnode_port_hash_insert( - ipc_port_t name_port, - vnode_pager_t rec) -{ - register vnode_port_entry_t new_entry; - new_entry = (vnode_port_entry_t) zalloc(vnode_port_hash_zone); /* - * TODO: Delete the following check once MAX_VNODE is removed + * The vm_map call takes both named entry ports and raw memory + * objects in the same parameter. We need to make sure that + * vm_map does not see this object as a named entry port. So, + * we reserve the second word in the object for a fake ip_kotype + * setting - that will tell vm_map to use it as a memory object. */ - if (!new_entry) - panic("vnode_port_hash_insert: no space"); - new_entry->name = name_port; - new_entry->pager_rec = rec; - - simple_lock(&vnode_port_hash_lock); - queue_enter(&vnode_port_hashtable[vnode_port_hash(name_port)], - new_entry, vnode_port_entry_t, links); - simple_unlock(&vnode_port_hash_lock); + vnode_object->pager = &vnode_pager_workaround; + vnode_object->pager_ikot = IKOT_MEMORY_OBJECT; + vnode_object->ref_count = 1; + vnode_object->control_handle = MEMORY_OBJECT_CONTROL_NULL; + vnode_object->vnode_handle = vp; + + return(vnode_object); } /* * */ vnode_pager_t -vnode_port_hash_lookup( - ipc_port_t name_port) +vnode_pager_lookup( + memory_object_t name) { - register queue_t bucket; - register vnode_port_entry_t entry; - vnode_pager_t rec; - - bucket = (queue_t) &vnode_port_hashtable[vnode_port_hash(name_port)]; - - simple_lock(&vnode_port_hash_lock); - entry = (vnode_port_entry_t) queue_first(bucket); - while (!queue_end(bucket,&entry->links)) { - if (entry->name == name_port) { - rec = entry->pager_rec; - simple_unlock(&vnode_port_hash_lock); - return(rec); - } - entry = (vnode_port_entry_t)queue_next(&entry->links); - } - simple_unlock(&vnode_port_hash_lock); - return(VNODE_PAGER_NULL); -} + vnode_pager_t vnode_object; -/* - * - */ -void -vnode_port_hash_delete( - ipc_port_t name_port) -{ - register queue_t bucket; - register vnode_port_entry_t entry; - - bucket = (queue_t) &vnode_port_hashtable[vnode_port_hash(name_port)]; - - simple_lock(&vnode_port_hash_lock); - entry = (vnode_port_entry_t) queue_first(bucket); - while (!queue_end(bucket,&entry->links)) { - if (entry->name == name_port) { - queue_remove(bucket, entry, vnode_port_entry_t,links); - simple_unlock(&vnode_port_hash_lock); - zfree(vnode_port_hash_zone, (vm_offset_t) entry); - return; - } - entry = (vnode_port_entry_t)queue_next(&entry->links); - } - simple_unlock(&vnode_port_hash_lock); + vnode_object = (vnode_pager_t)name; + assert(vnode_object->pager == &vnode_pager_workaround); + return (vnode_object); } +