X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/9bccf70c0258c7cac2dcb80011b2a964d884c552..3e170ce000f1506b7b5d2c5c7faec85ceabb573d:/osfmk/vm/bsd_vm.c?ds=sidebyside diff --git a/osfmk/vm/bsd_vm.c b/osfmk/vm/bsd_vm.c index c7136d470..f7e485bde 100644 --- a/osfmk/vm/bsd_vm.c +++ b/osfmk/vm/bsd_vm.c @@ -1,42 +1,61 @@ /* - * Copyright (c) 2000-2001 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * The contents of this file constitute Original Code as defined in and - * are subject to the Apple Public Source License Version 1.1 (the - * "License"). You may not use this file except in compliance with the - * License. Please obtain a copy of the License at - * http://www.apple.com/publicsource and read it before using this file. + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. * - * This Original Code and all software distributed under the License are - * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the - * License for the specific language governing rights and limitations - * under the License. + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include -#include + #include -#include -#include -#include +#include +#include #include +#include #include #include #include +#include +#include + +#include +#include +#include + #include #include -#include + +#include +#include + +#include +#include #include #include +#include +#include -#include /* BSD VM COMPONENT INTERFACES */ int @@ -61,150 +80,79 @@ get_map_nentries( return(map->hdr.nentries); } -/* - * - */ -vm_offset_t -get_map_start( - vm_map_t map) +mach_vm_offset_t +mach_get_vm_start(vm_map_t map) { - return(vm_map_first_entry(map)->vme_start); + return( vm_map_first_entry(map)->vme_start); } -/* - * - */ -vm_offset_t -get_map_end( - vm_map_t map) +mach_vm_offset_t +mach_get_vm_end(vm_map_t map) { - return(vm_map_last_entry(map)->vme_end); + return( vm_map_last_entry(map)->vme_end); } /* * BSD VNODE PAGER */ -/* until component support available */ -int vnode_pager_workaround; - -typedef int vnode_port_t; +const struct memory_object_pager_ops vnode_pager_ops = { + vnode_pager_reference, + vnode_pager_deallocate, + vnode_pager_init, + vnode_pager_terminate, + vnode_pager_data_request, + vnode_pager_data_return, + vnode_pager_data_initialize, + vnode_pager_data_unlock, + vnode_pager_synchronize, + vnode_pager_map, + vnode_pager_last_unmap, + NULL, /* data_reclaim */ + "vnode pager" +}; typedef struct vnode_pager { - int *pager; /* pager workaround pointer */ - unsigned int pager_ikot; /* JMM: fake ip_kotype() */ + struct ipc_object_header pager_header; /* fake ip_kotype() */ + memory_object_pager_ops_t pager_ops; /* == &vnode_pager_ops */ unsigned int ref_count; /* reference count */ memory_object_control_t control_handle; /* mem object control handle */ - vnode_port_t vnode_handle; /* vnode handle */ + struct vnode *vnode_handle; /* vnode handle */ } *vnode_pager_t; +#define pager_ikot pager_header.io_bits + ipc_port_t -trigger_name_to_port( +trigger_name_to_port( /* forward */ mach_port_t); -void -vnode_pager_bootstrap( - void); - -void -vnode_pager_alloc_map( - void); - -memory_object_t -vnode_pager_setup( - vnode_port_t, - memory_object_t); - - -kern_return_t -vnode_pager_init( - memory_object_t, - memory_object_control_t, - vm_size_t); - -kern_return_t -vnode_pager_get_object_size( - memory_object_t, - memory_object_offset_t *); - kern_return_t -vnode_pager_data_request( - memory_object_t, - memory_object_offset_t, - vm_size_t, - vm_prot_t); - -kern_return_t -vnode_pager_data_return( - memory_object_t, - memory_object_offset_t, - vm_size_t, - boolean_t, - boolean_t); - -kern_return_t -vnode_pager_data_initialize( - memory_object_t, - memory_object_offset_t, - vm_size_t); - -void -vnode_pager_deallocate( - memory_object_t); - -kern_return_t -vnode_pager_terminate( - memory_object_t); - -kern_return_t -vnode_pager_cluster_read( +vnode_pager_cluster_read( /* forward */ vnode_pager_t, - vm_object_offset_t, + vm_object_offset_t, + vm_object_offset_t, + uint32_t, vm_size_t); void -vnode_pager_cluster_write( +vnode_pager_cluster_write( /* forward */ vnode_pager_t, vm_object_offset_t, - vm_size_t); - + vm_size_t, + vm_object_offset_t *, + int *, + int); -int -vnode_pagein( - vnode_port_t, - upl_t, - vm_offset_t, - vm_object_offset_t, - int, - int, - int *); -int -vnode_pageout( - vnode_port_t, - upl_t, - vm_offset_t, - vm_object_offset_t, - int, - int, - int *); - -vm_object_offset_t -vnode_pager_get_filesize( - vnode_port_t); vnode_pager_t -vnode_object_create( - vnode_port_t vp); +vnode_object_create( /* forward */ + struct vnode *); vnode_pager_t -vnode_pager_lookup( +vnode_pager_lookup( /* forward */ memory_object_t); -void -vnode_pager_release_from_cache( - int *cnt); - zone_t vnode_pager_zone; @@ -229,19 +177,26 @@ int pagerdebug=0; #define PAGER_DEBUG(LEVEL, A) #endif +extern int proc_resetpcontrol(int); + +#if DEVELOPMENT || DEBUG +extern unsigned long vm_cs_validated_resets; +#endif + /* - * Routine: macx_triggers + * Routine: mach_macx_triggers * Function: * Syscall interface to set the call backs for low and * high water marks. */ int -macx_triggers( - int hi_water, - int low_water, - int flags, - mach_port_t trigger_name) +mach_macx_triggers( + struct macx_triggers_args *args) { + int hi_water = args->hi_water; + int low_water = args->low_water; + int flags = args->flags; + mach_port_t trigger_name = args->alert_port; kern_return_t kr; memory_object_default_t default_pager; ipc_port_t trigger_port; @@ -252,6 +207,59 @@ macx_triggers( if(kr != KERN_SUCCESS) { return EINVAL; } + + if (((flags & SWAP_ENCRYPT_ON) && (flags & SWAP_ENCRYPT_OFF)) || + ((flags & SWAP_COMPACT_ENABLE) && (flags & SWAP_COMPACT_DISABLE))) { + /* can't have it both ways */ + return EINVAL; + } + + if (default_pager_init_flag == 0) { + start_def_pager(NULL); + default_pager_init_flag = 1; + } + + if (flags & SWAP_ENCRYPT_ON) { + /* ENCRYPTED SWAP: tell default_pager to encrypt */ + default_pager_triggers(default_pager, + 0, 0, + SWAP_ENCRYPT_ON, + IP_NULL); + } else if (flags & SWAP_ENCRYPT_OFF) { + /* ENCRYPTED SWAP: tell default_pager not to encrypt */ + default_pager_triggers(default_pager, + 0, 0, + SWAP_ENCRYPT_OFF, + IP_NULL); + } + + if (flags & USE_EMERGENCY_SWAP_FILE_FIRST) { + /* + * Time to switch to the emergency segment. + */ + return default_pager_triggers(default_pager, + 0, 0, + USE_EMERGENCY_SWAP_FILE_FIRST, + IP_NULL); + } + + if (flags & SWAP_FILE_CREATION_ERROR) { + /* + * For some reason, the dynamic pager failed to create a swap file. + */ + trigger_port = trigger_name_to_port(trigger_name); + if(trigger_port == NULL) { + return EINVAL; + } + /* trigger_port is locked and active */ + ipc_port_make_send_locked(trigger_port); + ip_unlock(trigger_port); + default_pager_triggers(default_pager, + 0, 0, + SWAP_FILE_CREATION_ERROR, + trigger_port); + } + if (flags & HI_WAT_ALERT) { trigger_port = trigger_name_to_port(trigger_name); if(trigger_port == NULL) { @@ -259,7 +267,7 @@ macx_triggers( } /* trigger_port is locked and active */ ipc_port_make_send_locked(trigger_port); - /* now unlocked */ + ip_unlock(trigger_port); default_pager_triggers(default_pager, hi_water, low_water, HI_WAT_ALERT, trigger_port); @@ -272,32 +280,56 @@ macx_triggers( } /* trigger_port is locked and active */ ipc_port_make_send_locked(trigger_port); - /* and now its unlocked */ + ip_unlock(trigger_port); default_pager_triggers(default_pager, hi_water, low_water, LO_WAT_ALERT, trigger_port); } + + if (flags & PROC_RESUME) { + + /* + * For this call, hi_water is used to pass in the pid of the process we want to resume + * or unthrottle. This is of course restricted to the superuser (checked inside of + * proc_resetpcontrol). + */ + + return proc_resetpcontrol(hi_water); + } + /* * Set thread scheduling priority and policy for the current thread * it is assumed for the time being that the thread setting the alert - * is the same one which will be servicing it. + * is the same one which will be servicing it. + * + * XXX This does not belong in the kernel XXX */ - { - struct policy_timeshare_base fifo_base; - struct policy_timeshare_limit fifo_limit; - policy_base_t base; - processor_set_t pset; - policy_limit_t limit; - - pset = (current_thread())->processor_set; - base = (policy_base_t) &fifo_base; - limit = (policy_limit_t) &fifo_limit; - fifo_limit.max_priority = fifo_base.base_priority = MAXPRI_STANDARD; - thread_set_policy((current_thread())->top_act, pset, POLICY_FIFO, base, POLICY_TIMESHARE_BASE_COUNT, limit, POLICY_TIMESHARE_LIMIT_COUNT); + if (flags & HI_WAT_ALERT) { + thread_precedence_policy_data_t pre; + thread_extended_policy_data_t ext; + + ext.timeshare = FALSE; + pre.importance = INT32_MAX; + + thread_policy_set(current_thread(), + THREAD_EXTENDED_POLICY, + (thread_policy_t)&ext, + THREAD_EXTENDED_POLICY_COUNT); + + thread_policy_set(current_thread(), + THREAD_PRECEDENCE_POLICY, + (thread_policy_t)&pre, + THREAD_PRECEDENCE_POLICY_COUNT); + + current_thread()->options |= TH_OPT_VMPRIV; } - current_thread()->vm_privilege = TRUE; + if (flags & (SWAP_COMPACT_DISABLE | SWAP_COMPACT_ENABLE)) { + return macx_backing_store_compaction(flags & (SWAP_COMPACT_DISABLE | SWAP_COMPACT_ENABLE)); + } + + return 0; } /* @@ -314,12 +346,183 @@ trigger_name_to_port( return (NULL); space = current_space(); - if(ipc_port_translate_receive(space, (mach_port_name_t)trigger_name, + if(ipc_port_translate_receive(space, CAST_MACH_PORT_TO_NAME(trigger_name), &trigger_port) != KERN_SUCCESS) return (NULL); return trigger_port; } + +extern int uiomove64(addr64_t, int, void *); +#define MAX_RUN 32 + +int +memory_object_control_uiomove( + memory_object_control_t control, + memory_object_offset_t offset, + void * uio, + int start_offset, + int io_requested, + int mark_dirty, + int take_reference) +{ + vm_object_t object; + vm_page_t dst_page; + int xsize; + int retval = 0; + int cur_run; + int cur_needed; + int i; + int orig_offset; + vm_page_t page_run[MAX_RUN]; + + object = memory_object_control_to_vm_object(control); + if (object == VM_OBJECT_NULL) { + return (0); + } + assert(!object->internal); + + vm_object_lock(object); + + if (mark_dirty && object->copy != VM_OBJECT_NULL) { + /* + * We can't modify the pages without honoring + * copy-on-write obligations first, so fall off + * this optimized path and fall back to the regular + * path. + */ + vm_object_unlock(object); + return 0; + } + orig_offset = start_offset; + + while (io_requested && retval == 0) { + + cur_needed = (start_offset + io_requested + (PAGE_SIZE - 1)) / PAGE_SIZE; + + if (cur_needed > MAX_RUN) + cur_needed = MAX_RUN; + + for (cur_run = 0; cur_run < cur_needed; ) { + + if ((dst_page = vm_page_lookup(object, offset)) == VM_PAGE_NULL) + break; + + + if (dst_page->busy || dst_page->cleaning) { + /* + * someone else is playing with the page... if we've + * already collected pages into this run, go ahead + * and process now, we can't block on this + * page while holding other pages in the BUSY state + * otherwise we will wait + */ + if (cur_run) + break; + PAGE_SLEEP(object, dst_page, THREAD_UNINT); + continue; + } + if (dst_page->laundry) { + dst_page->pageout = FALSE; + + vm_pageout_steal_laundry(dst_page, FALSE); + } + /* + * this routine is only called when copying + * to/from real files... no need to consider + * encrypted swap pages + */ + assert(!dst_page->encrypted); + + if (mark_dirty) { + SET_PAGE_DIRTY(dst_page, FALSE); + if (dst_page->cs_validated && + !dst_page->cs_tainted) { + /* + * CODE SIGNING: + * We're modifying a code-signed + * page: force revalidate + */ + dst_page->cs_validated = FALSE; +#if DEVELOPMENT || DEBUG + vm_cs_validated_resets++; +#endif + pmap_disconnect(dst_page->phys_page); + } + } + dst_page->busy = TRUE; + + page_run[cur_run++] = dst_page; + + offset += PAGE_SIZE_64; + } + if (cur_run == 0) + /* + * we hit a 'hole' in the cache or + * a page we don't want to try to handle, + * so bail at this point + * we'll unlock the object below + */ + break; + vm_object_unlock(object); + + for (i = 0; i < cur_run; i++) { + + dst_page = page_run[i]; + + if ((xsize = PAGE_SIZE - start_offset) > io_requested) + xsize = io_requested; + + if ( (retval = uiomove64((addr64_t)(((addr64_t)(dst_page->phys_page) << PAGE_SHIFT) + start_offset), xsize, uio)) ) + break; + + io_requested -= xsize; + start_offset = 0; + } + vm_object_lock(object); + + /* + * if we have more than 1 page to work on + * in the current run, or the original request + * started at offset 0 of the page, or we're + * processing multiple batches, we will move + * the pages to the tail of the inactive queue + * to implement an LRU for read/write accesses + * + * the check for orig_offset == 0 is there to + * mitigate the cost of small (< page_size) requests + * to the same page (this way we only move it once) + */ + if (take_reference && (cur_run > 1 || orig_offset == 0)) { + + vm_page_lockspin_queues(); + + for (i = 0; i < cur_run; i++) + vm_page_lru(page_run[i]); + + vm_page_unlock_queues(); + } + for (i = 0; i < cur_run; i++) { + dst_page = page_run[i]; + + /* + * someone is explicitly referencing this page... + * update clustered and speculative state + * + */ + if (dst_page->clustered) + VM_PAGE_CONSUME_CLUSTERED(dst_page); + + PAGE_WAKEUP_DONE(dst_page); + } + orig_offset = 0; + } + vm_object_unlock(object); + + return (retval); +} + + /* * */ @@ -331,6 +534,14 @@ vnode_pager_bootstrap(void) size = (vm_size_t) sizeof(struct vnode_pager); vnode_pager_zone = zinit(size, (vm_size_t) MAX_VNODE*size, PAGE_SIZE, "vnode pager structures"); + zone_change(vnode_pager_zone, Z_CALLERACCT, FALSE); + zone_change(vnode_pager_zone, Z_NOENCRYPT, TRUE); + + +#if CONFIG_CODE_DECRYPTION + apple_protect_pager_bootstrap(); +#endif /* CONFIG_CODE_DECRYPTION */ + swapfile_pager_bootstrap(); return; } @@ -339,8 +550,8 @@ vnode_pager_bootstrap(void) */ memory_object_t vnode_pager_setup( - vnode_port_t vp, - memory_object_t pager) + struct vnode *vp, + __unused memory_object_t pager) { vnode_pager_t vnode_object; @@ -356,14 +567,17 @@ vnode_pager_setup( kern_return_t vnode_pager_init(memory_object_t mem_obj, memory_object_control_t control, - vm_size_t pg_size) +#if !DEBUG + __unused +#endif + memory_object_cluster_size_t pg_size) { vnode_pager_t vnode_object; kern_return_t kr; memory_object_attr_info_data_t attributes; - PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %x, %x, %x\n", pager, pager_request, pg_size)); + PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %p, %p, %lx\n", mem_obj, control, (unsigned long)pg_size)); if (control == MEMORY_OBJECT_CONTROL_NULL) return KERN_INVALID_ARGUMENT; @@ -371,6 +585,7 @@ vnode_pager_init(memory_object_t mem_obj, vnode_object = vnode_pager_lookup(mem_obj); memory_object_control_reference(control); + vnode_object->control_handle = control; attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; @@ -397,38 +612,92 @@ kern_return_t vnode_pager_data_return( memory_object_t mem_obj, memory_object_offset_t offset, - vm_size_t data_cnt, - boolean_t dirty, - boolean_t kernel_copy) + memory_object_cluster_size_t data_cnt, + memory_object_offset_t *resid_offset, + int *io_error, + __unused boolean_t dirty, + __unused boolean_t kernel_copy, + int upl_flags) { register vnode_pager_t vnode_object; vnode_object = vnode_pager_lookup(mem_obj); - vnode_pager_cluster_write(vnode_object, offset, data_cnt); + vnode_pager_cluster_write(vnode_object, offset, data_cnt, resid_offset, io_error, upl_flags); return KERN_SUCCESS; } kern_return_t vnode_pager_data_initialize( - memory_object_t mem_obj, - memory_object_offset_t offset, - vm_size_t data_cnt) + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_cluster_size_t data_cnt) { + panic("vnode_pager_data_initialize"); return KERN_FAILURE; } kern_return_t vnode_pager_data_unlock( - memory_object_t mem_obj, - memory_object_offset_t offset, - vm_size_t size, - vm_prot_t desired_access) + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_size_t size, + __unused vm_prot_t desired_access) { return KERN_FAILURE; } +kern_return_t +vnode_pager_get_isinuse( + memory_object_t mem_obj, + uint32_t *isinuse) +{ + vnode_pager_t vnode_object; + + if (mem_obj->mo_pager_ops != &vnode_pager_ops) { + *isinuse = 1; + return KERN_INVALID_ARGUMENT; + } + + vnode_object = vnode_pager_lookup(mem_obj); + + *isinuse = vnode_pager_isinuse(vnode_object->vnode_handle); + return KERN_SUCCESS; +} + +kern_return_t +vnode_pager_get_throttle_io_limit( + memory_object_t mem_obj, + uint32_t *limit) +{ + vnode_pager_t vnode_object; + + if (mem_obj->mo_pager_ops != &vnode_pager_ops) + return KERN_INVALID_ARGUMENT; + + vnode_object = vnode_pager_lookup(mem_obj); + + (void)vnode_pager_return_throttle_io_limit(vnode_object->vnode_handle, limit); + return KERN_SUCCESS; +} + +kern_return_t +vnode_pager_get_isSSD( + memory_object_t mem_obj, + boolean_t *isSSD) +{ + vnode_pager_t vnode_object; + + if (mem_obj->mo_pager_ops != &vnode_pager_ops) + return KERN_INVALID_ARGUMENT; + + vnode_object = vnode_pager_lookup(mem_obj); + + *isSSD = vnode_pager_isSSD(vnode_object->vnode_handle); + return KERN_SUCCESS; +} + kern_return_t vnode_pager_get_object_size( memory_object_t mem_obj, @@ -436,12 +705,98 @@ vnode_pager_get_object_size( { vnode_pager_t vnode_object; + if (mem_obj->mo_pager_ops != &vnode_pager_ops) { + *length = 0; + return KERN_INVALID_ARGUMENT; + } + vnode_object = vnode_pager_lookup(mem_obj); *length = vnode_pager_get_filesize(vnode_object->vnode_handle); return KERN_SUCCESS; } +kern_return_t +vnode_pager_get_object_name( + memory_object_t mem_obj, + char *pathname, + vm_size_t pathname_len, + char *filename, + vm_size_t filename_len, + boolean_t *truncated_path_p) +{ + vnode_pager_t vnode_object; + + if (mem_obj->mo_pager_ops != &vnode_pager_ops) { + return KERN_INVALID_ARGUMENT; + } + + vnode_object = vnode_pager_lookup(mem_obj); + + return vnode_pager_get_name(vnode_object->vnode_handle, + pathname, + pathname_len, + filename, + filename_len, + truncated_path_p); +} + +kern_return_t +vnode_pager_get_object_mtime( + memory_object_t mem_obj, + struct timespec *mtime, + struct timespec *cs_mtime) +{ + vnode_pager_t vnode_object; + + if (mem_obj->mo_pager_ops != &vnode_pager_ops) { + return KERN_INVALID_ARGUMENT; + } + + vnode_object = vnode_pager_lookup(mem_obj); + + return vnode_pager_get_mtime(vnode_object->vnode_handle, + mtime, + cs_mtime); +} + +kern_return_t +vnode_pager_get_object_cs_blobs( + memory_object_t mem_obj, + void **blobs) +{ + vnode_pager_t vnode_object; + + if (mem_obj == MEMORY_OBJECT_NULL || + mem_obj->mo_pager_ops != &vnode_pager_ops) { + return KERN_INVALID_ARGUMENT; + } + + vnode_object = vnode_pager_lookup(mem_obj); + + return vnode_pager_get_cs_blobs(vnode_object->vnode_handle, + blobs); +} + +#if CHECK_CS_VALIDATION_BITMAP +kern_return_t +vnode_pager_cs_check_validation_bitmap( + memory_object_t mem_obj, + memory_object_offset_t offset, + int optype ) +{ + vnode_pager_t vnode_object; + + if (mem_obj == MEMORY_OBJECT_NULL || + mem_obj->mo_pager_ops != &vnode_pager_ops) { + return KERN_INVALID_ARGUMENT; + } + + vnode_object = vnode_pager_lookup(mem_obj); + return ubc_cs_check_validation_bitmap( vnode_object->vnode_handle, offset, optype ); +} +#endif /* CHECK_CS_VALIDATION_BITMAP */ + /* * */ @@ -449,20 +804,27 @@ kern_return_t vnode_pager_data_request( memory_object_t mem_obj, memory_object_offset_t offset, - vm_size_t length, - vm_prot_t protection_required) + __unused memory_object_cluster_size_t length, + __unused vm_prot_t desired_access, + memory_object_fault_info_t fault_info) { - register vnode_pager_t vnode_object; - - PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x\n", mem_obj, offset, length, protection_required)); + vnode_pager_t vnode_object; + memory_object_offset_t base_offset; + vm_size_t size; + uint32_t io_streaming = 0; vnode_object = vnode_pager_lookup(mem_obj); - PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, vnode_object %x\n", mem_obj, offset, length, protection_required, vnode_object)); - - vnode_pager_cluster_read(vnode_object, offset, length); + size = MAX_UPL_TRANSFER_BYTES; + base_offset = offset; - return KERN_SUCCESS; + if (memory_object_cluster_size(vnode_object->control_handle, &base_offset, &size, &io_streaming, fault_info) != KERN_SUCCESS) + size = PAGE_SIZE; + + assert(offset >= base_offset && + offset < base_offset + size); + + return vnode_pager_cluster_read(vnode_object, base_offset, offset, io_streaming, size); } /* @@ -489,15 +851,15 @@ vnode_pager_deallocate( { register vnode_pager_t vnode_object; - PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %x\n", mem_obj)); + PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %p\n", mem_obj)); vnode_object = vnode_pager_lookup(mem_obj); if (hw_atomic_sub(&vnode_object->ref_count, 1) == 0) { - if (vnode_object->vnode_handle != (vnode_port_t) NULL) { + if (vnode_object->vnode_handle != NULL) { vnode_pager_vrele(vnode_object->vnode_handle); } - zfree(vnode_pager_zone, (vm_offset_t) vnode_object); + zfree(vnode_pager_zone, vnode_object); } return; } @@ -507,9 +869,12 @@ vnode_pager_deallocate( */ kern_return_t vnode_pager_terminate( +#if !DEBUG + __unused +#endif memory_object_t mem_obj) { - PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x\n", mem_obj)); + PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %p\n", mem_obj)); return(KERN_SUCCESS); } @@ -521,12 +886,12 @@ kern_return_t vnode_pager_synchronize( memory_object_t mem_obj, memory_object_offset_t offset, - vm_size_t length, - vm_sync_t sync_flags) + memory_object_size_t length, + __unused vm_sync_t sync_flags) { register vnode_pager_t vnode_object; - PAGER_DEBUG(PAGER_ALL, ("vnode_pager_synchronize: %x\n", mem_obj)); + PAGER_DEBUG(PAGER_ALL, ("vnode_pager_synchronize: %p\n", mem_obj)); vnode_object = vnode_pager_lookup(mem_obj); @@ -539,12 +904,36 @@ vnode_pager_synchronize( * */ kern_return_t -vnode_pager_unmap( +vnode_pager_map( + memory_object_t mem_obj, + vm_prot_t prot) +{ + vnode_pager_t vnode_object; + int ret; + kern_return_t kr; + + PAGER_DEBUG(PAGER_ALL, ("vnode_pager_map: %p %x\n", mem_obj, prot)); + + vnode_object = vnode_pager_lookup(mem_obj); + + ret = ubc_map(vnode_object->vnode_handle, prot); + + if (ret != 0) { + kr = KERN_FAILURE; + } else { + kr = KERN_SUCCESS; + } + + return kr; +} + +kern_return_t +vnode_pager_last_unmap( memory_object_t mem_obj) { register vnode_pager_t vnode_object; - PAGER_DEBUG(PAGER_ALL, ("vnode_pager_unmap: %x\n", mem_obj)); + PAGER_DEBUG(PAGER_ALL, ("vnode_pager_last_unmap: %p\n", mem_obj)); vnode_object = vnode_pager_lookup(mem_obj); @@ -553,6 +942,7 @@ vnode_pager_unmap( } + /* * */ @@ -560,66 +950,75 @@ void vnode_pager_cluster_write( vnode_pager_t vnode_object, vm_object_offset_t offset, - vm_size_t cnt) + vm_size_t cnt, + vm_object_offset_t * resid_offset, + int * io_error, + int upl_flags) { - int error = 0; - int local_error = 0; - int kret; - int size; + vm_size_t size; + int errno; - if (cnt & PAGE_MASK) { - panic("vs_cluster_write: cnt not a multiple of PAGE_SIZE"); - } - size = (cnt < (PAGE_SIZE*32)) ? cnt : (PAGE_SIZE*32); /* effective min */ - - while (cnt) { + if (upl_flags & UPL_MSYNC) { - kret = vnode_pageout(vnode_object->vnode_handle, - (upl_t )NULL, (vm_offset_t)NULL, - offset, size, 0, &local_error); -/* - if(kret == PAGER_ABSENT) { - Need to work out the defs here, 1 corresponds to - PAGER_ABSENT defined in bsd/vm/vm_pager.h However, - we should not be including that file here it is a - layering violation. -*/ - if(kret == 1) { - int uplflags; - upl_t upl = NULL; - int count = 0; - kern_return_t kr; - - uplflags = (UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | - UPL_SET_INTERNAL | UPL_COPYOUT_FROM); - count = 0; - kr = memory_object_upl_request( - vnode_object->control_handle, - offset, size, &upl, NULL, &count, uplflags); - if(kr != KERN_SUCCESS) { - panic("vnode_pager_cluster_write: upl request failed\n"); - } - upl_abort(upl, 0); - upl_deallocate(upl); + upl_flags |= UPL_VNODE_PAGER; - error = 0; - local_error = 0; - } + if ( (upl_flags & UPL_IOSYNC) && io_error) + upl_flags |= UPL_KEEPCACHED; + + while (cnt) { + size = (cnt < MAX_UPL_TRANSFER_BYTES) ? cnt : MAX_UPL_TRANSFER_BYTES; /* effective max */ - if (local_error != 0) { - error = local_error; - local_error = 0; + assert((upl_size_t) size == size); + vnode_pageout(vnode_object->vnode_handle, + NULL, (upl_offset_t)0, offset, (upl_size_t)size, upl_flags, &errno); + + if ( (upl_flags & UPL_KEEPCACHED) ) { + if ( (*io_error = errno) ) + break; + } + cnt -= size; + offset += size; + } + if (resid_offset) + *resid_offset = offset; + + } else { + vm_object_offset_t vnode_size; + vm_object_offset_t base_offset; + + /* + * this is the pageout path + */ + vnode_size = vnode_pager_get_filesize(vnode_object->vnode_handle); + + if (vnode_size > (offset + PAGE_SIZE)) { + /* + * preset the maximum size of the cluster + * and put us on a nice cluster boundary... + * and then clip the size to insure we + * don't request past the end of the underlying file + */ + size = MAX_UPL_TRANSFER_BYTES; + base_offset = offset & ~((signed)(size - 1)); + + if ((base_offset + size) > vnode_size) + size = round_page(((vm_size_t)(vnode_size - base_offset))); + } else { + /* + * we've been requested to page out a page beyond the current + * end of the 'file'... don't try to cluster in this case... + * we still need to send this page through because it might + * be marked precious and the underlying filesystem may need + * to do something with it (besides page it out)... + */ + base_offset = offset; + size = PAGE_SIZE; } - cnt -= size; - offset += size; - size = (cnt < (PAGE_SIZE*32)) ? cnt : (PAGE_SIZE*32); /* effective min */ + assert((upl_size_t) size == size); + vnode_pageout(vnode_object->vnode_handle, + NULL, (upl_offset_t)(offset - base_offset), base_offset, (upl_size_t) size, + (upl_flags & UPL_IOSYNC) | UPL_VNODE_PAGER, NULL); } -#if 0 - if (error != 0) - return(KERN_FAILURE); - - return(KERN_SUCCESS); -#endif /* 0 */ } @@ -629,49 +1028,65 @@ vnode_pager_cluster_write( kern_return_t vnode_pager_cluster_read( vnode_pager_t vnode_object, + vm_object_offset_t base_offset, vm_object_offset_t offset, + uint32_t io_streaming, vm_size_t cnt) { - int error = 0; int local_error = 0; int kret; + int flags = 0; - if(cnt & PAGE_MASK) { - panic("vs_cluster_read: cnt not a multiple of PAGE_SIZE"); - } + assert(! (cnt & PAGE_MASK)); + + if (io_streaming) + flags |= UPL_IOSTREAMING; - kret = vnode_pagein(vnode_object->vnode_handle, (upl_t)NULL, (vm_offset_t)NULL, offset, cnt, 0, &local_error); + assert((upl_size_t) cnt == cnt); + kret = vnode_pagein(vnode_object->vnode_handle, + (upl_t) NULL, + (upl_offset_t) (offset - base_offset), + base_offset, + (upl_size_t) cnt, + flags, + &local_error); /* if(kret == PAGER_ABSENT) { Need to work out the defs here, 1 corresponds to PAGER_ABSENT defined in bsd/vm/vm_pager.h However, we should not be including that file here it is a layering violation. */ - if(kret == 1) { - int uplflags; - upl_t upl = NULL; - int count = 0; - kern_return_t kr; - - uplflags = (UPL_NO_SYNC | - UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL); - count = 0; - kr = memory_object_upl_request( - vnode_object->control_handle, offset, cnt, - &upl, NULL, &count, uplflags); - if(kr != KERN_SUCCESS) { - panic("vnode_pager_cluster_read: upl request failed\n"); - } + if (kret == 1) { + int uplflags; + upl_t upl = NULL; + unsigned int count = 0; + kern_return_t kr; + + uplflags = (UPL_NO_SYNC | + UPL_CLEAN_IN_PLACE | + UPL_SET_INTERNAL); + count = 0; + assert((upl_size_t) cnt == cnt); + kr = memory_object_upl_request(vnode_object->control_handle, + base_offset, (upl_size_t) cnt, + &upl, NULL, &count, uplflags); + if (kr == KERN_SUCCESS) { upl_abort(upl, 0); upl_deallocate(upl); + } else { + /* + * We couldn't gather the page list, probably + * because the memory object doesn't have a link + * to a VM object anymore (forced unmount, for + * example). Just return an error to the vm_fault() + * path and let it handle it. + */ + } - error = 1; + return KERN_FAILURE; } - if (error != 0) - return(KERN_FAILURE); - - return(KERN_SUCCESS); + return KERN_SUCCESS; } @@ -684,7 +1099,7 @@ vnode_pager_release_from_cache( int *cnt) { memory_object_free_from_cache( - &realhost, &vnode_pager_workaround, cnt); + &realhost, &vnode_pager_ops, cnt); } /* @@ -692,7 +1107,7 @@ vnode_pager_release_from_cache( */ vnode_pager_t vnode_object_create( - vnode_port_t vp) + struct vnode *vp) { register vnode_pager_t vnode_object; @@ -704,10 +1119,10 @@ vnode_object_create( * The vm_map call takes both named entry ports and raw memory * objects in the same parameter. We need to make sure that * vm_map does not see this object as a named entry port. So, - * we reserve the second word in the object for a fake ip_kotype + * we reserve the first word in the object for a fake ip_kotype * setting - that will tell vm_map to use it as a memory object. */ - vnode_object->pager = &vnode_pager_workaround; + vnode_object->pager_ops = &vnode_pager_ops; vnode_object->pager_ikot = IKOT_MEMORY_OBJECT; vnode_object->ref_count = 1; vnode_object->control_handle = MEMORY_OBJECT_CONTROL_NULL; @@ -726,7 +1141,353 @@ vnode_pager_lookup( vnode_pager_t vnode_object; vnode_object = (vnode_pager_t)name; - assert(vnode_object->pager == &vnode_pager_workaround); + assert(vnode_object->pager_ops == &vnode_pager_ops); return (vnode_object); } + +/*********************** proc_info implementation *************/ + +#include + +static int fill_vnodeinfoforaddr( vm_map_entry_t entry, uintptr_t * vnodeaddr, uint32_t * vid); + + +int +fill_procregioninfo(task_t task, uint64_t arg, struct proc_regioninfo_internal *pinfo, uintptr_t *vnodeaddr, uint32_t *vid) +{ + + vm_map_t map; + vm_map_offset_t address = (vm_map_offset_t )arg; + vm_map_entry_t tmp_entry; + vm_map_entry_t entry; + vm_map_offset_t start; + vm_region_extended_info_data_t extended; + vm_region_top_info_data_t top; + + task_lock(task); + map = task->map; + if (map == VM_MAP_NULL) + { + task_unlock(task); + return(0); + } + vm_map_reference(map); + task_unlock(task); + + vm_map_lock_read(map); + + start = address; + if (!vm_map_lookup_entry(map, start, &tmp_entry)) { + if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) { + vm_map_unlock_read(map); + vm_map_deallocate(map); + return(0); + } + } else { + entry = tmp_entry; + } + + start = entry->vme_start; + + pinfo->pri_offset = VME_OFFSET(entry); + pinfo->pri_protection = entry->protection; + pinfo->pri_max_protection = entry->max_protection; + pinfo->pri_inheritance = entry->inheritance; + pinfo->pri_behavior = entry->behavior; + pinfo->pri_user_wired_count = entry->user_wired_count; + pinfo->pri_user_tag = VME_ALIAS(entry); + + if (entry->is_sub_map) { + pinfo->pri_flags |= PROC_REGION_SUBMAP; + } else { + if (entry->is_shared) + pinfo->pri_flags |= PROC_REGION_SHARED; + } + + + extended.protection = entry->protection; + extended.user_tag = VME_ALIAS(entry); + extended.pages_resident = 0; + extended.pages_swapped_out = 0; + extended.pages_shared_now_private = 0; + extended.pages_dirtied = 0; + extended.external_pager = 0; + extended.shadow_depth = 0; + + vm_map_region_walk(map, start, entry, VME_OFFSET(entry), entry->vme_end - start, &extended); + + if (extended.external_pager && extended.ref_count == 2 && extended.share_mode == SM_SHARED) + extended.share_mode = SM_PRIVATE; + + top.private_pages_resident = 0; + top.shared_pages_resident = 0; + vm_map_region_top_walk(entry, &top); + + + pinfo->pri_pages_resident = extended.pages_resident; + pinfo->pri_pages_shared_now_private = extended.pages_shared_now_private; + pinfo->pri_pages_swapped_out = extended.pages_swapped_out; + pinfo->pri_pages_dirtied = extended.pages_dirtied; + pinfo->pri_ref_count = extended.ref_count; + pinfo->pri_shadow_depth = extended.shadow_depth; + pinfo->pri_share_mode = extended.share_mode; + + pinfo->pri_private_pages_resident = top.private_pages_resident; + pinfo->pri_shared_pages_resident = top.shared_pages_resident; + pinfo->pri_obj_id = top.obj_id; + + pinfo->pri_address = (uint64_t)start; + pinfo->pri_size = (uint64_t)(entry->vme_end - start); + pinfo->pri_depth = 0; + + if ((vnodeaddr != 0) && (entry->is_sub_map == 0)) { + *vnodeaddr = (uintptr_t)0; + + if (fill_vnodeinfoforaddr(entry, vnodeaddr, vid) ==0) { + vm_map_unlock_read(map); + vm_map_deallocate(map); + return(1); + } + } + + vm_map_unlock_read(map); + vm_map_deallocate(map); + return(1); +} + +int +fill_procregioninfo_onlymappedvnodes(task_t task, uint64_t arg, struct proc_regioninfo_internal *pinfo, uintptr_t *vnodeaddr, uint32_t *vid) +{ + + vm_map_t map; + vm_map_offset_t address = (vm_map_offset_t )arg; + vm_map_entry_t tmp_entry; + vm_map_entry_t entry; + + task_lock(task); + map = task->map; + if (map == VM_MAP_NULL) + { + task_unlock(task); + return(0); + } + vm_map_reference(map); + task_unlock(task); + + vm_map_lock_read(map); + + if (!vm_map_lookup_entry(map, address, &tmp_entry)) { + if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) { + vm_map_unlock_read(map); + vm_map_deallocate(map); + return(0); + } + } else { + entry = tmp_entry; + } + + while (entry != vm_map_to_entry(map)) { + *vnodeaddr = 0; + *vid = 0; + + if (entry->is_sub_map == 0) { + if (fill_vnodeinfoforaddr(entry, vnodeaddr, vid)) { + + pinfo->pri_offset = VME_OFFSET(entry); + pinfo->pri_protection = entry->protection; + pinfo->pri_max_protection = entry->max_protection; + pinfo->pri_inheritance = entry->inheritance; + pinfo->pri_behavior = entry->behavior; + pinfo->pri_user_wired_count = entry->user_wired_count; + pinfo->pri_user_tag = VME_ALIAS(entry); + + if (entry->is_shared) + pinfo->pri_flags |= PROC_REGION_SHARED; + + pinfo->pri_pages_resident = 0; + pinfo->pri_pages_shared_now_private = 0; + pinfo->pri_pages_swapped_out = 0; + pinfo->pri_pages_dirtied = 0; + pinfo->pri_ref_count = 0; + pinfo->pri_shadow_depth = 0; + pinfo->pri_share_mode = 0; + + pinfo->pri_private_pages_resident = 0; + pinfo->pri_shared_pages_resident = 0; + pinfo->pri_obj_id = 0; + + pinfo->pri_address = (uint64_t)entry->vme_start; + pinfo->pri_size = (uint64_t)(entry->vme_end - entry->vme_start); + pinfo->pri_depth = 0; + + vm_map_unlock_read(map); + vm_map_deallocate(map); + return(1); + } + } + + /* Keep searching for a vnode-backed mapping */ + entry = entry->vme_next; + } + + vm_map_unlock_read(map); + vm_map_deallocate(map); + return(0); +} + +static int +fill_vnodeinfoforaddr( + vm_map_entry_t entry, + uintptr_t * vnodeaddr, + uint32_t * vid) +{ + vm_object_t top_object, object; + memory_object_t memory_object; + memory_object_pager_ops_t pager_ops; + kern_return_t kr; + int shadow_depth; + + + if (entry->is_sub_map) { + return(0); + } else { + /* + * The last object in the shadow chain has the + * relevant pager information. + */ + top_object = VME_OBJECT(entry); + if (top_object == VM_OBJECT_NULL) { + object = VM_OBJECT_NULL; + shadow_depth = 0; + } else { + vm_object_lock(top_object); + for (object = top_object, shadow_depth = 0; + object->shadow != VM_OBJECT_NULL; + object = object->shadow, shadow_depth++) { + vm_object_lock(object->shadow); + vm_object_unlock(object); + } + } + } + + if (object == VM_OBJECT_NULL) { + return(0); + } else if (object->internal) { + vm_object_unlock(object); + return(0); + } else if (! object->pager_ready || + object->terminating || + ! object->alive) { + vm_object_unlock(object); + return(0); + } else { + memory_object = object->pager; + pager_ops = memory_object->mo_pager_ops; + if (pager_ops == &vnode_pager_ops) { + kr = vnode_pager_get_object_vnode( + memory_object, + vnodeaddr, vid); + if (kr != KERN_SUCCESS) { + vm_object_unlock(object); + return(0); + } + } else { + vm_object_unlock(object); + return(0); + } + } + vm_object_unlock(object); + return(1); +} + +kern_return_t +vnode_pager_get_object_vnode ( + memory_object_t mem_obj, + uintptr_t * vnodeaddr, + uint32_t * vid) +{ + vnode_pager_t vnode_object; + + vnode_object = vnode_pager_lookup(mem_obj); + if (vnode_object->vnode_handle) { + *vnodeaddr = (uintptr_t)vnode_object->vnode_handle; + *vid = (uint32_t)vnode_vid((void *)vnode_object->vnode_handle); + + return(KERN_SUCCESS); + } + + return(KERN_FAILURE); +} + +#if CONFIG_IOSCHED +kern_return_t +vnode_pager_get_object_devvp( + memory_object_t mem_obj, + uintptr_t *devvp) +{ + struct vnode *vp; + uint32_t vid; + + if(vnode_pager_get_object_vnode(mem_obj, (uintptr_t *)&vp, (uint32_t *)&vid) != KERN_SUCCESS) + return (KERN_FAILURE); + *devvp = (uintptr_t)vnode_mountdevvp(vp); + if (*devvp) + return (KERN_SUCCESS); + return (KERN_FAILURE); +} +#endif + +/* + * Find the underlying vnode object for the given vm_map_entry. If found, return with the + * object locked, otherwise return NULL with nothing locked. + */ + +vm_object_t +find_vnode_object( + vm_map_entry_t entry +) +{ + vm_object_t top_object, object; + memory_object_t memory_object; + memory_object_pager_ops_t pager_ops; + + if (!entry->is_sub_map) { + + /* + * The last object in the shadow chain has the + * relevant pager information. + */ + + top_object = VME_OBJECT(entry); + + if (top_object) { + vm_object_lock(top_object); + + for (object = top_object; object->shadow != VM_OBJECT_NULL; object = object->shadow) { + vm_object_lock(object->shadow); + vm_object_unlock(object); + } + + if (object && !object->internal && object->pager_ready && !object->terminating && + object->alive) { + memory_object = object->pager; + pager_ops = memory_object->mo_pager_ops; + + /* + * If this object points to the vnode_pager_ops, then we found what we're + * looking for. Otherwise, this vm_map_entry doesn't have an underlying + * vnode and so we fall through to the bottom and return NULL. + */ + + if (pager_ops == &vnode_pager_ops) + return object; /* we return with the object locked */ + } + + vm_object_unlock(object); + } + + } + + return(VM_OBJECT_NULL); +}