X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/ff6e181ae92fc6f1e89841290f461d1f2f9badd9..36401178fd6817c043cc00b0c00c7f723e58efae:/osfmk/vm/memory_object.c diff --git a/osfmk/vm/memory_object.c b/osfmk/vm/memory_object.c index 42e6df398..c84e776c6 100644 --- a/osfmk/vm/memory_object.c +++ b/osfmk/vm/memory_object.c @@ -1,14 +1,19 @@ /* - * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER @@ -18,7 +23,7 @@ * Please see the License for the specific language governing rights and * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ @@ -95,6 +100,7 @@ #include /* For pmap_clear_modify */ #include /* For kernel_map, vm_move */ #include /* For vm_map_pageable */ +#include /* Needed by some vm_page.h macros */ #if MACH_PAGEMAP #include @@ -104,7 +110,6 @@ memory_object_default_t memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL; -vm_size_t memory_manager_default_cluster = 0; decl_mutex_data(, memory_manager_default_lock) @@ -200,37 +205,20 @@ memory_object_lock_page( assert(!m->fictitious); - if (m->wire_count != 0) { - /* - * If no change would take place - * anyway, return successfully. - * - * No change means: - * Not flushing AND - * No change to page lock [2 checks] AND - * Should not return page - * - * XXX This doesn't handle sending a copy of a wired - * XXX page to the pager, but that will require some - * XXX significant surgery. - */ - if (!should_flush && - (m->page_lock == prot || prot == VM_PROT_NO_CHANGE) && - ! memory_object_should_return_page(m, should_return)) { - - /* - * Restart page unlock requests, - * even though no change took place. - * [Memory managers may be expecting - * to see new requests.] - */ - m->unlock_request = VM_PROT_NONE; - PAGE_WAKEUP(m); + /* + * If the page is wired, just clean or return the page if needed. + * Wired pages don't get flushed or disconnected from the pmap. + */ - return(MEMORY_OBJECT_LOCK_RESULT_DONE); + if (m->wire_count != 0) { + if (memory_object_should_return_page(m, should_return)) { + if (m->dirty) + return(MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN); + else + return(MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN); } - return(MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK); + return(MEMORY_OBJECT_LOCK_RESULT_DONE); } /* @@ -250,36 +238,14 @@ memory_object_lock_page( */ if (prot != VM_PROT_NO_CHANGE) { - if ((m->page_lock ^ prot) & prot) { - pmap_page_protect(m->phys_page, VM_PROT_ALL & ~prot); - } -#if 0 - /* code associated with the vestigial - * memory_object_data_unlock - */ - m->page_lock = prot; - m->lock_supplied = TRUE; - if (prot != VM_PROT_NONE) - m->unusual = TRUE; - else - m->unusual = FALSE; - - /* - * Restart any past unlock requests, even if no - * change resulted. If the manager explicitly - * requested no protection change, then it is assumed - * to be remembering past requests. - */ + pmap_page_protect(m->phys_page, VM_PROT_ALL & ~prot); - m->unlock_request = VM_PROT_NONE; -#endif /* 0 */ PAGE_WAKEUP(m); } /* * Handle page returning. */ - if (memory_object_should_return_page(m, should_return)) { /* @@ -291,7 +257,7 @@ memory_object_lock_page( * maps now. */ - vm_page_lock_queues(); + vm_page_lockspin_queues(); VM_PAGE_QUEUES_REMOVE(m); vm_page_unlock_queues(); @@ -307,7 +273,6 @@ memory_object_lock_page( /* * Handle flushing */ - if (should_flush) { VM_PAGE_FREE(m); } else { @@ -334,24 +299,29 @@ memory_object_lock_page( MACRO_BEGIN \ \ register int upl_flags; \ + memory_object_t pager; \ \ - vm_object_unlock(object); \ + if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) { \ + vm_object_paging_begin(object); \ + vm_object_unlock(object); \ \ - if (iosync) \ - upl_flags = UPL_MSYNC | UPL_IOSYNC; \ - else \ - upl_flags = UPL_MSYNC; \ + if (iosync) \ + upl_flags = UPL_MSYNC | UPL_IOSYNC; \ + else \ + upl_flags = UPL_MSYNC; \ \ - (void) memory_object_data_return(object->pager, \ - po, \ - data_cnt, \ - ro, \ - ioerr, \ - (action == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN), \ - !should_flush, \ - upl_flags); \ + (void) memory_object_data_return(pager, \ + po, \ + data_cnt, \ + ro, \ + ioerr, \ + (action) == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN,\ + !should_flush, \ + upl_flags); \ \ - vm_object_lock(object); \ + vm_object_lock(object); \ + vm_object_paging_end(object); \ + } \ MACRO_END /* @@ -659,7 +629,7 @@ vm_object_update_extent( data_cnt += PAGE_SIZE; last_offset = offset + PAGE_SIZE_64; - vm_page_lock_queues(); + vm_page_lockspin_queues(); /* * Clean */ @@ -714,7 +684,7 @@ vm_object_update( int flags, vm_prot_t protection) { - vm_object_t copy_object; + vm_object_t copy_object = VM_OBJECT_NULL; boolean_t data_returned = FALSE; boolean_t update_cow; boolean_t should_flush = (flags & MEMORY_OBJECT_DATA_FLUSH) ? TRUE : FALSE; @@ -752,9 +722,39 @@ vm_object_update( !(flags & MEMORY_OBJECT_DATA_PURGE))) || (flags & MEMORY_OBJECT_COPY_SYNC); + if (update_cow || (flags & (MEMORY_OBJECT_DATA_PURGE | MEMORY_OBJECT_DATA_SYNC))) { + int collisions = 0; + + while ((copy_object = object->copy) != VM_OBJECT_NULL) { + /* + * need to do a try here since we're swimming upstream + * against the normal lock ordering... however, we need + * to hold the object stable until we gain control of the + * copy object so we have to be careful how we approach this + */ + if (vm_object_lock_try(copy_object)) { + /* + * we 'won' the lock on the copy object... + * no need to hold the object lock any longer... + * take a real reference on the copy object because + * we're going to call vm_fault_page on it which may + * under certain conditions drop the lock and the paging + * reference we're about to take... the reference + * will keep the copy object from going away if that happens + */ + vm_object_unlock(object); + vm_object_reference_locked(copy_object); + break; + } + vm_object_unlock(object); - if((((copy_object = object->copy) != NULL) && update_cow) || - (flags & MEMORY_OBJECT_DATA_SYNC)) { + collisions++; + mutex_pause(collisions); + + vm_object_lock(object); + } + } + if ((copy_object != VM_OBJECT_NULL && update_cow) || (flags & MEMORY_OBJECT_DATA_SYNC)) { vm_map_size_t i; vm_map_size_t copy_size; vm_map_offset_t copy_offset; @@ -762,81 +762,83 @@ vm_object_update( vm_page_t page; vm_page_t top_page; kern_return_t error = 0; + struct vm_object_fault_info fault_info; + + if (copy_object != VM_OBJECT_NULL) { + /* + * translate offset with respect to shadow's offset + */ + copy_offset = (offset >= copy_object->shadow_offset) ? + (vm_map_offset_t)(offset - copy_object->shadow_offset) : + (vm_map_offset_t) 0; + + if (copy_offset > copy_object->size) + copy_offset = copy_object->size; + + /* + * clip size with respect to shadow offset + */ + if (offset >= copy_object->shadow_offset) { + copy_size = size; + } else if (size >= copy_object->shadow_offset - offset) { + copy_size = size - (copy_object->shadow_offset - offset); + } else { + copy_size = 0; + } + + if (copy_offset + copy_size > copy_object->size) { + if (copy_object->size >= copy_offset) { + copy_size = copy_object->size - copy_offset; + } else { + copy_size = 0; + } + } + copy_size+=copy_offset; - if(copy_object != NULL) { - /* translate offset with respect to shadow's offset */ - copy_offset = (offset >= copy_object->shadow_offset)? - (vm_map_offset_t)(offset - copy_object->shadow_offset) : - (vm_map_offset_t) 0; - if(copy_offset > copy_object->size) - copy_offset = copy_object->size; - - /* clip size with respect to shadow offset */ - if (offset >= copy_object->shadow_offset) { - copy_size = size; - } else if (size >= copy_object->shadow_offset - offset) { - copy_size = size - - (copy_object->shadow_offset - offset); - } else { - copy_size = 0; - } - - if (copy_offset + copy_size > copy_object->size) { - if (copy_object->size >= copy_offset) { - copy_size = copy_object->size - copy_offset; - } else { - copy_size = 0; - } - } - - copy_size+=copy_offset; - - vm_object_unlock(object); - vm_object_lock(copy_object); } else { copy_object = object; copy_size = offset + size; copy_offset = offset; } + fault_info.interruptible = THREAD_UNINT; + fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL; + fault_info.user_tag = 0; + fault_info.lo_offset = copy_offset; + fault_info.hi_offset = copy_size; + fault_info.no_cache = FALSE; vm_object_paging_begin(copy_object); - for (i=copy_offset; iobject, top_page); - PAGE_WAKEUP_DONE(page); - vm_page_lock_queues(); - if (!page->active && !page->inactive) - vm_page_activate(page); - vm_page_unlock_queues(); vm_object_lock(copy_object); vm_object_paging_begin(copy_object); - } else { - PAGE_WAKEUP_DONE(page); - vm_page_lock_queues(); - if (!page->active && !page->inactive) - vm_page_activate(page); - vm_page_unlock_queues(); } + vm_page_lock_queues(); + if (!page->active && !page->inactive) + vm_page_deactivate(page); + vm_page_unlock_queues(); + + PAGE_WAKEUP_DONE(page); break; case VM_FAULT_RETRY: prot = VM_PROT_WRITE|VM_PROT_READ; @@ -861,27 +863,36 @@ vm_object_update( vm_object_paging_begin(copy_object); goto RETRY_COW_OF_LOCK_REQUEST; case VM_FAULT_MEMORY_ERROR: + if (object != copy_object) + vm_object_deallocate(copy_object); vm_object_lock(object); goto BYPASS_COW_COPYIN; } } vm_object_paging_end(copy_object); - if(copy_object != object) { + } + if ((flags & (MEMORY_OBJECT_DATA_SYNC | MEMORY_OBJECT_COPY_SYNC))) { + if (copy_object != VM_OBJECT_NULL && copy_object != object) { vm_object_unlock(copy_object); + vm_object_deallocate(copy_object); vm_object_lock(object); } + return KERN_SUCCESS; } - if((flags & (MEMORY_OBJECT_DATA_SYNC | MEMORY_OBJECT_COPY_SYNC))) { - return KERN_SUCCESS; - } - if(((copy_object = object->copy) != NULL) && - (flags & MEMORY_OBJECT_DATA_PURGE)) { - copy_object->shadow_severed = TRUE; - copy_object->shadowed = FALSE; - copy_object->shadow = NULL; - /* delete the ref the COW was holding on the target object */ - vm_object_deallocate(object); + if (copy_object != VM_OBJECT_NULL && copy_object != object) { + if ((flags & MEMORY_OBJECT_DATA_PURGE)) { + copy_object->shadow_severed = TRUE; + copy_object->shadowed = FALSE; + copy_object->shadow = NULL; + /* + * delete the ref the COW was holding on the target object + */ + vm_object_deallocate(object); + } + vm_object_unlock(copy_object); + vm_object_deallocate(copy_object); + vm_object_lock(object); } BYPASS_COW_COPYIN: @@ -1077,7 +1088,6 @@ vm_object_set_attributes_common( boolean_t may_cache, memory_object_copy_strategy_t copy_strategy, boolean_t temporary, - memory_object_cluster_size_t cluster_size, boolean_t silent_overwrite, boolean_t advisory_pageout) { @@ -1111,17 +1121,6 @@ vm_object_set_attributes_common( may_cache = TRUE; if (temporary) temporary = TRUE; - if (cluster_size != 0) { - int pages_per_cluster; - pages_per_cluster = atop_32(cluster_size); - /* - * Cluster size must be integral multiple of page size, - * and be a power of 2 number of pages. - */ - if ((cluster_size & (PAGE_SIZE-1)) || - ((pages_per_cluster-1) & pages_per_cluster)) - return KERN_INVALID_ARGUMENT; - } vm_object_lock(object); @@ -1135,12 +1134,6 @@ vm_object_set_attributes_common( object->temporary = temporary; object->silent_overwrite = silent_overwrite; object->advisory_pageout = advisory_pageout; - if (cluster_size == 0) - cluster_size = PAGE_SIZE; - object->cluster_size = cluster_size; - - assert(cluster_size >= PAGE_SIZE && - cluster_size % PAGE_SIZE == 0); /* * Wake up anyone waiting for the ready attribute @@ -1177,7 +1170,6 @@ memory_object_change_attributes( boolean_t temporary; boolean_t may_cache; boolean_t invalidate; - memory_object_cluster_size_t cluster_size; memory_object_copy_strategy_t copy_strategy; boolean_t silent_overwrite; boolean_t advisory_pageout; @@ -1196,7 +1188,6 @@ memory_object_change_attributes( #if notyet invalidate = object->invalidate; #endif - cluster_size = object->cluster_size; vm_object_unlock(object); switch (flavor) { @@ -1249,7 +1240,6 @@ memory_object_change_attributes( perf = (memory_object_perf_info_t) attributes; may_cache = perf->may_cache; - cluster_size = round_page_32(perf->cluster_size); break; } @@ -1267,7 +1257,6 @@ memory_object_change_attributes( may_cache = attr->may_cache; copy_strategy = attr->copy_strategy; - cluster_size = page_size; break; } @@ -1285,7 +1274,6 @@ memory_object_change_attributes( copy_strategy = attr->copy_strategy; may_cache = attr->may_cache_object; - cluster_size = attr->cluster_size; temporary = attr->temporary; break; @@ -1314,7 +1302,6 @@ memory_object_change_attributes( may_cache, copy_strategy, temporary, - cluster_size, silent_overwrite, advisory_pageout)); } @@ -1391,7 +1378,7 @@ memory_object_get_attributes( } perf = (memory_object_perf_info_t) attributes; - perf->cluster_size = object->cluster_size; + perf->cluster_size = PAGE_SIZE; perf->may_cache = object->can_persist; *count = MEMORY_OBJECT_PERF_INFO_COUNT; @@ -1426,7 +1413,7 @@ memory_object_get_attributes( attr = (memory_object_attr_info_t) attributes; attr->copy_strategy = object->copy_strategy; - attr->cluster_size = object->cluster_size; + attr->cluster_size = PAGE_SIZE; attr->may_cache_object = object->can_persist; attr->temporary = object->temporary; @@ -1542,15 +1529,17 @@ memory_object_iopl_request( vm_object_reference(object); named_entry_unlock(named_entry); } - } else { + } else if (ip_kotype(port) == IKOT_MEM_OBJ_CONTROL) { memory_object_control_t control; - control = (memory_object_control_t)port->ip_kobject; + control = (memory_object_control_t) port; if (control == NULL) return (KERN_INVALID_ARGUMENT); object = memory_object_control_to_vm_object(control); if (object == VM_OBJECT_NULL) return (KERN_INVALID_ARGUMENT); vm_object_reference(object); + } else { + return KERN_INVALID_ARGUMENT; } if (object == VM_OBJECT_NULL) return (KERN_INVALID_ARGUMENT); @@ -1650,6 +1639,27 @@ memory_object_super_upl_request( cntrl_flags); } +kern_return_t +memory_object_cluster_size(memory_object_control_t control, memory_object_offset_t *start, + vm_size_t *length, memory_object_fault_info_t fault_info) +{ + vm_object_t object; + + object = memory_object_control_to_vm_object(control); + + if (object == VM_OBJECT_NULL || object->paging_offset > *start) + return (KERN_INVALID_ARGUMENT); + + *start -= object->paging_offset; + + vm_object_cluster_size(object, (vm_object_offset_t *)start, length, (vm_object_fault_info_t)fault_info); + + *start += object->paging_offset; + + return (KERN_SUCCESS); +} + + int vm_stat_discard_cleared_reply = 0; int vm_stat_discard_cleared_unset = 0; int vm_stat_discard_cleared_too_late = 0; @@ -1668,11 +1678,12 @@ kern_return_t host_default_memory_manager( host_priv_t host_priv, memory_object_default_t *default_manager, - memory_object_cluster_size_t cluster_size) + __unused memory_object_cluster_size_t cluster_size) { memory_object_default_t current_manager; memory_object_default_t new_manager; memory_object_default_t returned_manager; + kern_return_t result = KERN_SUCCESS; if (host_priv == HOST_PRIV_NULL) return(KERN_INVALID_HOST); @@ -1682,14 +1693,26 @@ host_default_memory_manager( new_manager = *default_manager; mutex_lock(&memory_manager_default_lock); current_manager = memory_manager_default; + returned_manager = MEMORY_OBJECT_DEFAULT_NULL; if (new_manager == MEMORY_OBJECT_DEFAULT_NULL) { /* * Retrieve the current value. */ - memory_object_default_reference(current_manager); returned_manager = current_manager; + memory_object_default_reference(returned_manager); } else { + + /* + * If this is the first non-null manager, start + * up the internal pager support. + */ + if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) { + result = vm_pageout_internal_start(); + if (result != KERN_SUCCESS) + goto out; + } + /* * Retrieve the current value, * and replace it with the supplied value. @@ -1697,21 +1720,10 @@ host_default_memory_manager( * but we have to take a reference on the new * one. */ - returned_manager = current_manager; memory_manager_default = new_manager; memory_object_default_reference(new_manager); - if (cluster_size % PAGE_SIZE != 0) { -#if 0 - mutex_unlock(&memory_manager_default_lock); - return KERN_INVALID_ARGUMENT; -#else - cluster_size = round_page_32(cluster_size); -#endif - } - memory_manager_default_cluster = cluster_size; - /* * In case anyone's been waiting for a memory * manager to be established, wake them up. @@ -1719,11 +1731,11 @@ host_default_memory_manager( thread_wakeup((event_t) &memory_manager_default); } - + out: mutex_unlock(&memory_manager_default_lock); *default_manager = returned_manager; - return(KERN_SUCCESS); + return(result); } /* @@ -1735,8 +1747,7 @@ host_default_memory_manager( */ __private_extern__ memory_object_default_t -memory_manager_default_reference( - memory_object_cluster_size_t *cluster_size) +memory_manager_default_reference(void) { memory_object_default_t current_manager; @@ -1752,7 +1763,6 @@ memory_manager_default_reference( current_manager = memory_manager_default; } memory_object_default_reference(current_manager); - *cluster_size = memory_manager_default_cluster; mutex_unlock(&memory_manager_default_lock); return current_manager; @@ -1812,152 +1822,12 @@ memory_object_page_op( int *flags) { vm_object_t object; - vm_page_t dst_page; - object = memory_object_control_to_vm_object(control); if (object == VM_OBJECT_NULL) return (KERN_INVALID_ARGUMENT); - vm_object_lock(object); - - if(ops & UPL_POP_PHYSICAL) { - if(object->phys_contiguous) { - if (phys_entry) { - *phys_entry = (ppnum_t) - (object->shadow_offset >> 12); - } - vm_object_unlock(object); - return KERN_SUCCESS; - } else { - vm_object_unlock(object); - return KERN_INVALID_OBJECT; - } - } - if(object->phys_contiguous) { - vm_object_unlock(object); - return KERN_INVALID_OBJECT; - } - - while(TRUE) { - if((dst_page = vm_page_lookup(object,offset)) == VM_PAGE_NULL) { - vm_object_unlock(object); - return KERN_FAILURE; - } - - /* Sync up on getting the busy bit */ - if((dst_page->busy || dst_page->cleaning) && - (((ops & UPL_POP_SET) && - (ops & UPL_POP_BUSY)) || (ops & UPL_POP_DUMP))) { - /* someone else is playing with the page, we will */ - /* have to wait */ - PAGE_SLEEP(object, dst_page, THREAD_UNINT); - continue; - } - - if (ops & UPL_POP_DUMP) { - vm_page_lock_queues(); - - if (dst_page->no_isync == FALSE) - pmap_disconnect(dst_page->phys_page); - vm_page_free(dst_page); - - vm_page_unlock_queues(); - break; - } - - if (flags) { - *flags = 0; - - /* Get the condition of flags before requested ops */ - /* are undertaken */ - - if(dst_page->dirty) *flags |= UPL_POP_DIRTY; - if(dst_page->pageout) *flags |= UPL_POP_PAGEOUT; - if(dst_page->precious) *flags |= UPL_POP_PRECIOUS; - if(dst_page->absent) *flags |= UPL_POP_ABSENT; - if(dst_page->busy) *flags |= UPL_POP_BUSY; - } - - /* The caller should have made a call either contingent with */ - /* or prior to this call to set UPL_POP_BUSY */ - if(ops & UPL_POP_SET) { - /* The protection granted with this assert will */ - /* not be complete. If the caller violates the */ - /* convention and attempts to change page state */ - /* without first setting busy we may not see it */ - /* because the page may already be busy. However */ - /* if such violations occur we will assert sooner */ - /* or later. */ - assert(dst_page->busy || (ops & UPL_POP_BUSY)); - if (ops & UPL_POP_DIRTY) dst_page->dirty = TRUE; - if (ops & UPL_POP_PAGEOUT) dst_page->pageout = TRUE; - if (ops & UPL_POP_PRECIOUS) dst_page->precious = TRUE; - if (ops & UPL_POP_ABSENT) dst_page->absent = TRUE; - if (ops & UPL_POP_BUSY) dst_page->busy = TRUE; - } - - if(ops & UPL_POP_CLR) { - assert(dst_page->busy); - if (ops & UPL_POP_DIRTY) dst_page->dirty = FALSE; - if (ops & UPL_POP_PAGEOUT) dst_page->pageout = FALSE; - if (ops & UPL_POP_PRECIOUS) dst_page->precious = FALSE; - if (ops & UPL_POP_ABSENT) dst_page->absent = FALSE; - if (ops & UPL_POP_BUSY) { - dst_page->busy = FALSE; - PAGE_WAKEUP(dst_page); - } - } - - if (dst_page->encrypted) { - /* - * ENCRYPTED SWAP: - * We need to decrypt this encrypted page before the - * caller can access its contents. - * But if the caller really wants to access the page's - * contents, they have to keep the page "busy". - * Otherwise, the page could get recycled or re-encrypted - * at any time. - */ - if ((ops & UPL_POP_SET) && (ops & UPL_POP_BUSY) && - dst_page->busy) { - /* - * The page is stable enough to be accessed by - * the caller, so make sure its contents are - * not encrypted. - */ - vm_page_decrypt(dst_page, 0); - } else { - /* - * The page is not busy, so don't bother - * decrypting it, since anything could - * happen to it between now and when the - * caller wants to access it. - * We should not give the caller access - * to this page. - */ - assert(!phys_entry); - } - } - - if (phys_entry) { - /* - * The physical page number will remain valid - * only if the page is kept busy. - * ENCRYPTED SWAP: make sure we don't let the - * caller access an encrypted page. - */ - assert(dst_page->busy); - assert(!dst_page->encrypted); - *phys_entry = dst_page->phys_page; - } - - break; - } - - vm_object_unlock(object); - return KERN_SUCCESS; - + return vm_object_page_op(object, offset, ops, phys_entry, flags); } /* @@ -1978,71 +1848,17 @@ memory_object_range_op( int ops, int *range) { - memory_object_offset_t offset; vm_object_t object; - vm_page_t dst_page; object = memory_object_control_to_vm_object(control); if (object == VM_OBJECT_NULL) return (KERN_INVALID_ARGUMENT); - if (object->resident_page_count == 0) { - if (range) { - if (ops & UPL_ROP_PRESENT) - *range = 0; - else - *range = offset_end - offset_beg; - } - return KERN_SUCCESS; - } - vm_object_lock(object); - - if (object->phys_contiguous) { - vm_object_unlock(object); - return KERN_INVALID_OBJECT; - } - - offset = offset_beg; - - while (offset < offset_end) { - dst_page = vm_page_lookup(object, offset); - if (dst_page != VM_PAGE_NULL) { - if (ops & UPL_ROP_DUMP) { - if (dst_page->busy || dst_page->cleaning) { - /* - * someone else is playing with the - * page, we will have to wait - */ - PAGE_SLEEP(object, - dst_page, THREAD_UNINT); - /* - * need to relook the page up since it's - * state may have changed while we slept - * it might even belong to a different object - * at this point - */ - continue; - } - vm_page_lock_queues(); - - if (dst_page->no_isync == FALSE) - pmap_disconnect(dst_page->phys_page); - vm_page_free(dst_page); - - vm_page_unlock_queues(); - } else if (ops & UPL_ROP_ABSENT) - break; - } else if (ops & UPL_ROP_PRESENT) - break; - - offset += PAGE_SIZE; - } - vm_object_unlock(object); - - if (range) - *range = offset - offset_beg; - - return KERN_SUCCESS; + return vm_object_range_op(object, + offset_beg, + offset_end, + ops, + range); } @@ -2065,6 +1881,23 @@ memory_object_pages_resident( return (KERN_SUCCESS); } +kern_return_t +memory_object_signed( + memory_object_control_t control, + boolean_t is_signed) +{ + vm_object_t object; + + object = memory_object_control_to_vm_object(control); + if (object == VM_OBJECT_NULL) + return KERN_INVALID_ARGUMENT; + + vm_object_lock(object); + object->code_signed = is_signed; + vm_object_unlock(object); + + return KERN_SUCCESS; +} static zone_t mem_obj_control_zone; @@ -2085,8 +1918,10 @@ memory_object_control_allocate( memory_object_control_t control; control = (memory_object_control_t)zalloc(mem_obj_control_zone); - if (control != MEMORY_OBJECT_CONTROL_NULL) - control->object = object; + if (control != MEMORY_OBJECT_CONTROL_NULL) { + control->moc_object = object; + control->moc_ikot = IKOT_MEM_OBJ_CONTROL; /* fake ip_kotype */ + } return (control); } @@ -2095,19 +1930,20 @@ memory_object_control_collapse( memory_object_control_t control, vm_object_t object) { - assert((control->object != VM_OBJECT_NULL) && - (control->object != object)); - control->object = object; + assert((control->moc_object != VM_OBJECT_NULL) && + (control->moc_object != object)); + control->moc_object = object; } __private_extern__ vm_object_t memory_object_control_to_vm_object( memory_object_control_t control) { - if (control == MEMORY_OBJECT_CONTROL_NULL) + if (control == MEMORY_OBJECT_CONTROL_NULL || + control->moc_ikot != IKOT_MEM_OBJ_CONTROL) return VM_OBJECT_NULL; - return (control->object); + return (control->moc_object); } memory_object_control_t @@ -2148,8 +1984,8 @@ void memory_object_control_disable( memory_object_control_t control) { - assert(control->object != VM_OBJECT_NULL); - control->object = VM_OBJECT_NULL; + assert(control->moc_object != VM_OBJECT_NULL); + control->moc_object = VM_OBJECT_NULL; } void @@ -2186,30 +2022,16 @@ convert_memory_object_to_port( void memory_object_reference( memory_object_t memory_object) { - -#ifdef MACH_BSD - if (memory_object->pager == &vnode_pager_workaround) { - vnode_pager_reference(memory_object); - } else if (memory_object->pager == &device_pager_workaround) { - device_pager_reference(memory_object); - } else -#endif - dp_memory_object_reference(memory_object); + (memory_object->mo_pager_ops->memory_object_reference)( + memory_object); } /* Routine memory_object_deallocate */ void memory_object_deallocate( memory_object_t memory_object) { - -#ifdef MACH_BSD - if (memory_object->pager == &vnode_pager_workaround) { - vnode_pager_deallocate(memory_object); - } else if (memory_object->pager == &device_pager_workaround) { - device_pager_deallocate(memory_object); - } else -#endif - dp_memory_object_deallocate(memory_object); + (memory_object->mo_pager_ops->memory_object_deallocate)( + memory_object); } @@ -2221,20 +2043,10 @@ kern_return_t memory_object_init memory_object_cluster_size_t memory_object_page_size ) { -#ifdef MACH_BSD - if (memory_object->pager == &vnode_pager_workaround) { - return vnode_pager_init(memory_object, - memory_control, - memory_object_page_size); - } else if (memory_object->pager == &device_pager_workaround) { - return device_pager_init(memory_object, - memory_control, - memory_object_page_size); - } else -#endif - return dp_memory_object_init(memory_object, - memory_control, - memory_object_page_size); + return (memory_object->mo_pager_ops->memory_object_init)( + memory_object, + memory_control, + memory_object_page_size); } /* Routine memory_object_terminate */ @@ -2243,14 +2055,8 @@ kern_return_t memory_object_terminate memory_object_t memory_object ) { -#ifdef MACH_BSD - if (memory_object->pager == &vnode_pager_workaround) { - return vnode_pager_terminate(memory_object); - } else if (memory_object->pager == &device_pager_workaround) { - return device_pager_terminate(memory_object); - } else -#endif - return dp_memory_object_terminate(memory_object); + return (memory_object->mo_pager_ops->memory_object_terminate)( + memory_object); } /* Routine memory_object_data_request */ @@ -2259,26 +2065,16 @@ kern_return_t memory_object_data_request memory_object_t memory_object, memory_object_offset_t offset, memory_object_cluster_size_t length, - vm_prot_t desired_access + vm_prot_t desired_access, + memory_object_fault_info_t fault_info ) { -#ifdef MACH_BSD - if (memory_object->pager == &vnode_pager_workaround) { - return vnode_pager_data_request(memory_object, - offset, - length, - desired_access); - } else if (memory_object->pager == &device_pager_workaround) { - return device_pager_data_request(memory_object, - offset, - length, - desired_access); - } else -#endif - return dp_memory_object_data_request(memory_object, - offset, - length, - desired_access); + return (memory_object->mo_pager_ops->memory_object_data_request)( + memory_object, + offset, + length, + desired_access, + fault_info); } /* Routine memory_object_data_return */ @@ -2294,37 +2090,15 @@ kern_return_t memory_object_data_return int upl_flags ) { -#ifdef MACH_BSD - if (memory_object->pager == &vnode_pager_workaround) { - return vnode_pager_data_return(memory_object, - offset, - size, - resid_offset, - io_error, - dirty, - kernel_copy, - upl_flags); - } else if (memory_object->pager == &device_pager_workaround) { - - return device_pager_data_return(memory_object, - offset, - size, - dirty, - kernel_copy, - upl_flags); - } - else -#endif - { - return dp_memory_object_data_return(memory_object, - offset, - size, - NULL, - NULL, - dirty, - kernel_copy, - upl_flags); - } + return (memory_object->mo_pager_ops->memory_object_data_return)( + memory_object, + offset, + size, + resid_offset, + io_error, + dirty, + kernel_copy, + upl_flags); } /* Routine memory_object_data_initialize */ @@ -2335,20 +2109,10 @@ kern_return_t memory_object_data_initialize vm_size_t size ) { -#ifdef MACH_BSD - if (memory_object->pager == &vnode_pager_workaround) { - return vnode_pager_data_initialize(memory_object, - offset, - size); - } else if (memory_object->pager == &device_pager_workaround) { - return device_pager_data_initialize(memory_object, - offset, - size); - } else -#endif - return dp_memory_object_data_initialize(memory_object, - offset, - size); + return (memory_object->mo_pager_ops->memory_object_data_initialize)( + memory_object, + offset, + size); } /* Routine memory_object_data_unlock */ @@ -2360,23 +2124,11 @@ kern_return_t memory_object_data_unlock vm_prot_t desired_access ) { -#ifdef MACH_BSD - if (memory_object->pager == &vnode_pager_workaround) { - return vnode_pager_data_unlock(memory_object, - offset, - size, - desired_access); - } else if (memory_object->pager == &device_pager_workaround) { - return device_pager_data_unlock(memory_object, - offset, - size, - desired_access); - } else -#endif - return dp_memory_object_data_unlock(memory_object, - offset, - size, - desired_access); + return (memory_object->mo_pager_ops->memory_object_data_unlock)( + memory_object, + offset, + size, + desired_access); } /* Routine memory_object_synchronize */ @@ -2388,23 +2140,11 @@ kern_return_t memory_object_synchronize vm_sync_t sync_flags ) { -#ifdef MACH_BSD - if (memory_object->pager == &vnode_pager_workaround) { - return vnode_pager_synchronize(memory_object, - offset, - size, - sync_flags); - } else if (memory_object->pager == &device_pager_workaround) { - return device_pager_synchronize(memory_object, - offset, - size, - sync_flags); - } else -#endif - return dp_memory_object_synchronize(memory_object, - offset, - size, - sync_flags); + return (memory_object->mo_pager_ops->memory_object_synchronize)( + memory_object, + offset, + size, + sync_flags); } /* Routine memory_object_unmap */ @@ -2413,14 +2153,8 @@ kern_return_t memory_object_unmap memory_object_t memory_object ) { -#ifdef MACH_BSD - if (memory_object->pager == &vnode_pager_workaround) { - return vnode_pager_unmap(memory_object); - } else if (memory_object->pager == &device_pager_workaround) { - return device_pager_unmap(memory_object); - } else -#endif - return dp_memory_object_unmap(memory_object); + return (memory_object->mo_pager_ops->memory_object_unmap)( + memory_object); } /* Routine memory_object_create */