X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/1c79356b52d46aa6b508fb032f5ae709b1f2897b..e5568f75972dfc723778653c11cb6b4dc825716a:/osfmk/vm/memory_object.c diff --git a/osfmk/vm/memory_object.c b/osfmk/vm/memory_object.c index 6c51bdcb9..2bc1d624f 100644 --- a/osfmk/vm/memory_object.c +++ b/osfmk/vm/memory_object.c @@ -56,11 +56,6 @@ * External memory management interface control functions. */ -#ifdef MACH_BSD -/* THIS code should be removed when the component merge is completed */ -extern int vnode_pager_workaround; -#endif - #include /* @@ -70,50 +65,49 @@ extern int vnode_pager_workaround; #include /* For pointer_t */ #include +#include #include #include #include #include -#include +#include #include #include #include -#include -#include /* * Implementation dependencies: */ #include /* For memcpy() */ +#include +#include +#include /* For current_thread() */ +#include +#include + +#include +#include #include #include #include #include /* For pmap_clear_modify */ -#include -#include /* For current_thread() */ -#include #include /* For kernel_map, vm_move */ #include /* For vm_map_pageable */ -#include -#include - -#include #if MACH_PAGEMAP #include #endif /* MACH_PAGEMAP */ - -ipc_port_t memory_manager_default = IP_NULL; -vm_size_t memory_manager_default_cluster = 0; -decl_mutex_data(,memory_manager_default_lock) +memory_object_default_t memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL; +vm_size_t memory_manager_default_cluster = 0; +decl_mutex_data(, memory_manager_default_lock) /* * Forward ref to file-local function: */ boolean_t -memory_object_update(vm_object_t, vm_object_offset_t, +vm_object_update(vm_object_t, vm_object_offset_t, vm_size_t, memory_object_return_t, int, vm_prot_t); @@ -137,7 +131,7 @@ memory_object_update(vm_object_t, vm_object_offset_t, #define memory_object_should_return_page(m, should_return) \ (should_return != MEMORY_OBJECT_RETURN_NONE && \ - (((m)->dirty || ((m)->dirty = pmap_is_modified((m)->phys_addr))) || \ + (((m)->dirty || ((m)->dirty = pmap_is_modified((m)->phys_page))) || \ ((m)->precious && (should_return) == MEMORY_OBJECT_RETURN_ALL) || \ (should_return) == MEMORY_OBJECT_RETURN_ANYTHING)) @@ -193,8 +187,19 @@ memory_object_lock_page( * does not have any data. */ - if (m->absent || m->error || m->restart) - return(MEMORY_OBJECT_LOCK_RESULT_DONE); + if (m->absent || m->error || m->restart) { + if(m->error && should_flush) { + /* dump the page, pager wants us to */ + /* clean it up and there is no */ + /* relevant data to return */ + if(m->wire_count == 0) { + VM_PAGE_FREE(m); + return(MEMORY_OBJECT_LOCK_RESULT_DONE); + } + } else { + return(MEMORY_OBJECT_LOCK_RESULT_DONE); + } + } assert(!m->fictitious); @@ -248,13 +253,13 @@ memory_object_lock_page( */ if (prot != VM_PROT_NO_CHANGE) { + if ((m->page_lock ^ prot) & prot) { + pmap_page_protect(m->phys_page, VM_PROT_ALL & ~prot); + } #if 0 /* code associated with the vestigial * memory_object_data_unlock */ - if ((m->page_lock ^ prot) & prot) { - pmap_page_protect(m->phys_addr, VM_PROT_ALL & ~prot); - } m->page_lock = prot; m->lock_supplied = TRUE; if (prot != VM_PROT_NONE) @@ -294,7 +299,7 @@ memory_object_lock_page( vm_page_unlock_queues(); if (!should_flush) - pmap_page_protect(m->phys_addr, VM_PROT_NONE); + pmap_page_protect(m->phys_page, VM_PROT_NONE); if (m->dirty) return(MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN); @@ -329,6 +334,7 @@ memory_object_lock_page( return(MEMORY_OBJECT_LOCK_RESULT_DONE); } + #define LIST_REQ_PAGEOUT_PAGES(object, data_cnt, action, po) \ MACRO_BEGIN \ \ @@ -337,107 +343,15 @@ MACRO_BEGIN \ \ vm_object_unlock(object); \ \ - if(((rpc_subsystem_t)pager_mux_hash_lookup(object->pager)) == \ - ((rpc_subsystem_t) &vnode_pager_workaround)) { \ - (void) vnode_pager_data_return(object->pager, \ - object->pager_request, \ - po, \ - POINTER_T(0), \ - data_cnt, \ - (action == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN), \ - !should_flush); \ - } else { \ (void) memory_object_data_return(object->pager, \ - object->pager_request, \ po, \ - POINTER_T(0), \ data_cnt, \ (action == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN), \ !should_flush); \ - } \ \ vm_object_lock(object); \ - \ MACRO_END -#ifdef MACH_BSD -#define PAGEOUT_PAGES(object, new_object, new_offset, action, po) \ -MACRO_BEGIN \ - \ - vm_map_copy_t copy; \ - register int i; \ - register vm_page_t hp; \ - \ - vm_object_unlock(object); \ - \ - (void) vm_map_copyin_object(new_object, 0, new_offset, ©); \ - \ - if(((rpc_subsystem_t)pager_mux_hash_lookup(object->pager)) == \ - ((rpc_subsystem_t) &vnode_pager_workaround)) { \ - (void) vnode_pager_data_return(object->pager, \ - object->pager_request, \ - po, \ - POINTER_T(copy), \ - new_offset, \ - (action == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN), \ - !should_flush); \ - } else { \ - (void) memory_object_data_return(object->pager, \ - object->pager_request, \ - po, \ - POINTER_T(copy), \ - new_offset, \ - (action == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN), \ - !should_flush); \ - } \ - \ - vm_object_lock(object); \ - \ - for (i = 0; i < atop(new_offset); i++) { \ - hp = holding_pages[i]; \ - if (hp != VM_PAGE_NULL) { \ - vm_object_paging_end(object); \ - VM_PAGE_FREE(hp); \ - } \ - } \ - \ - new_object = VM_OBJECT_NULL; \ -MACRO_END -#else -#define PAGEOUT_PAGES(object, new_object, new_offset, action, po) \ -MACRO_BEGIN \ - \ - vm_map_copy_t copy; \ - register int i; \ - register vm_page_t hp; \ - \ - vm_object_unlock(object); \ - \ - (void) vm_map_copyin_object(new_object, 0, new_offset, ©); \ - \ - (void) memory_object_data_return( \ - object->pager, \ - object->pager_request, \ - po, \ - POINTER_T(copy), \ - new_offset, \ - (action == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN), \ - !should_flush); \ - \ - vm_object_lock(object); \ - \ - for (i = 0; i < atop(new_offset); i++) { \ - hp = holding_pages[i]; \ - if (hp != VM_PAGE_NULL) { \ - vm_object_paging_end(object); \ - VM_PAGE_FREE(hp); \ - } \ - } \ - \ - new_object = VM_OBJECT_NULL; \ -MACRO_END -#endif - /* * Routine: memory_object_lock_request [user interface] * @@ -465,68 +379,107 @@ MACRO_END kern_return_t memory_object_lock_request( - register vm_object_t object, - register vm_object_offset_t offset, - register vm_object_size_t size, + memory_object_control_t control, + memory_object_offset_t offset, + memory_object_size_t size, memory_object_return_t should_return, int flags, - vm_prot_t prot, - ipc_port_t reply_to, - mach_msg_type_name_t reply_to_type) + vm_prot_t prot) { + vm_object_t object; vm_object_offset_t original_offset = offset; boolean_t should_flush=flags & MEMORY_OBJECT_DATA_FLUSH; XPR(XPR_MEMORY_OBJECT, - "m_o_lock_request, obj 0x%X off 0x%X size 0x%X flags %X prot %X\n", - (integer_t)object, offset, size, + "m_o_lock_request, control 0x%X off 0x%X size 0x%X flags %X prot %X\n", + (integer_t)control, offset, size, (((should_return&1)<<1)|should_flush), prot); /* * Check for bogus arguments. */ + object = memory_object_control_to_vm_object(control); if (object == VM_OBJECT_NULL) return (KERN_INVALID_ARGUMENT); - if ((prot & ~VM_PROT_ALL) != 0 && prot != VM_PROT_NO_CHANGE) { - vm_object_deallocate(object); + if ((prot & ~VM_PROT_ALL) != 0 && prot != VM_PROT_NO_CHANGE) return (KERN_INVALID_ARGUMENT); - } - size = round_page(size); + size = round_page_64(size); /* * Lock the object, and acquire a paging reference to - * prevent the memory_object and control ports from - * being destroyed. + * prevent the memory_object reference from being released. */ - vm_object_lock(object); vm_object_paging_begin(object); offset -= object->paging_offset; - (void)memory_object_update(object, + (void)vm_object_update(object, offset, size, should_return, flags, prot); - if (IP_VALID(reply_to)) { - vm_object_unlock(object); - - /* consumes our naked send-once/send right for reply_to */ - (void) memory_object_lock_completed(reply_to, reply_to_type, - object->pager_request, original_offset, size); - - vm_object_lock(object); - } - vm_object_paging_end(object); vm_object_unlock(object); - vm_object_deallocate(object); return (KERN_SUCCESS); } /* - * Routine: memory_object_sync + * memory_object_release_name: [interface] + * + * Enforces name semantic on memory_object reference count decrement + * This routine should not be called unless the caller holds a name + * reference gained through the memory_object_named_create or the + * memory_object_rename call. + * If the TERMINATE_IDLE flag is set, the call will return if the + * reference count is not 1. i.e. idle with the only remaining reference + * being the name. + * If the decision is made to proceed the name field flag is set to + * false and the reference count is decremented. If the RESPECT_CACHE + * flag is set and the reference count has gone to zero, the + * memory_object is checked to see if it is cacheable otherwise when + * the reference count is zero, it is simply terminated. + */ + +kern_return_t +memory_object_release_name( + memory_object_control_t control, + int flags) +{ + vm_object_t object; + + object = memory_object_control_to_vm_object(control); + if (object == VM_OBJECT_NULL) + return (KERN_INVALID_ARGUMENT); + + return vm_object_release_name(object, flags); +} + + + +/* + * Routine: memory_object_destroy [user interface] + * Purpose: + * Shut down a memory object, despite the + * presence of address map (or other) references + * to the vm_object. + */ +kern_return_t +memory_object_destroy( + memory_object_control_t control, + kern_return_t reason) +{ + vm_object_t object; + + object = memory_object_control_to_vm_object(control); + if (object == VM_OBJECT_NULL) + return (KERN_INVALID_ARGUMENT); + + return (vm_object_destroy(object, reason)); +} + +/* + * Routine: vm_object_sync * * Kernel internal function to synch out pages in a given * range within an object to its memory manager. Much the @@ -550,17 +503,17 @@ memory_object_lock_request( */ boolean_t -memory_object_sync( +vm_object_sync( vm_object_t object, vm_object_offset_t offset, - vm_object_size_t size, + vm_size_t size, boolean_t should_flush, boolean_t should_return) { boolean_t rv; - XPR(XPR_MEMORY_OBJECT, - "m_o_sync, object 0x%X, offset 0x%X size 0x%x flush %d rtn %d\n", + XPR(XPR_VM_OBJECT, + "vm_o_sync, object 0x%X, offset 0x%X size 0x%x flush %d rtn %d\n", (integer_t)object, offset, size, should_flush, should_return); /* @@ -571,7 +524,7 @@ memory_object_sync( vm_object_lock(object); vm_object_paging_begin(object); - rv = memory_object_update(object, offset, size, + rv = vm_object_update(object, offset, size, (should_return) ? MEMORY_OBJECT_RETURN_ALL : MEMORY_OBJECT_RETURN_NONE, @@ -586,14 +539,14 @@ memory_object_sync( } /* - * Routine: memory_object_update + * Routine: vm_object_update * Description: - * Work function for m_o_lock_request(), m_o_sync(). + * Work function for m_o_lock_request(), vm_o_sync(). * * Called with object locked and paging ref taken. */ kern_return_t -memory_object_update( +vm_object_update( register vm_object_t object, register vm_object_offset_t offset, register vm_size_t size, @@ -613,9 +566,7 @@ memory_object_update( boolean_t data_returned = FALSE; boolean_t update_cow; boolean_t should_flush = flags & MEMORY_OBJECT_DATA_FLUSH; -#ifndef NOT_LIST_REQ boolean_t pending_pageout = FALSE; -#endif /* * To avoid blocking while scanning for pages, save @@ -674,7 +625,7 @@ memory_object_update( if(copy_size < 0) copy_size = 0; - copy_size+=offset; + copy_size+=copy_offset; vm_object_unlock(object); vm_object_lock(copy_object); @@ -702,7 +653,7 @@ memory_object_update( (int *)0, &error, FALSE, - FALSE)) { + FALSE, NULL, 0)) { case VM_FAULT_SUCCESS: if(top_page) { @@ -824,10 +775,7 @@ BYPASS_COW_COPYIN: continue; } - PAGE_ASSERT_WAIT(m, THREAD_UNINT); - vm_object_unlock(object); - thread_block((void (*)(void))0); - vm_object_lock(object); + PAGE_SLEEP(object, m, THREAD_UNINT); continue; case MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN: @@ -860,9 +808,7 @@ BYPASS_COW_COPYIN: m->busy = FALSE; holding_page = VM_PAGE_NULL; if(m->cleaning) { - PAGE_ASSERT_WAIT(m, THREAD_UNINT); - vm_object_unlock(object); - thread_block((void (*)(void))0); + PAGE_SLEEP(object, m, THREAD_UNINT); continue; } if(!pending_pageout) { @@ -906,17 +852,10 @@ BYPASS_COW_COPYIN: * We have completed the scan for applicable pages. * Clean any pages that have been saved. */ -#ifdef NOT_LIST_REQ - if (new_object != VM_OBJECT_NULL) { - PAGEOUT_PAGES(object, new_object, new_offset, pageout_action, - paging_offset); - } -#else if (pending_pageout) { LIST_REQ_PAGEOUT_PAGES(object, data_cnt, pageout_action, paging_offset); } -#endif return (data_returned); } @@ -933,11 +872,12 @@ BYPASS_COW_COPYIN: kern_return_t memory_object_synchronize_completed( - vm_object_t object, - vm_object_offset_t offset, - vm_offset_t length) + memory_object_control_t control, + memory_object_offset_t offset, + vm_offset_t length) { - msync_req_t msr; + vm_object_t object; + msync_req_t msr; XPR(XPR_MEMORY_OBJECT, "m_o_sync_completed, object 0x%X, offset 0x%X length 0x%X\n", @@ -947,9 +887,9 @@ memory_object_synchronize_completed( * Look for bogus arguments */ - if (object == VM_OBJECT_NULL) { - return KERN_INVALID_ARGUMENT; - } + object = memory_object_control_to_vm_object(control); + if (object == VM_OBJECT_NULL) + return (KERN_INVALID_ARGUMENT); vm_object_lock(object); @@ -965,7 +905,6 @@ memory_object_synchronize_completed( if (queue_end(&object->msr_q, (queue_entry_t)msr)) { vm_object_unlock(object); - vm_object_deallocate(object); return KERN_INVALID_ARGUMENT; } @@ -974,13 +913,12 @@ memory_object_synchronize_completed( msr->flag = VM_MSYNC_DONE; msr_unlock(msr); thread_wakeup((event_t) msr); - vm_object_deallocate(object); return KERN_SUCCESS; }/* memory_object_synchronize_completed */ - -kern_return_t -memory_object_set_attributes_common( + +static kern_return_t +vm_object_set_attributes_common( vm_object_t object, boolean_t may_cache, memory_object_copy_strategy_t copy_strategy, @@ -1007,15 +945,13 @@ memory_object_set_attributes_common( case MEMORY_OBJECT_COPY_DELAY: break; default: - vm_object_deallocate(object); return(KERN_INVALID_ARGUMENT); } #if !ADVISORY_PAGEOUT - if (silent_overwrite || advisory_pageout) { - vm_object_deallocate(object); + if (silent_overwrite || advisory_pageout) return(KERN_INVALID_ARGUMENT); - } + #endif /* !ADVISORY_PAGEOUT */ if (may_cache) may_cache = TRUE; @@ -1023,16 +959,14 @@ memory_object_set_attributes_common( temporary = TRUE; if (cluster_size != 0) { int pages_per_cluster; - pages_per_cluster = atop(cluster_size); + pages_per_cluster = atop_32(cluster_size); /* * Cluster size must be integral multiple of page size, * and be a power of 2 number of pages. */ if ((cluster_size & (PAGE_SIZE-1)) || - ((pages_per_cluster-1) & pages_per_cluster)) { - vm_object_deallocate(object); + ((pages_per_cluster-1) & pages_per_cluster)) return KERN_INVALID_ARGUMENT; - } } vm_object_lock(object); @@ -1066,8 +1000,6 @@ memory_object_set_attributes_common( vm_object_unlock(object); - vm_object_deallocate(object); - return(KERN_SUCCESS); } @@ -1076,31 +1008,32 @@ memory_object_set_attributes_common( * * XXX This routine cannot be completed until the vm_msync, clean * in place, and cluster work is completed. See ifdef notyet - * below and note that memory_object_set_attributes_common() + * below and note that vm_object_set_attributes_common() * may have to be expanded. */ kern_return_t memory_object_change_attributes( - vm_object_t object, - memory_object_flavor_t flavor, - memory_object_info_t attributes, - mach_msg_type_number_t count, - ipc_port_t reply_to, - mach_msg_type_name_t reply_to_type) + memory_object_control_t control, + memory_object_flavor_t flavor, + memory_object_info_t attributes, + mach_msg_type_number_t count) { - kern_return_t result = KERN_SUCCESS; - boolean_t temporary; - boolean_t may_cache; - boolean_t invalidate; + vm_object_t object; + kern_return_t result = KERN_SUCCESS; + boolean_t temporary; + boolean_t may_cache; + boolean_t invalidate; vm_size_t cluster_size; memory_object_copy_strategy_t copy_strategy; - boolean_t silent_overwrite; + boolean_t silent_overwrite; boolean_t advisory_pageout; + object = memory_object_control_to_vm_object(control); if (object == VM_OBJECT_NULL) - return(KERN_INVALID_ARGUMENT); + return (KERN_INVALID_ARGUMENT); vm_object_lock(object); + temporary = object->temporary; may_cache = object->can_persist; copy_strategy = object->copy_strategy; @@ -1162,7 +1095,7 @@ memory_object_change_attributes( perf = (memory_object_perf_info_t) attributes; may_cache = perf->may_cache; - cluster_size = round_page(perf->cluster_size); + cluster_size = round_page_32(perf->cluster_size); break; } @@ -1209,10 +1142,8 @@ memory_object_change_attributes( break; } - if (result != KERN_SUCCESS) { - vm_object_deallocate(object); + if (result != KERN_SUCCESS) return(result); - } if (copy_strategy == MEMORY_OBJECT_COPY_TEMPORARY) { copy_strategy = MEMORY_OBJECT_COPY_DELAY; @@ -1222,45 +1153,31 @@ memory_object_change_attributes( } /* - * Do the work and throw away our object reference. It - * is important that the object reference be deallocated - * BEFORE sending the reply. The whole point of the reply - * is that it shows up after the terminate message that - * may be generated by setting the object uncacheable. - * * XXX may_cache may become a tri-valued variable to handle * XXX uncache if not in use. */ - result = memory_object_set_attributes_common(object, + return (vm_object_set_attributes_common(object, may_cache, copy_strategy, temporary, cluster_size, silent_overwrite, - advisory_pageout); - - if (IP_VALID(reply_to)) { - /* consumes our naked send-once/send right for reply_to */ - (void) memory_object_change_completed(reply_to, reply_to_type, - object->alive ? - object->pager_request : PAGER_REQUEST_NULL, - flavor); - } - - return(result); + advisory_pageout)); } kern_return_t memory_object_get_attributes( - vm_object_t object, + memory_object_control_t control, memory_object_flavor_t flavor, memory_object_info_t attributes, /* pointer to OUT array */ mach_msg_type_number_t *count) /* IN/OUT */ { - kern_return_t ret = KERN_SUCCESS; + kern_return_t ret = KERN_SUCCESS; + vm_object_t object; - if (object == VM_OBJECT_NULL) - return(KERN_INVALID_ARGUMENT); + object = memory_object_control_to_vm_object(control); + if (object == VM_OBJECT_NULL) + return (KERN_INVALID_ARGUMENT); vm_object_lock(object); @@ -1370,30 +1287,212 @@ memory_object_get_attributes( vm_object_unlock(object); - vm_object_deallocate(object); - return(ret); } -int vm_stat_discard_cleared_reply = 0; -int vm_stat_discard_cleared_unset = 0; -int vm_stat_discard_cleared_too_late = 0; +kern_return_t +memory_object_iopl_request( + ipc_port_t port, + memory_object_offset_t offset, + vm_size_t *upl_size, + upl_t *upl_ptr, + upl_page_info_array_t user_page_list, + unsigned int *page_list_count, + int *flags) +{ + vm_object_t object; + kern_return_t ret; + int caller_flags; + + caller_flags = *flags; -/* - * vm_set_default_memory_manager(): - * [Obsolete] + if (ip_kotype(port) == IKOT_NAMED_ENTRY) { + vm_named_entry_t named_entry; + + named_entry = (vm_named_entry_t)port->ip_kobject; + /* a few checks to make sure user is obeying rules */ + if(*upl_size == 0) { + if(offset >= named_entry->size) + return(KERN_INVALID_RIGHT); + *upl_size = named_entry->size - offset; + } + if(caller_flags & UPL_COPYOUT_FROM) { + if((named_entry->protection & VM_PROT_READ) + != VM_PROT_READ) { + return(KERN_INVALID_RIGHT); + } + } else { + if((named_entry->protection & + (VM_PROT_READ | VM_PROT_WRITE)) + != (VM_PROT_READ | VM_PROT_WRITE)) { + return(KERN_INVALID_RIGHT); + } + } + if(named_entry->size < (offset + *upl_size)) + return(KERN_INVALID_ARGUMENT); + + /* the callers parameter offset is defined to be the */ + /* offset from beginning of named entry offset in object */ + offset = offset + named_entry->offset; + + if(named_entry->is_sub_map) + return (KERN_INVALID_ARGUMENT); + + named_entry_lock(named_entry); + + if(named_entry->object) { + /* This is the case where we are going to map */ + /* an already mapped object. If the object is */ + /* not ready it is internal. An external */ + /* object cannot be mapped until it is ready */ + /* we can therefore avoid the ready check */ + /* in this case. */ + vm_object_reference(named_entry->object); + object = named_entry->object; + named_entry_unlock(named_entry); + } else { + object = vm_object_enter(named_entry->backing.pager, + named_entry->offset + named_entry->size, + named_entry->internal, + FALSE, + FALSE); + if (object == VM_OBJECT_NULL) { + named_entry_unlock(named_entry); + return(KERN_INVALID_OBJECT); + } + vm_object_lock(object); + + /* create an extra reference for the named entry */ + vm_object_reference_locked(object); + named_entry->object = object; + named_entry_unlock(named_entry); + + /* wait for object to be ready */ + while (!object->pager_ready) { + vm_object_wait(object, + VM_OBJECT_EVENT_PAGER_READY, + THREAD_UNINT); + vm_object_lock(object); + } + vm_object_unlock(object); + } + } else { + memory_object_control_t control; + control = (memory_object_control_t)port->ip_kobject; + if (control == NULL) + return (KERN_INVALID_ARGUMENT); + object = memory_object_control_to_vm_object(control); + if (object == VM_OBJECT_NULL) + return (KERN_INVALID_ARGUMENT); + vm_object_reference(object); + } + if (object == VM_OBJECT_NULL) + return (KERN_INVALID_ARGUMENT); + + if (!object->private) { + if (*upl_size > (MAX_UPL_TRANSFER*PAGE_SIZE)) + *upl_size = (MAX_UPL_TRANSFER*PAGE_SIZE); + if (object->phys_contiguous) { + *flags = UPL_PHYS_CONTIG; + } else { + *flags = 0; + } + } else { + *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG; + } + + ret = vm_object_iopl_request(object, + offset, + *upl_size, + upl_ptr, + user_page_list, + page_list_count, + caller_flags); + vm_object_deallocate(object); + return ret; +} + +/* + * Routine: memory_object_upl_request [interface] + * Purpose: + * Cause the population of a portion of a vm_object. + * Depending on the nature of the request, the pages + * returned may be contain valid data or be uninitialized. + * */ + kern_return_t -vm_set_default_memory_manager( - host_t host, - ipc_port_t *default_manager) +memory_object_upl_request( + memory_object_control_t control, + memory_object_offset_t offset, + vm_size_t size, + upl_t *upl_ptr, + upl_page_info_array_t user_page_list, + unsigned int *page_list_count, + int cntrl_flags) { - return(host_default_memory_manager(host_priv_self(), default_manager, 4*PAGE_SIZE)); + vm_object_t object; + + object = memory_object_control_to_vm_object(control); + if (object == VM_OBJECT_NULL) + return (KERN_INVALID_ARGUMENT); + + return vm_object_upl_request(object, + offset, + size, + upl_ptr, + user_page_list, + page_list_count, + cntrl_flags); } +/* + * Routine: memory_object_super_upl_request [interface] + * Purpose: + * Cause the population of a portion of a vm_object + * in much the same way as memory_object_upl_request. + * Depending on the nature of the request, the pages + * returned may be contain valid data or be uninitialized. + * However, the region may be expanded up to the super + * cluster size provided. + */ + +kern_return_t +memory_object_super_upl_request( + memory_object_control_t control, + memory_object_offset_t offset, + vm_size_t size, + vm_size_t super_cluster, + upl_t *upl, + upl_page_info_t *user_page_list, + unsigned int *page_list_count, + int cntrl_flags) +{ + vm_object_t object; + + object = memory_object_control_to_vm_object(control); + if (object == VM_OBJECT_NULL) + return (KERN_INVALID_ARGUMENT); + + return vm_object_super_upl_request(object, + offset, + size, + super_cluster, + upl, + user_page_list, + page_list_count, + cntrl_flags); +} + +int vm_stat_discard_cleared_reply = 0; +int vm_stat_discard_cleared_unset = 0; +int vm_stat_discard_cleared_too_late = 0; + + + /* - * Routine: host_default_memory_manager + * Routine: host_default_memory_manager [interface] * Purpose: * set/get the default memory manager port and default cluster * size. @@ -1402,13 +1501,13 @@ vm_set_default_memory_manager( */ kern_return_t host_default_memory_manager( - host_priv_t host_priv, - ipc_port_t *default_manager, - vm_size_t cluster_size) + host_priv_t host_priv, + memory_object_default_t *default_manager, + vm_size_t cluster_size) { - ipc_port_t current_manager; - ipc_port_t new_manager; - ipc_port_t returned_manager; + memory_object_default_t current_manager; + memory_object_default_t new_manager; + memory_object_default_t returned_manager; if (host_priv == HOST_PRIV_NULL) return(KERN_INVALID_HOST); @@ -1419,27 +1518,31 @@ host_default_memory_manager( mutex_lock(&memory_manager_default_lock); current_manager = memory_manager_default; - if (new_manager == IP_NULL) { + if (new_manager == MEMORY_OBJECT_DEFAULT_NULL) { /* * Retrieve the current value. */ - - returned_manager = ipc_port_copy_send(current_manager); + memory_object_default_reference(current_manager); + returned_manager = current_manager; } else { /* * Retrieve the current value, * and replace it with the supplied value. - * We consume the supplied naked send right. + * We return the old reference to the caller + * but we have to take a reference on the new + * one. */ returned_manager = current_manager; memory_manager_default = new_manager; + memory_object_default_reference(new_manager); + if (cluster_size % PAGE_SIZE != 0) { #if 0 mutex_unlock(&memory_manager_default_lock); return KERN_INVALID_ARGUMENT; #else - cluster_size = round_page(cluster_size); + cluster_size = round_page_32(cluster_size); #endif } memory_manager_default_cluster = cluster_size; @@ -1466,62 +1569,30 @@ host_default_memory_manager( * valid (not IP_NULL or IP_DEAD). */ -ipc_port_t +__private_extern__ memory_object_default_t memory_manager_default_reference( vm_size_t *cluster_size) { - ipc_port_t current_manager; + memory_object_default_t current_manager; mutex_lock(&memory_manager_default_lock); - - while (current_manager = ipc_port_copy_send(memory_manager_default), - !IP_VALID(current_manager)) { - thread_sleep_mutex((event_t) &memory_manager_default, - &memory_manager_default_lock, THREAD_UNINT); - mutex_lock(&memory_manager_default_lock); + current_manager = memory_manager_default; + while (current_manager == MEMORY_OBJECT_DEFAULT_NULL) { + wait_result_t res; + + res = thread_sleep_mutex((event_t) &memory_manager_default, + &memory_manager_default_lock, + THREAD_UNINT); + assert(res == THREAD_AWAKENED); + current_manager = memory_manager_default; } + memory_object_default_reference(current_manager); *cluster_size = memory_manager_default_cluster; - mutex_unlock(&memory_manager_default_lock); return current_manager; } -/* - * Routine: memory_manager_default_port - * Purpose: - * Returns true if the receiver for the port - * is the default memory manager. - * - * This is a hack to let ds_read_done - * know when it should keep memory wired. - */ - -boolean_t -memory_manager_default_port( - ipc_port_t port) -{ - ipc_port_t current; - boolean_t result; - - mutex_lock(&memory_manager_default_lock); - current = memory_manager_default; - if (IP_VALID(current)) { - /* - * There is no point in bothering to lock - * both ports, which would be painful to do. - * If the receive rights are moving around, - * we might be inaccurate. - */ - - result = port->ip_receiver == current->ip_receiver; - } else - result = FALSE; - mutex_unlock(&memory_manager_default_lock); - - return result; -} - /* * Routine: memory_manager_default_check * @@ -1534,14 +1605,14 @@ memory_manager_default_port( * but only the first time. * */ -kern_return_t +__private_extern__ kern_return_t memory_manager_default_check(void) { - ipc_port_t current; + memory_object_default_t current; mutex_lock(&memory_manager_default_lock); current = memory_manager_default; - if (!IP_VALID(current)) { + if (current == MEMORY_OBJECT_DEFAULT_NULL) { static boolean_t logged; /* initialized to 0 */ boolean_t complain = !logged; logged = TRUE; @@ -1555,10 +1626,10 @@ memory_manager_default_check(void) } } -void +__private_extern__ void memory_manager_default_init(void) { - memory_manager_default = IP_NULL; + memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL; mutex_init(&memory_manager_default_lock, ETAP_VM_MEMMAN); } @@ -1598,17 +1669,25 @@ memory_object_deactivate_pages( if ((m->wire_count == 0) && (!m->private) && (!m->gobbled) && (!m->busy)) { m->reference = FALSE; - pmap_clear_reference(m->phys_addr); + pmap_clear_reference(m->phys_page); if ((kill_page) && (object->internal)) { m->precious = FALSE; m->dirty = FALSE; - pmap_clear_modify(m->phys_addr); + pmap_clear_modify(m->phys_page); vm_external_state_clr(object->existence_map, offset); } VM_PAGE_QUEUES_REMOVE(m); - queue_enter_first(&vm_page_queue_inactive, m, vm_page_t, pageq); + if(m->zero_fill) { + queue_enter_first( + &vm_page_queue_zf, + m, vm_page_t, pageq); + } else { + queue_enter_first( + &vm_page_queue_inactive, + m, vm_page_t, pageq); + } m->inactive = TRUE; if (!m->fictitious) @@ -1646,16 +1725,40 @@ memory_object_deactivate_pages( kern_return_t memory_object_page_op( - vm_object_t object, - vm_object_offset_t offset, - int ops, - vm_offset_t *phys_entry, - int *flags) + memory_object_control_t control, + memory_object_offset_t offset, + int ops, + ppnum_t *phys_entry, + int *flags) { - vm_page_t dst_page; + vm_object_t object; + vm_page_t dst_page; + + + object = memory_object_control_to_vm_object(control); + if (object == VM_OBJECT_NULL) + return (KERN_INVALID_ARGUMENT); vm_object_lock(object); + if(ops & UPL_POP_PHYSICAL) { + if(object->phys_contiguous) { + if (phys_entry) { + *phys_entry = (ppnum_t) + (object->shadow_offset >> 12); + } + vm_object_unlock(object); + return KERN_SUCCESS; + } else { + vm_object_unlock(object); + return KERN_INVALID_OBJECT; + } + } + if(object->phys_contiguous) { + vm_object_unlock(object); + return KERN_INVALID_OBJECT; + } + while(TRUE) { if((dst_page = vm_page_lookup(object,offset)) == VM_PAGE_NULL) { vm_object_unlock(object); @@ -1664,21 +1767,23 @@ memory_object_page_op( /* Sync up on getting the busy bit */ if((dst_page->busy || dst_page->cleaning) && - (((ops & UPL_POP_SET) && (ops & UPL_POP_BUSY)) || (ops & UPL_POP_DUMP))) { + (((ops & UPL_POP_SET) && + (ops & UPL_POP_BUSY)) || (ops & UPL_POP_DUMP))) { /* someone else is playing with the page, we will */ /* have to wait */ - PAGE_ASSERT_WAIT(dst_page, THREAD_UNINT); - vm_object_unlock(object); - thread_block((void(*)(void))0); - vm_object_lock(object); + PAGE_SLEEP(object, dst_page, THREAD_UNINT); continue; } if (ops & UPL_POP_DUMP) { - vm_page_lock_queues(); - vm_page_free(dst_page); - vm_page_unlock_queues(); - break; + vm_page_lock_queues(); + + if (dst_page->no_isync == FALSE) + pmap_page_protect(dst_page->phys_page, VM_PROT_NONE); + vm_page_free(dst_page); + + vm_page_unlock_queues(); + break; } if (flags) { @@ -1694,7 +1799,7 @@ memory_object_page_op( if(dst_page->busy) *flags |= UPL_POP_BUSY; } if (phys_entry) - *phys_entry = dst_page->phys_addr; + *phys_entry = dst_page->phys_page; /* The caller should have made a call either contingent with */ /* or prior to this call to set UPL_POP_BUSY */ @@ -1733,4 +1838,543 @@ memory_object_page_op( } +/* + * memory_object_range_op offers performance enhancement over + * memory_object_page_op for page_op functions which do not require page + * level state to be returned from the call. Page_op was created to provide + * a low-cost alternative to page manipulation via UPLs when only a single + * page was involved. The range_op call establishes the ability in the _op + * family of functions to work on multiple pages where the lack of page level + * state handling allows the caller to avoid the overhead of the upl structures. + */ + +kern_return_t +memory_object_range_op( + memory_object_control_t control, + memory_object_offset_t offset_beg, + memory_object_offset_t offset_end, + int ops, + int *range) +{ + memory_object_offset_t offset; + vm_object_t object; + vm_page_t dst_page; + + object = memory_object_control_to_vm_object(control); + if (object == VM_OBJECT_NULL) + return (KERN_INVALID_ARGUMENT); + + if (object->resident_page_count == 0) { + if (range) { + if (ops & UPL_ROP_PRESENT) + *range = 0; + else + *range = offset_end - offset_beg; + } + return KERN_SUCCESS; + } + vm_object_lock(object); + + if (object->phys_contiguous) + return KERN_INVALID_OBJECT; + + offset = offset_beg; + + while (offset < offset_end) { + if (dst_page = vm_page_lookup(object, offset)) { + if (ops & UPL_ROP_DUMP) { + if (dst_page->busy || dst_page->cleaning) { + /* + * someone else is playing with the + * page, we will have to wait + */ + PAGE_SLEEP(object, + dst_page, THREAD_UNINT); + /* + * need to relook the page up since it's + * state may have changed while we slept + * it might even belong to a different object + * at this point + */ + continue; + } + vm_page_lock_queues(); + + if (dst_page->no_isync == FALSE) + pmap_page_protect(dst_page->phys_page, VM_PROT_NONE); + vm_page_free(dst_page); + + vm_page_unlock_queues(); + } else if (ops & UPL_ROP_ABSENT) + break; + } else if (ops & UPL_ROP_PRESENT) + break; + + offset += PAGE_SIZE; + } + vm_object_unlock(object); + + if (range) + *range = offset - offset_beg; + + return KERN_SUCCESS; +} + +static zone_t mem_obj_control_zone; + +__private_extern__ void +memory_object_control_bootstrap(void) +{ + int i; + + i = (vm_size_t) sizeof (struct memory_object_control); + mem_obj_control_zone = zinit (i, 8192*i, 4096, "mem_obj_control"); + return; +} + +__private_extern__ memory_object_control_t +memory_object_control_allocate( + vm_object_t object) +{ + memory_object_control_t control; + + control = (memory_object_control_t)zalloc(mem_obj_control_zone); + if (control != MEMORY_OBJECT_CONTROL_NULL) + control->object = object; + return (control); +} + +__private_extern__ void +memory_object_control_collapse( + memory_object_control_t control, + vm_object_t object) +{ + assert((control->object != VM_OBJECT_NULL) && + (control->object != object)); + control->object = object; +} + +__private_extern__ vm_object_t +memory_object_control_to_vm_object( + memory_object_control_t control) +{ + if (control == MEMORY_OBJECT_CONTROL_NULL) + return VM_OBJECT_NULL; + + return (control->object); +} + +memory_object_control_t +convert_port_to_mo_control( + mach_port_t port) +{ + return MEMORY_OBJECT_CONTROL_NULL; +} + + +mach_port_t +convert_mo_control_to_port( + memory_object_control_t control) +{ + return MACH_PORT_NULL; +} + +void +memory_object_control_reference( + memory_object_control_t control) +{ + return; +} + +/* + * We only every issue one of these references, so kill it + * when that gets released (should switch the real reference + * counting in true port-less EMMI). + */ +void +memory_object_control_deallocate( + memory_object_control_t control) +{ + zfree(mem_obj_control_zone, (vm_offset_t)control); +} + +void +memory_object_control_disable( + memory_object_control_t control) +{ + assert(control->object != VM_OBJECT_NULL); + control->object = VM_OBJECT_NULL; +} + +void +memory_object_default_reference( + memory_object_default_t dmm) +{ + ipc_port_make_send(dmm); +} + +void +memory_object_default_deallocate( + memory_object_default_t dmm) +{ + ipc_port_release_send(dmm); +} + +memory_object_t +convert_port_to_memory_object( + mach_port_t port) +{ + return (MEMORY_OBJECT_NULL); +} + + +mach_port_t +convert_memory_object_to_port( + memory_object_t object) +{ + return (MACH_PORT_NULL); +} + +#ifdef MACH_BSD +/* remove after component interface available */ +extern int vnode_pager_workaround; +extern int device_pager_workaround; +#endif + + +/* Routine memory_object_reference */ +void memory_object_reference( + memory_object_t memory_object) +{ +extern void dp_memory_object_reference(memory_object_t); + +#ifdef MACH_BSD + extern void vnode_pager_reference(memory_object_t); + extern void device_pager_reference(memory_object_t); + + if(memory_object->pager == &vnode_pager_workaround) { + vnode_pager_reference(memory_object); + } else if(memory_object->pager == &device_pager_workaround) { + device_pager_reference(memory_object); + } else +#endif + dp_memory_object_reference(memory_object); +} + +/* Routine memory_object_deallocate */ +void memory_object_deallocate( + memory_object_t memory_object) +{ +extern void dp_memory_object_deallocate(memory_object_t); + +#ifdef MACH_BSD + extern void vnode_pager_deallocate(memory_object_t); + extern void device_pager_deallocate(memory_object_t); + + if(memory_object->pager == &vnode_pager_workaround) { + vnode_pager_deallocate(memory_object); + } else if(memory_object->pager == &device_pager_workaround) { + device_pager_deallocate(memory_object); + } else +#endif + dp_memory_object_deallocate(memory_object); +} + + +/* Routine memory_object_init */ +kern_return_t memory_object_init +( + memory_object_t memory_object, + memory_object_control_t memory_control, + vm_size_t memory_object_page_size +) +{ +extern kern_return_t dp_memory_object_init(memory_object_t, + memory_object_control_t, + vm_size_t); +#ifdef MACH_BSD +extern kern_return_t vnode_pager_init(memory_object_t, + memory_object_control_t, + vm_size_t); +extern kern_return_t device_pager_init(memory_object_t, + memory_object_control_t, + vm_size_t); + + if(memory_object->pager == &vnode_pager_workaround) { + return vnode_pager_init(memory_object, + memory_control, + memory_object_page_size); + } else if(memory_object->pager == &device_pager_workaround) { + return device_pager_init(memory_object, + memory_control, + memory_object_page_size); + } else +#endif + return dp_memory_object_init(memory_object, + memory_control, + memory_object_page_size); +} + +/* Routine memory_object_terminate */ +kern_return_t memory_object_terminate +( + memory_object_t memory_object +) +{ +extern kern_return_t dp_memory_object_terminate(memory_object_t); + +#ifdef MACH_BSD +extern kern_return_t vnode_pager_terminate(memory_object_t); +extern kern_return_t device_pager_terminate(memory_object_t); + + if(memory_object->pager == &vnode_pager_workaround) { + return vnode_pager_terminate(memory_object); + } else if(memory_object->pager == &device_pager_workaround) { + return device_pager_terminate(memory_object); + } else +#endif + return dp_memory_object_terminate(memory_object); +} + +/* Routine memory_object_data_request */ +kern_return_t memory_object_data_request +( + memory_object_t memory_object, + memory_object_offset_t offset, + vm_size_t length, + vm_prot_t desired_access +) +{ +extern kern_return_t dp_memory_object_data_request(memory_object_t, + memory_object_offset_t, vm_size_t, vm_prot_t); + +#ifdef MACH_BSD +extern kern_return_t vnode_pager_data_request(memory_object_t, + memory_object_offset_t, vm_size_t, vm_prot_t); +extern kern_return_t device_pager_data_request(memory_object_t, + memory_object_offset_t, vm_size_t, vm_prot_t); + + if (memory_object->pager == &vnode_pager_workaround) { + return vnode_pager_data_request(memory_object, + offset, + length, + desired_access); + } else if (memory_object->pager == &device_pager_workaround) { + return device_pager_data_request(memory_object, + offset, + length, + desired_access); + } else +#endif + return dp_memory_object_data_request(memory_object, + offset, + length, + desired_access); +} + +/* Routine memory_object_data_return */ +kern_return_t memory_object_data_return +( + memory_object_t memory_object, + memory_object_offset_t offset, + vm_size_t size, + boolean_t dirty, + boolean_t kernel_copy +) +{ + extern kern_return_t dp_memory_object_data_return(memory_object_t, + memory_object_offset_t, + vm_size_t, + boolean_t, + boolean_t); +#ifdef MACH_BSD + extern kern_return_t vnode_pager_data_return(memory_object_t, + memory_object_offset_t, + vm_size_t, + boolean_t, + boolean_t); + extern kern_return_t device_pager_data_return(memory_object_t, + memory_object_offset_t, + vm_size_t, + boolean_t, + boolean_t); + + if (memory_object->pager == &vnode_pager_workaround) { + return vnode_pager_data_return(memory_object, + offset, + size, + dirty, + kernel_copy); + } else if (memory_object->pager == &device_pager_workaround) { + return device_pager_data_return(memory_object, + offset, + size, + dirty, + kernel_copy); + } else +#endif + return dp_memory_object_data_return(memory_object, + offset, + size, + dirty, + kernel_copy); +} + +/* Routine memory_object_data_initialize */ +kern_return_t memory_object_data_initialize +( + memory_object_t memory_object, + memory_object_offset_t offset, + vm_size_t size +) +{ + + extern kern_return_t dp_memory_object_data_initialize(memory_object_t, + memory_object_offset_t, + vm_size_t); +#ifdef MACH_BSD + extern kern_return_t vnode_pager_data_initialize(memory_object_t, + memory_object_offset_t, + vm_size_t); + extern kern_return_t device_pager_data_initialize(memory_object_t, + memory_object_offset_t, + vm_size_t); + + if (memory_object->pager == &vnode_pager_workaround) { + return vnode_pager_data_initialize(memory_object, + offset, + size); + } else if (memory_object->pager == &device_pager_workaround) { + return device_pager_data_initialize(memory_object, + offset, + size); + } else +#endif + return dp_memory_object_data_initialize(memory_object, + offset, + size); +} + +/* Routine memory_object_data_unlock */ +kern_return_t memory_object_data_unlock +( + memory_object_t memory_object, + memory_object_offset_t offset, + vm_size_t size, + vm_prot_t desired_access +) +{ + extern kern_return_t dp_memory_object_data_unlock(memory_object_t, + memory_object_offset_t, + vm_size_t, + vm_prot_t); +#ifdef MACH_BSD + extern kern_return_t vnode_pager_data_unlock(memory_object_t, + memory_object_offset_t, + vm_size_t, + vm_prot_t); + extern kern_return_t device_pager_data_unlock(memory_object_t, + memory_object_offset_t, + vm_size_t, + vm_prot_t); + + if (memory_object->pager == &vnode_pager_workaround) { + return vnode_pager_data_unlock(memory_object, + offset, + size, + desired_access); + } else if (memory_object->pager == &device_pager_workaround) { + return device_pager_data_unlock(memory_object, + offset, + size, + desired_access); + } else +#endif + return dp_memory_object_data_unlock(memory_object, + offset, + size, + desired_access); + +} + +/* Routine memory_object_synchronize */ +kern_return_t memory_object_synchronize +( + memory_object_t memory_object, + memory_object_offset_t offset, + vm_size_t size, + vm_sync_t sync_flags +) +{ + extern kern_return_t dp_memory_object_data_synchronize(memory_object_t, + memory_object_offset_t, + vm_size_t, + vm_sync_t); +#ifdef MACH_BSD + extern kern_return_t vnode_pager_data_synchronize(memory_object_t, + memory_object_offset_t, + vm_size_t, + vm_sync_t); + extern kern_return_t device_pager_data_synchronize(memory_object_t, + memory_object_offset_t, + vm_size_t, + vm_sync_t); + + if (memory_object->pager == &vnode_pager_workaround) { + return vnode_pager_synchronize( + memory_object, + offset, + size, + sync_flags); + } else if (memory_object->pager == &device_pager_workaround) { + return device_pager_synchronize( + memory_object, + offset, + size, + sync_flags); + } else +#endif + return dp_memory_object_synchronize( + memory_object, + offset, + size, + sync_flags); +} + +/* Routine memory_object_unmap */ +kern_return_t memory_object_unmap +( + memory_object_t memory_object +) +{ + extern kern_return_t dp_memory_object_unmap(memory_object_t); +#ifdef MACH_BSD + extern kern_return_t vnode_pager_unmap(memory_object_t); + extern kern_return_t device_pager_unmap(memory_object_t); + + if (memory_object->pager == &vnode_pager_workaround) { + return vnode_pager_unmap(memory_object); + } else if (memory_object->pager == &device_pager_workaround) { + return device_pager_unmap(memory_object); + } else +#endif + return dp_memory_object_unmap(memory_object); +} + +/* Routine memory_object_create */ +kern_return_t memory_object_create +( + memory_object_default_t default_memory_manager, + vm_size_t new_memory_object_size, + memory_object_t *new_memory_object +) +{ +extern kern_return_t default_pager_memory_object_create(memory_object_default_t, + vm_size_t, + memory_object_t *); + + return default_pager_memory_object_create(default_memory_manager, + new_memory_object_size, + new_memory_object); +}