X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/5ba3f43ea354af8ad55bea84372a2bc834d8757c..f427ee49d309d8fc33ebf3042c3a775f2f530ded:/osfmk/vm/memory_object.c diff --git a/osfmk/vm/memory_object.c b/osfmk/vm/memory_object.c index d37eb4224..e4013fd82 100644 --- a/osfmk/vm/memory_object.c +++ b/osfmk/vm/memory_object.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2000-2008 Apple Inc. All rights reserved. + * Copyright (c) 2000-2020 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,34 +22,34 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ -/* +/* * Mach Operating System * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University * All Rights Reserved. - * + * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. - * + * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. - * + * * Carnegie Mellon requests users of this software to return to - * + * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 - * + * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ @@ -66,7 +66,7 @@ * Interface dependencies: */ -#include /* For pointer_t */ +#include /* For pointer_t */ #include #include @@ -82,11 +82,10 @@ /* * Implementation dependencies: */ -#include /* For memcpy() */ +#include /* For memcpy() */ -#include #include -#include /* For current_thread() */ +#include /* For current_thread() */ #include #include @@ -95,18 +94,18 @@ #include #include #include -#include /* For pmap_clear_modify */ -#include /* For kernel_map, vm_move */ -#include /* For vm_map_pageable */ -#include /* Needed by some vm_page.h macros */ +#include /* For pmap_clear_modify */ +#include /* For kernel_map, vm_move */ +#include /* For vm_map_pageable */ +#include /* Needed by some vm_page.h macros */ #include #include #include -memory_object_default_t memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL; -decl_lck_mtx_data(, memory_manager_default_lock) +memory_object_default_t memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL; +LCK_MTX_EARLY_DECLARE(memory_manager_default_lock, &vm_object_lck_grp); /* @@ -122,29 +121,29 @@ decl_lck_mtx_data(, memory_manager_default_lock) * 2. Page is precious and should_return is RETURN_ALL. * 3. Should_return is RETURN_ANYTHING. * - * As a side effect, m->dirty will be made consistent + * As a side effect, m->vmp_dirty will be made consistent * with pmap_is_modified(m), if should_return is not * MEMORY_OBJECT_RETURN_NONE. */ -#define memory_object_should_return_page(m, should_return) \ +#define memory_object_should_return_page(m, should_return) \ (should_return != MEMORY_OBJECT_RETURN_NONE && \ - (((m)->dirty || ((m)->dirty = pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m)))) || \ - ((m)->precious && (should_return) == MEMORY_OBJECT_RETURN_ALL) || \ + (((m)->vmp_dirty || ((m)->vmp_dirty = pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m)))) || \ + ((m)->vmp_precious && (should_return) == MEMORY_OBJECT_RETURN_ALL) || \ (should_return) == MEMORY_OBJECT_RETURN_ANYTHING)) -typedef int memory_object_lock_result_t; +typedef int memory_object_lock_result_t; -#define MEMORY_OBJECT_LOCK_RESULT_DONE 0 -#define MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK 1 -#define MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN 2 -#define MEMORY_OBJECT_LOCK_RESULT_MUST_FREE 3 +#define MEMORY_OBJECT_LOCK_RESULT_DONE 0 +#define MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK 1 +#define MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN 2 +#define MEMORY_OBJECT_LOCK_RESULT_MUST_FREE 3 memory_object_lock_result_t memory_object_lock_page( - vm_page_t m, - memory_object_return_t should_return, - boolean_t should_flush, - vm_prot_t prot); + vm_page_t m, + memory_object_return_t should_return, + boolean_t should_flush, + vm_prot_t prot); /* * Routine: memory_object_lock_page @@ -161,53 +160,51 @@ memory_object_lock_result_t memory_object_lock_page( */ memory_object_lock_result_t memory_object_lock_page( - vm_page_t m, - memory_object_return_t should_return, - boolean_t should_flush, - vm_prot_t prot) + vm_page_t m, + memory_object_return_t should_return, + boolean_t should_flush, + vm_prot_t prot) { - XPR(XPR_MEMORY_OBJECT, - "m_o_lock_page, page 0x%X rtn %d flush %d prot %d\n", - m, should_return, should_flush, prot, 0); - - - if (m->busy || m->cleaning) - return (MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK); + if (m->vmp_busy || m->vmp_cleaning) { + return MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK; + } - if (m->laundry) + if (m->vmp_laundry) { vm_pageout_steal_laundry(m, FALSE); + } /* * Don't worry about pages for which the kernel * does not have any data. */ - if (m->absent || m->error || m->restart) { - if (m->error && should_flush && !VM_PAGE_WIRED(m)) { + if (m->vmp_absent || m->vmp_error || m->vmp_restart) { + if (m->vmp_error && should_flush && !VM_PAGE_WIRED(m)) { /* * dump the page, pager wants us to * clean it up and there is no * relevant data to return */ - return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE); + return MEMORY_OBJECT_LOCK_RESULT_MUST_FREE; } - return (MEMORY_OBJECT_LOCK_RESULT_DONE); + return MEMORY_OBJECT_LOCK_RESULT_DONE; } - assert(!m->fictitious); + assert(!m->vmp_fictitious); if (VM_PAGE_WIRED(m)) { /* * The page is wired... just clean or return the page if needed. * Wired pages don't get flushed or disconnected from the pmap. */ - if (memory_object_should_return_page(m, should_return)) - return (MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN); + if (memory_object_should_return_page(m, should_return)) { + return MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN; + } - return (MEMORY_OBJECT_LOCK_RESULT_DONE); - } + return MEMORY_OBJECT_LOCK_RESULT_DONE; + } if (should_flush) { /* - * must do the pmap_disconnect before determining the + * must do the pmap_disconnect before determining the * need to return the page... otherwise it's possible * for the page to go from the clean to the dirty state * after we've made our decision @@ -221,8 +218,9 @@ memory_object_lock_page( * let the fault handler take care of increases * (pmap_page_protect may not increase protection). */ - if (prot != VM_PROT_NO_CHANGE) + if (prot != VM_PROT_NO_CHANGE) { pmap_page_protect(VM_PAGE_GET_PHYS_PAGE(m), VM_PROT_ALL & ~prot); + } } /* * Handle returning dirty or precious pages @@ -240,14 +238,15 @@ memory_object_lock_page( * if (!should_flush) * pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)); */ - return (MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN); + return MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN; } /* * Handle flushing clean pages */ - if (should_flush) - return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE); + if (should_flush) { + return MEMORY_OBJECT_LOCK_RESULT_MUST_FREE; + } /* * we use to deactivate clean pages at this point, @@ -265,7 +264,7 @@ memory_object_lock_page( * return (MEMORY_OBJECT_LOCK_RESULT_MUST_DEACTIVATE); */ - return (MEMORY_OBJECT_LOCK_RESULT_DONE); + return MEMORY_OBJECT_LOCK_RESULT_DONE; } @@ -281,7 +280,7 @@ memory_object_lock_page( * forms specified by "prot"); * 2) return data to the manager (if "should_return" * is RETURN_DIRTY and the page is dirty, or - * "should_return" is RETURN_ALL and the page + * "should_return" is RETURN_ALL and the page * is either dirty or precious); and, * 3) flush the cached copy (if "should_flush" * is asserted). @@ -297,26 +296,28 @@ memory_object_lock_page( kern_return_t memory_object_lock_request( - memory_object_control_t control, - memory_object_offset_t offset, - memory_object_size_t size, - memory_object_offset_t * resid_offset, - int * io_errno, - memory_object_return_t should_return, - int flags, - vm_prot_t prot) + memory_object_control_t control, + memory_object_offset_t offset, + memory_object_size_t size, + memory_object_offset_t * resid_offset, + int * io_errno, + memory_object_return_t should_return, + int flags, + vm_prot_t prot) { - vm_object_t object; + vm_object_t object; - /* + /* * Check for bogus arguments. */ object = memory_object_control_to_vm_object(control); - if (object == VM_OBJECT_NULL) - return (KERN_INVALID_ARGUMENT); + if (object == VM_OBJECT_NULL) { + return KERN_INVALID_ARGUMENT; + } - if ((prot & ~VM_PROT_ALL) != 0 && prot != VM_PROT_NO_CHANGE) - return (KERN_INVALID_ARGUMENT); + if ((prot & ~VM_PROT_ALL) != 0 && prot != VM_PROT_NO_CHANGE) { + return KERN_INVALID_ARGUMENT; + } size = round_page_64(size); @@ -335,16 +336,17 @@ memory_object_lock_request( } offset -= object->paging_offset; - if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL) + if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL) { vm_object_reap_pages(object, REAP_DATA_FLUSH); - else + } else { (void)vm_object_update(object, offset, size, resid_offset, - io_errno, should_return, flags, prot); + io_errno, should_return, flags, prot); + } vm_object_paging_end(object); vm_object_unlock(object); - return (KERN_SUCCESS); + return KERN_SUCCESS; } /* @@ -359,21 +361,22 @@ memory_object_lock_request( * being the name. * If the decision is made to proceed the name field flag is set to * false and the reference count is decremented. If the RESPECT_CACHE - * flag is set and the reference count has gone to zero, the + * flag is set and the reference count has gone to zero, the * memory_object is checked to see if it is cacheable otherwise when * the reference count is zero, it is simply terminated. */ kern_return_t memory_object_release_name( - memory_object_control_t control, - int flags) + memory_object_control_t control, + int flags) { - vm_object_t object; + vm_object_t object; object = memory_object_control_to_vm_object(control); - if (object == VM_OBJECT_NULL) - return (KERN_INVALID_ARGUMENT); + if (object == VM_OBJECT_NULL) { + return KERN_INVALID_ARGUMENT; + } return vm_object_release_name(object, flags); } @@ -389,16 +392,17 @@ memory_object_release_name( */ kern_return_t memory_object_destroy( - memory_object_control_t control, - kern_return_t reason) + memory_object_control_t control, + kern_return_t reason) { - vm_object_t object; + vm_object_t object; object = memory_object_control_to_vm_object(control); - if (object == VM_OBJECT_NULL) - return (KERN_INVALID_ARGUMENT); + if (object == VM_OBJECT_NULL) { + return KERN_INVALID_ARGUMENT; + } - return (vm_object_destroy(object, reason)); + return vm_object_destroy(object, reason); } /* @@ -427,20 +431,16 @@ memory_object_destroy( boolean_t vm_object_sync( - vm_object_t object, - vm_object_offset_t offset, - vm_object_size_t size, - boolean_t should_flush, - boolean_t should_return, - boolean_t should_iosync) + vm_object_t object, + vm_object_offset_t offset, + vm_object_size_t size, + boolean_t should_flush, + boolean_t should_return, + boolean_t should_iosync) { - boolean_t rv; + boolean_t rv; int flags; - XPR(XPR_VM_OBJECT, - "vm_o_sync, object 0x%X, offset 0x%X size 0x%x flush %d rtn %d\n", - object, offset, size, should_flush, should_return); - /* * Lock the object, and acquire a paging reference to * prevent the memory_object and control ports from @@ -450,7 +450,7 @@ vm_object_sync( vm_object_paging_begin(object); if (should_flush) { - flags = MEMORY_OBJECT_DATA_FLUSH; + flags = MEMORY_OBJECT_DATA_FLUSH; /* * This flush is from an msync(), not a truncate(), so the * contents of the file are not affected. @@ -459,18 +459,20 @@ vm_object_sync( * push the old contents to a copy object. */ flags |= MEMORY_OBJECT_DATA_NO_CHANGE; - } else - flags = 0; + } else { + flags = 0; + } - if (should_iosync) - flags |= MEMORY_OBJECT_IO_SYNC; + if (should_iosync) { + flags |= MEMORY_OBJECT_IO_SYNC; + } rv = vm_object_update(object, offset, (vm_object_size_t)size, NULL, NULL, - (should_return) ? - MEMORY_OBJECT_RETURN_ALL : - MEMORY_OBJECT_RETURN_NONE, - flags, - VM_PROT_NO_CHANGE); + (should_return) ? + MEMORY_OBJECT_RETURN_ALL : + MEMORY_OBJECT_RETURN_NONE, + flags, + VM_PROT_NO_CHANGE); vm_object_paging_end(object); @@ -481,36 +483,32 @@ vm_object_sync( #define LIST_REQ_PAGEOUT_PAGES(object, data_cnt, po, ro, ioerr, iosync) \ -MACRO_BEGIN \ - \ - int upl_flags; \ - memory_object_t pager; \ - \ - if (object->object_slid) { \ - panic("Objects with slid pages not allowed\n"); \ - } \ - \ - if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) { \ - vm_object_paging_begin(object); \ - vm_object_unlock(object); \ - \ - if (iosync) \ - upl_flags = UPL_MSYNC | UPL_IOSYNC; \ - else \ - upl_flags = UPL_MSYNC; \ - \ - (void) memory_object_data_return(pager, \ - po, \ - (memory_object_cluster_size_t)data_cnt, \ +MACRO_BEGIN \ + \ + int upl_flags; \ + memory_object_t pager; \ + \ + if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) { \ + vm_object_paging_begin(object); \ + vm_object_unlock(object); \ + \ + if (iosync) \ + upl_flags = UPL_MSYNC | UPL_IOSYNC; \ + else \ + upl_flags = UPL_MSYNC; \ + \ + (void) memory_object_data_return(pager, \ + po, \ + (memory_object_cluster_size_t)data_cnt, \ ro, \ ioerr, \ - FALSE, \ - FALSE, \ - upl_flags); \ - \ - vm_object_lock(object); \ - vm_object_paging_end(object); \ - } \ + FALSE, \ + FALSE, \ + upl_flags); \ + \ + vm_object_lock(object); \ + vm_object_paging_end(object); \ + } \ MACRO_END extern struct vnode * @@ -518,58 +516,64 @@ vnode_pager_lookup_vnode(memory_object_t); static int vm_object_update_extent( - vm_object_t object, - vm_object_offset_t offset, - vm_object_offset_t offset_end, - vm_object_offset_t *offset_resid, - int *io_errno, - boolean_t should_flush, - memory_object_return_t should_return, - boolean_t should_iosync, - vm_prot_t prot) + vm_object_t object, + vm_object_offset_t offset, + vm_object_offset_t offset_end, + vm_object_offset_t *offset_resid, + int *io_errno, + boolean_t should_flush, + memory_object_return_t should_return, + boolean_t should_iosync, + vm_prot_t prot) { - vm_page_t m; - int retval = 0; - vm_object_offset_t paging_offset = 0; - vm_object_offset_t next_offset = offset; - memory_object_lock_result_t page_lock_result; - memory_object_cluster_size_t data_cnt = 0; - struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT]; - struct vm_page_delayed_work *dwp; - int dw_count; - int dw_limit; - int dirty_count; - - dwp = &dw_array[0]; - dw_count = 0; + vm_page_t m; + int retval = 0; + vm_object_offset_t paging_offset = 0; + vm_object_offset_t next_offset = offset; + memory_object_lock_result_t page_lock_result; + memory_object_cluster_size_t data_cnt = 0; + struct vm_page_delayed_work dw_array; + struct vm_page_delayed_work *dwp, *dwp_start; + bool dwp_finish_ctx = TRUE; + int dw_count; + int dw_limit; + int dirty_count; + + dwp_start = dwp = NULL; + dw_count = 0; dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT); + dwp_start = vm_page_delayed_work_get_ctx(); + if (dwp_start == NULL) { + dwp_start = &dw_array; + dw_limit = 1; + dwp_finish_ctx = FALSE; + } + dwp = dwp_start; + dirty_count = 0; for (; - offset < offset_end && object->resident_page_count; - offset += PAGE_SIZE_64) { - - /* + offset < offset_end && object->resident_page_count; + offset += PAGE_SIZE_64) { + /* * Limit the number of pages to be cleaned at once to a contiguous * run, or at most MAX_UPL_TRANSFER_BYTES */ if (data_cnt) { if ((data_cnt >= MAX_UPL_TRANSFER_BYTES) || (next_offset != offset)) { - if (dw_count) { - vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count); - dwp = &dw_array[0]; + vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, dwp_start, dw_count); + dwp = dwp_start; dw_count = 0; } LIST_REQ_PAGEOUT_PAGES(object, data_cnt, - paging_offset, offset_resid, io_errno, should_iosync); + paging_offset, offset_resid, io_errno, should_iosync); data_cnt = 0; } } while ((m = vm_page_lookup(object, offset)) != VM_PAGE_NULL) { - dwp->dw_mask = 0; - + page_lock_result = memory_object_lock_page(m, should_return, should_flush, prot); if (data_cnt && page_lock_result != MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN) { @@ -577,12 +581,12 @@ vm_object_update_extent( * End of a run of dirty/precious pages. */ if (dw_count) { - vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count); - dwp = &dw_array[0]; + vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, dwp_start, dw_count); + dwp = dwp_start; dw_count = 0; } LIST_REQ_PAGEOUT_PAGES(object, data_cnt, - paging_offset, offset_resid, io_errno, should_iosync); + paging_offset, offset_resid, io_errno, should_iosync); /* * LIST_REQ_PAGEOUT_PAGES will drop the object lock which will * allow the state of page 'm' to change... we need to re-lookup @@ -593,13 +597,13 @@ vm_object_update_extent( } switch (page_lock_result) { - case MEMORY_OBJECT_LOCK_RESULT_DONE: break; case MEMORY_OBJECT_LOCK_RESULT_MUST_FREE: - if (m->dirty == TRUE) + if (m->vmp_dirty == TRUE) { dirty_count++; + } dwp->dw_mask |= DW_vm_page_free; break; @@ -608,8 +612,9 @@ vm_object_update_extent( continue; case MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN: - if (data_cnt == 0) + if (data_cnt == 0) { paging_offset = offset; + } data_cnt += PAGE_SIZE; next_offset = offset + PAGE_SIZE_64; @@ -620,12 +625,11 @@ vm_object_update_extent( * no need to remove them */ if (!VM_PAGE_WIRED(m)) { - if (should_flush) { /* * add additional state for the flush */ - m->free_when_done = TRUE; + m->vmp_free_when_done = TRUE; } /* * we use to remove the page from the queues at this @@ -643,29 +647,37 @@ vm_object_update_extent( VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count); if (dw_count >= dw_limit) { - vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count); - dwp = &dw_array[0]; + vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, dwp_start, dw_count); + dwp = dwp_start; dw_count = 0; } } break; } } - - if (object->pager) + + if (object->pager) { task_update_logical_writes(current_task(), (dirty_count * PAGE_SIZE), TASK_WRITE_INVALIDATED, vnode_pager_lookup_vnode(object->pager)); + } /* * We have completed the scan for applicable pages. * Clean any pages that have been saved. */ - if (dw_count) - vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count); + if (dw_count) { + vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, dwp_start, dw_count); + } if (data_cnt) { - LIST_REQ_PAGEOUT_PAGES(object, data_cnt, - paging_offset, offset_resid, io_errno, should_iosync); + LIST_REQ_PAGEOUT_PAGES(object, data_cnt, + paging_offset, offset_resid, io_errno, should_iosync); + } + + if (dwp_start && dwp_finish_ctx) { + vm_page_delayed_work_finish_ctx(dwp_start); + dwp_start = dwp = NULL; } - return (retval); + + return retval; } @@ -679,30 +691,30 @@ vm_object_update_extent( */ kern_return_t vm_object_update( - vm_object_t object, - vm_object_offset_t offset, - vm_object_size_t size, - vm_object_offset_t *resid_offset, - int *io_errno, - memory_object_return_t should_return, - int flags, - vm_prot_t protection) + vm_object_t object, + vm_object_offset_t offset, + vm_object_size_t size, + vm_object_offset_t *resid_offset, + int *io_errno, + memory_object_return_t should_return, + int flags, + vm_prot_t protection) { - vm_object_t copy_object = VM_OBJECT_NULL; - boolean_t data_returned = FALSE; - boolean_t update_cow; - boolean_t should_flush = (flags & MEMORY_OBJECT_DATA_FLUSH) ? TRUE : FALSE; - boolean_t should_iosync = (flags & MEMORY_OBJECT_IO_SYNC) ? TRUE : FALSE; - vm_fault_return_t result; - int num_of_extents; - int n; -#define MAX_EXTENTS 8 -#define EXTENT_SIZE (1024 * 1024 * 256) -#define RESIDENT_LIMIT (1024 * 32) + vm_object_t copy_object = VM_OBJECT_NULL; + boolean_t data_returned = FALSE; + boolean_t update_cow; + boolean_t should_flush = (flags & MEMORY_OBJECT_DATA_FLUSH) ? TRUE : FALSE; + boolean_t should_iosync = (flags & MEMORY_OBJECT_IO_SYNC) ? TRUE : FALSE; + vm_fault_return_t result; + int num_of_extents; + int n; +#define MAX_EXTENTS 8 +#define EXTENT_SIZE (1024 * 1024 * 256) +#define RESIDENT_LIMIT (1024 * 32) struct extent { - vm_object_offset_t e_base; - vm_object_offset_t e_min; - vm_object_offset_t e_max; + vm_object_offset_t e_base; + vm_object_offset_t e_min; + vm_object_offset_t e_max; } extents[MAX_EXTENTS]; /* @@ -722,34 +734,34 @@ vm_object_update( * XXX coalescing implications before doing so. */ - update_cow = ((flags & MEMORY_OBJECT_DATA_FLUSH) - && (!(flags & MEMORY_OBJECT_DATA_NO_CHANGE) && - !(flags & MEMORY_OBJECT_DATA_PURGE))) - || (flags & MEMORY_OBJECT_COPY_SYNC); - + update_cow = ((flags & MEMORY_OBJECT_DATA_FLUSH) + && (!(flags & MEMORY_OBJECT_DATA_NO_CHANGE) && + !(flags & MEMORY_OBJECT_DATA_PURGE))) + || (flags & MEMORY_OBJECT_COPY_SYNC); + if (update_cow || (flags & (MEMORY_OBJECT_DATA_PURGE | MEMORY_OBJECT_DATA_SYNC))) { - int collisions = 0; + int collisions = 0; - while ((copy_object = object->copy) != VM_OBJECT_NULL) { - /* + while ((copy_object = object->copy) != VM_OBJECT_NULL) { + /* * need to do a try here since we're swimming upstream * against the normal lock ordering... however, we need * to hold the object stable until we gain control of the * copy object so we have to be careful how we approach this */ - if (vm_object_lock_try(copy_object)) { - /* - * we 'won' the lock on the copy object... - * no need to hold the object lock any longer... - * take a real reference on the copy object because - * we're going to call vm_fault_page on it which may - * under certain conditions drop the lock and the paging - * reference we're about to take... the reference - * will keep the copy object from going away if that happens - */ - vm_object_unlock(object); - vm_object_reference_locked(copy_object); - break; + if (vm_object_lock_try(copy_object)) { + /* + * we 'won' the lock on the copy object... + * no need to hold the object lock any longer... + * take a real reference on the copy object because + * we're going to call vm_fault_page on it which may + * under certain conditions drop the lock and the paging + * reference we're about to take... the reference + * will keep the copy object from going away if that happens + */ + vm_object_unlock(object); + vm_object_reference_locked(copy_object); + break; } vm_object_unlock(object); @@ -760,46 +772,45 @@ vm_object_update( } } if ((copy_object != VM_OBJECT_NULL && update_cow) || (flags & MEMORY_OBJECT_DATA_SYNC)) { - vm_map_size_t i; - vm_map_size_t copy_size; - vm_map_offset_t copy_offset; - vm_prot_t prot; - vm_page_t page; - vm_page_t top_page; - kern_return_t error = 0; - struct vm_object_fault_info fault_info; + vm_object_offset_t i; + vm_object_size_t copy_size; + vm_object_offset_t copy_offset; + vm_prot_t prot; + vm_page_t page; + vm_page_t top_page; + kern_return_t error = 0; + struct vm_object_fault_info fault_info = {}; if (copy_object != VM_OBJECT_NULL) { - /* + /* * translate offset with respect to shadow's offset */ - copy_offset = (offset >= copy_object->vo_shadow_offset) ? - (vm_map_offset_t)(offset - copy_object->vo_shadow_offset) : - (vm_map_offset_t) 0; + copy_offset = (offset >= copy_object->vo_shadow_offset) ? + (offset - copy_object->vo_shadow_offset) : 0; - if (copy_offset > copy_object->vo_size) - copy_offset = copy_object->vo_size; + if (copy_offset > copy_object->vo_size) { + copy_offset = copy_object->vo_size; + } /* * clip size with respect to shadow offset */ if (offset >= copy_object->vo_shadow_offset) { - copy_size = size; + copy_size = size; } else if (size >= copy_object->vo_shadow_offset - offset) { - copy_size = size - (copy_object->vo_shadow_offset - offset); + copy_size = (size - (copy_object->vo_shadow_offset - offset)); } else { - copy_size = 0; + copy_size = 0; } - + if (copy_offset + copy_size > copy_object->vo_size) { - if (copy_object->vo_size >= copy_offset) { - copy_size = copy_object->vo_size - copy_offset; + if (copy_object->vo_size >= copy_offset) { + copy_size = copy_object->vo_size - copy_offset; } else { - copy_size = 0; + copy_size = 0; } } - copy_size+=copy_offset; - + copy_size += copy_offset; } else { copy_object = object; @@ -808,37 +819,32 @@ vm_object_update( } fault_info.interruptible = THREAD_UNINT; fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL; - fault_info.user_tag = 0; - fault_info.pmap_options = 0; fault_info.lo_offset = copy_offset; fault_info.hi_offset = copy_size; - fault_info.no_cache = FALSE; fault_info.stealth = TRUE; - fault_info.io_sync = FALSE; - fault_info.cs_bypass = FALSE; - fault_info.mark_zf_absent = FALSE; - fault_info.batch_pmap_op = FALSE; + assert(fault_info.cs_bypass == FALSE); + assert(fault_info.pmap_cs_associated == FALSE); vm_object_paging_begin(copy_object); for (i = copy_offset; i < copy_size; i += PAGE_SIZE) { - RETRY_COW_OF_LOCK_REQUEST: +RETRY_COW_OF_LOCK_REQUEST: fault_info.cluster_size = (vm_size_t) (copy_size - i); assert(fault_info.cluster_size == copy_size - i); - prot = VM_PROT_WRITE|VM_PROT_READ; + prot = VM_PROT_WRITE | VM_PROT_READ; page = VM_PAGE_NULL; - result = vm_fault_page(copy_object, i, - VM_PROT_WRITE|VM_PROT_READ, - FALSE, - FALSE, /* page not looked up */ - &prot, - &page, - &top_page, - (int *)0, - &error, - FALSE, - FALSE, &fault_info); + result = vm_fault_page(copy_object, i, + VM_PROT_WRITE | VM_PROT_READ, + FALSE, + FALSE, /* page not looked up */ + &prot, + &page, + &top_page, + (int *)0, + &error, + FALSE, + FALSE, &fault_info); switch (result) { case VM_FAULT_SUCCESS: @@ -848,11 +854,10 @@ vm_object_update( vm_object_lock(copy_object); vm_object_paging_begin(copy_object); } - if (( !VM_PAGE_NON_SPECULATIVE_PAGEABLE(page))) { - + if ((!VM_PAGE_NON_SPECULATIVE_PAGEABLE(page))) { vm_page_lockspin_queues(); - - if (( !VM_PAGE_NON_SPECULATIVE_PAGEABLE(page))) { + + if ((!VM_PAGE_NON_SPECULATIVE_PAGEABLE(page))) { vm_page_deactivate(page); } vm_page_unlock_queues(); @@ -860,18 +865,18 @@ vm_object_update( PAGE_WAKEUP_DONE(page); break; case VM_FAULT_RETRY: - prot = VM_PROT_WRITE|VM_PROT_READ; + prot = VM_PROT_WRITE | VM_PROT_READ; vm_object_lock(copy_object); vm_object_paging_begin(copy_object); goto RETRY_COW_OF_LOCK_REQUEST; case VM_FAULT_INTERRUPTED: - prot = VM_PROT_WRITE|VM_PROT_READ; + prot = VM_PROT_WRITE | VM_PROT_READ; vm_object_lock(copy_object); vm_object_paging_begin(copy_object); goto RETRY_COW_OF_LOCK_REQUEST; case VM_FAULT_MEMORY_SHORTAGE: VM_PAGE_WAIT(); - prot = VM_PROT_WRITE|VM_PROT_READ; + prot = VM_PROT_WRITE | VM_PROT_READ; vm_object_lock(copy_object); vm_object_paging_begin(copy_object); goto RETRY_COW_OF_LOCK_REQUEST; @@ -879,32 +884,32 @@ vm_object_update( /* success but no VM page: fail */ vm_object_paging_end(copy_object); vm_object_unlock(copy_object); - /*FALLTHROUGH*/ + OS_FALLTHROUGH; case VM_FAULT_MEMORY_ERROR: - if (object != copy_object) - vm_object_deallocate(copy_object); + if (object != copy_object) { + vm_object_deallocate(copy_object); + } vm_object_lock(object); goto BYPASS_COW_COPYIN; default: panic("vm_object_update: unexpected error 0x%x" - " from vm_fault_page()\n", result); + " from vm_fault_page()\n", result); } - } vm_object_paging_end(copy_object); } if ((flags & (MEMORY_OBJECT_DATA_SYNC | MEMORY_OBJECT_COPY_SYNC))) { - if (copy_object != VM_OBJECT_NULL && copy_object != object) { + if (copy_object != VM_OBJECT_NULL && copy_object != object) { vm_object_unlock(copy_object); - vm_object_deallocate(copy_object); + vm_object_deallocate(copy_object); vm_object_lock(object); } return KERN_SUCCESS; } if (copy_object != VM_OBJECT_NULL && copy_object != object) { - if ((flags & MEMORY_OBJECT_DATA_PURGE)) { + if ((flags & MEMORY_OBJECT_DATA_PURGE)) { vm_object_lock_assert_exclusive(copy_object); - copy_object->shadow_severed = TRUE; + copy_object->shadow_severed = TRUE; copy_object->shadowed = FALSE; copy_object->shadow = NULL; /* @@ -913,7 +918,7 @@ vm_object_update( vm_object_deallocate(object); } vm_object_unlock(copy_object); - vm_object_deallocate(copy_object); + vm_object_deallocate(copy_object); vm_object_lock(object); } BYPASS_COW_COPYIN: @@ -926,9 +931,9 @@ BYPASS_COW_COPYIN: * the page which means the resident queue can change which * means we can't walk the queue as we process the pages * we also want to do the processing in offset order to allow - * 'runs' of pages to be collected if we're being told to + * 'runs' of pages to be collected if we're being told to * flush to disk... the resident page queue is NOT ordered. - * + * * a temporary solution (until we figure out how to deal with * large address spaces more generically) is to pre-flight * the resident page queue (if it's small enough) and develop @@ -942,12 +947,12 @@ BYPASS_COW_COPYIN: * is not a theoretical problem */ - if ((object->resident_page_count < RESIDENT_LIMIT) && - (atop_64(size) > (unsigned)(object->resident_page_count/(8 * MAX_EXTENTS)))) { - vm_page_t next; - vm_object_offset_t start; - vm_object_offset_t end; - vm_object_size_t e_mask; + if ((object->resident_page_count < RESIDENT_LIMIT) && + (atop_64(size) > (unsigned)(object->resident_page_count / (8 * MAX_EXTENTS)))) { + vm_page_t next; + vm_object_offset_t start; + vm_object_offset_t end; + vm_object_size_t e_mask; vm_page_t m; start = offset; @@ -958,57 +963,60 @@ BYPASS_COW_COPYIN: m = (vm_page_t) vm_page_queue_first(&object->memq); while (!vm_page_queue_end(&object->memq, (vm_page_queue_entry_t) m)) { - next = (vm_page_t) vm_page_queue_next(&m->listq); + next = (vm_page_t) vm_page_queue_next(&m->vmp_listq); - if ((m->offset >= start) && (m->offset < end)) { - /* + if ((m->vmp_offset >= start) && (m->vmp_offset < end)) { + /* * this is a page we're interested in * try to fit it into a current extent */ - for (n = 0; n < num_of_extents; n++) { - if ((m->offset & e_mask) == extents[n].e_base) { - /* + for (n = 0; n < num_of_extents; n++) { + if ((m->vmp_offset & e_mask) == extents[n].e_base) { + /* * use (PAGE_SIZE - 1) to determine the * max offset so that we don't wrap if * we're at the last page of the space */ - if (m->offset < extents[n].e_min) - extents[n].e_min = m->offset; - else if ((m->offset + (PAGE_SIZE - 1)) > extents[n].e_max) - extents[n].e_max = m->offset + (PAGE_SIZE - 1); - break; + if (m->vmp_offset < extents[n].e_min) { + extents[n].e_min = m->vmp_offset; + } else if ((m->vmp_offset + (PAGE_SIZE - 1)) > extents[n].e_max) { + extents[n].e_max = m->vmp_offset + (PAGE_SIZE - 1); + } + break; } } if (n == num_of_extents) { - /* + /* * didn't find a current extent that can encompass * this page */ - if (n < MAX_EXTENTS) { - /* - * if we still have room, + if (n < MAX_EXTENTS) { + /* + * if we still have room, * create a new extent */ - extents[n].e_base = m->offset & e_mask; - extents[n].e_min = m->offset; - extents[n].e_max = m->offset + (PAGE_SIZE - 1); + extents[n].e_base = m->vmp_offset & e_mask; + extents[n].e_min = m->vmp_offset; + extents[n].e_max = m->vmp_offset + (PAGE_SIZE - 1); num_of_extents++; } else { /* * no room to create a new extent... * fall back to a single extent based - * on the min and max page offsets + * on the min and max page offsets * we find in the range we're interested in... * first, look through the extent list and * develop the overall min and max for the * pages we've looked at up to this point - */ - for (n = 1; n < num_of_extents; n++) { - if (extents[n].e_min < extents[0].e_min) - extents[0].e_min = extents[n].e_min; - if (extents[n].e_max > extents[0].e_max) - extents[0].e_max = extents[n].e_max; + */ + for (n = 1; n < num_of_extents; n++) { + if (extents[n].e_min < extents[0].e_min) { + extents[0].e_min = extents[n].e_min; + } + if (extents[n].e_max > extents[0].e_max) { + extents[0].e_max = extents[n].e_max; + } } /* * now setup to run through the remaining pages @@ -1031,49 +1039,48 @@ BYPASS_COW_COPYIN: m = next; } } else { - extents[0].e_min = offset; + extents[0].e_min = offset; extents[0].e_max = offset + (size - 1); num_of_extents = 1; } for (n = 0; n < num_of_extents; n++) { - if (vm_object_update_extent(object, extents[n].e_min, extents[n].e_max, resid_offset, io_errno, - should_flush, should_return, should_iosync, protection)) - data_returned = TRUE; + if (vm_object_update_extent(object, extents[n].e_min, extents[n].e_max, resid_offset, io_errno, + should_flush, should_return, should_iosync, protection)) { + data_returned = TRUE; + } } - return (data_returned); + return data_returned; } static kern_return_t vm_object_set_attributes_common( - vm_object_t object, - boolean_t may_cache, + vm_object_t object, + boolean_t may_cache, memory_object_copy_strategy_t copy_strategy) { - boolean_t object_became_ready; + boolean_t object_became_ready; - XPR(XPR_MEMORY_OBJECT, - "m_o_set_attr_com, object 0x%X flg %x strat %d\n", - object, (may_cache&1), copy_strategy, 0, 0); - - if (object == VM_OBJECT_NULL) - return(KERN_INVALID_ARGUMENT); + if (object == VM_OBJECT_NULL) { + return KERN_INVALID_ARGUMENT; + } /* * Verify the attributes of importance */ - switch(copy_strategy) { - case MEMORY_OBJECT_COPY_NONE: - case MEMORY_OBJECT_COPY_DELAY: - break; - default: - return(KERN_INVALID_ARGUMENT); + switch (copy_strategy) { + case MEMORY_OBJECT_COPY_NONE: + case MEMORY_OBJECT_COPY_DELAY: + break; + default: + return KERN_INVALID_ARGUMENT; } - if (may_cache) + if (may_cache) { may_cache = TRUE; + } vm_object_lock(object); @@ -1097,45 +1104,46 @@ vm_object_set_attributes_common( vm_object_unlock(object); - return(KERN_SUCCESS); + return KERN_SUCCESS; } kern_return_t memory_object_synchronize_completed( - __unused memory_object_control_t control, - __unused memory_object_offset_t offset, - __unused memory_object_size_t length) + __unused memory_object_control_t control, + __unused memory_object_offset_t offset, + __unused memory_object_size_t length) { - panic("memory_object_synchronize_completed no longer supported\n"); - return(KERN_FAILURE); + panic("memory_object_synchronize_completed no longer supported\n"); + return KERN_FAILURE; } /* * Set the memory object attribute as provided. * - * XXX This routine cannot be completed until the vm_msync, clean + * XXX This routine cannot be completed until the vm_msync, clean * in place, and cluster work is completed. See ifdef notyet * below and note that vm_object_set_attributes_common() * may have to be expanded. */ kern_return_t memory_object_change_attributes( - memory_object_control_t control, - memory_object_flavor_t flavor, - memory_object_info_t attributes, - mach_msg_type_number_t count) + memory_object_control_t control, + memory_object_flavor_t flavor, + memory_object_info_t attributes, + mach_msg_type_number_t count) { - vm_object_t object; - kern_return_t result = KERN_SUCCESS; - boolean_t may_cache; - boolean_t invalidate; - memory_object_copy_strategy_t copy_strategy; + vm_object_t object; + kern_return_t result = KERN_SUCCESS; + boolean_t may_cache; + boolean_t invalidate; + memory_object_copy_strategy_t copy_strategy; object = memory_object_control_to_vm_object(control); - if (object == VM_OBJECT_NULL) - return (KERN_INVALID_ARGUMENT); + if (object == VM_OBJECT_NULL) { + return KERN_INVALID_ARGUMENT; + } vm_object_lock(object); @@ -1144,99 +1152,100 @@ memory_object_change_attributes( #if notyet invalidate = object->invalidate; #endif - vm_object_unlock(object); + vm_object_unlock(object); switch (flavor) { - case OLD_MEMORY_OBJECT_BEHAVIOR_INFO: - { - old_memory_object_behave_info_t behave; + case OLD_MEMORY_OBJECT_BEHAVIOR_INFO: + { + old_memory_object_behave_info_t behave; - if (count != OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) { - result = KERN_INVALID_ARGUMENT; - break; - } + if (count != OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) { + result = KERN_INVALID_ARGUMENT; + break; + } - behave = (old_memory_object_behave_info_t) attributes; + behave = (old_memory_object_behave_info_t) attributes; invalidate = behave->invalidate; copy_strategy = behave->copy_strategy; break; - } + } - case MEMORY_OBJECT_BEHAVIOR_INFO: - { - memory_object_behave_info_t behave; + case MEMORY_OBJECT_BEHAVIOR_INFO: + { + memory_object_behave_info_t behave; - if (count != MEMORY_OBJECT_BEHAVE_INFO_COUNT) { - result = KERN_INVALID_ARGUMENT; - break; - } + if (count != MEMORY_OBJECT_BEHAVE_INFO_COUNT) { + result = KERN_INVALID_ARGUMENT; + break; + } - behave = (memory_object_behave_info_t) attributes; + behave = (memory_object_behave_info_t) attributes; invalidate = behave->invalidate; copy_strategy = behave->copy_strategy; break; - } + } - case MEMORY_OBJECT_PERFORMANCE_INFO: - { - memory_object_perf_info_t perf; + case MEMORY_OBJECT_PERFORMANCE_INFO: + { + memory_object_perf_info_t perf; - if (count != MEMORY_OBJECT_PERF_INFO_COUNT) { - result = KERN_INVALID_ARGUMENT; - break; - } + if (count != MEMORY_OBJECT_PERF_INFO_COUNT) { + result = KERN_INVALID_ARGUMENT; + break; + } - perf = (memory_object_perf_info_t) attributes; + perf = (memory_object_perf_info_t) attributes; may_cache = perf->may_cache; break; - } + } - case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO: - { - old_memory_object_attr_info_t attr; + case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO: + { + old_memory_object_attr_info_t attr; - if (count != OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) { - result = KERN_INVALID_ARGUMENT; - break; - } + if (count != OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) { + result = KERN_INVALID_ARGUMENT; + break; + } attr = (old_memory_object_attr_info_t) attributes; - may_cache = attr->may_cache; - copy_strategy = attr->copy_strategy; + may_cache = attr->may_cache; + copy_strategy = attr->copy_strategy; break; - } + } - case MEMORY_OBJECT_ATTRIBUTE_INFO: - { - memory_object_attr_info_t attr; + case MEMORY_OBJECT_ATTRIBUTE_INFO: + { + memory_object_attr_info_t attr; - if (count != MEMORY_OBJECT_ATTR_INFO_COUNT) { - result = KERN_INVALID_ARGUMENT; - break; - } + if (count != MEMORY_OBJECT_ATTR_INFO_COUNT) { + result = KERN_INVALID_ARGUMENT; + break; + } attr = (memory_object_attr_info_t) attributes; copy_strategy = attr->copy_strategy; - may_cache = attr->may_cache_object; + may_cache = attr->may_cache_object; break; - } + } - default: + default: result = KERN_INVALID_ARGUMENT; break; } - if (result != KERN_SUCCESS) - return(result); + if (result != KERN_SUCCESS) { + return result; + } if (copy_strategy == MEMORY_OBJECT_COPY_TEMPORARY) { copy_strategy = MEMORY_OBJECT_COPY_DELAY; @@ -1246,31 +1255,32 @@ memory_object_change_attributes( * XXX may_cache may become a tri-valued variable to handle * XXX uncache if not in use. */ - return (vm_object_set_attributes_common(object, - may_cache, - copy_strategy)); + return vm_object_set_attributes_common(object, + may_cache, + copy_strategy); } kern_return_t memory_object_get_attributes( - memory_object_control_t control, - memory_object_flavor_t flavor, - memory_object_info_t attributes, /* pointer to OUT array */ - mach_msg_type_number_t *count) /* IN/OUT */ + memory_object_control_t control, + memory_object_flavor_t flavor, + memory_object_info_t attributes, /* pointer to OUT array */ + mach_msg_type_number_t *count) /* IN/OUT */ { - kern_return_t ret = KERN_SUCCESS; - vm_object_t object; + kern_return_t ret = KERN_SUCCESS; + vm_object_t object; object = memory_object_control_to_vm_object(control); - if (object == VM_OBJECT_NULL) - return (KERN_INVALID_ARGUMENT); + if (object == VM_OBJECT_NULL) { + return KERN_INVALID_ARGUMENT; + } - vm_object_lock(object); + vm_object_lock(object); switch (flavor) { - case OLD_MEMORY_OBJECT_BEHAVIOR_INFO: - { - old_memory_object_behave_info_t behave; + case OLD_MEMORY_OBJECT_BEHAVIOR_INFO: + { + old_memory_object_behave_info_t behave; if (*count < OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) { ret = KERN_INVALID_ARGUMENT; @@ -1280,42 +1290,42 @@ memory_object_get_attributes( behave = (old_memory_object_behave_info_t) attributes; behave->copy_strategy = object->copy_strategy; behave->temporary = FALSE; -#if notyet /* remove when vm_msync complies and clean in place fini */ - behave->invalidate = object->invalidate; +#if notyet /* remove when vm_msync complies and clean in place fini */ + behave->invalidate = object->invalidate; #else behave->invalidate = FALSE; #endif *count = OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT; break; - } + } - case MEMORY_OBJECT_BEHAVIOR_INFO: - { - memory_object_behave_info_t behave; + case MEMORY_OBJECT_BEHAVIOR_INFO: + { + memory_object_behave_info_t behave; if (*count < MEMORY_OBJECT_BEHAVE_INFO_COUNT) { - ret = KERN_INVALID_ARGUMENT; - break; - } + ret = KERN_INVALID_ARGUMENT; + break; + } - behave = (memory_object_behave_info_t) attributes; - behave->copy_strategy = object->copy_strategy; + behave = (memory_object_behave_info_t) attributes; + behave->copy_strategy = object->copy_strategy; behave->temporary = FALSE; -#if notyet /* remove when vm_msync complies and clean in place fini */ - behave->invalidate = object->invalidate; +#if notyet /* remove when vm_msync complies and clean in place fini */ + behave->invalidate = object->invalidate; #else behave->invalidate = FALSE; #endif behave->advisory_pageout = FALSE; behave->silent_overwrite = FALSE; - *count = MEMORY_OBJECT_BEHAVE_INFO_COUNT; + *count = MEMORY_OBJECT_BEHAVE_INFO_COUNT; break; - } + } - case MEMORY_OBJECT_PERFORMANCE_INFO: - { - memory_object_perf_info_t perf; + case MEMORY_OBJECT_PERFORMANCE_INFO: + { + memory_object_perf_info_t perf; if (*count < MEMORY_OBJECT_PERF_INFO_COUNT) { ret = KERN_INVALID_ARGUMENT; @@ -1328,69 +1338,69 @@ memory_object_get_attributes( *count = MEMORY_OBJECT_PERF_INFO_COUNT; break; - } + } - case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO: - { - old_memory_object_attr_info_t attr; + case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO: + { + old_memory_object_attr_info_t attr; - if (*count < OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) { - ret = KERN_INVALID_ARGUMENT; - break; - } + if (*count < OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) { + ret = KERN_INVALID_ARGUMENT; + break; + } - attr = (old_memory_object_attr_info_t) attributes; - attr->may_cache = object->can_persist; - attr->copy_strategy = object->copy_strategy; + attr = (old_memory_object_attr_info_t) attributes; + attr->may_cache = object->can_persist; + attr->copy_strategy = object->copy_strategy; - *count = OLD_MEMORY_OBJECT_ATTR_INFO_COUNT; - break; - } + *count = OLD_MEMORY_OBJECT_ATTR_INFO_COUNT; + break; + } - case MEMORY_OBJECT_ATTRIBUTE_INFO: - { - memory_object_attr_info_t attr; + case MEMORY_OBJECT_ATTRIBUTE_INFO: + { + memory_object_attr_info_t attr; - if (*count < MEMORY_OBJECT_ATTR_INFO_COUNT) { - ret = KERN_INVALID_ARGUMENT; - break; - } + if (*count < MEMORY_OBJECT_ATTR_INFO_COUNT) { + ret = KERN_INVALID_ARGUMENT; + break; + } - attr = (memory_object_attr_info_t) attributes; - attr->copy_strategy = object->copy_strategy; + attr = (memory_object_attr_info_t) attributes; + attr->copy_strategy = object->copy_strategy; attr->cluster_size = PAGE_SIZE; - attr->may_cache_object = object->can_persist; + attr->may_cache_object = object->can_persist; attr->temporary = FALSE; - *count = MEMORY_OBJECT_ATTR_INFO_COUNT; - break; - } + *count = MEMORY_OBJECT_ATTR_INFO_COUNT; + break; + } - default: + default: ret = KERN_INVALID_ARGUMENT; break; } - vm_object_unlock(object); + vm_object_unlock(object); - return(ret); + return ret; } kern_return_t memory_object_iopl_request( - ipc_port_t port, - memory_object_offset_t offset, - upl_size_t *upl_size, - upl_t *upl_ptr, - upl_page_info_array_t user_page_list, - unsigned int *page_list_count, - upl_control_flags_t *flags, - vm_tag_t tag) + ipc_port_t port, + memory_object_offset_t offset, + upl_size_t *upl_size, + upl_t *upl_ptr, + upl_page_info_array_t user_page_list, + unsigned int *page_list_count, + upl_control_flags_t *flags, + vm_tag_t tag) { - vm_object_t object; - kern_return_t ret; - upl_control_flags_t caller_flags; + vm_object_t object; + kern_return_t ret; + upl_control_flags_t caller_flags; caller_flags = *flags; @@ -1403,59 +1413,71 @@ memory_object_iopl_request( } if (ip_kotype(port) == IKOT_NAMED_ENTRY) { - vm_named_entry_t named_entry; + vm_named_entry_t named_entry; - named_entry = (vm_named_entry_t)port->ip_kobject; + named_entry = (vm_named_entry_t) ip_get_kobject(port); /* a few checks to make sure user is obeying rules */ - if(*upl_size == 0) { - if(offset >= named_entry->size) - return(KERN_INVALID_RIGHT); + if (*upl_size == 0) { + if (offset >= named_entry->size) { + return KERN_INVALID_RIGHT; + } *upl_size = (upl_size_t)(named_entry->size - offset); - if (*upl_size != named_entry->size - offset) + if (*upl_size != named_entry->size - offset) { return KERN_INVALID_ARGUMENT; + } } - if(caller_flags & UPL_COPYOUT_FROM) { - if((named_entry->protection & VM_PROT_READ) - != VM_PROT_READ) { - return(KERN_INVALID_RIGHT); + if (caller_flags & UPL_COPYOUT_FROM) { + if ((named_entry->protection & VM_PROT_READ) + != VM_PROT_READ) { + return KERN_INVALID_RIGHT; } } else { - if((named_entry->protection & - (VM_PROT_READ | VM_PROT_WRITE)) - != (VM_PROT_READ | VM_PROT_WRITE)) { - return(KERN_INVALID_RIGHT); + if ((named_entry->protection & + (VM_PROT_READ | VM_PROT_WRITE)) + != (VM_PROT_READ | VM_PROT_WRITE)) { + return KERN_INVALID_RIGHT; } } - if(named_entry->size < (offset + *upl_size)) - return(KERN_INVALID_ARGUMENT); + if (named_entry->size < (offset + *upl_size)) { + return KERN_INVALID_ARGUMENT; + } /* the callers parameter offset is defined to be the */ /* offset from beginning of named entry offset in object */ offset = offset + named_entry->offset; + offset += named_entry->data_offset; if (named_entry->is_sub_map || - named_entry->is_copy) + named_entry->is_copy) { + return KERN_INVALID_ARGUMENT; + } + if (!named_entry->is_object) { return KERN_INVALID_ARGUMENT; - + } + named_entry_lock(named_entry); - object = named_entry->backing.object; + object = vm_named_entry_to_vm_object(named_entry); + assert(object != VM_OBJECT_NULL); vm_object_reference(object); named_entry_unlock(named_entry); } else if (ip_kotype(port) == IKOT_MEM_OBJ_CONTROL) { - memory_object_control_t control; + memory_object_control_t control; control = (memory_object_control_t) port; - if (control == NULL) - return (KERN_INVALID_ARGUMENT); + if (control == NULL) { + return KERN_INVALID_ARGUMENT; + } object = memory_object_control_to_vm_object(control); - if (object == VM_OBJECT_NULL) - return (KERN_INVALID_ARGUMENT); + if (object == VM_OBJECT_NULL) { + return KERN_INVALID_ARGUMENT; + } vm_object_reference(object); } else { return KERN_INVALID_ARGUMENT; } - if (object == VM_OBJECT_NULL) - return (KERN_INVALID_ARGUMENT); + if (object == VM_OBJECT_NULL) { + return KERN_INVALID_ARGUMENT; + } if (!object->private) { if (object->phys_contiguous) { @@ -1468,18 +1490,18 @@ memory_object_iopl_request( } ret = vm_object_iopl_request(object, - offset, - *upl_size, - upl_ptr, - user_page_list, - page_list_count, - caller_flags, - tag); + offset, + *upl_size, + upl_ptr, + user_page_list, + page_list_count, + caller_flags, + tag); vm_object_deallocate(object); return ret; } -/* +/* * Routine: memory_object_upl_request [interface] * Purpose: * Cause the population of a portion of a vm_object. @@ -1490,32 +1512,35 @@ memory_object_iopl_request( kern_return_t memory_object_upl_request( - memory_object_control_t control, - memory_object_offset_t offset, - upl_size_t size, - upl_t *upl_ptr, - upl_page_info_array_t user_page_list, - unsigned int *page_list_count, - int cntrl_flags, - int tag) + memory_object_control_t control, + memory_object_offset_t offset, + upl_size_t size, + upl_t *upl_ptr, + upl_page_info_array_t user_page_list, + unsigned int *page_list_count, + int cntrl_flags, + int tag) { - vm_object_t object; + vm_object_t object; + vm_tag_t vmtag = (vm_tag_t)tag; + assert(vmtag == tag); object = memory_object_control_to_vm_object(control); - if (object == VM_OBJECT_NULL) - return (KERN_TERMINATED); + if (object == VM_OBJECT_NULL) { + return KERN_TERMINATED; + } return vm_object_upl_request(object, - offset, - size, - upl_ptr, - user_page_list, - page_list_count, - (upl_control_flags_t)(unsigned int) cntrl_flags, - tag); + offset, + size, + upl_ptr, + user_page_list, + page_list_count, + (upl_control_flags_t)(unsigned int) cntrl_flags, + vmtag); } -/* +/* * Routine: memory_object_super_upl_request [interface] * Purpose: * Cause the population of a portion of a vm_object @@ -1529,50 +1554,64 @@ memory_object_upl_request( kern_return_t memory_object_super_upl_request( memory_object_control_t control, - memory_object_offset_t offset, - upl_size_t size, - upl_size_t super_cluster, - upl_t *upl, - upl_page_info_t *user_page_list, - unsigned int *page_list_count, - int cntrl_flags, - int tag) + memory_object_offset_t offset, + upl_size_t size, + upl_size_t super_cluster, + upl_t *upl, + upl_page_info_t *user_page_list, + unsigned int *page_list_count, + int cntrl_flags, + int tag) { - vm_object_t object; + vm_object_t object; + vm_tag_t vmtag = (vm_tag_t)tag; + assert(vmtag == tag); object = memory_object_control_to_vm_object(control); - if (object == VM_OBJECT_NULL) - return (KERN_INVALID_ARGUMENT); + if (object == VM_OBJECT_NULL) { + return KERN_INVALID_ARGUMENT; + } return vm_object_super_upl_request(object, - offset, - size, - super_cluster, - upl, - user_page_list, - page_list_count, - (upl_control_flags_t)(unsigned int) cntrl_flags, - tag); + offset, + size, + super_cluster, + upl, + user_page_list, + page_list_count, + (upl_control_flags_t)(unsigned int) cntrl_flags, + vmtag); } kern_return_t -memory_object_cluster_size(memory_object_control_t control, memory_object_offset_t *start, - vm_size_t *length, uint32_t *io_streaming, memory_object_fault_info_t fault_info) +memory_object_cluster_size( + memory_object_control_t control, + memory_object_offset_t *start, + vm_size_t *length, + uint32_t *io_streaming, + memory_object_fault_info_t mo_fault_info) { - vm_object_t object; + vm_object_t object; + vm_object_fault_info_t fault_info; object = memory_object_control_to_vm_object(control); - if (object == VM_OBJECT_NULL || object->paging_offset > *start) - return (KERN_INVALID_ARGUMENT); + if (object == VM_OBJECT_NULL || object->paging_offset > *start) { + return KERN_INVALID_ARGUMENT; + } *start -= object->paging_offset; - vm_object_cluster_size(object, (vm_object_offset_t *)start, length, (vm_object_fault_info_t)fault_info, io_streaming); + fault_info = (vm_object_fault_info_t)(uintptr_t) mo_fault_info; + vm_object_cluster_size(object, + (vm_object_offset_t *)start, + length, + fault_info, + io_streaming); *start += object->paging_offset; - return (KERN_SUCCESS); + return KERN_SUCCESS; } @@ -1586,8 +1625,8 @@ memory_object_cluster_size(memory_object_control_t control, memory_object_offset */ kern_return_t host_default_memory_manager( - host_priv_t host_priv, - memory_object_default_t *default_manager, + host_priv_t host_priv, + memory_object_default_t *default_manager, __unused memory_object_cluster_size_t cluster_size) { memory_object_default_t current_manager; @@ -1595,8 +1634,9 @@ host_default_memory_manager( memory_object_default_t returned_manager; kern_return_t result = KERN_SUCCESS; - if (host_priv == HOST_PRIV_NULL) - return(KERN_INVALID_HOST); + if (host_priv == HOST_PRIV_NULL) { + return KERN_INVALID_HOST; + } assert(host_priv == &realhost); @@ -1627,8 +1667,9 @@ host_default_memory_manager( */ if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) { result = vm_pageout_internal_start(); - if (result != KERN_SUCCESS) + if (result != KERN_SUCCESS) { goto out; + } } /* @@ -1654,16 +1695,15 @@ host_default_memory_manager( * reactivate all the throttled pages (i.e. dirty pages with * no pager). */ - if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) - { + if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) { vm_page_reactivate_all_throttled(); } } - out: +out: lck_mtx_unlock(&memory_manager_default_lock); *default_manager = returned_manager; - return(result); + return result; } /* @@ -1685,9 +1725,9 @@ memory_manager_default_reference(void) wait_result_t res; res = lck_mtx_sleep(&memory_manager_default_lock, - LCK_SLEEP_DEFAULT, - (event_t) &memory_manager_default, - THREAD_UNINT); + LCK_SLEEP_DEFAULT, + (event_t) &memory_manager_default, + THREAD_UNINT); assert(res == THREAD_AWAKENED); current_manager = memory_manager_default; } @@ -1717,110 +1757,108 @@ memory_manager_default_check(void) lck_mtx_lock(&memory_manager_default_lock); current = memory_manager_default; if (current == MEMORY_OBJECT_DEFAULT_NULL) { - static boolean_t logged; /* initialized to 0 */ - boolean_t complain = !logged; + static boolean_t logged; /* initialized to 0 */ + boolean_t complain = !logged; logged = TRUE; lck_mtx_unlock(&memory_manager_default_lock); - if (complain) + if (complain) { printf("Warning: No default memory manager\n"); - return(KERN_FAILURE); + } + return KERN_FAILURE; } else { lck_mtx_unlock(&memory_manager_default_lock); - return(KERN_SUCCESS); + return KERN_SUCCESS; } } -__private_extern__ void -memory_manager_default_init(void) -{ - memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL; - lck_mtx_init(&memory_manager_default_lock, &vm_object_lck_grp, &vm_object_lck_attr); -} - - - /* Allow manipulation of individual page state. This is actually part of */ /* the UPL regimen but takes place on the object rather than on a UPL */ kern_return_t memory_object_page_op( - memory_object_control_t control, - memory_object_offset_t offset, - int ops, - ppnum_t *phys_entry, - int *flags) + memory_object_control_t control, + memory_object_offset_t offset, + int ops, + ppnum_t *phys_entry, + int *flags) { - vm_object_t object; + vm_object_t object; object = memory_object_control_to_vm_object(control); - if (object == VM_OBJECT_NULL) - return (KERN_INVALID_ARGUMENT); + if (object == VM_OBJECT_NULL) { + return KERN_INVALID_ARGUMENT; + } return vm_object_page_op(object, offset, ops, phys_entry, flags); } /* - * memory_object_range_op offers performance enhancement over - * memory_object_page_op for page_op functions which do not require page - * level state to be returned from the call. Page_op was created to provide - * a low-cost alternative to page manipulation via UPLs when only a single - * page was involved. The range_op call establishes the ability in the _op + * memory_object_range_op offers performance enhancement over + * memory_object_page_op for page_op functions which do not require page + * level state to be returned from the call. Page_op was created to provide + * a low-cost alternative to page manipulation via UPLs when only a single + * page was involved. The range_op call establishes the ability in the _op * family of functions to work on multiple pages where the lack of page level * state handling allows the caller to avoid the overhead of the upl structures. */ kern_return_t memory_object_range_op( - memory_object_control_t control, - memory_object_offset_t offset_beg, - memory_object_offset_t offset_end, + memory_object_control_t control, + memory_object_offset_t offset_beg, + memory_object_offset_t offset_end, int ops, int *range) { - vm_object_t object; + vm_object_t object; object = memory_object_control_to_vm_object(control); - if (object == VM_OBJECT_NULL) - return (KERN_INVALID_ARGUMENT); + if (object == VM_OBJECT_NULL) { + return KERN_INVALID_ARGUMENT; + } return vm_object_range_op(object, - offset_beg, - offset_end, - ops, - (uint32_t *) range); + offset_beg, + offset_end, + ops, + (uint32_t *) range); } void memory_object_mark_used( - memory_object_control_t control) + memory_object_control_t control) { - vm_object_t object; + vm_object_t object; - if (control == NULL) + if (control == NULL) { return; + } object = memory_object_control_to_vm_object(control); - if (object != VM_OBJECT_NULL) + if (object != VM_OBJECT_NULL) { vm_object_cache_remove(object); + } } void memory_object_mark_unused( - memory_object_control_t control, - __unused boolean_t rage) + memory_object_control_t control, + __unused boolean_t rage) { - vm_object_t object; + vm_object_t object; - if (control == NULL) + if (control == NULL) { return; + } object = memory_object_control_to_vm_object(control); - if (object != VM_OBJECT_NULL) + if (object != VM_OBJECT_NULL) { vm_object_cache_add(object); + } } void @@ -1829,8 +1867,9 @@ memory_object_mark_io_tracking( { vm_object_t object; - if (control == NULL) + if (control == NULL) { return; + } object = memory_object_control_to_vm_object(control); if (object != VM_OBJECT_NULL) { @@ -1840,16 +1879,35 @@ memory_object_mark_io_tracking( } } +void +memory_object_mark_trusted( + memory_object_control_t control) +{ + vm_object_t object; + + if (control == NULL) { + return; + } + object = memory_object_control_to_vm_object(control); + + if (object != VM_OBJECT_NULL) { + vm_object_lock(object); + object->pager_trusted = TRUE; + vm_object_unlock(object); + } +} + #if CONFIG_SECLUDED_MEMORY void memory_object_mark_eligible_for_secluded( memory_object_control_t control, - boolean_t eligible_for_secluded) + boolean_t eligible_for_secluded) { vm_object_t object; - if (control == NULL) + if (control == NULL) { return; + } object = memory_object_control_to_vm_object(control); if (object == VM_OBJECT_NULL) { @@ -1863,7 +1921,7 @@ memory_object_mark_eligible_for_secluded( object->eligible_for_secluded = TRUE; vm_page_secluded.eligible_for_secluded += object->resident_page_count; } else if (!eligible_for_secluded && - object->eligible_for_secluded) { + object->eligible_for_secluded) { object->eligible_for_secluded = FALSE; vm_page_secluded.eligible_for_secluded -= object->resident_page_count; if (object->resident_page_count) { @@ -1877,33 +1935,36 @@ memory_object_mark_eligible_for_secluded( kern_return_t memory_object_pages_resident( - memory_object_control_t control, - boolean_t * has_pages_resident) + memory_object_control_t control, + boolean_t * has_pages_resident) { - vm_object_t object; + vm_object_t object; *has_pages_resident = FALSE; object = memory_object_control_to_vm_object(control); - if (object == VM_OBJECT_NULL) - return (KERN_INVALID_ARGUMENT); + if (object == VM_OBJECT_NULL) { + return KERN_INVALID_ARGUMENT; + } - if (object->resident_page_count) + if (object->resident_page_count) { *has_pages_resident = TRUE; - - return (KERN_SUCCESS); + } + + return KERN_SUCCESS; } kern_return_t memory_object_signed( - memory_object_control_t control, - boolean_t is_signed) + memory_object_control_t control, + boolean_t is_signed) { - vm_object_t object; + vm_object_t object; object = memory_object_control_to_vm_object(control); - if (object == VM_OBJECT_NULL) + if (object == VM_OBJECT_NULL) { return KERN_INVALID_ARGUMENT; + } vm_object_lock(object); object->code_signed = is_signed; @@ -1914,14 +1975,15 @@ memory_object_signed( boolean_t memory_object_is_signed( - memory_object_control_t control) + memory_object_control_t control) { - boolean_t is_signed; - vm_object_t object; + boolean_t is_signed; + vm_object_t object; object = memory_object_control_to_vm_object(control); - if (object == VM_OBJECT_NULL) + if (object == VM_OBJECT_NULL) { return FALSE; + } vm_object_lock_shared(object); is_signed = object->code_signed; @@ -1931,36 +1993,26 @@ memory_object_is_signed( } boolean_t -memory_object_is_slid( - memory_object_control_t control) +memory_object_is_shared_cache( + memory_object_control_t control) { - vm_object_t object = VM_OBJECT_NULL; + vm_object_t object = VM_OBJECT_NULL; object = memory_object_control_to_vm_object(control); - if (object == VM_OBJECT_NULL) + if (object == VM_OBJECT_NULL) { return FALSE; + } - return object->object_slid; + return object->object_is_shared_cache; } -static zone_t mem_obj_control_zone; - -__private_extern__ void -memory_object_control_bootstrap(void) -{ - int i; - - i = (vm_size_t) sizeof (struct memory_object_control); - mem_obj_control_zone = zinit (i, 8192*i, 4096, "mem_obj_control"); - zone_change(mem_obj_control_zone, Z_CALLERACCT, FALSE); - zone_change(mem_obj_control_zone, Z_NOENCRYPT, TRUE); - return; -} +static ZONE_DECLARE(mem_obj_control_zone, "mem_obj_control", + sizeof(struct memory_object_control), ZC_NOENCRYPT); __private_extern__ memory_object_control_t memory_object_control_allocate( - vm_object_t object) -{ + vm_object_t object) +{ memory_object_control_t control; control = (memory_object_control_t)zalloc(mem_obj_control_zone); @@ -1968,28 +2020,29 @@ memory_object_control_allocate( control->moc_object = object; control->moc_ikot = IKOT_MEM_OBJ_CONTROL; /* fake ip_kotype */ } - return (control); + return control; } __private_extern__ void memory_object_control_collapse( - memory_object_control_t control, - vm_object_t object) -{ + memory_object_control_t control, + vm_object_t object) +{ assert((control->moc_object != VM_OBJECT_NULL) && - (control->moc_object != object)); + (control->moc_object != object)); control->moc_object = object; } __private_extern__ vm_object_t memory_object_control_to_vm_object( - memory_object_control_t control) + memory_object_control_t control) { if (control == MEMORY_OBJECT_CONTROL_NULL || - control->moc_ikot != IKOT_MEM_OBJ_CONTROL) + control->moc_ikot != IKOT_MEM_OBJ_CONTROL) { return VM_OBJECT_NULL; + } - return (control->moc_object); + return control->moc_object; } __private_extern__ vm_object_t @@ -2010,7 +2063,7 @@ memory_object_to_vm_object( memory_object_control_t convert_port_to_mo_control( - __unused mach_port_t port) + __unused mach_port_t port) { return MEMORY_OBJECT_CONTROL_NULL; } @@ -2018,14 +2071,14 @@ convert_port_to_mo_control( mach_port_t convert_mo_control_to_port( - __unused memory_object_control_t control) + __unused memory_object_control_t control) { return MACH_PORT_NULL; } void memory_object_control_reference( - __unused memory_object_control_t control) + __unused memory_object_control_t control) { return; } @@ -2037,14 +2090,14 @@ memory_object_control_reference( */ void memory_object_control_deallocate( - memory_object_control_t control) + memory_object_control_t control) { zfree(mem_obj_control_zone, control); } void memory_object_control_disable( - memory_object_control_t control) + memory_object_control_t control) { assert(control->moc_object != VM_OBJECT_NULL); control->moc_object = VM_OBJECT_NULL; @@ -2066,22 +2119,23 @@ memory_object_default_deallocate( memory_object_t convert_port_to_memory_object( - __unused mach_port_t port) + __unused mach_port_t port) { - return (MEMORY_OBJECT_NULL); + return MEMORY_OBJECT_NULL; } mach_port_t convert_memory_object_to_port( - __unused memory_object_t object) + __unused memory_object_t object) { - return (MACH_PORT_NULL); + return MACH_PORT_NULL; } /* Routine memory_object_reference */ -void memory_object_reference( +void +memory_object_reference( memory_object_t memory_object) { (memory_object->mo_pager_ops->memory_object_reference)( @@ -2089,16 +2143,18 @@ void memory_object_reference( } /* Routine memory_object_deallocate */ -void memory_object_deallocate( +void +memory_object_deallocate( memory_object_t memory_object) { (memory_object->mo_pager_ops->memory_object_deallocate)( - memory_object); + memory_object); } /* Routine memory_object_init */ -kern_return_t memory_object_init +kern_return_t +memory_object_init ( memory_object_t memory_object, memory_object_control_t memory_control, @@ -2112,7 +2168,8 @@ kern_return_t memory_object_init } /* Routine memory_object_terminate */ -kern_return_t memory_object_terminate +kern_return_t +memory_object_terminate ( memory_object_t memory_object ) @@ -2122,7 +2179,8 @@ kern_return_t memory_object_terminate } /* Routine memory_object_data_request */ -kern_return_t memory_object_data_request +kern_return_t +memory_object_data_request ( memory_object_t memory_object, memory_object_offset_t offset, @@ -2133,23 +2191,24 @@ kern_return_t memory_object_data_request { return (memory_object->mo_pager_ops->memory_object_data_request)( memory_object, - offset, + offset, length, desired_access, fault_info); } /* Routine memory_object_data_return */ -kern_return_t memory_object_data_return +kern_return_t +memory_object_data_return ( memory_object_t memory_object, memory_object_offset_t offset, memory_object_cluster_size_t size, memory_object_offset_t *resid_offset, - int *io_error, + int *io_error, boolean_t dirty, boolean_t kernel_copy, - int upl_flags + int upl_flags ) { return (memory_object->mo_pager_ops->memory_object_data_return)( @@ -2164,7 +2223,8 @@ kern_return_t memory_object_data_return } /* Routine memory_object_data_initialize */ -kern_return_t memory_object_data_initialize +kern_return_t +memory_object_data_initialize ( memory_object_t memory_object, memory_object_offset_t offset, @@ -2178,7 +2238,8 @@ kern_return_t memory_object_data_initialize } /* Routine memory_object_data_unlock */ -kern_return_t memory_object_data_unlock +kern_return_t +memory_object_data_unlock ( memory_object_t memory_object, memory_object_offset_t offset, @@ -2194,7 +2255,8 @@ kern_return_t memory_object_data_unlock } /* Routine memory_object_synchronize */ -kern_return_t memory_object_synchronize +kern_return_t +memory_object_synchronize ( memory_object_t memory_object, memory_object_offset_t offset, @@ -2202,7 +2264,7 @@ kern_return_t memory_object_synchronize vm_sync_t sync_flags ) { - panic("memory_object_syncrhonize no longer supported\n"); + panic("memory_object_syncrhonize no longer supported\n"); return (memory_object->mo_pager_ops->memory_object_synchronize)( memory_object, @@ -2216,7 +2278,7 @@ kern_return_t memory_object_synchronize * memory_object_map() is called by VM (in vm_map_enter() and its variants) * each time a "named" VM object gets mapped directly or indirectly * (copy-on-write mapping). A "named" VM object has an extra reference held - * by the pager to keep it alive until the pager decides that the + * by the pager to keep it alive until the pager decides that the * memory object (and its VM object) can be reclaimed. * VM calls memory_object_last_unmap() (in vm_object_deallocate()) when all * the mappings of that memory object have been removed. @@ -2231,7 +2293,8 @@ kern_return_t memory_object_synchronize */ /* Routine memory_object_map */ -kern_return_t memory_object_map +kern_return_t +memory_object_map ( memory_object_t memory_object, vm_prot_t prot @@ -2243,7 +2306,8 @@ kern_return_t memory_object_map } /* Routine memory_object_last_unmap */ -kern_return_t memory_object_last_unmap +kern_return_t +memory_object_last_unmap ( memory_object_t memory_object ) @@ -2253,14 +2317,16 @@ kern_return_t memory_object_last_unmap } /* Routine memory_object_data_reclaim */ -kern_return_t memory_object_data_reclaim +kern_return_t +memory_object_data_reclaim ( memory_object_t memory_object, - boolean_t reclaim_backing_store + boolean_t reclaim_backing_store ) { - if (memory_object->mo_pager_ops->memory_object_data_reclaim == NULL) + if (memory_object->mo_pager_ops->memory_object_data_reclaim == NULL) { return KERN_NOT_SUPPORTED; + } return (memory_object->mo_pager_ops->memory_object_data_reclaim)( memory_object, reclaim_backing_store); @@ -2268,34 +2334,34 @@ kern_return_t memory_object_data_reclaim upl_t convert_port_to_upl( - ipc_port_t port) + ipc_port_t port) { upl_t upl; ip_lock(port); if (!ip_active(port) || (ip_kotype(port) != IKOT_UPL)) { - ip_unlock(port); - return (upl_t)NULL; + ip_unlock(port); + return (upl_t)NULL; } - upl = (upl_t) port->ip_kobject; + upl = (upl_t) ip_get_kobject(port); ip_unlock(port); upl_lock(upl); - upl->ref_count+=1; + upl->ref_count += 1; upl_unlock(upl); return upl; } mach_port_t convert_upl_to_port( - __unused upl_t upl) + __unused upl_t upl) { return MACH_PORT_NULL; } __private_extern__ void upl_no_senders( - __unused ipc_port_t port, - __unused mach_port_mscount_t mscount) + __unused ipc_port_t port, + __unused mach_port_mscount_t mscount) { return; }