/*
- * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* External memory management interface control functions.
*/
-#include <advisory_pageout.h>
-
/*
* Interface dependencies:
*/
#include <vm/vm_kern.h> /* For kernel_map, vm_move */
#include <vm/vm_map.h> /* For vm_map_pageable */
#include <vm/vm_purgeable_internal.h> /* Needed by some vm_page.h macros */
+#include <vm/vm_shared_region.h>
-#if MACH_PAGEMAP
#include <vm/vm_external.h>
-#endif /* MACH_PAGEMAP */
#include <vm/vm_protos.h>
-
memory_object_default_t memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL;
-decl_mutex_data(, memory_manager_default_lock)
+decl_lck_mtx_data(, memory_manager_default_lock)
/*
typedef int memory_object_lock_result_t;
-#define MEMORY_OBJECT_LOCK_RESULT_DONE 0
-#define MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK 1
-#define MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN 2
-#define MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN 3
+#define MEMORY_OBJECT_LOCK_RESULT_DONE 0
+#define MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK 1
+#define MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN 2
+#define MEMORY_OBJECT_LOCK_RESULT_MUST_FREE 3
memory_object_lock_result_t memory_object_lock_page(
vm_page_t m,
{
XPR(XPR_MEMORY_OBJECT,
"m_o_lock_page, page 0x%X rtn %d flush %d prot %d\n",
- (integer_t)m, should_return, should_flush, prot, 0);
+ m, should_return, should_flush, prot, 0);
- /*
- * If we cannot change access to the page,
- * either because a mapping is in progress
- * (busy page) or because a mapping has been
- * wired, then give up.
- */
if (m->busy || m->cleaning)
- return(MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK);
+ return (MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK);
+
+ if (m->laundry)
+ vm_pageout_steal_laundry(m, FALSE);
/*
* Don't worry about pages for which the kernel
* does not have any data.
*/
-
if (m->absent || m->error || m->restart) {
- if(m->error && should_flush) {
- /* dump the page, pager wants us to */
- /* clean it up and there is no */
- /* relevant data to return */
- if(m->wire_count == 0) {
- VM_PAGE_FREE(m);
- return(MEMORY_OBJECT_LOCK_RESULT_DONE);
- }
- } else {
- return(MEMORY_OBJECT_LOCK_RESULT_DONE);
+ if (m->error && should_flush && !VM_PAGE_WIRED(m)) {
+ /*
+ * dump the page, pager wants us to
+ * clean it up and there is no
+ * relevant data to return
+ */
+ return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE);
}
+ return (MEMORY_OBJECT_LOCK_RESULT_DONE);
}
-
assert(!m->fictitious);
- /*
- * If the page is wired, just clean or return the page if needed.
- * Wired pages don't get flushed or disconnected from the pmap.
- */
-
- if (m->wire_count != 0) {
- if (memory_object_should_return_page(m, should_return)) {
- if (m->dirty)
- return(MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN);
- else
- return(MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN);
- }
-
- return(MEMORY_OBJECT_LOCK_RESULT_DONE);
- }
-
- /*
- * If the page is to be flushed, allow
- * that to be done as part of the protection.
- */
-
- if (should_flush)
- prot = VM_PROT_ALL;
-
- /*
- * Set the page lock.
- *
- * If we are decreasing permission, do it now;
- * let the fault handler take care of increases
- * (pmap_page_protect may not increase protection).
- */
+ if (VM_PAGE_WIRED(m)) {
+ /*
+ * The page is wired... just clean or return the page if needed.
+ * Wired pages don't get flushed or disconnected from the pmap.
+ */
+ if (memory_object_should_return_page(m, should_return))
+ return (MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN);
- if (prot != VM_PROT_NO_CHANGE) {
- pmap_page_protect(m->phys_page, VM_PROT_ALL & ~prot);
+ return (MEMORY_OBJECT_LOCK_RESULT_DONE);
+ }
- PAGE_WAKEUP(m);
+ if (should_flush) {
+ /*
+ * must do the pmap_disconnect before determining the
+ * need to return the page... otherwise it's possible
+ * for the page to go from the clean to the dirty state
+ * after we've made our decision
+ */
+ if (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(m, FALSE);
+ }
+ } else {
+ /*
+ * If we are decreasing permission, do it now;
+ * let the fault handler take care of increases
+ * (pmap_page_protect may not increase protection).
+ */
+ if (prot != VM_PROT_NO_CHANGE)
+ pmap_page_protect(m->phys_page, VM_PROT_ALL & ~prot);
}
-
/*
- * Handle page returning.
+ * Handle returning dirty or precious pages
*/
if (memory_object_should_return_page(m, should_return)) {
-
/*
- * If we weren't planning
- * to flush the page anyway,
- * we may need to remove the
- * page from the pageout
- * system and from physical
- * maps now.
+ * we use to do a pmap_disconnect here in support
+ * of memory_object_lock_request, but that routine
+ * no longer requires this... in any event, in
+ * our world, it would turn into a big noop since
+ * we don't lock the page in any way and as soon
+ * as we drop the object lock, the page can be
+ * faulted back into an address space
+ *
+ * if (!should_flush)
+ * pmap_disconnect(m->phys_page);
*/
-
- vm_page_lockspin_queues();
- VM_PAGE_QUEUES_REMOVE(m);
- vm_page_unlock_queues();
-
- if (!should_flush)
- pmap_disconnect(m->phys_page);
-
- if (m->dirty)
- return(MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN);
- else
- return(MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN);
+ return (MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN);
}
/*
- * Handle flushing
+ * Handle flushing clean pages
*/
- if (should_flush) {
- VM_PAGE_FREE(m);
- } else {
- /*
- * XXX Make clean but not flush a paging hint,
- * and deactivate the pages. This is a hack
- * because it overloads flush/clean with
- * implementation-dependent meaning. This only
- * happens to pages that are already clean.
- */
+ if (should_flush)
+ return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE);
- if (vm_page_deactivate_hint &&
- (should_return != MEMORY_OBJECT_RETURN_NONE)) {
- vm_page_lock_queues();
- vm_page_deactivate(m);
- vm_page_unlock_queues();
- }
- }
+ /*
+ * we use to deactivate clean pages at this point,
+ * but we do not believe that an msync should change
+ * the 'age' of a page in the cache... here is the
+ * original comment and code concerning this...
+ *
+ * XXX Make clean but not flush a paging hint,
+ * and deactivate the pages. This is a hack
+ * because it overloads flush/clean with
+ * implementation-dependent meaning. This only
+ * happens to pages that are already clean.
+ *
+ * if (vm_page_deactivate_hint && (should_return != MEMORY_OBJECT_RETURN_NONE))
+ * return (MEMORY_OBJECT_LOCK_RESULT_MUST_DEACTIVATE);
+ */
- return(MEMORY_OBJECT_LOCK_RESULT_DONE);
+ return (MEMORY_OBJECT_LOCK_RESULT_DONE);
}
-#define LIST_REQ_PAGEOUT_PAGES(object, data_cnt, action, po, ro, ioerr, iosync) \
-MACRO_BEGIN \
- \
- register int upl_flags; \
- memory_object_t pager; \
- \
- if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) { \
- vm_object_paging_begin(object); \
- vm_object_unlock(object); \
- \
- if (iosync) \
- upl_flags = UPL_MSYNC | UPL_IOSYNC; \
- else \
- upl_flags = UPL_MSYNC; \
- \
- (void) memory_object_data_return(pager, \
- po, \
- data_cnt, \
- ro, \
- ioerr, \
- (action) == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN,\
- !should_flush, \
- upl_flags); \
- \
- vm_object_lock(object); \
- vm_object_paging_end(object); \
- } \
-MACRO_END
+
/*
* Routine: memory_object_lock_request [user interface]
vm_prot_t prot)
{
vm_object_t object;
- __unused boolean_t should_flush;
- should_flush = flags & MEMORY_OBJECT_DATA_FLUSH;
-
- XPR(XPR_MEMORY_OBJECT,
- "m_o_lock_request, control 0x%X off 0x%X size 0x%X flags %X prot %X\n",
- (integer_t)control, offset, size,
- (((should_return&1)<<1)|should_flush), prot);
-
- /*
+ /*
* Check for bogus arguments.
*/
object = memory_object_control_to_vm_object(control);
*/
vm_object_lock(object);
vm_object_paging_begin(object);
+
+ if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL) {
+ if ((should_return != MEMORY_OBJECT_RETURN_NONE) || offset || object->copy) {
+ flags &= ~MEMORY_OBJECT_DATA_FLUSH_ALL;
+ flags |= MEMORY_OBJECT_DATA_FLUSH;
+ }
+ }
offset -= object->paging_offset;
- (void)vm_object_update(object,
- offset, size, resid_offset, io_errno, should_return, flags, prot);
+ if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL)
+ vm_object_reap_pages(object, REAP_DATA_FLUSH);
+ else
+ (void)vm_object_update(object, offset, size, resid_offset,
+ io_errno, should_return, flags, prot);
vm_object_paging_end(object);
vm_object_unlock(object);
XPR(XPR_VM_OBJECT,
"vm_o_sync, object 0x%X, offset 0x%X size 0x%x flush %d rtn %d\n",
- (integer_t)object, offset, size, should_flush, should_return);
+ object, offset, size, should_flush, should_return);
/*
* Lock the object, and acquire a paging reference to
vm_object_lock(object);
vm_object_paging_begin(object);
- if (should_flush)
+ if (should_flush) {
flags = MEMORY_OBJECT_DATA_FLUSH;
- else
+ /*
+ * This flush is from an msync(), not a truncate(), so the
+ * contents of the file are not affected.
+ * MEMORY_OBECT_DATA_NO_CHANGE lets vm_object_update() know
+ * that the data is not changed and that there's no need to
+ * push the old contents to a copy object.
+ */
+ flags |= MEMORY_OBJECT_DATA_NO_CHANGE;
+ } else
flags = 0;
if (should_iosync)
+#define LIST_REQ_PAGEOUT_PAGES(object, data_cnt, po, ro, ioerr, iosync) \
+MACRO_BEGIN \
+ \
+ int upl_flags; \
+ memory_object_t pager; \
+ \
+ if (object->object_slid) { \
+ panic("Objects with slid pages not allowed\n"); \
+ } \
+ \
+ if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) { \
+ vm_object_paging_begin(object); \
+ vm_object_unlock(object); \
+ \
+ if (iosync) \
+ upl_flags = UPL_MSYNC | UPL_IOSYNC; \
+ else \
+ upl_flags = UPL_MSYNC; \
+ \
+ (void) memory_object_data_return(pager, \
+ po, \
+ (memory_object_cluster_size_t)data_cnt, \
+ ro, \
+ ioerr, \
+ FALSE, \
+ FALSE, \
+ upl_flags); \
+ \
+ vm_object_lock(object); \
+ vm_object_paging_end(object); \
+ } \
+MACRO_END
+
+
static int
vm_object_update_extent(
{
vm_page_t m;
int retval = 0;
- vm_size_t data_cnt = 0;
vm_object_offset_t paging_offset = 0;
- vm_object_offset_t last_offset = offset;
+ vm_object_offset_t next_offset = offset;
memory_object_lock_result_t page_lock_result;
- memory_object_lock_result_t pageout_action;
-
- pageout_action = MEMORY_OBJECT_LOCK_RESULT_DONE;
+ memory_object_cluster_size_t data_cnt = 0;
+ struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT];
+ struct vm_page_delayed_work *dwp;
+ int dw_count;
+ int dw_limit;
+
+ dwp = &dw_array[0];
+ dw_count = 0;
+ dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
for (;
offset < offset_end && object->resident_page_count;
offset += PAGE_SIZE_64) {
/*
- * Limit the number of pages to be cleaned at once.
+ * Limit the number of pages to be cleaned at once to a contiguous
+ * run, or at most MAX_UPL_TRANSFER_BYTES
*/
- if (data_cnt >= PAGE_SIZE * MAX_UPL_TRANSFER) {
- LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
- pageout_action, paging_offset, offset_resid, io_errno, should_iosync);
- data_cnt = 0;
- }
+ if (data_cnt) {
+ if ((data_cnt >= MAX_UPL_TRANSFER_BYTES) || (next_offset != offset)) {
+ if (dw_count) {
+ vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);
+ dwp = &dw_array[0];
+ dw_count = 0;
+ }
+ LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
+ paging_offset, offset_resid, io_errno, should_iosync);
+ data_cnt = 0;
+ }
+ }
while ((m = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
- page_lock_result = memory_object_lock_page(m, should_return, should_flush, prot);
-
- XPR(XPR_MEMORY_OBJECT,
- "m_o_update: lock_page, obj 0x%X offset 0x%X result %d\n",
- (integer_t)object, offset, page_lock_result, 0, 0);
-
- switch (page_lock_result)
- {
- case MEMORY_OBJECT_LOCK_RESULT_DONE:
- /*
- * End of a cluster of dirty pages.
- */
- if (data_cnt) {
- LIST_REQ_PAGEOUT_PAGES(object,
- data_cnt, pageout_action,
- paging_offset, offset_resid, io_errno, should_iosync);
- data_cnt = 0;
- continue;
- }
- break;
-
- case MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK:
- /*
- * Since it is necessary to block,
- * clean any dirty pages now.
- */
- if (data_cnt) {
- LIST_REQ_PAGEOUT_PAGES(object,
- data_cnt, pageout_action,
- paging_offset, offset_resid, io_errno, should_iosync);
- data_cnt = 0;
- continue;
- }
- PAGE_SLEEP(object, m, THREAD_UNINT);
- continue;
-
- case MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN:
- case MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN:
- /*
- * The clean and return cases are similar.
- *
- * if this would form a discontiguous block,
- * clean the old pages and start anew.
- *
- * Mark the page busy since we will unlock the
- * object if we issue the LIST_REQ_PAGEOUT
- */
- m->busy = TRUE;
- if (data_cnt &&
- ((last_offset != offset) || (pageout_action != page_lock_result))) {
- LIST_REQ_PAGEOUT_PAGES(object,
- data_cnt, pageout_action,
- paging_offset, offset_resid, io_errno, should_iosync);
- data_cnt = 0;
- }
- m->busy = FALSE;
-
- if (m->cleaning) {
- PAGE_SLEEP(object, m, THREAD_UNINT);
- continue;
- }
- if (data_cnt == 0) {
- pageout_action = page_lock_result;
- paging_offset = offset;
- }
- data_cnt += PAGE_SIZE;
- last_offset = offset + PAGE_SIZE_64;
-
- vm_page_lockspin_queues();
- /*
- * Clean
- */
- m->list_req_pending = TRUE;
- m->cleaning = TRUE;
-
- if (should_flush &&
- /* let's no flush a wired page... */
- !m->wire_count) {
- /*
- * and add additional state
- * for the flush
- */
- m->busy = TRUE;
- m->pageout = TRUE;
- vm_page_wire(m);
- }
- vm_page_unlock_queues();
-
- retval = 1;
- break;
+
+ dwp->dw_mask = 0;
+
+ page_lock_result = memory_object_lock_page(m, should_return, should_flush, prot);
+
+ if (data_cnt && page_lock_result != MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN) {
+ /*
+ * End of a run of dirty/precious pages.
+ */
+ if (dw_count) {
+ vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);
+ dwp = &dw_array[0];
+ dw_count = 0;
+ }
+ LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
+ paging_offset, offset_resid, io_errno, should_iosync);
+ /*
+ * LIST_REQ_PAGEOUT_PAGES will drop the object lock which will
+ * allow the state of page 'm' to change... we need to re-lookup
+ * the current offset
+ */
+ data_cnt = 0;
+ continue;
+ }
+
+ switch (page_lock_result) {
+
+ case MEMORY_OBJECT_LOCK_RESULT_DONE:
+ break;
+
+ case MEMORY_OBJECT_LOCK_RESULT_MUST_FREE:
+ dwp->dw_mask |= DW_vm_page_free;
+ break;
+
+ case MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK:
+ PAGE_SLEEP(object, m, THREAD_UNINT);
+ continue;
+
+ case MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN:
+ if (data_cnt == 0)
+ paging_offset = offset;
+
+ data_cnt += PAGE_SIZE;
+ next_offset = offset + PAGE_SIZE_64;
+
+ /*
+ * wired pages shouldn't be flushed and
+ * since they aren't on any queue,
+ * no need to remove them
+ */
+ if (!VM_PAGE_WIRED(m)) {
+
+ if (should_flush) {
+ /*
+ * add additional state for the flush
+ */
+ m->pageout = TRUE;
+ }
+ /*
+ * we use to remove the page from the queues at this
+ * point, but we do not believe that an msync
+ * should cause the 'age' of a page to be changed
+ *
+ * else
+ * dwp->dw_mask |= DW_VM_PAGE_QUEUES_REMOVE;
+ */
+ }
+ retval = 1;
+ break;
+ }
+ if (dwp->dw_mask) {
+ VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count);
+
+ if (dw_count >= dw_limit) {
+ vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);
+ dwp = &dw_array[0];
+ dw_count = 0;
+ }
}
break;
}
* We have completed the scan for applicable pages.
* Clean any pages that have been saved.
*/
+ if (dw_count)
+ vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);
+
if (data_cnt) {
- LIST_REQ_PAGEOUT_PAGES(object,
- data_cnt, pageout_action, paging_offset, offset_resid, io_errno, should_iosync);
+ LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
+ paging_offset, offset_resid, io_errno, should_iosync);
}
return (retval);
}
*/
kern_return_t
vm_object_update(
- register vm_object_t object,
- register vm_object_offset_t offset,
- register vm_object_size_t size,
- register vm_object_offset_t *resid_offset,
- int *io_errno,
- memory_object_return_t should_return,
- int flags,
- vm_prot_t protection)
+ vm_object_t object,
+ vm_object_offset_t offset,
+ vm_object_size_t size,
+ vm_object_offset_t *resid_offset,
+ int *io_errno,
+ memory_object_return_t should_return,
+ int flags,
+ vm_prot_t protection)
{
vm_object_t copy_object = VM_OBJECT_NULL;
boolean_t data_returned = FALSE;
boolean_t update_cow;
boolean_t should_flush = (flags & MEMORY_OBJECT_DATA_FLUSH) ? TRUE : FALSE;
boolean_t should_iosync = (flags & MEMORY_OBJECT_IO_SYNC) ? TRUE : FALSE;
+ vm_fault_return_t result;
int num_of_extents;
int n;
#define MAX_EXTENTS 8
/*
* translate offset with respect to shadow's offset
*/
- copy_offset = (offset >= copy_object->shadow_offset) ?
- (vm_map_offset_t)(offset - copy_object->shadow_offset) :
+ copy_offset = (offset >= copy_object->vo_shadow_offset) ?
+ (vm_map_offset_t)(offset - copy_object->vo_shadow_offset) :
(vm_map_offset_t) 0;
- if (copy_offset > copy_object->size)
- copy_offset = copy_object->size;
+ if (copy_offset > copy_object->vo_size)
+ copy_offset = copy_object->vo_size;
/*
* clip size with respect to shadow offset
*/
- if (offset >= copy_object->shadow_offset) {
+ if (offset >= copy_object->vo_shadow_offset) {
copy_size = size;
- } else if (size >= copy_object->shadow_offset - offset) {
- copy_size = size - (copy_object->shadow_offset - offset);
+ } else if (size >= copy_object->vo_shadow_offset - offset) {
+ copy_size = size - (copy_object->vo_shadow_offset - offset);
} else {
copy_size = 0;
}
- if (copy_offset + copy_size > copy_object->size) {
- if (copy_object->size >= copy_offset) {
- copy_size = copy_object->size - copy_offset;
+ if (copy_offset + copy_size > copy_object->vo_size) {
+ if (copy_object->vo_size >= copy_offset) {
+ copy_size = copy_object->vo_size - copy_offset;
} else {
copy_size = 0;
}
fault_info.interruptible = THREAD_UNINT;
fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL;
fault_info.user_tag = 0;
+ fault_info.pmap_options = 0;
fault_info.lo_offset = copy_offset;
fault_info.hi_offset = copy_size;
fault_info.no_cache = FALSE;
+ fault_info.stealth = TRUE;
+ fault_info.io_sync = FALSE;
+ fault_info.cs_bypass = FALSE;
+ fault_info.mark_zf_absent = FALSE;
+ fault_info.batch_pmap_op = FALSE;
vm_object_paging_begin(copy_object);
for (i = copy_offset; i < copy_size; i += PAGE_SIZE) {
RETRY_COW_OF_LOCK_REQUEST:
- fault_info.cluster_size = copy_size - i;
+ fault_info.cluster_size = (vm_size_t) (copy_size - i);
+ assert(fault_info.cluster_size == copy_size - i);
prot = VM_PROT_WRITE|VM_PROT_READ;
- switch (vm_fault_page(copy_object, i,
- VM_PROT_WRITE|VM_PROT_READ,
- FALSE,
- &prot,
- &page,
- &top_page,
- (int *)0,
- &error,
- FALSE,
- FALSE, &fault_info)) {
-
+ page = VM_PAGE_NULL;
+ result = vm_fault_page(copy_object, i,
+ VM_PROT_WRITE|VM_PROT_READ,
+ FALSE,
+ FALSE, /* page not looked up */
+ &prot,
+ &page,
+ &top_page,
+ (int *)0,
+ &error,
+ FALSE,
+ FALSE, &fault_info);
+
+ switch (result) {
case VM_FAULT_SUCCESS:
if (top_page) {
vm_fault_cleanup(
vm_object_lock(copy_object);
vm_object_paging_begin(copy_object);
}
- vm_page_lock_queues();
- if (!page->active && !page->inactive)
- vm_page_deactivate(page);
- vm_page_unlock_queues();
-
+ if (!page->active &&
+ !page->inactive &&
+ !page->throttled) {
+ vm_page_lockspin_queues();
+ if (!page->active &&
+ !page->inactive &&
+ !page->throttled)
+ vm_page_deactivate(page);
+ vm_page_unlock_queues();
+ }
PAGE_WAKEUP_DONE(page);
break;
case VM_FAULT_RETRY:
vm_object_lock(copy_object);
vm_object_paging_begin(copy_object);
goto RETRY_COW_OF_LOCK_REQUEST;
- case VM_FAULT_FICTITIOUS_SHORTAGE:
- vm_page_more_fictitious();
- prot = VM_PROT_WRITE|VM_PROT_READ;
- vm_object_lock(copy_object);
- vm_object_paging_begin(copy_object);
- goto RETRY_COW_OF_LOCK_REQUEST;
+ case VM_FAULT_SUCCESS_NO_VM_PAGE:
+ /* success but no VM page: fail */
+ vm_object_paging_end(copy_object);
+ vm_object_unlock(copy_object);
+ /*FALLTHROUGH*/
case VM_FAULT_MEMORY_ERROR:
if (object != copy_object)
vm_object_deallocate(copy_object);
vm_object_lock(object);
goto BYPASS_COW_COPYIN;
+ default:
+ panic("vm_object_update: unexpected error 0x%x"
+ " from vm_fault_page()\n", result);
}
}
memory_object_synchronize_completed(
memory_object_control_t control,
memory_object_offset_t offset,
- vm_offset_t length)
+ memory_object_size_t length)
{
vm_object_t object;
msync_req_t msr;
XPR(XPR_MEMORY_OBJECT,
"m_o_sync_completed, object 0x%X, offset 0x%X length 0x%X\n",
- (integer_t)object, offset, length, 0, 0);
+ object, offset, length, 0, 0);
/*
* Look for bogus arguments
boolean_t may_cache,
memory_object_copy_strategy_t copy_strategy,
boolean_t temporary,
- boolean_t silent_overwrite,
+ __unused boolean_t silent_overwrite,
boolean_t advisory_pageout)
{
boolean_t object_became_ready;
XPR(XPR_MEMORY_OBJECT,
"m_o_set_attr_com, object 0x%X flg %x strat %d\n",
- (integer_t)object, (may_cache&1)|((temporary&1)<1), copy_strategy, 0, 0);
+ object, (may_cache&1)|((temporary&1)<1), copy_strategy, 0, 0);
if (object == VM_OBJECT_NULL)
return(KERN_INVALID_ARGUMENT);
return(KERN_INVALID_ARGUMENT);
}
-#if !ADVISORY_PAGEOUT
- if (silent_overwrite || advisory_pageout)
- return(KERN_INVALID_ARGUMENT);
-
-#endif /* !ADVISORY_PAGEOUT */
if (may_cache)
may_cache = TRUE;
if (temporary)
object->copy_strategy = copy_strategy;
object->can_persist = may_cache;
object->temporary = temporary;
- object->silent_overwrite = silent_overwrite;
+// object->silent_overwrite = silent_overwrite;
object->advisory_pageout = advisory_pageout;
/*
temporary = object->temporary;
may_cache = object->can_persist;
copy_strategy = object->copy_strategy;
- silent_overwrite = object->silent_overwrite;
+// silent_overwrite = object->silent_overwrite;
+ silent_overwrite = FALSE;
advisory_pageout = object->advisory_pageout;
#if notyet
invalidate = object->invalidate;
behave->invalidate = FALSE;
#endif
behave->advisory_pageout = object->advisory_pageout;
- behave->silent_overwrite = object->silent_overwrite;
+// behave->silent_overwrite = object->silent_overwrite;
+ behave->silent_overwrite = FALSE;
*count = MEMORY_OBJECT_BEHAVE_INFO_COUNT;
break;
}
upl_t *upl_ptr,
upl_page_info_array_t user_page_list,
unsigned int *page_list_count,
- int *flags)
+ upl_control_flags_t *flags)
{
vm_object_t object;
kern_return_t ret;
- int caller_flags;
+ upl_control_flags_t caller_flags;
caller_flags = *flags;
if(*upl_size == 0) {
if(offset >= named_entry->size)
return(KERN_INVALID_RIGHT);
- *upl_size = named_entry->size - offset;
+ *upl_size = (upl_size_t)(named_entry->size - offset);
+ if (*upl_size != named_entry->size - offset)
+ return KERN_INVALID_ARGUMENT;
}
if(caller_flags & UPL_COPYOUT_FROM) {
if((named_entry->protection & VM_PROT_READ)
/* offset from beginning of named entry offset in object */
offset = offset + named_entry->offset;
- if(named_entry->is_sub_map)
- return (KERN_INVALID_ARGUMENT);
+ if (named_entry->is_sub_map ||
+ named_entry->is_copy)
+ return KERN_INVALID_ARGUMENT;
named_entry_lock(named_entry);
return (KERN_INVALID_ARGUMENT);
if (!object->private) {
- if (*upl_size > (MAX_UPL_TRANSFER*PAGE_SIZE))
- *upl_size = (MAX_UPL_TRANSFER*PAGE_SIZE);
if (object->phys_contiguous) {
*flags = UPL_PHYS_CONTIG;
} else {
object = memory_object_control_to_vm_object(control);
if (object == VM_OBJECT_NULL)
- return (KERN_INVALID_ARGUMENT);
+ return (KERN_TERMINATED);
return vm_object_upl_request(object,
offset,
upl_ptr,
user_page_list,
page_list_count,
- cntrl_flags);
+ (upl_control_flags_t)(unsigned int) cntrl_flags);
}
/*
upl,
user_page_list,
page_list_count,
- cntrl_flags);
+ (upl_control_flags_t)(unsigned int) cntrl_flags);
}
kern_return_t
memory_object_cluster_size(memory_object_control_t control, memory_object_offset_t *start,
- vm_size_t *length, memory_object_fault_info_t fault_info)
+ vm_size_t *length, uint32_t *io_streaming, memory_object_fault_info_t fault_info)
{
vm_object_t object;
*start -= object->paging_offset;
- vm_object_cluster_size(object, (vm_object_offset_t *)start, length, (vm_object_fault_info_t)fault_info);
+ vm_object_cluster_size(object, (vm_object_offset_t *)start, length, (vm_object_fault_info_t)fault_info, io_streaming);
*start += object->paging_offset;
assert(host_priv == &realhost);
new_manager = *default_manager;
- mutex_lock(&memory_manager_default_lock);
+ lck_mtx_lock(&memory_manager_default_lock);
current_manager = memory_manager_default;
returned_manager = MEMORY_OBJECT_DEFAULT_NULL;
returned_manager = current_manager;
memory_object_default_reference(returned_manager);
} else {
+ /*
+ * Only allow the kernel to change the value.
+ */
+ extern task_t kernel_task;
+ if (current_task() != kernel_task) {
+ result = KERN_NO_ACCESS;
+ goto out;
+ }
/*
* If this is the first non-null manager, start
*/
thread_wakeup((event_t) &memory_manager_default);
+
+ /*
+ * Now that we have a default pager for anonymous memory,
+ * reactivate all the throttled pages (i.e. dirty pages with
+ * no pager).
+ */
+ if (current_manager == MEMORY_OBJECT_DEFAULT_NULL)
+ {
+ vm_page_reactivate_all_throttled();
+ }
}
out:
- mutex_unlock(&memory_manager_default_lock);
+ lck_mtx_unlock(&memory_manager_default_lock);
*default_manager = returned_manager;
return(result);
{
memory_object_default_t current_manager;
- mutex_lock(&memory_manager_default_lock);
+ lck_mtx_lock(&memory_manager_default_lock);
current_manager = memory_manager_default;
while (current_manager == MEMORY_OBJECT_DEFAULT_NULL) {
wait_result_t res;
- res = thread_sleep_mutex((event_t) &memory_manager_default,
- &memory_manager_default_lock,
- THREAD_UNINT);
+ res = lck_mtx_sleep(&memory_manager_default_lock,
+ LCK_SLEEP_DEFAULT,
+ (event_t) &memory_manager_default,
+ THREAD_UNINT);
assert(res == THREAD_AWAKENED);
current_manager = memory_manager_default;
}
memory_object_default_reference(current_manager);
- mutex_unlock(&memory_manager_default_lock);
+ lck_mtx_unlock(&memory_manager_default_lock);
return current_manager;
}
{
memory_object_default_t current;
- mutex_lock(&memory_manager_default_lock);
+ lck_mtx_lock(&memory_manager_default_lock);
current = memory_manager_default;
if (current == MEMORY_OBJECT_DEFAULT_NULL) {
static boolean_t logged; /* initialized to 0 */
boolean_t complain = !logged;
logged = TRUE;
- mutex_unlock(&memory_manager_default_lock);
+ lck_mtx_unlock(&memory_manager_default_lock);
if (complain)
printf("Warning: No default memory manager\n");
return(KERN_FAILURE);
} else {
- mutex_unlock(&memory_manager_default_lock);
+ lck_mtx_unlock(&memory_manager_default_lock);
return(KERN_SUCCESS);
}
}
memory_manager_default_init(void)
{
memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL;
- mutex_init(&memory_manager_default_lock, 0);
+ lck_mtx_init(&memory_manager_default_lock, &vm_object_lck_grp, &vm_object_lck_attr);
}
offset_beg,
offset_end,
ops,
- range);
+ (uint32_t *) range);
}
+void
+memory_object_mark_used(
+ memory_object_control_t control)
+{
+ vm_object_t object;
+
+ if (control == NULL)
+ return;
+
+ object = memory_object_control_to_vm_object(control);
+
+ if (object != VM_OBJECT_NULL)
+ vm_object_cache_remove(object);
+}
+
+
+void
+memory_object_mark_unused(
+ memory_object_control_t control,
+ __unused boolean_t rage)
+{
+ vm_object_t object;
+
+ if (control == NULL)
+ return;
+
+ object = memory_object_control_to_vm_object(control);
+
+ if (object != VM_OBJECT_NULL)
+ vm_object_cache_add(object);
+}
+
+void
+memory_object_mark_io_tracking(
+ memory_object_control_t control)
+{
+ vm_object_t object;
+
+ if (control == NULL)
+ return;
+ object = memory_object_control_to_vm_object(control);
+
+ if (object != VM_OBJECT_NULL) {
+ vm_object_lock(object);
+ object->io_tracking = TRUE;
+ vm_object_unlock(object);
+ }
+}
+
kern_return_t
memory_object_pages_resident(
memory_object_control_t control,
return KERN_SUCCESS;
}
+boolean_t
+memory_object_is_signed(
+ memory_object_control_t control)
+{
+ boolean_t is_signed;
+ vm_object_t object;
+
+ object = memory_object_control_to_vm_object(control);
+ if (object == VM_OBJECT_NULL)
+ return FALSE;
+
+ vm_object_lock_shared(object);
+ is_signed = object->code_signed;
+ vm_object_unlock(object);
+
+ return is_signed;
+}
+
+boolean_t
+memory_object_is_slid(
+ memory_object_control_t control)
+{
+ vm_object_t object = VM_OBJECT_NULL;
+
+ object = memory_object_control_to_vm_object(control);
+ if (object == VM_OBJECT_NULL)
+ return FALSE;
+
+ return object->object_slid;
+}
+
static zone_t mem_obj_control_zone;
__private_extern__ void
i = (vm_size_t) sizeof (struct memory_object_control);
mem_obj_control_zone = zinit (i, 8192*i, 4096, "mem_obj_control");
+ zone_change(mem_obj_control_zone, Z_CALLERACCT, FALSE);
+ zone_change(mem_obj_control_zone, Z_NOENCRYPT, TRUE);
return;
}
(
memory_object_t memory_object,
memory_object_offset_t offset,
- vm_size_t size,
+ memory_object_cluster_size_t size,
memory_object_offset_t *resid_offset,
int *io_error,
boolean_t dirty,
(
memory_object_t memory_object,
memory_object_offset_t offset,
- vm_size_t size
+ memory_object_cluster_size_t size
)
{
return (memory_object->mo_pager_ops->memory_object_data_initialize)(
(
memory_object_t memory_object,
memory_object_offset_t offset,
- vm_size_t size,
+ memory_object_size_t size,
vm_prot_t desired_access
)
{
(
memory_object_t memory_object,
memory_object_offset_t offset,
- vm_size_t size,
+ memory_object_size_t size,
vm_sync_t sync_flags
)
{
memory_object);
}
+/* Routine memory_object_data_reclaim */
+kern_return_t memory_object_data_reclaim
+(
+ memory_object_t memory_object,
+ boolean_t reclaim_backing_store
+)
+{
+ if (memory_object->mo_pager_ops->memory_object_data_reclaim == NULL)
+ return KERN_NOT_SUPPORTED;
+ return (memory_object->mo_pager_ops->memory_object_data_reclaim)(
+ memory_object,
+ reclaim_backing_store);
+}
+
/* Routine memory_object_create */
kern_return_t memory_object_create
(
projects / apple / xnu.git / blobdiff