/*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
*
- * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. The rights granted to you under the
- * License may not be used to create, or enable the creation or
- * redistribution of, unlawful or unlicensed copies of an Apple operating
- * system, or to circumvent, violate, or enable the circumvention or
- * violation of, any terms of an Apple operating system software license
- * agreement.
- *
- * Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
- *
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
* limitations under the License.
- *
- * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* @OSF_COPYRIGHT@
#include <vm/pmap.h> /* For pmap_clear_modify */
#include <vm/vm_kern.h> /* For kernel_map, vm_move */
#include <vm/vm_map.h> /* For vm_map_pageable */
+#include <vm/vm_purgeable_internal.h> /* Needed by some vm_page.h macros */
#if MACH_PAGEMAP
#include <vm/vm_external.h>
memory_object_default_t memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL;
-vm_size_t memory_manager_default_cluster = 0;
decl_mutex_data(, memory_manager_default_lock)
assert(!m->fictitious);
- if (m->wire_count != 0) {
- /*
- * If no change would take place
- * anyway, return successfully.
- *
- * No change means:
- * Not flushing AND
- * No change to page lock [2 checks] AND
- * Should not return page
- *
- * XXX This doesn't handle sending a copy of a wired
- * XXX page to the pager, but that will require some
- * XXX significant surgery.
- */
- if (!should_flush &&
- (m->page_lock == prot || prot == VM_PROT_NO_CHANGE) &&
- ! memory_object_should_return_page(m, should_return)) {
-
- /*
- * Restart page unlock requests,
- * even though no change took place.
- * [Memory managers may be expecting
- * to see new requests.]
- */
- m->unlock_request = VM_PROT_NONE;
- PAGE_WAKEUP(m);
+ /*
+ * If the page is wired, just clean or return the page if needed.
+ * Wired pages don't get flushed or disconnected from the pmap.
+ */
- return(MEMORY_OBJECT_LOCK_RESULT_DONE);
+ if (m->wire_count != 0) {
+ if (memory_object_should_return_page(m, should_return)) {
+ if (m->dirty)
+ return(MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN);
+ else
+ return(MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN);
}
- return(MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK);
+ return(MEMORY_OBJECT_LOCK_RESULT_DONE);
}
/*
*/
if (prot != VM_PROT_NO_CHANGE) {
- if ((m->page_lock ^ prot) & prot) {
- pmap_page_protect(m->phys_page, VM_PROT_ALL & ~prot);
- }
-#if 0
- /* code associated with the vestigial
- * memory_object_data_unlock
- */
- m->page_lock = prot;
- m->lock_supplied = TRUE;
- if (prot != VM_PROT_NONE)
- m->unusual = TRUE;
- else
- m->unusual = FALSE;
-
- /*
- * Restart any past unlock requests, even if no
- * change resulted. If the manager explicitly
- * requested no protection change, then it is assumed
- * to be remembering past requests.
- */
+ pmap_page_protect(m->phys_page, VM_PROT_ALL & ~prot);
- m->unlock_request = VM_PROT_NONE;
-#endif /* 0 */
PAGE_WAKEUP(m);
}
/*
* Handle page returning.
*/
-
if (memory_object_should_return_page(m, should_return)) {
/*
* maps now.
*/
- vm_page_lock_queues();
+ vm_page_lockspin_queues();
VM_PAGE_QUEUES_REMOVE(m);
vm_page_unlock_queues();
/*
* Handle flushing
*/
-
if (should_flush) {
VM_PAGE_FREE(m);
} else {
MACRO_BEGIN \
\
register int upl_flags; \
+ memory_object_t pager; \
\
- vm_object_unlock(object); \
+ if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) { \
+ vm_object_paging_begin(object); \
+ vm_object_unlock(object); \
\
- if (iosync) \
- upl_flags = UPL_MSYNC | UPL_IOSYNC; \
- else \
- upl_flags = UPL_MSYNC; \
+ if (iosync) \
+ upl_flags = UPL_MSYNC | UPL_IOSYNC; \
+ else \
+ upl_flags = UPL_MSYNC; \
\
- (void) memory_object_data_return(object->pager, \
- po, \
- data_cnt, \
- ro, \
- ioerr, \
- (action == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN), \
- !should_flush, \
- upl_flags); \
+ (void) memory_object_data_return(pager, \
+ po, \
+ data_cnt, \
+ ro, \
+ ioerr, \
+ (action) == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN,\
+ !should_flush, \
+ upl_flags); \
\
- vm_object_lock(object); \
+ vm_object_lock(object); \
+ vm_object_paging_end(object); \
+ } \
MACRO_END
/*
data_cnt += PAGE_SIZE;
last_offset = offset + PAGE_SIZE_64;
- vm_page_lock_queues();
+ vm_page_lockspin_queues();
/*
* Clean
*/
int flags,
vm_prot_t protection)
{
- vm_object_t copy_object;
+ vm_object_t copy_object = VM_OBJECT_NULL;
boolean_t data_returned = FALSE;
boolean_t update_cow;
boolean_t should_flush = (flags & MEMORY_OBJECT_DATA_FLUSH) ? TRUE : FALSE;
!(flags & MEMORY_OBJECT_DATA_PURGE)))
|| (flags & MEMORY_OBJECT_COPY_SYNC);
+ if (update_cow || (flags & (MEMORY_OBJECT_DATA_PURGE | MEMORY_OBJECT_DATA_SYNC))) {
+ int collisions = 0;
+
+ while ((copy_object = object->copy) != VM_OBJECT_NULL) {
+ /*
+ * need to do a try here since we're swimming upstream
+ * against the normal lock ordering... however, we need
+ * to hold the object stable until we gain control of the
+ * copy object so we have to be careful how we approach this
+ */
+ if (vm_object_lock_try(copy_object)) {
+ /*
+ * we 'won' the lock on the copy object...
+ * no need to hold the object lock any longer...
+ * take a real reference on the copy object because
+ * we're going to call vm_fault_page on it which may
+ * under certain conditions drop the lock and the paging
+ * reference we're about to take... the reference
+ * will keep the copy object from going away if that happens
+ */
+ vm_object_unlock(object);
+ vm_object_reference_locked(copy_object);
+ break;
+ }
+ vm_object_unlock(object);
+
+ collisions++;
+ mutex_pause(collisions);
- if((((copy_object = object->copy) != NULL) && update_cow) ||
- (flags & MEMORY_OBJECT_DATA_SYNC)) {
+ vm_object_lock(object);
+ }
+ }
+ if ((copy_object != VM_OBJECT_NULL && update_cow) || (flags & MEMORY_OBJECT_DATA_SYNC)) {
vm_map_size_t i;
vm_map_size_t copy_size;
vm_map_offset_t copy_offset;
vm_page_t page;
vm_page_t top_page;
kern_return_t error = 0;
+ struct vm_object_fault_info fault_info;
+
+ if (copy_object != VM_OBJECT_NULL) {
+ /*
+ * translate offset with respect to shadow's offset
+ */
+ copy_offset = (offset >= copy_object->shadow_offset) ?
+ (vm_map_offset_t)(offset - copy_object->shadow_offset) :
+ (vm_map_offset_t) 0;
+
+ if (copy_offset > copy_object->size)
+ copy_offset = copy_object->size;
+
+ /*
+ * clip size with respect to shadow offset
+ */
+ if (offset >= copy_object->shadow_offset) {
+ copy_size = size;
+ } else if (size >= copy_object->shadow_offset - offset) {
+ copy_size = size - (copy_object->shadow_offset - offset);
+ } else {
+ copy_size = 0;
+ }
+
+ if (copy_offset + copy_size > copy_object->size) {
+ if (copy_object->size >= copy_offset) {
+ copy_size = copy_object->size - copy_offset;
+ } else {
+ copy_size = 0;
+ }
+ }
+ copy_size+=copy_offset;
- if(copy_object != NULL) {
- /* translate offset with respect to shadow's offset */
- copy_offset = (offset >= copy_object->shadow_offset)?
- (vm_map_offset_t)(offset - copy_object->shadow_offset) :
- (vm_map_offset_t) 0;
- if(copy_offset > copy_object->size)
- copy_offset = copy_object->size;
-
- /* clip size with respect to shadow offset */
- if (offset >= copy_object->shadow_offset) {
- copy_size = size;
- } else if (size >= copy_object->shadow_offset - offset) {
- copy_size = size -
- (copy_object->shadow_offset - offset);
- } else {
- copy_size = 0;
- }
-
- if (copy_offset + copy_size > copy_object->size) {
- if (copy_object->size >= copy_offset) {
- copy_size = copy_object->size - copy_offset;
- } else {
- copy_size = 0;
- }
- }
-
- copy_size+=copy_offset;
-
- vm_object_unlock(object);
- vm_object_lock(copy_object);
} else {
copy_object = object;
copy_size = offset + size;
copy_offset = offset;
}
+ fault_info.interruptible = THREAD_UNINT;
+ fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL;
+ fault_info.user_tag = 0;
+ fault_info.lo_offset = copy_offset;
+ fault_info.hi_offset = copy_size;
+ fault_info.no_cache = FALSE;
vm_object_paging_begin(copy_object);
- for (i=copy_offset; i<copy_size; i+=PAGE_SIZE) {
+
+ for (i = copy_offset; i < copy_size; i += PAGE_SIZE) {
RETRY_COW_OF_LOCK_REQUEST:
+ fault_info.cluster_size = copy_size - i;
+
prot = VM_PROT_WRITE|VM_PROT_READ;
switch (vm_fault_page(copy_object, i,
VM_PROT_WRITE|VM_PROT_READ,
FALSE,
- THREAD_UNINT,
- copy_offset,
- copy_offset+copy_size,
- VM_BEHAVIOR_SEQUENTIAL,
&prot,
&page,
&top_page,
(int *)0,
&error,
FALSE,
- FALSE, NULL, 0)) {
+ FALSE, &fault_info)) {
case VM_FAULT_SUCCESS:
- if(top_page) {
+ if (top_page) {
vm_fault_cleanup(
page->object, top_page);
- PAGE_WAKEUP_DONE(page);
- vm_page_lock_queues();
- if (!page->active && !page->inactive)
- vm_page_activate(page);
- vm_page_unlock_queues();
vm_object_lock(copy_object);
vm_object_paging_begin(copy_object);
- } else {
- PAGE_WAKEUP_DONE(page);
- vm_page_lock_queues();
- if (!page->active && !page->inactive)
- vm_page_activate(page);
- vm_page_unlock_queues();
}
+ vm_page_lock_queues();
+ if (!page->active && !page->inactive)
+ vm_page_deactivate(page);
+ vm_page_unlock_queues();
+
+ PAGE_WAKEUP_DONE(page);
break;
case VM_FAULT_RETRY:
prot = VM_PROT_WRITE|VM_PROT_READ;
vm_object_paging_begin(copy_object);
goto RETRY_COW_OF_LOCK_REQUEST;
case VM_FAULT_MEMORY_ERROR:
+ if (object != copy_object)
+ vm_object_deallocate(copy_object);
vm_object_lock(object);
goto BYPASS_COW_COPYIN;
}
}
vm_object_paging_end(copy_object);
- if(copy_object != object) {
+ }
+ if ((flags & (MEMORY_OBJECT_DATA_SYNC | MEMORY_OBJECT_COPY_SYNC))) {
+ if (copy_object != VM_OBJECT_NULL && copy_object != object) {
vm_object_unlock(copy_object);
+ vm_object_deallocate(copy_object);
vm_object_lock(object);
}
+ return KERN_SUCCESS;
}
- if((flags & (MEMORY_OBJECT_DATA_SYNC | MEMORY_OBJECT_COPY_SYNC))) {
- return KERN_SUCCESS;
- }
- if(((copy_object = object->copy) != NULL) &&
- (flags & MEMORY_OBJECT_DATA_PURGE)) {
- copy_object->shadow_severed = TRUE;
- copy_object->shadowed = FALSE;
- copy_object->shadow = NULL;
- /* delete the ref the COW was holding on the target object */
- vm_object_deallocate(object);
+ if (copy_object != VM_OBJECT_NULL && copy_object != object) {
+ if ((flags & MEMORY_OBJECT_DATA_PURGE)) {
+ copy_object->shadow_severed = TRUE;
+ copy_object->shadowed = FALSE;
+ copy_object->shadow = NULL;
+ /*
+ * delete the ref the COW was holding on the target object
+ */
+ vm_object_deallocate(object);
+ }
+ vm_object_unlock(copy_object);
+ vm_object_deallocate(copy_object);
+ vm_object_lock(object);
}
BYPASS_COW_COPYIN:
boolean_t may_cache,
memory_object_copy_strategy_t copy_strategy,
boolean_t temporary,
- memory_object_cluster_size_t cluster_size,
boolean_t silent_overwrite,
boolean_t advisory_pageout)
{
may_cache = TRUE;
if (temporary)
temporary = TRUE;
- if (cluster_size != 0) {
- int pages_per_cluster;
- pages_per_cluster = atop_32(cluster_size);
- /*
- * Cluster size must be integral multiple of page size,
- * and be a power of 2 number of pages.
- */
- if ((cluster_size & (PAGE_SIZE-1)) ||
- ((pages_per_cluster-1) & pages_per_cluster))
- return KERN_INVALID_ARGUMENT;
- }
vm_object_lock(object);
object->temporary = temporary;
object->silent_overwrite = silent_overwrite;
object->advisory_pageout = advisory_pageout;
- if (cluster_size == 0)
- cluster_size = PAGE_SIZE;
- object->cluster_size = cluster_size;
-
- assert(cluster_size >= PAGE_SIZE &&
- cluster_size % PAGE_SIZE == 0);
/*
* Wake up anyone waiting for the ready attribute
boolean_t temporary;
boolean_t may_cache;
boolean_t invalidate;
- memory_object_cluster_size_t cluster_size;
memory_object_copy_strategy_t copy_strategy;
boolean_t silent_overwrite;
boolean_t advisory_pageout;
#if notyet
invalidate = object->invalidate;
#endif
- cluster_size = object->cluster_size;
vm_object_unlock(object);
switch (flavor) {
perf = (memory_object_perf_info_t) attributes;
may_cache = perf->may_cache;
- cluster_size = round_page_32(perf->cluster_size);
break;
}
may_cache = attr->may_cache;
copy_strategy = attr->copy_strategy;
- cluster_size = page_size;
break;
}
copy_strategy = attr->copy_strategy;
may_cache = attr->may_cache_object;
- cluster_size = attr->cluster_size;
temporary = attr->temporary;
break;
may_cache,
copy_strategy,
temporary,
- cluster_size,
silent_overwrite,
advisory_pageout));
}
}
perf = (memory_object_perf_info_t) attributes;
- perf->cluster_size = object->cluster_size;
+ perf->cluster_size = PAGE_SIZE;
perf->may_cache = object->can_persist;
*count = MEMORY_OBJECT_PERF_INFO_COUNT;
attr = (memory_object_attr_info_t) attributes;
attr->copy_strategy = object->copy_strategy;
- attr->cluster_size = object->cluster_size;
+ attr->cluster_size = PAGE_SIZE;
attr->may_cache_object = object->can_persist;
attr->temporary = object->temporary;
vm_object_reference(object);
named_entry_unlock(named_entry);
}
- } else {
+ } else if (ip_kotype(port) == IKOT_MEM_OBJ_CONTROL) {
memory_object_control_t control;
- control = (memory_object_control_t)port->ip_kobject;
+ control = (memory_object_control_t) port;
if (control == NULL)
return (KERN_INVALID_ARGUMENT);
object = memory_object_control_to_vm_object(control);
if (object == VM_OBJECT_NULL)
return (KERN_INVALID_ARGUMENT);
vm_object_reference(object);
+ } else {
+ return KERN_INVALID_ARGUMENT;
}
if (object == VM_OBJECT_NULL)
return (KERN_INVALID_ARGUMENT);
cntrl_flags);
}
+kern_return_t
+memory_object_cluster_size(memory_object_control_t control, memory_object_offset_t *start,
+ vm_size_t *length, memory_object_fault_info_t fault_info)
+{
+ vm_object_t object;
+
+ object = memory_object_control_to_vm_object(control);
+
+ if (object == VM_OBJECT_NULL || object->paging_offset > *start)
+ return (KERN_INVALID_ARGUMENT);
+
+ *start -= object->paging_offset;
+
+ vm_object_cluster_size(object, (vm_object_offset_t *)start, length, (vm_object_fault_info_t)fault_info);
+
+ *start += object->paging_offset;
+
+ return (KERN_SUCCESS);
+}
+
+
int vm_stat_discard_cleared_reply = 0;
int vm_stat_discard_cleared_unset = 0;
int vm_stat_discard_cleared_too_late = 0;
host_default_memory_manager(
host_priv_t host_priv,
memory_object_default_t *default_manager,
- memory_object_cluster_size_t cluster_size)
+ __unused memory_object_cluster_size_t cluster_size)
{
memory_object_default_t current_manager;
memory_object_default_t new_manager;
memory_object_default_t returned_manager;
+ kern_return_t result = KERN_SUCCESS;
if (host_priv == HOST_PRIV_NULL)
return(KERN_INVALID_HOST);
new_manager = *default_manager;
mutex_lock(&memory_manager_default_lock);
current_manager = memory_manager_default;
+ returned_manager = MEMORY_OBJECT_DEFAULT_NULL;
if (new_manager == MEMORY_OBJECT_DEFAULT_NULL) {
/*
* Retrieve the current value.
*/
- memory_object_default_reference(current_manager);
returned_manager = current_manager;
+ memory_object_default_reference(returned_manager);
} else {
+
+ /*
+ * If this is the first non-null manager, start
+ * up the internal pager support.
+ */
+ if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) {
+ result = vm_pageout_internal_start();
+ if (result != KERN_SUCCESS)
+ goto out;
+ }
+
/*
* Retrieve the current value,
* and replace it with the supplied value.
* but we have to take a reference on the new
* one.
*/
-
returned_manager = current_manager;
memory_manager_default = new_manager;
memory_object_default_reference(new_manager);
- if (cluster_size % PAGE_SIZE != 0) {
-#if 0
- mutex_unlock(&memory_manager_default_lock);
- return KERN_INVALID_ARGUMENT;
-#else
- cluster_size = round_page_32(cluster_size);
-#endif
- }
- memory_manager_default_cluster = cluster_size;
-
/*
* In case anyone's been waiting for a memory
* manager to be established, wake them up.
thread_wakeup((event_t) &memory_manager_default);
}
-
+ out:
mutex_unlock(&memory_manager_default_lock);
*default_manager = returned_manager;
- return(KERN_SUCCESS);
+ return(result);
}
/*
*/
__private_extern__ memory_object_default_t
-memory_manager_default_reference(
- memory_object_cluster_size_t *cluster_size)
+memory_manager_default_reference(void)
{
memory_object_default_t current_manager;
current_manager = memory_manager_default;
}
memory_object_default_reference(current_manager);
- *cluster_size = memory_manager_default_cluster;
mutex_unlock(&memory_manager_default_lock);
return current_manager;
int *flags)
{
vm_object_t object;
- vm_page_t dst_page;
-
object = memory_object_control_to_vm_object(control);
if (object == VM_OBJECT_NULL)
return (KERN_INVALID_ARGUMENT);
- vm_object_lock(object);
-
- if(ops & UPL_POP_PHYSICAL) {
- if(object->phys_contiguous) {
- if (phys_entry) {
- *phys_entry = (ppnum_t)
- (object->shadow_offset >> 12);
- }
- vm_object_unlock(object);
- return KERN_SUCCESS;
- } else {
- vm_object_unlock(object);
- return KERN_INVALID_OBJECT;
- }
- }
- if(object->phys_contiguous) {
- vm_object_unlock(object);
- return KERN_INVALID_OBJECT;
- }
-
- while(TRUE) {
- if((dst_page = vm_page_lookup(object,offset)) == VM_PAGE_NULL) {
- vm_object_unlock(object);
- return KERN_FAILURE;
- }
-
- /* Sync up on getting the busy bit */
- if((dst_page->busy || dst_page->cleaning) &&
- (((ops & UPL_POP_SET) &&
- (ops & UPL_POP_BUSY)) || (ops & UPL_POP_DUMP))) {
- /* someone else is playing with the page, we will */
- /* have to wait */
- PAGE_SLEEP(object, dst_page, THREAD_UNINT);
- continue;
- }
-
- if (ops & UPL_POP_DUMP) {
- vm_page_lock_queues();
-
- if (dst_page->no_isync == FALSE)
- pmap_disconnect(dst_page->phys_page);
- vm_page_free(dst_page);
-
- vm_page_unlock_queues();
- break;
- }
-
- if (flags) {
- *flags = 0;
-
- /* Get the condition of flags before requested ops */
- /* are undertaken */
-
- if(dst_page->dirty) *flags |= UPL_POP_DIRTY;
- if(dst_page->pageout) *flags |= UPL_POP_PAGEOUT;
- if(dst_page->precious) *flags |= UPL_POP_PRECIOUS;
- if(dst_page->absent) *flags |= UPL_POP_ABSENT;
- if(dst_page->busy) *flags |= UPL_POP_BUSY;
- }
-
- /* The caller should have made a call either contingent with */
- /* or prior to this call to set UPL_POP_BUSY */
- if(ops & UPL_POP_SET) {
- /* The protection granted with this assert will */
- /* not be complete. If the caller violates the */
- /* convention and attempts to change page state */
- /* without first setting busy we may not see it */
- /* because the page may already be busy. However */
- /* if such violations occur we will assert sooner */
- /* or later. */
- assert(dst_page->busy || (ops & UPL_POP_BUSY));
- if (ops & UPL_POP_DIRTY) dst_page->dirty = TRUE;
- if (ops & UPL_POP_PAGEOUT) dst_page->pageout = TRUE;
- if (ops & UPL_POP_PRECIOUS) dst_page->precious = TRUE;
- if (ops & UPL_POP_ABSENT) dst_page->absent = TRUE;
- if (ops & UPL_POP_BUSY) dst_page->busy = TRUE;
- }
-
- if(ops & UPL_POP_CLR) {
- assert(dst_page->busy);
- if (ops & UPL_POP_DIRTY) dst_page->dirty = FALSE;
- if (ops & UPL_POP_PAGEOUT) dst_page->pageout = FALSE;
- if (ops & UPL_POP_PRECIOUS) dst_page->precious = FALSE;
- if (ops & UPL_POP_ABSENT) dst_page->absent = FALSE;
- if (ops & UPL_POP_BUSY) {
- dst_page->busy = FALSE;
- PAGE_WAKEUP(dst_page);
- }
- }
-
- if (dst_page->encrypted) {
- /*
- * ENCRYPTED SWAP:
- * We need to decrypt this encrypted page before the
- * caller can access its contents.
- * But if the caller really wants to access the page's
- * contents, they have to keep the page "busy".
- * Otherwise, the page could get recycled or re-encrypted
- * at any time.
- */
- if ((ops & UPL_POP_SET) && (ops & UPL_POP_BUSY) &&
- dst_page->busy) {
- /*
- * The page is stable enough to be accessed by
- * the caller, so make sure its contents are
- * not encrypted.
- */
- vm_page_decrypt(dst_page, 0);
- } else {
- /*
- * The page is not busy, so don't bother
- * decrypting it, since anything could
- * happen to it between now and when the
- * caller wants to access it.
- * We should not give the caller access
- * to this page.
- */
- assert(!phys_entry);
- }
- }
-
- if (phys_entry) {
- /*
- * The physical page number will remain valid
- * only if the page is kept busy.
- * ENCRYPTED SWAP: make sure we don't let the
- * caller access an encrypted page.
- */
- assert(dst_page->busy);
- assert(!dst_page->encrypted);
- *phys_entry = dst_page->phys_page;
- }
-
- break;
- }
-
- vm_object_unlock(object);
- return KERN_SUCCESS;
-
+ return vm_object_page_op(object, offset, ops, phys_entry, flags);
}
/*
int ops,
int *range)
{
- memory_object_offset_t offset;
vm_object_t object;
- vm_page_t dst_page;
object = memory_object_control_to_vm_object(control);
if (object == VM_OBJECT_NULL)
return (KERN_INVALID_ARGUMENT);
- if (object->resident_page_count == 0) {
- if (range) {
- if (ops & UPL_ROP_PRESENT)
- *range = 0;
- else
- *range = offset_end - offset_beg;
- }
- return KERN_SUCCESS;
- }
- vm_object_lock(object);
-
- if (object->phys_contiguous) {
- vm_object_unlock(object);
- return KERN_INVALID_OBJECT;
- }
-
- offset = offset_beg;
-
- while (offset < offset_end) {
- dst_page = vm_page_lookup(object, offset);
- if (dst_page != VM_PAGE_NULL) {
- if (ops & UPL_ROP_DUMP) {
- if (dst_page->busy || dst_page->cleaning) {
- /*
- * someone else is playing with the
- * page, we will have to wait
- */
- PAGE_SLEEP(object,
- dst_page, THREAD_UNINT);
- /*
- * need to relook the page up since it's
- * state may have changed while we slept
- * it might even belong to a different object
- * at this point
- */
- continue;
- }
- vm_page_lock_queues();
-
- if (dst_page->no_isync == FALSE)
- pmap_disconnect(dst_page->phys_page);
- vm_page_free(dst_page);
-
- vm_page_unlock_queues();
- } else if (ops & UPL_ROP_ABSENT)
- break;
- } else if (ops & UPL_ROP_PRESENT)
- break;
-
- offset += PAGE_SIZE;
- }
- vm_object_unlock(object);
-
- if (range)
- *range = offset - offset_beg;
-
- return KERN_SUCCESS;
+ return vm_object_range_op(object,
+ offset_beg,
+ offset_end,
+ ops,
+ range);
}
return (KERN_SUCCESS);
}
+kern_return_t
+memory_object_signed(
+ memory_object_control_t control,
+ boolean_t is_signed)
+{
+ vm_object_t object;
+
+ object = memory_object_control_to_vm_object(control);
+ if (object == VM_OBJECT_NULL)
+ return KERN_INVALID_ARGUMENT;
+
+ vm_object_lock(object);
+ object->code_signed = is_signed;
+ vm_object_unlock(object);
+
+ return KERN_SUCCESS;
+}
static zone_t mem_obj_control_zone;
memory_object_control_t control;
control = (memory_object_control_t)zalloc(mem_obj_control_zone);
- if (control != MEMORY_OBJECT_CONTROL_NULL)
- control->object = object;
+ if (control != MEMORY_OBJECT_CONTROL_NULL) {
+ control->moc_object = object;
+ control->moc_ikot = IKOT_MEM_OBJ_CONTROL; /* fake ip_kotype */
+ }
return (control);
}
memory_object_control_t control,
vm_object_t object)
{
- assert((control->object != VM_OBJECT_NULL) &&
- (control->object != object));
- control->object = object;
+ assert((control->moc_object != VM_OBJECT_NULL) &&
+ (control->moc_object != object));
+ control->moc_object = object;
}
__private_extern__ vm_object_t
memory_object_control_to_vm_object(
memory_object_control_t control)
{
- if (control == MEMORY_OBJECT_CONTROL_NULL)
+ if (control == MEMORY_OBJECT_CONTROL_NULL ||
+ control->moc_ikot != IKOT_MEM_OBJ_CONTROL)
return VM_OBJECT_NULL;
- return (control->object);
+ return (control->moc_object);
}
memory_object_control_t
memory_object_control_disable(
memory_object_control_t control)
{
- assert(control->object != VM_OBJECT_NULL);
- control->object = VM_OBJECT_NULL;
+ assert(control->moc_object != VM_OBJECT_NULL);
+ control->moc_object = VM_OBJECT_NULL;
}
void
void memory_object_reference(
memory_object_t memory_object)
{
-
-#ifdef MACH_BSD
- if (memory_object->pager == &vnode_pager_workaround) {
- vnode_pager_reference(memory_object);
- } else if (memory_object->pager == &device_pager_workaround) {
- device_pager_reference(memory_object);
- } else
-#endif
- dp_memory_object_reference(memory_object);
+ (memory_object->mo_pager_ops->memory_object_reference)(
+ memory_object);
}
/* Routine memory_object_deallocate */
void memory_object_deallocate(
memory_object_t memory_object)
{
-
-#ifdef MACH_BSD
- if (memory_object->pager == &vnode_pager_workaround) {
- vnode_pager_deallocate(memory_object);
- } else if (memory_object->pager == &device_pager_workaround) {
- device_pager_deallocate(memory_object);
- } else
-#endif
- dp_memory_object_deallocate(memory_object);
+ (memory_object->mo_pager_ops->memory_object_deallocate)(
+ memory_object);
}
memory_object_cluster_size_t memory_object_page_size
)
{
-#ifdef MACH_BSD
- if (memory_object->pager == &vnode_pager_workaround) {
- return vnode_pager_init(memory_object,
- memory_control,
- memory_object_page_size);
- } else if (memory_object->pager == &device_pager_workaround) {
- return device_pager_init(memory_object,
- memory_control,
- memory_object_page_size);
- } else
-#endif
- return dp_memory_object_init(memory_object,
- memory_control,
- memory_object_page_size);
+ return (memory_object->mo_pager_ops->memory_object_init)(
+ memory_object,
+ memory_control,
+ memory_object_page_size);
}
/* Routine memory_object_terminate */
memory_object_t memory_object
)
{
-#ifdef MACH_BSD
- if (memory_object->pager == &vnode_pager_workaround) {
- return vnode_pager_terminate(memory_object);
- } else if (memory_object->pager == &device_pager_workaround) {
- return device_pager_terminate(memory_object);
- } else
-#endif
- return dp_memory_object_terminate(memory_object);
+ return (memory_object->mo_pager_ops->memory_object_terminate)(
+ memory_object);
}
/* Routine memory_object_data_request */
memory_object_t memory_object,
memory_object_offset_t offset,
memory_object_cluster_size_t length,
- vm_prot_t desired_access
+ vm_prot_t desired_access,
+ memory_object_fault_info_t fault_info
)
{
-#ifdef MACH_BSD
- if (memory_object->pager == &vnode_pager_workaround) {
- return vnode_pager_data_request(memory_object,
- offset,
- length,
- desired_access);
- } else if (memory_object->pager == &device_pager_workaround) {
- return device_pager_data_request(memory_object,
- offset,
- length,
- desired_access);
- } else
-#endif
- return dp_memory_object_data_request(memory_object,
- offset,
- length,
- desired_access);
+ return (memory_object->mo_pager_ops->memory_object_data_request)(
+ memory_object,
+ offset,
+ length,
+ desired_access,
+ fault_info);
}
/* Routine memory_object_data_return */
int upl_flags
)
{
-#ifdef MACH_BSD
- if (memory_object->pager == &vnode_pager_workaround) {
- return vnode_pager_data_return(memory_object,
- offset,
- size,
- resid_offset,
- io_error,
- dirty,
- kernel_copy,
- upl_flags);
- } else if (memory_object->pager == &device_pager_workaround) {
-
- return device_pager_data_return(memory_object,
- offset,
- size,
- dirty,
- kernel_copy,
- upl_flags);
- }
- else
-#endif
- {
- return dp_memory_object_data_return(memory_object,
- offset,
- size,
- NULL,
- NULL,
- dirty,
- kernel_copy,
- upl_flags);
- }
+ return (memory_object->mo_pager_ops->memory_object_data_return)(
+ memory_object,
+ offset,
+ size,
+ resid_offset,
+ io_error,
+ dirty,
+ kernel_copy,
+ upl_flags);
}
/* Routine memory_object_data_initialize */
vm_size_t size
)
{
-#ifdef MACH_BSD
- if (memory_object->pager == &vnode_pager_workaround) {
- return vnode_pager_data_initialize(memory_object,
- offset,
- size);
- } else if (memory_object->pager == &device_pager_workaround) {
- return device_pager_data_initialize(memory_object,
- offset,
- size);
- } else
-#endif
- return dp_memory_object_data_initialize(memory_object,
- offset,
- size);
+ return (memory_object->mo_pager_ops->memory_object_data_initialize)(
+ memory_object,
+ offset,
+ size);
}
/* Routine memory_object_data_unlock */
vm_prot_t desired_access
)
{
-#ifdef MACH_BSD
- if (memory_object->pager == &vnode_pager_workaround) {
- return vnode_pager_data_unlock(memory_object,
- offset,
- size,
- desired_access);
- } else if (memory_object->pager == &device_pager_workaround) {
- return device_pager_data_unlock(memory_object,
- offset,
- size,
- desired_access);
- } else
-#endif
- return dp_memory_object_data_unlock(memory_object,
- offset,
- size,
- desired_access);
+ return (memory_object->mo_pager_ops->memory_object_data_unlock)(
+ memory_object,
+ offset,
+ size,
+ desired_access);
}
/* Routine memory_object_synchronize */
vm_sync_t sync_flags
)
{
-#ifdef MACH_BSD
- if (memory_object->pager == &vnode_pager_workaround) {
- return vnode_pager_synchronize(memory_object,
- offset,
- size,
- sync_flags);
- } else if (memory_object->pager == &device_pager_workaround) {
- return device_pager_synchronize(memory_object,
- offset,
- size,
- sync_flags);
- } else
-#endif
- return dp_memory_object_synchronize(memory_object,
- offset,
- size,
- sync_flags);
+ return (memory_object->mo_pager_ops->memory_object_synchronize)(
+ memory_object,
+ offset,
+ size,
+ sync_flags);
}
/* Routine memory_object_unmap */
memory_object_t memory_object
)
{
-#ifdef MACH_BSD
- if (memory_object->pager == &vnode_pager_workaround) {
- return vnode_pager_unmap(memory_object);
- } else if (memory_object->pager == &device_pager_workaround) {
- return device_pager_unmap(memory_object);
- } else
-#endif
- return dp_memory_object_unmap(memory_object);
+ return (memory_object->mo_pager_ops->memory_object_unmap)(
+ memory_object);
}
/* Routine memory_object_create */
projects / apple / xnu.git / blobdiff