/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
*
* @APPLE_LICENSE_HEADER_START@
*
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
+#include <vm/vm_kern.h>
#include <vm/pmap.h>
#include <vm/vm_pageout.h>
#include <mach/vm_param.h>
int vm_object_absent_max = 50;
int vm_fault_debug = 0;
-boolean_t vm_page_deactivate_behind = TRUE;
-
#if !VM_FAULT_STATIC_CONFIG
boolean_t vm_fault_dirty_handling = FALSE;
boolean_t vm_allow_clustered_pagein = FALSE;
int vm_pagein_cluster_used = 0;
+#define ALIGNED(x) (((x) & (PAGE_SIZE_64 - 1)) == 0)
+
+
+boolean_t vm_page_deactivate_behind = TRUE;
/*
* Prepage default sizes given VM_BEHAVIOR_DEFAULT reference behavior
*/
-int vm_default_ahead = 1; /* Number of pages to prepage ahead */
-int vm_default_behind = 0; /* Number of pages to prepage behind */
+int vm_default_ahead = 0;
+int vm_default_behind = MAX_UPL_TRANSFER;
+
+/*
+ * vm_page_deactivate_behind
+ *
+ * Determine if sequential access is in progress
+ * in accordance with the behavior specified. If
+ * so, compute a potential page to deactive and
+ * deactivate it.
+ *
+ * The object must be locked.
+ */
+static
+boolean_t
+vm_fault_deactivate_behind(
+ vm_object_t object,
+ vm_offset_t offset,
+ vm_behavior_t behavior)
+{
+ vm_page_t m;
+
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF0018, (unsigned int) object, (unsigned int) vm_fault_deactivate_behind); /* (TEST/DEBUG) */
+#endif
+
+ switch (behavior) {
+ case VM_BEHAVIOR_RANDOM:
+ object->sequential = PAGE_SIZE_64;
+ m = VM_PAGE_NULL;
+ break;
+ case VM_BEHAVIOR_SEQUENTIAL:
+ if (offset &&
+ object->last_alloc == offset - PAGE_SIZE_64) {
+ object->sequential += PAGE_SIZE_64;
+ m = vm_page_lookup(object, offset - PAGE_SIZE_64);
+ } else {
+ object->sequential = PAGE_SIZE_64; /* reset */
+ m = VM_PAGE_NULL;
+ }
+ break;
+ case VM_BEHAVIOR_RSEQNTL:
+ if (object->last_alloc &&
+ object->last_alloc == offset + PAGE_SIZE_64) {
+ object->sequential += PAGE_SIZE_64;
+ m = vm_page_lookup(object, offset + PAGE_SIZE_64);
+ } else {
+ object->sequential = PAGE_SIZE_64; /* reset */
+ m = VM_PAGE_NULL;
+ }
+ break;
+ case VM_BEHAVIOR_DEFAULT:
+ default:
+ if (offset &&
+ object->last_alloc == offset - PAGE_SIZE_64) {
+ vm_object_offset_t behind = vm_default_behind * PAGE_SIZE_64;
+
+ object->sequential += PAGE_SIZE_64;
+ m = (offset >= behind &&
+ object->sequential >= behind) ?
+ vm_page_lookup(object, offset - behind) :
+ VM_PAGE_NULL;
+ } else if (object->last_alloc &&
+ object->last_alloc == offset + PAGE_SIZE_64) {
+ vm_object_offset_t behind = vm_default_behind * PAGE_SIZE_64;
+
+ object->sequential += PAGE_SIZE_64;
+ m = (offset < -behind &&
+ object->sequential >= behind) ?
+ vm_page_lookup(object, offset + behind) :
+ VM_PAGE_NULL;
+ } else {
+ object->sequential = PAGE_SIZE_64;
+ m = VM_PAGE_NULL;
+ }
+ break;
+ }
+
+ object->last_alloc = offset;
+
+ if (m) {
+ if (!m->busy) {
+ vm_page_lock_queues();
+ vm_page_deactivate(m);
+ vm_page_unlock_queues();
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF0019, (unsigned int) object, (unsigned int) m); /* (TEST/DEBUG) */
+#endif
+ }
+ return TRUE;
+ }
+ return FALSE;
+}
-#define ALIGNED(x) (((x) & (PAGE_SIZE_64 - 1)) == 0)
/*
* Routine: vm_fault_page
* need to allocate a real page.
*/
if (VM_PAGE_THROTTLED() ||
- (real_m = vm_page_grab()) == VM_PAGE_NULL) {
- vm_fault_cleanup(object, first_m);
- thread_interrupt_level(interruptible_state);
- return(VM_FAULT_MEMORY_SHORTAGE);
+ (real_m = vm_page_grab())
+ == VM_PAGE_NULL) {
+ vm_fault_cleanup(
+ object, first_m);
+ thread_interrupt_level(
+ interruptible_state);
+ return(
+ VM_FAULT_MEMORY_SHORTAGE);
+ }
+
+ /*
+ * are we protecting the system from
+ * backing store exhaustion. If so
+ * sleep unless we are privileged.
+ */
+
+ if(vm_backing_store_low) {
+ if(!(current_task()->priv_flags
+ & VM_BACKING_STORE_PRIV)) {
+ assert_wait((event_t)
+ &vm_backing_store_low,
+ THREAD_UNINT);
+ vm_fault_cleanup(object,
+ first_m);
+ thread_block((void(*)(void)) 0);
+ thread_interrupt_level(
+ interruptible_state);
+ return(VM_FAULT_RETRY);
+ }
}
if (!no_zero_fill) {
vm_object_unlock(object);
vm_page_zero_fill(m);
- if (type_of_fault)
- *type_of_fault = DBG_ZERO_FILL_FAULT;
- VM_STAT(zero_fill_count++);
-
- if (bumped_pagein == TRUE) {
- VM_STAT(pageins--);
- current_task()->pageins--;
- }
vm_object_lock(object);
}
+ if (type_of_fault)
+ *type_of_fault = DBG_ZERO_FILL_FAULT;
+ VM_STAT(zero_fill_count++);
+
+ if (bumped_pagein == TRUE) {
+ VM_STAT(pageins--);
+ current_task()->pageins--;
+ }
+#if 0
pmap_clear_modify(m->phys_page);
+#endif
vm_page_lock_queues();
VM_PAGE_QUEUES_REMOVE(m);
m->page_ticket = vm_page_ticket;
assert(m->object == object);
first_m = VM_PAGE_NULL;
+ if(m == VM_PAGE_NULL) {
+ m = vm_page_grab();
+ if (m == VM_PAGE_NULL) {
+ vm_fault_cleanup(
+ object, VM_PAGE_NULL);
+ thread_interrupt_level(
+ interruptible_state);
+ return(VM_FAULT_MEMORY_SHORTAGE);
+ }
+ vm_page_insert(
+ m, object, offset);
+ }
+
if (object->shadow_severed) {
VM_PAGE_FREE(m);
vm_fault_cleanup(object, VM_PAGE_NULL);
return VM_FAULT_MEMORY_ERROR;
}
+ /*
+ * are we protecting the system from
+ * backing store exhaustion. If so
+ * sleep unless we are privileged.
+ */
+
+ if(vm_backing_store_low) {
+ if(!(current_task()->priv_flags
+ & VM_BACKING_STORE_PRIV)) {
+ assert_wait((event_t)
+ &vm_backing_store_low,
+ THREAD_UNINT);
+ VM_PAGE_FREE(m);
+ vm_fault_cleanup(object, VM_PAGE_NULL);
+ thread_block((void (*)(void)) 0);
+ thread_interrupt_level(
+ interruptible_state);
+ return(VM_FAULT_RETRY);
+ }
+ }
+
if (VM_PAGE_THROTTLED() ||
(m->fictitious && !vm_page_convert(m))) {
VM_PAGE_FREE(m);
if (!no_zero_fill) {
vm_object_unlock(object);
vm_page_zero_fill(m);
- if (type_of_fault)
- *type_of_fault = DBG_ZERO_FILL_FAULT;
- VM_STAT(zero_fill_count++);
-
- if (bumped_pagein == TRUE) {
- VM_STAT(pageins--);
- current_task()->pageins--;
- }
vm_object_lock(object);
}
+ if (type_of_fault)
+ *type_of_fault = DBG_ZERO_FILL_FAULT;
+ VM_STAT(zero_fill_count++);
+
+ if (bumped_pagein == TRUE) {
+ VM_STAT(pageins--);
+ current_task()->pageins--;
+ }
+
vm_page_lock_queues();
VM_PAGE_QUEUES_REMOVE(m);
if(m->object->size > 0x80000) {
m->inactive = TRUE;
vm_page_inactive_count++;
vm_page_unlock_queues();
+#if 0
pmap_clear_modify(m->phys_page);
+#endif
break;
}
else {
assert(!must_be_resident);
+ /*
+ * are we protecting the system from
+ * backing store exhaustion. If so
+ * sleep unless we are privileged.
+ */
+
+ if(vm_backing_store_low) {
+ if(!(current_task()->priv_flags
+ & VM_BACKING_STORE_PRIV)) {
+ assert_wait((event_t)
+ &vm_backing_store_low,
+ THREAD_UNINT);
+ RELEASE_PAGE(m);
+ vm_fault_cleanup(object, first_m);
+ thread_block((void (*)(void)) 0);
+ thread_interrupt_level(
+ interruptible_state);
+ return(VM_FAULT_RETRY);
+ }
+ }
+
/*
* If we try to collapse first_object at this
* point, we may deadlock when we try to get
*/
vm_object_paging_end(object);
- vm_object_collapse(object);
+ vm_object_collapse(object, offset);
vm_object_paging_begin(object);
}
* We must copy the page to the copy object.
*/
+ /*
+ * are we protecting the system from
+ * backing store exhaustion. If so
+ * sleep unless we are privileged.
+ */
+
+ if(vm_backing_store_low) {
+ if(!(current_task()->priv_flags
+ & VM_BACKING_STORE_PRIV)) {
+ assert_wait((event_t)
+ &vm_backing_store_low,
+ THREAD_UNINT);
+ RELEASE_PAGE(m);
+ VM_OBJ_RES_DECR(copy_object);
+ copy_object->ref_count--;
+ assert(copy_object->ref_count > 0);
+ vm_object_unlock(copy_object);
+ vm_fault_cleanup(object, first_m);
+ thread_block((void (*)(void)) 0);
+ thread_interrupt_level(
+ interruptible_state);
+ return(VM_FAULT_RETRY);
+ }
+ }
+
/*
* Allocate a page for the copy
*/
* mark read-only data as dirty.]
*/
+
+ if(m != VM_PAGE_NULL) {
#if !VM_FAULT_STATIC_CONFIG
- if (vm_fault_dirty_handling && (*protection & VM_PROT_WRITE) &&
- (m != VM_PAGE_NULL)) {
- m->dirty = TRUE;
- }
-#endif
-#if TRACEFAULTPAGE
- dbgTrace(0xBEEF0018, (unsigned int) object, (unsigned int) vm_page_deactivate_behind); /* (TEST/DEBUG) */
-#endif
- if (vm_page_deactivate_behind) {
- if (offset && /* don't underflow */
- (object->last_alloc == (offset - PAGE_SIZE_64))) {
- m = vm_page_lookup(object, object->last_alloc);
- if ((m != VM_PAGE_NULL) && !m->busy) {
- vm_page_lock_queues();
- vm_page_deactivate(m);
- vm_page_unlock_queues();
- }
-#if TRACEFAULTPAGE
- dbgTrace(0xBEEF0019, (unsigned int) object, (unsigned int) m); /* (TEST/DEBUG) */
+ if (vm_fault_dirty_handling && (*protection & VM_PROT_WRITE))
+ m->dirty = TRUE;
#endif
- }
- object->last_alloc = offset;
+ if (vm_page_deactivate_behind)
+ vm_fault_deactivate_behind(object, offset, behavior);
+ } else {
+ vm_object_unlock(object);
}
+ thread_interrupt_level(interruptible_state);
+
#if TRACEFAULTPAGE
dbgTrace(0xBEEF001A, (unsigned int) VM_FAULT_SUCCESS, 0); /* (TEST/DEBUG) */
#endif
- thread_interrupt_level(interruptible_state);
- if(*result_page == VM_PAGE_NULL) {
- vm_object_unlock(object);
- }
return(VM_FAULT_SUCCESS);
#if 0
#undef RELEASE_PAGE
}
+/*
+ * Routine: vm_fault_tws_insert
+ * Purpose:
+ * Add fault information to the task working set.
+ * Implementation:
+ * We always insert the base object/offset pair
+ * rather the actual object/offset.
+ * Assumptions:
+ * Map and pmap_map locked.
+ * Object locked and referenced.
+ * Returns:
+ * TRUE if startup file should be written.
+ * With object locked and still referenced.
+ * But we may drop the object lock temporarily.
+ */
+static boolean_t
+vm_fault_tws_insert(
+ vm_map_t map,
+ vm_map_t pmap_map,
+ vm_offset_t vaddr,
+ vm_object_t object,
+ vm_object_offset_t offset)
+{
+ tws_hash_line_t line;
+ task_t task;
+ kern_return_t kr;
+ boolean_t result = FALSE;
+ extern vm_map_t kalloc_map;
+
+ /* Avoid possible map lock deadlock issues */
+ if (map == kernel_map || map == kalloc_map ||
+ pmap_map == kernel_map || pmap_map == kalloc_map)
+ return result;
+
+ task = current_task();
+ if (task->dynamic_working_set != 0) {
+ vm_object_t base_object;
+ vm_object_t base_shadow;
+ vm_object_offset_t base_offset;
+ base_object = object;
+ base_offset = offset;
+ while(base_shadow = base_object->shadow) {
+ vm_object_lock(base_shadow);
+ vm_object_unlock(base_object);
+ base_offset +=
+ base_object->shadow_offset;
+ base_object = base_shadow;
+ }
+ kr = tws_lookup((tws_hash_t)
+ task->dynamic_working_set,
+ base_offset, base_object,
+ &line);
+ if (kr == KERN_OPERATION_TIMED_OUT){
+ result = TRUE;
+ if (base_object != object) {
+ vm_object_unlock(base_object);
+ vm_object_lock(object);
+ }
+ } else if (kr != KERN_SUCCESS) {
+ if(base_object != object)
+ vm_object_reference_locked(base_object);
+ kr = tws_insert((tws_hash_t)
+ task->dynamic_working_set,
+ base_offset, base_object,
+ vaddr, pmap_map);
+ if(base_object != object) {
+ vm_object_unlock(base_object);
+ vm_object_deallocate(base_object);
+ }
+ if(kr == KERN_NO_SPACE) {
+ if (base_object == object)
+ vm_object_unlock(object);
+ tws_expand_working_set(
+ task->dynamic_working_set,
+ TWS_HASH_LINE_COUNT,
+ FALSE);
+ if (base_object == object)
+ vm_object_lock(object);
+ } else if(kr == KERN_OPERATION_TIMED_OUT) {
+ result = TRUE;
+ }
+ if(base_object != object)
+ vm_object_lock(object);
+ } else if (base_object != object) {
+ vm_object_unlock(base_object);
+ vm_object_lock(object);
+ }
+ }
+ return result;
+}
+
/*
* Routine: vm_fault
* Purpose:
unsigned int cache_attr;
int write_startup_file = 0;
vm_prot_t full_fault_type;
-
+
+ if (get_preemption_level() != 0)
+ return (KERN_FAILURE);
KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 0)) | DBG_FUNC_START,
vaddr,
while (TRUE) {
m = vm_page_lookup(cur_object, cur_offset);
if (m != VM_PAGE_NULL) {
- if (m->busy) {
+ if (m->busy) {
wait_result_t result;
if (object != cur_object)
goto FastMapInFault;
if ((fault_type & VM_PROT_WRITE) == 0) {
+ boolean_t sequential;
prot &= ~VM_PROT_WRITE;
#endif /* MACH_KDB */
#endif /* STATIC_CONFIG */
cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK;
- if ((m->no_isync == TRUE) ||
- (cache_attr != VM_WIMG_DEFAULT)) {
- pmap_sync_caches_phys(m->phys_page);
+
+ sequential = FALSE;
+ if (m->no_isync == TRUE) {
m->no_isync = FALSE;
+ pmap_sync_caches_phys(m->phys_page);
+ if (type_of_fault == DBG_CACHE_HIT_FAULT) {
+ /*
+ * found it in the cache, but this
+ * is the first fault-in of the page (no_isync == TRUE)
+ * so it must have come in as part of
+ * a cluster... account 1 pagein against it
+ */
+ VM_STAT(pageins++);
+ current_task()->pageins++;
+ type_of_fault = DBG_PAGEIN_FAULT;
+ sequential = TRUE;
+ }
+ } else if (cache_attr != VM_WIMG_DEFAULT) {
+ pmap_sync_caches_phys(m->phys_page);
}
if(caller_pmap) {
}
/*
- * Grab the queues lock to manipulate
+ * Hold queues lock to manipulate
* the page queues. Change wiring
* case is obvious. In soft ref bits
* case activate page only if it fell
* queue. This code doesn't.
*/
vm_page_lock_queues();
-
if (m->clustered) {
vm_pagein_cluster_used++;
m->clustered = FALSE;
* That's it, clean up and return.
*/
PAGE_WAKEUP_DONE(m);
- vm_object_paging_end(object);
- {
- tws_hash_line_t line;
- task_t task;
-
- task = current_task();
- if((map != NULL) &&
- (task->dynamic_working_set != 0) &&
- !(object->private)) {
- kern_return_t kr;
- vm_object_t base_object;
- vm_object_offset_t base_offset;
- base_object = object;
- base_offset = cur_offset;
- while(base_object->shadow) {
- base_offset +=
- base_object->shadow_offset;
- base_object =
- base_object->shadow;
- }
- kr = tws_lookup((tws_hash_t)
- task->dynamic_working_set,
- base_offset, base_object,
- &line);
- if(kr == KERN_OPERATION_TIMED_OUT){
- write_startup_file = 1;
- } else if (kr != KERN_SUCCESS) {
- kr = tws_insert((tws_hash_t)
- task->dynamic_working_set,
- base_offset, base_object,
- vaddr, pmap_map);
- if(kr == KERN_NO_SPACE) {
- vm_object_unlock(object);
-
- tws_expand_working_set(
- task->dynamic_working_set,
- TWS_HASH_LINE_COUNT,
- FALSE);
-
- vm_object_lock(object);
- }
- if(kr ==
- KERN_OPERATION_TIMED_OUT) {
- write_startup_file = 1;
- }
- }
- }
+ sequential = (sequential && vm_page_deactivate_behind) ?
+ vm_fault_deactivate_behind(object, cur_offset, behavior) :
+ FALSE;
+
+ /*
+ * Add non-sequential pages to the working set.
+ * The sequential pages will be brought in through
+ * normal clustering behavior.
+ */
+ if (!sequential && !object->private) {
+ write_startup_file =
+ vm_fault_tws_insert(map, pmap_map, vaddr,
+ object, cur_offset);
}
+
+ vm_object_paging_end(object);
vm_object_unlock(object);
vm_map_unlock_read(map);
*/
vm_object_paging_end(object);
- vm_object_collapse(object);
+ vm_object_collapse(object, offset);
vm_object_paging_begin(object);
goto FastPmapEnter;
* page, then drop any lower lock.
* Give up if no page.
*/
- if ((vm_page_free_target -
- ((vm_page_free_target-vm_page_free_min)>>2))
- > vm_page_free_count) {
+ if (VM_PAGE_THROTTLED()) {
+ break;
+ }
+
+ /*
+ * are we protecting the system from
+ * backing store exhaustion. If so
+ * sleep unless we are privileged.
+ */
+ if(vm_backing_store_low) {
+ if(!(current_task()->priv_flags
+ & VM_BACKING_STORE_PRIV))
break;
}
m = vm_page_alloc(object, offset);
vm_object_paging_begin(object);
XPR(XPR_VM_FAULT,"vm_fault -> vm_fault_page\n",0,0,0,0,0);
- {
- tws_hash_line_t line;
- task_t task;
- kern_return_t kr;
-
- task = current_task();
- if((map != NULL) &&
- (task->dynamic_working_set != 0)
- && !(object->private)) {
- vm_object_t base_object;
- vm_object_offset_t base_offset;
- base_object = object;
- base_offset = offset;
- while(base_object->shadow) {
- base_offset +=
- base_object->shadow_offset;
- base_object =
- base_object->shadow;
- }
- kr = tws_lookup((tws_hash_t)
- task->dynamic_working_set,
- base_offset, base_object,
- &line);
- if(kr == KERN_OPERATION_TIMED_OUT){
- write_startup_file = 1;
- } else if (kr != KERN_SUCCESS) {
- tws_insert((tws_hash_t)
- task->dynamic_working_set,
- base_offset, base_object,
- vaddr, pmap_map);
- kr = tws_insert((tws_hash_t)
- task->dynamic_working_set,
- base_offset, base_object,
- vaddr, pmap_map);
- if(kr == KERN_NO_SPACE) {
- vm_object_unlock(object);
- tws_expand_working_set(
- task->dynamic_working_set,
- TWS_HASH_LINE_COUNT,
- FALSE);
- vm_object_lock(object);
- }
- if(kr == KERN_OPERATION_TIMED_OUT) {
- write_startup_file = 1;
- }
- }
- }
+
+ if (!object->private) {
+ write_startup_file =
+ vm_fault_tws_insert(map, pmap_map, vaddr, object, offset);
}
+
kr = vm_fault_page(object, offset, fault_type,
(change_wiring && !wired),
interruptible,
if (m->no_isync == TRUE) {
pmap_sync_caches_phys(m->phys_page);
+ if (type_of_fault == DBG_CACHE_HIT_FAULT) {
+ /*
+ * found it in the cache, but this
+ * is the first fault-in of the page (no_isync == TRUE)
+ * so it must have come in as part of
+ * a cluster... account 1 pagein against it
+ */
+ VM_STAT(pageins++);
+ current_task()->pageins++;
+
+ type_of_fault = DBG_PAGEIN_FAULT;
+ }
m->no_isync = FALSE;
}
-
cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK;
if(caller_pmap) {
PMAP_ENTER(pmap, vaddr, m,
prot, cache_attr, wired);
}
- {
- tws_hash_line_t line;
- task_t task;
- kern_return_t kr;
-
- task = current_task();
- if((map != NULL) &&
- (task->dynamic_working_set != 0)
- && (object->private)) {
- vm_object_t base_object;
- vm_object_offset_t base_offset;
- base_object = m->object;
- base_offset = m->offset;
- while(base_object->shadow) {
- base_offset +=
- base_object->shadow_offset;
- base_object =
- base_object->shadow;
- }
- kr = tws_lookup((tws_hash_t)
- task->dynamic_working_set,
- base_offset, base_object, &line);
- if(kr == KERN_OPERATION_TIMED_OUT){
- write_startup_file = 1;
- } else if (kr != KERN_SUCCESS) {
- tws_insert((tws_hash_t)
- task->dynamic_working_set,
- base_offset, base_object,
- vaddr, pmap_map);
- kr = tws_insert((tws_hash_t)
- task->dynamic_working_set,
- base_offset, base_object,
- vaddr, pmap_map);
- if(kr == KERN_NO_SPACE) {
- vm_object_unlock(m->object);
- tws_expand_working_set(
- task->dynamic_working_set,
- TWS_HASH_LINE_COUNT,
- FALSE);
- vm_object_lock(m->object);
- }
- if(kr == KERN_OPERATION_TIMED_OUT) {
- write_startup_file = 1;
- }
- }
- }
+
+ /*
+ * Add working set information for private objects here.
+ */
+ if (m->object->private) {
+ write_startup_file =
+ vm_fault_tws_insert(map, pmap_map, vaddr,
+ m->object, m->offset);
}
} else {
projects / apple / xnu.git / blobdiff