X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/1c79356b52d46aa6b508fb032f5ae709b1f2897b..143cc14e17b26a90f1f4060725df7ea635161581:/osfmk/vm/vm_fault.c diff --git a/osfmk/vm/vm_fault.c b/osfmk/vm/vm_fault.c index 77ec5f18c..14c503dbb 100644 --- a/osfmk/vm/vm_fault.c +++ b/osfmk/vm/vm_fault.c @@ -1,3 +1,4 @@ + /* * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. * @@ -58,6 +59,7 @@ #ifdef MACH_BSD /* remove after component interface available */ extern int vnode_pager_workaround; +extern int device_pager_workaround; #endif #include @@ -74,6 +76,9 @@ extern int vnode_pager_workaround; #include #include #include +#include +#include +#include #include #include #include @@ -100,7 +105,6 @@ int vm_object_absent_max = 50; int vm_fault_debug = 0; boolean_t vm_page_deactivate_behind = TRUE; -vm_machine_attribute_val_t mv_cache_sync = MATTR_VAL_CACHE_SYNC; #if !VM_FAULT_STATIC_CONFIG boolean_t vm_fault_dirty_handling = FALSE; @@ -117,7 +121,8 @@ extern kern_return_t vm_fault_wire_fast( vm_map_t map, vm_offset_t va, vm_map_entry_t entry, - pmap_t pmap); + pmap_t pmap, + vm_offset_t pmap_addr); extern void vm_fault_continue(void); @@ -274,9 +279,11 @@ vm_fault_page( /* More arguments: */ kern_return_t *error_code, /* code if page is in error */ boolean_t no_zero_fill, /* don't zero fill absent pages */ - boolean_t data_supply) /* treat as data_supply if + boolean_t data_supply, /* treat as data_supply if * it is a write fault and a full * page is provided */ + vm_map_t map, + vm_offset_t vaddr) { register vm_page_t m; @@ -297,13 +304,9 @@ vm_fault_page( CLUSTER_STAT(int pages_at_higher_offsets;) CLUSTER_STAT(int pages_at_lower_offsets;) kern_return_t wait_result; - thread_t cur_thread; boolean_t interruptible_state; + boolean_t bumped_pagein = FALSE; -#ifdef MACH_BSD - kern_return_t vnode_pager_data_request(ipc_port_t, - ipc_port_t, vm_object_offset_t, vm_size_t, vm_prot_t); -#endif #if MACH_PAGEMAP /* @@ -428,11 +431,7 @@ vm_fault_page( #endif /* MACH_KDB */ #endif /* STATIC_CONFIG */ - cur_thread = current_thread(); - - interruptible_state = cur_thread->interruptible; - if (interruptible == THREAD_UNINT) - cur_thread->interruptible = FALSE; + interruptible_state = thread_interrupt_level(interruptible); /* * INVARIANTS (through entire routine): @@ -489,7 +488,7 @@ vm_fault_page( #endif if (!object->alive) { vm_fault_cleanup(object, first_m); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return(VM_FAULT_MEMORY_ERROR); } m = vm_page_lookup(object, offset); @@ -520,19 +519,16 @@ vm_fault_page( #if TRACEFAULTPAGE dbgTrace(0xBEEF0005, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */ #endif - PAGE_ASSERT_WAIT(m, interruptible); - vm_object_unlock(object); + wait_result = PAGE_SLEEP(object, m, interruptible); XPR(XPR_VM_FAULT, "vm_f_page: block busy obj 0x%X, offset 0x%X, page 0x%X\n", (integer_t)object, offset, (integer_t)m, 0, 0); counter(c_vm_fault_page_block_busy_kernel++); - wait_result = thread_block((void (*)(void))0); - vm_object_lock(object); if (wait_result != THREAD_AWAKENED) { vm_fault_cleanup(object, first_m); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); if (wait_result == THREAD_RESTART) { return(VM_FAULT_RETRY); @@ -557,7 +553,7 @@ vm_fault_page( *error_code = m->page_error; VM_PAGE_FREE(m); vm_fault_cleanup(object, first_m); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return(VM_FAULT_MEMORY_ERROR); } @@ -574,7 +570,7 @@ vm_fault_page( #endif VM_PAGE_FREE(m); vm_fault_cleanup(object, first_m); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return(VM_FAULT_RETRY); } @@ -602,7 +598,7 @@ vm_fault_page( if (object->shadow_severed) { vm_fault_cleanup( object, first_m); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return VM_FAULT_MEMORY_ERROR; } @@ -616,7 +612,7 @@ vm_fault_page( if (VM_PAGE_THROTTLED() || (real_m = vm_page_grab()) == VM_PAGE_NULL) { vm_fault_cleanup(object, first_m); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return(VM_FAULT_MEMORY_SHORTAGE); } @@ -650,19 +646,46 @@ vm_fault_page( * newly allocated -- in both cases * it can't be page locked by a pager. */ + m->no_isync = FALSE; + if (!no_zero_fill) { vm_object_unlock(object); vm_page_zero_fill(m); if (type_of_fault) *type_of_fault = DBG_ZERO_FILL_FAULT; VM_STAT(zero_fill_count++); + + if (bumped_pagein == TRUE) { + VM_STAT(pageins--); + current_task()->pageins--; + } vm_object_lock(object); } pmap_clear_modify(m->phys_addr); vm_page_lock_queues(); VM_PAGE_QUEUES_REMOVE(m); - queue_enter(&vm_page_queue_inactive, + m->page_ticket = vm_page_ticket; + if(m->object->size > 0x80000) { + m->zero_fill = TRUE; + /* depends on the queues lock */ + vm_zf_count += 1; + queue_enter(&vm_page_queue_zf, m, vm_page_t, pageq); + } else { + queue_enter( + &vm_page_queue_inactive, + m, vm_page_t, pageq); + } + vm_page_ticket_roll++; + if(vm_page_ticket_roll == + VM_PAGE_TICKETS_IN_ROLL) { + vm_page_ticket_roll = 0; + if(vm_page_ticket == + VM_PAGE_TICKET_ROLL_IDS) + vm_page_ticket= 0; + else + vm_page_ticket++; + } m->inactive = TRUE; vm_page_inactive_count++; vm_page_unlock_queues(); @@ -734,13 +757,13 @@ vm_fault_page( if (m != VM_PAGE_NULL && m->cleaning) { PAGE_ASSERT_WAIT(m, interruptible); vm_object_unlock(object); - wait_result = thread_block((void (*)(void)) 0); + wait_result = thread_block(THREAD_CONTINUE_NULL); vm_object_deallocate(object); goto backoff; } else { vm_object_unlock(object); vm_object_deallocate(object); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return VM_FAULT_RETRY; } } @@ -774,18 +797,19 @@ vm_fault_page( vm_object_lock(object); assert(object->ref_count > 0); if (!object->pager_ready) { - vm_object_assert_wait( + wait_result = vm_object_assert_wait( object, VM_OBJECT_EVENT_PAGER_READY, interruptible); vm_object_unlock(object); - wait_result = thread_block((void (*)(void))0); + if (wait_result == THREAD_WAITING) + wait_result = thread_block(THREAD_CONTINUE_NULL); vm_object_deallocate(object); goto backoff; } else { vm_object_unlock(object); vm_object_deallocate(object); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return VM_FAULT_RETRY; } } @@ -799,7 +823,6 @@ vm_fault_page( (integer_t)m, new_unlock_request, 0); if ((rc = memory_object_data_unlock( object->pager, - object->pager_request, offset + object->paging_offset, PAGE_SIZE, new_unlock_request)) @@ -808,7 +831,7 @@ vm_fault_page( printf("vm_fault: memory_object_data_unlock failed\n"); vm_object_lock(object); vm_fault_cleanup(object, first_m); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return((rc == MACH_SEND_INTERRUPTED) ? VM_FAULT_INTERRUPTED : VM_FAULT_MEMORY_ERROR); @@ -835,13 +858,13 @@ vm_fault_page( !((access_required & m->unlock_request) != access_required)) { PAGE_ASSERT_WAIT(m, interruptible); vm_object_unlock(object); - wait_result = thread_block((void (*)(void)) 0); + wait_result = thread_block(THREAD_CONTINUE_NULL); vm_object_deallocate(object); goto backoff; } else { vm_object_unlock(object); vm_object_deallocate(object); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return VM_FAULT_RETRY; } } @@ -884,7 +907,8 @@ vm_fault_page( dbgTrace(0xBEEF000C, (unsigned int) look_for_page, (unsigned int) object); /* (TEST/DEBUG) */ #endif if ((look_for_page || (object == first_object)) - && !must_be_resident) { + && !must_be_resident + && !(object->phys_contiguous)) { /* * Allocate a new page for this object/offset * pair. @@ -896,13 +920,13 @@ vm_fault_page( #endif if (m == VM_PAGE_NULL) { vm_fault_cleanup(object, first_m); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return(VM_FAULT_FICTITIOUS_SHORTAGE); } vm_page_insert(m, object, offset); } - if (look_for_page && !must_be_resident) { + if ((look_for_page && !must_be_resident)) { kern_return_t rc; /* @@ -913,7 +937,8 @@ vm_fault_page( #if TRACEFAULTPAGE dbgTrace(0xBEEF000E, (unsigned int) 0, (unsigned int) 0); /* (TEST/DEBUG) */ #endif - VM_PAGE_FREE(m); + if(m != VM_PAGE_NULL) + VM_PAGE_FREE(m); XPR(XPR_VM_FAULT, "vm_f_page: ready wait obj 0x%X, offset 0x%X\n", (integer_t)object, offset, 0, 0, 0); @@ -926,21 +951,29 @@ vm_fault_page( vm_object_lock(object); assert(object->ref_count > 0); if (!object->pager_ready) { - vm_object_assert_wait(object, + wait_result = vm_object_assert_wait(object, VM_OBJECT_EVENT_PAGER_READY, interruptible); vm_object_unlock(object); - wait_result = thread_block((void (*)(void))0); + if (wait_result == THREAD_WAITING) + wait_result = thread_block(THREAD_CONTINUE_NULL); vm_object_deallocate(object); goto backoff; } else { vm_object_unlock(object); vm_object_deallocate(object); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return VM_FAULT_RETRY; } } + if(object->phys_contiguous) { + if(m != VM_PAGE_NULL) { + VM_PAGE_FREE(m); + m = VM_PAGE_NULL; + } + goto no_clustering; + } if (object->internal) { /* * Requests to the default pager @@ -958,7 +991,7 @@ vm_fault_page( if (m->fictitious && !vm_page_convert(m)) { VM_PAGE_FREE(m); vm_fault_cleanup(object, first_m); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return(VM_FAULT_MEMORY_SHORTAGE); } } else if (object->absent_count > @@ -972,7 +1005,8 @@ vm_fault_page( #if TRACEFAULTPAGE dbgTrace(0xBEEF0010, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */ #endif - VM_PAGE_FREE(m); + if(m != VM_PAGE_NULL) + VM_PAGE_FREE(m); /* take an extra ref so object won't die */ assert(object->ref_count > 0); object->ref_count++; @@ -985,13 +1019,13 @@ vm_fault_page( vm_object_absent_assert_wait(object, interruptible); vm_object_unlock(object); - wait_result = thread_block((void (*)(void))0); + wait_result = thread_block(THREAD_CONTINUE_NULL); vm_object_deallocate(object); goto backoff; } else { vm_object_unlock(object); vm_object_deallocate(object); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return VM_FAULT_RETRY; } } @@ -1001,233 +1035,39 @@ vm_fault_page( * from the memory manager. */ - m->list_req_pending = TRUE; - m->absent = TRUE; - m->unusual = TRUE; - object->absent_count++; - - cluster_start = offset; - length = PAGE_SIZE; - cluster_size = object->cluster_size; - - /* - * Skip clustered pagein if it is globally disabled - * or random page reference behavior is expected - * for the address range containing the faulting - * address or the object paging block size is - * equal to the page size. - */ - if (!vm_allow_clustered_pagein || - behavior == VM_BEHAVIOR_RANDOM || - cluster_size == PAGE_SIZE) { - cluster_start = trunc_page_64(cluster_start); - goto no_clustering; - } - - assert(offset >= lo_offset); - assert(offset < hi_offset); - assert(ALIGNED(object->paging_offset)); - assert(cluster_size >= PAGE_SIZE); - -#if TRACEFAULTPAGE - dbgTrace(0xBEEF0011, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */ -#endif - /* - * Decide whether to scan ahead or behind for - * additional pages contiguous to the faulted - * page in the same paging block. The decision - * is based on system wide globals and the - * expected page reference behavior of the - * address range contained the faulting address. - * First calculate some constants. - */ - paging_offset = offset + object->paging_offset; - cluster_offset = paging_offset & (cluster_size - 1); - align_offset = paging_offset&(PAGE_SIZE_64-1); - if (align_offset != 0) { - cluster_offset = trunc_page_64(cluster_offset); - } - -#define SPANS_CLUSTER(x) ((((x) - align_offset) & (vm_object_offset_t)(cluster_size - 1)) == 0) + if(m != VM_PAGE_NULL) { - /* - * Backward scan only if reverse sequential - * behavior has been specified - */ - CLUSTER_STAT(pages_at_lower_offsets = 0;) - if (((vm_default_behind != 0 && - behavior == VM_BEHAVIOR_DEFAULT) || - behavior == VM_BEHAVIOR_RSEQNTL) && offset) { - vm_object_offset_t cluster_bot; - - /* - * Calculate lower search boundary. - * Exclude pages that span a cluster boundary. - * Clip to start of map entry. - * For default page reference behavior, scan - * default pages behind. - */ - cluster_bot = (offset > cluster_offset) ? - offset - cluster_offset : offset; - if (align_offset != 0) { - if ((cluster_bot < offset) && - SPANS_CLUSTER(cluster_bot)) { - cluster_bot += PAGE_SIZE_64; - } - } - if (behavior == VM_BEHAVIOR_DEFAULT) { - vm_object_offset_t - bot = (vm_object_offset_t) - (vm_default_behind * PAGE_SIZE); - - if (cluster_bot < (offset - bot)) - cluster_bot = offset - bot; - } - if (lo_offset > cluster_bot) - cluster_bot = lo_offset; - - for ( cluster_start = offset - PAGE_SIZE_64; - (cluster_start >= cluster_bot) && - (cluster_start != - (align_offset - PAGE_SIZE_64)); - cluster_start -= PAGE_SIZE_64) { - assert(cluster_size > PAGE_SIZE_64); -retry_cluster_backw: - if (!LOOK_FOR(object, cluster_start) || - vm_page_lookup(object, cluster_start) - != VM_PAGE_NULL) { - break; - } - if (object->internal) { - /* - * need to acquire a real page in - * advance because this acts as - * a throttling mechanism for - * data_requests to the default - * pager. If this fails, give up - * trying to find any more pages - * in the cluster and send off the - * request for what we already have. - */ - if ((m = vm_page_grab()) - == VM_PAGE_NULL) { - cluster_start += PAGE_SIZE_64; - cluster_end = offset + PAGE_SIZE_64; - goto give_up; - } - } else if ((m = vm_page_grab_fictitious()) - == VM_PAGE_NULL) { - vm_object_unlock(object); - vm_page_more_fictitious(); - vm_object_lock(object); - goto retry_cluster_backw; - } + m->list_req_pending = TRUE; m->absent = TRUE; m->unusual = TRUE; - m->clustered = TRUE; - m->list_req_pending = TRUE; + object->absent_count++; - vm_page_insert(m, object, cluster_start); - CLUSTER_STAT(pages_at_lower_offsets++;) - object->absent_count++; - } - cluster_start += PAGE_SIZE_64; - assert(cluster_start >= cluster_bot); } - assert(cluster_start <= offset); - /* - * Forward scan if default or sequential behavior - * specified - */ - CLUSTER_STAT(pages_at_higher_offsets = 0;) - if ((behavior == VM_BEHAVIOR_DEFAULT && - vm_default_ahead != 0) || - behavior == VM_BEHAVIOR_SEQUENTIAL) { - vm_object_offset_t cluster_top; - - /* - * Calculate upper search boundary. - * Exclude pages that span a cluster boundary. - * Clip to end of map entry. - * For default page reference behavior, scan - * default pages ahead. - */ - cluster_top = (offset + cluster_size) - - cluster_offset; - if (align_offset != 0) { - if ((cluster_top > (offset + PAGE_SIZE_64)) && - SPANS_CLUSTER(cluster_top)) { - cluster_top -= PAGE_SIZE_64; - } - } - if (behavior == VM_BEHAVIOR_DEFAULT) { - vm_object_offset_t top = (vm_object_offset_t) - ((vm_default_ahead*PAGE_SIZE)+PAGE_SIZE); - - if (cluster_top > (offset + top)) - cluster_top = offset + top; - } - if (cluster_top > hi_offset) - cluster_top = hi_offset; - - for (cluster_end = offset + PAGE_SIZE_64; - cluster_end < cluster_top; - cluster_end += PAGE_SIZE_64) { - assert(cluster_size > PAGE_SIZE); -retry_cluster_forw: - if (!LOOK_FOR(object, cluster_end) || - vm_page_lookup(object, cluster_end) - != VM_PAGE_NULL) { - break; - } - if (object->internal) { - /* - * need to acquire a real page in - * advance because this acts as - * a throttling mechanism for - * data_requests to the default - * pager. If this fails, give up - * trying to find any more pages - * in the cluster and send off the - * request for what we already have. - */ - if ((m = vm_page_grab()) - == VM_PAGE_NULL) { - break; - } - } else if ((m = vm_page_grab_fictitious()) - == VM_PAGE_NULL) { - vm_object_unlock(object); - vm_page_more_fictitious(); - vm_object_lock(object); - goto retry_cluster_forw; - } - m->absent = TRUE; - m->unusual = TRUE; - m->clustered = TRUE; - m->list_req_pending = TRUE; +no_clustering: + cluster_start = offset; + length = PAGE_SIZE; - vm_page_insert(m, object, cluster_end); - CLUSTER_STAT(pages_at_higher_offsets++;) - object->absent_count++; - } - assert(cluster_end <= cluster_top); - } - else { + /* + * lengthen the cluster by the pages in the working set + */ + if((map != NULL) && + (current_task()->dynamic_working_set != 0)) { + cluster_end = cluster_start + length; + /* tws values for start and end are just a + * suggestions. Therefore, as long as + * build_cluster does not use pointers or + * take action based on values that + * could be affected by re-entrance we + * do not need to take the map lock. + */ cluster_end = offset + PAGE_SIZE_64; + tws_build_cluster((tws_hash_t) + current_task()->dynamic_working_set, + object, &cluster_start, + &cluster_end, 0x40000); + length = cluster_end - cluster_start; } -give_up: - assert(cluster_end >= offset + PAGE_SIZE_64); - length = cluster_end - cluster_start; - -#if MACH_CLUSTER_STATS - CLUSTER_STAT_HIGHER(pages_at_higher_offsets); - CLUSTER_STAT_LOWER(pages_at_lower_offsets); - CLUSTER_STAT_CLUSTER(length/PAGE_SIZE); -#endif /* MACH_CLUSTER_STATS */ - -no_clustering: #if TRACEFAULTPAGE dbgTrace(0xBEEF0012, (unsigned int) object, (unsigned int) 0); /* (TEST/DEBUG) */ #endif @@ -1242,9 +1082,10 @@ no_clustering: */ if (type_of_fault) - *type_of_fault = DBG_PAGEIN_FAULT; + *type_of_fault = (length << 8) | DBG_PAGEIN_FAULT; VM_STAT(pageins++); current_task()->pageins++; + bumped_pagein = TRUE; /* * If this object uses a copy_call strategy, @@ -1270,29 +1111,11 @@ no_clustering: (integer_t)object, offset, (integer_t)m, access_required | wants_copy_flag, 0); -#ifdef MACH_BSD - if (((rpc_subsystem_t)pager_mux_hash_lookup(object->pager)) == - ((rpc_subsystem_t) &vnode_pager_workaround)) { - rc = vnode_pager_data_request(object->pager, - object->pager_request, - cluster_start + object->paging_offset, - length, - access_required | wants_copy_flag); - } else { - rc = memory_object_data_request(object->pager, - object->pager_request, - cluster_start + object->paging_offset, - length, - access_required | wants_copy_flag); - } -#else rc = memory_object_data_request(object->pager, - object->pager_request, cluster_start + object->paging_offset, length, access_required | wants_copy_flag); -#endif #if TRACEFAULTPAGE dbgTrace(0xBEEF0013, (unsigned int) object, (unsigned int) rc); /* (TEST/DEBUG) */ @@ -1300,35 +1123,65 @@ no_clustering: if (rc != KERN_SUCCESS) { if (rc != MACH_SEND_INTERRUPTED && vm_fault_debug) - printf("%s(0x%x, 0x%x, 0x%x, 0x%x, 0x%x) failed, rc=%d, object=0x%x\n", + printf("%s(0x%x, 0x%x, 0x%x, 0x%x) failed, rc=%d\n", "memory_object_data_request", object->pager, - object->pager_request, cluster_start + object->paging_offset, - length, access_required, - rc, object); + length, access_required, rc); /* * Don't want to leave a busy page around, * but the data request may have blocked, * so check if it's still there and busy. */ - vm_object_lock(object); - for (; length; - length -= PAGE_SIZE, - cluster_start += PAGE_SIZE_64) { - vm_page_t p; - if ((p = vm_page_lookup(object, + if(!object->phys_contiguous) { + vm_object_lock(object); + for (; length; length -= PAGE_SIZE, + cluster_start += PAGE_SIZE_64) { + vm_page_t p; + if ((p = vm_page_lookup(object, cluster_start)) - && p->absent && p->busy - && p != first_m) { - VM_PAGE_FREE(m); - } + && p->absent && p->busy + && p != first_m) { + VM_PAGE_FREE(p); + } + } } vm_fault_cleanup(object, first_m); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return((rc == MACH_SEND_INTERRUPTED) ? VM_FAULT_INTERRUPTED : VM_FAULT_MEMORY_ERROR); + } else { +#ifdef notdefcdy + tws_hash_line_t line; + task_t task; + + task = current_task(); + + if((map != NULL) && + (task->dynamic_working_set != 0)) + && !(object->private)) { + vm_object_t base_object; + vm_object_offset_t base_offset; + base_object = object; + base_offset = offset; + while(base_object->shadow) { + base_offset += + base_object->shadow_offset; + base_object = + base_object->shadow; + } + if(tws_lookup + ((tws_hash_t) + task->dynamic_working_set, + base_offset, base_object, + &line) == KERN_SUCCESS) { + tws_line_signal((tws_hash_t) + task->dynamic_working_set, + map, line, vaddr); + } + } +#endif } /* @@ -1340,9 +1193,11 @@ no_clustering: if ((interruptible != THREAD_UNINT) && (current_thread()->state & TH_ABORT)) { vm_fault_cleanup(object, first_m); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return(VM_FAULT_INTERRUPTED); } + if(m == VM_PAGE_NULL) + break; continue; } @@ -1393,7 +1248,7 @@ no_clustering: if (object->shadow_severed) { VM_PAGE_FREE(m); vm_fault_cleanup(object, VM_PAGE_NULL); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return VM_FAULT_MEMORY_ERROR; } @@ -1401,9 +1256,10 @@ no_clustering: (m->fictitious && !vm_page_convert(m))) { VM_PAGE_FREE(m); vm_fault_cleanup(object, VM_PAGE_NULL); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return(VM_FAULT_MEMORY_SHORTAGE); } + m->no_isync = FALSE; if (!no_zero_fill) { vm_object_unlock(object); @@ -1411,12 +1267,36 @@ no_clustering: if (type_of_fault) *type_of_fault = DBG_ZERO_FILL_FAULT; VM_STAT(zero_fill_count++); + + if (bumped_pagein == TRUE) { + VM_STAT(pageins--); + current_task()->pageins--; + } vm_object_lock(object); } vm_page_lock_queues(); VM_PAGE_QUEUES_REMOVE(m); - queue_enter(&vm_page_queue_inactive, - m, vm_page_t, pageq); + if(m->object->size > 0x80000) { + m->zero_fill = TRUE; + /* depends on the queues lock */ + vm_zf_count += 1; + queue_enter(&vm_page_queue_zf, + m, vm_page_t, pageq); + } else { + queue_enter( + &vm_page_queue_inactive, + m, vm_page_t, pageq); + } + m->page_ticket = vm_page_ticket; + vm_page_ticket_roll++; + if(vm_page_ticket_roll == VM_PAGE_TICKETS_IN_ROLL) { + vm_page_ticket_roll = 0; + if(vm_page_ticket == + VM_PAGE_TICKET_ROLL_IDS) + vm_page_ticket= 0; + else + vm_page_ticket++; + } m->inactive = TRUE; vm_page_inactive_count++; vm_page_unlock_queues(); @@ -1459,10 +1339,12 @@ no_clustering: dbgTrace(0xBEEF0015, (unsigned int) object, (unsigned int) m); /* (TEST/DEBUG) */ #endif #if EXTRA_ASSERTIONS - assert(m->busy && !m->absent); - assert((first_m == VM_PAGE_NULL) || - (first_m->busy && !first_m->absent && - !first_m->active && !first_m->inactive)); + if(m != VM_PAGE_NULL) { + assert(m->busy && !m->absent); + assert((first_m == VM_PAGE_NULL) || + (first_m->busy && !first_m->absent && + !first_m->active && !first_m->inactive)); + } #endif /* EXTRA_ASSERTIONS */ XPR(XPR_VM_FAULT, @@ -1476,7 +1358,7 @@ no_clustering: * by the top-level object. */ - if (object != first_object) { + if ((object != first_object) && (m != VM_PAGE_NULL)) { /* * We only really need to copy if we * want to write it. @@ -1513,7 +1395,7 @@ no_clustering: if (copy_m == VM_PAGE_NULL) { RELEASE_PAGE(m); vm_fault_cleanup(object, first_m); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return(VM_FAULT_MEMORY_SHORTAGE); } @@ -1597,8 +1479,8 @@ no_clustering: * shadowed object, and one here to push it into the copy. */ - while (first_object->copy_strategy == MEMORY_OBJECT_COPY_DELAY && - (copy_object = first_object->copy) != VM_OBJECT_NULL) { + while ((copy_object = first_object->copy) != VM_OBJECT_NULL && + (m!= VM_PAGE_NULL)) { vm_object_offset_t copy_offset; vm_page_t copy_m; @@ -1678,13 +1560,13 @@ no_clustering: if (copy_m != VM_PAGE_NULL && copy_m->busy) { PAGE_ASSERT_WAIT(copy_m, interruptible); vm_object_unlock(copy_object); - wait_result = thread_block((void (*)(void))0); + wait_result = thread_block(THREAD_CONTINUE_NULL); vm_object_deallocate(copy_object); goto backoff; } else { vm_object_unlock(copy_object); vm_object_deallocate(copy_object); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return VM_FAULT_RETRY; } } @@ -1711,7 +1593,7 @@ no_clustering: assert(copy_object->ref_count > 0); vm_object_unlock(copy_object); vm_fault_cleanup(object, first_m); - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return(VM_FAULT_MEMORY_SHORTAGE); } @@ -1846,8 +1728,10 @@ no_clustering: */ #if !VM_FAULT_STATIC_CONFIG - if (vm_fault_dirty_handling && (*protection & VM_PROT_WRITE)) + if (vm_fault_dirty_handling && (*protection & VM_PROT_WRITE) && + (m != VM_PAGE_NULL)) { m->dirty = TRUE; + } #endif #if TRACEFAULTPAGE dbgTrace(0xBEEF0018, (unsigned int) object, (unsigned int) vm_page_deactivate_behind); /* (TEST/DEBUG) */ @@ -1870,7 +1754,10 @@ no_clustering: #if TRACEFAULTPAGE dbgTrace(0xBEEF001A, (unsigned int) VM_FAULT_SUCCESS, 0); /* (TEST/DEBUG) */ #endif - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); + if(*result_page == VM_PAGE_NULL) { + vm_object_unlock(object); + } return(VM_FAULT_SUCCESS); #if 0 @@ -1878,11 +1765,11 @@ no_clustering: vm_fault_cleanup(object, first_m); counter(c_vm_fault_page_block_backoff_kernel++); - thread_block((void (*)(void))0); + thread_block(THREAD_CONTINUE_NULL); #endif backoff: - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); if (wait_result == THREAD_INTERRUPTED) return VM_FAULT_INTERRUPTED; return VM_FAULT_RETRY; @@ -1910,7 +1797,9 @@ vm_fault( vm_offset_t vaddr, vm_prot_t fault_type, boolean_t change_wiring, - int interruptible) + int interruptible, + pmap_t caller_pmap, + vm_offset_t caller_pmap_addr) { vm_map_version_t version; /* Map version for verificiation */ boolean_t wired; /* Should mapping be wired down? */ @@ -1941,8 +1830,12 @@ vm_fault( funnel_t *curflock; thread_t cur_thread; boolean_t interruptible_state; + unsigned int cache_attr; + int write_startup_file = 0; + vm_prot_t full_fault_type; + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 0)) | DBG_FUNC_START, vaddr, 0, @@ -1951,10 +1844,16 @@ vm_fault( 0); cur_thread = current_thread(); + /* at present we do not fully check for execute permission */ + /* we generally treat it is read except in certain device */ + /* memory settings */ + full_fault_type = fault_type; + if(fault_type & VM_PROT_EXECUTE) { + fault_type &= ~VM_PROT_EXECUTE; + fault_type |= VM_PROT_READ; + } - interruptible_state = cur_thread->interruptible; - if (interruptible == THREAD_UNINT) - cur_thread->interruptible = FALSE; + interruptible_state = thread_interrupt_level(interruptible); /* * assume we will hit a page in the cache @@ -2056,13 +1955,39 @@ vm_fault( while (TRUE) { m = vm_page_lookup(cur_object, cur_offset); if (m != VM_PAGE_NULL) { - if (m->busy) - break; + if (m->busy) { + wait_result_t result; - if (m->unusual && (m->error || m->restart || - m->absent || (fault_type & m->page_lock))) { + if (object != cur_object) + vm_object_unlock(object); - /* + vm_map_unlock_read(map); + if (pmap_map != map) + vm_map_unlock(pmap_map); + +#if !VM_FAULT_STATIC_CONFIG + if (!vm_fault_interruptible) + interruptible = THREAD_UNINT; +#endif + result = PAGE_ASSERT_WAIT(m, interruptible); + + vm_object_unlock(cur_object); + + if (result == THREAD_WAITING) { + result = thread_block(THREAD_CONTINUE_NULL); + + counter(c_vm_fault_page_block_busy_kernel++); + } + if (result == THREAD_AWAKENED || result == THREAD_RESTART) + goto RetryFault; + + kr = KERN_ABORTED; + goto done; + } + if (m->unusual && (m->error || m->restart || m->private + || m->absent || (fault_type & m->page_lock))) { + + /* * Unusual case. Give up. */ break; @@ -2098,7 +2023,6 @@ FastMapInFault: m->busy = TRUE; vm_object_paging_begin(object); - vm_object_unlock(object); FastPmapEnter: /* @@ -2120,20 +2044,23 @@ FastPmapEnter: prot &= ~VM_PROT_WRITE; #endif /* MACH_KDB */ #endif /* STATIC_CONFIG */ - PMAP_ENTER(pmap, vaddr, m, prot, wired); - pmap_attribute(pmap, - vaddr, - PAGE_SIZE, - MATTR_CACHE, - &mv_cache_sync); - - if (m->clustered) { - vm_pagein_cluster_used++; - m->clustered = FALSE; + if (m->no_isync == TRUE) { + pmap_sync_caches_phys(m->phys_addr); + m->no_isync = FALSE; + } + cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK; + if(caller_pmap) { + PMAP_ENTER(caller_pmap, + caller_pmap_addr, m, + prot, cache_attr, wired); + } else { + PMAP_ENTER(pmap, vaddr, m, + prot, cache_attr, wired); } + /* - * Grab the object lock to manipulate + * Grab the queues lock to manipulate * the page queues. Change wiring * case is obvious. In soft ref bits * case activate page only if it fell @@ -2144,9 +2071,12 @@ FastPmapEnter: * move active page to back of active * queue. This code doesn't. */ - vm_object_lock(object); vm_page_lock_queues(); + if (m->clustered) { + vm_pagein_cluster_used++; + m->clustered = FALSE; + } m->reference = TRUE; if (change_wiring) { @@ -2176,23 +2106,76 @@ FastPmapEnter: */ PAGE_WAKEUP_DONE(m); vm_object_paging_end(object); + + { + tws_hash_line_t line; + task_t task; + + task = current_task(); + if((map != NULL) && + (task->dynamic_working_set != 0) && + !(object->private)) { + kern_return_t kr; + vm_object_t base_object; + vm_object_offset_t base_offset; + base_object = object; + base_offset = cur_offset; + while(base_object->shadow) { + base_offset += + base_object->shadow_offset; + base_object = + base_object->shadow; + } + kr = tws_lookup((tws_hash_t) + task->dynamic_working_set, + base_offset, base_object, + &line); + if(kr == KERN_OPERATION_TIMED_OUT){ + write_startup_file = 1; + } else if (kr != KERN_SUCCESS) { + kr = tws_insert((tws_hash_t) + task->dynamic_working_set, + base_offset, base_object, + vaddr, pmap_map); + if(kr == KERN_NO_SPACE) { + vm_object_unlock(object); + + tws_expand_working_set( + task->dynamic_working_set, + TWS_HASH_LINE_COUNT, + FALSE); + + vm_object_lock(object); + } + if(kr == + KERN_OPERATION_TIMED_OUT) { + write_startup_file = 1; + } + } + } + } vm_object_unlock(object); + vm_map_unlock_read(map); if(pmap_map != map) vm_map_unlock(pmap_map); - if (funnel_set) { + if(write_startup_file) + tws_send_startup_info(current_task()); + + if (funnel_set) thread_funnel_set( curflock, TRUE); - funnel_set = FALSE; - } - cur_thread->interruptible = interruptible_state; + + thread_interrupt_level(interruptible_state); + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 0)) | DBG_FUNC_END, vaddr, - type_of_fault, + type_of_fault & 0xff, KERN_SUCCESS, - 0, + type_of_fault >> 8, 0); + return KERN_SUCCESS; } @@ -2206,7 +2189,6 @@ FastPmapEnter: if (cur_object == object) break; - /* * This is now a shadow based copy on write * fault -- it requires a copy up the shadow @@ -2222,7 +2204,6 @@ FastPmapEnter: if (m == VM_PAGE_NULL) { break; } - /* * Now do the copy. Mark the source busy * and take out paging references on both @@ -2271,7 +2252,6 @@ FastPmapEnter: vm_object_paging_end(object); vm_object_collapse(object); vm_object_paging_begin(object); - vm_object_unlock(object); goto FastPmapEnter; } @@ -2286,7 +2266,6 @@ FastPmapEnter: /* * Have to talk to the pager. Give up. */ - break; } @@ -2300,11 +2279,15 @@ FastPmapEnter: if(pmap_map != map) vm_map_unlock(pmap_map); + if(write_startup_file) + tws_send_startup_info( + current_task()); + if (funnel_set) { thread_funnel_set( curflock, TRUE); funnel_set = FALSE; } - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); return VM_FAULT_MEMORY_ERROR; } @@ -2324,6 +2307,18 @@ FastPmapEnter: if (m == VM_PAGE_NULL) { break; } + /* + * This is a zero-fill or initial fill + * page fault. As such, we consider it + * undefined with respect to instruction + * execution. i.e. it is the responsibility + * of higher layers to call for an instruction + * sync after changing the contents and before + * sending a program into this area. We + * choose this approach for performance + */ + + m->no_isync = FALSE; if (cur_object != object) vm_object_unlock(cur_object); @@ -2348,11 +2343,35 @@ FastPmapEnter: } vm_page_lock_queues(); VM_PAGE_QUEUES_REMOVE(m); - queue_enter(&vm_page_queue_inactive, - m, vm_page_t, pageq); + + m->page_ticket = vm_page_ticket; + if(m->object->size > 0x80000) { + m->zero_fill = TRUE; + /* depends on the queues lock */ + vm_zf_count += 1; + queue_enter(&vm_page_queue_zf, + m, vm_page_t, pageq); + } else { + queue_enter( + &vm_page_queue_inactive, + m, vm_page_t, pageq); + } + vm_page_ticket_roll++; + if(vm_page_ticket_roll == + VM_PAGE_TICKETS_IN_ROLL) { + vm_page_ticket_roll = 0; + if(vm_page_ticket == + VM_PAGE_TICKET_ROLL_IDS) + vm_page_ticket= 0; + else + vm_page_ticket++; + } + m->inactive = TRUE; vm_page_inactive_count++; vm_page_unlock_queues(); + vm_object_lock(object); + goto FastPmapEnter; } @@ -2380,6 +2399,7 @@ FastPmapEnter: vm_object_unlock(cur_object); } vm_map_unlock_read(map); + if(pmap_map != map) vm_map_unlock(pmap_map); @@ -2397,13 +2417,61 @@ FastPmapEnter: vm_object_paging_begin(object); XPR(XPR_VM_FAULT,"vm_fault -> vm_fault_page\n",0,0,0,0,0); + { + tws_hash_line_t line; + task_t task; + kern_return_t kr; + + task = current_task(); + if((map != NULL) && + (task->dynamic_working_set != 0) + && !(object->private)) { + vm_object_t base_object; + vm_object_offset_t base_offset; + base_object = object; + base_offset = offset; + while(base_object->shadow) { + base_offset += + base_object->shadow_offset; + base_object = + base_object->shadow; + } + kr = tws_lookup((tws_hash_t) + task->dynamic_working_set, + base_offset, base_object, + &line); + if(kr == KERN_OPERATION_TIMED_OUT){ + write_startup_file = 1; + } else if (kr != KERN_SUCCESS) { + tws_insert((tws_hash_t) + task->dynamic_working_set, + base_offset, base_object, + vaddr, pmap_map); + kr = tws_insert((tws_hash_t) + task->dynamic_working_set, + base_offset, base_object, + vaddr, pmap_map); + if(kr == KERN_NO_SPACE) { + vm_object_unlock(object); + tws_expand_working_set( + task->dynamic_working_set, + TWS_HASH_LINE_COUNT, + FALSE); + vm_object_lock(object); + } + if(kr == KERN_OPERATION_TIMED_OUT) { + write_startup_file = 1; + } + } + } + } kr = vm_fault_page(object, offset, fault_type, (change_wiring && !wired), interruptible, lo_offset, hi_offset, behavior, &prot, &result_page, &top_page, &type_of_fault, - &error_code, map->no_zero_fill, FALSE); + &error_code, map->no_zero_fill, FALSE, map, vaddr); /* * If we didn't succeed, lose the object reference immediately. @@ -2443,9 +2511,11 @@ FastPmapEnter: m = result_page; - assert((change_wiring && !wired) ? - (top_page == VM_PAGE_NULL) : - ((top_page == VM_PAGE_NULL) == (m->object == object))); + if(m != VM_PAGE_NULL) { + assert((change_wiring && !wired) ? + (top_page == VM_PAGE_NULL) : + ((top_page == VM_PAGE_NULL) == (m->object == object))); + } /* * How to clean up the result of vm_fault_page. This @@ -2477,9 +2547,12 @@ FastPmapEnter: * since our last lookup. */ - old_copy_object = m->object->copy; - - vm_object_unlock(m->object); + if(m != VM_PAGE_NULL) { + old_copy_object = m->object->copy; + vm_object_unlock(m->object); + } else { + old_copy_object = VM_OBJECT_NULL; + } if ((map != original_map) || !vm_map_verify(map, &version)) { vm_object_t retry_object; vm_object_offset_t retry_offset; @@ -2505,22 +2578,34 @@ FastPmapEnter: if (kr != KERN_SUCCESS) { vm_map_unlock_read(map); - vm_object_lock(m->object); - RELEASE_PAGE(m); - UNLOCK_AND_DEALLOCATE; + if(m != VM_PAGE_NULL) { + vm_object_lock(m->object); + RELEASE_PAGE(m); + UNLOCK_AND_DEALLOCATE; + } else { + vm_object_deallocate(object); + } goto done; } vm_object_unlock(retry_object); - vm_object_lock(m->object); + if(m != VM_PAGE_NULL) { + vm_object_lock(m->object); + } else { + vm_object_lock(object); + } if ((retry_object != object) || (retry_offset != offset)) { vm_map_unlock_read(map); if(pmap_map != map) vm_map_unlock(pmap_map); - RELEASE_PAGE(m); - UNLOCK_AND_DEALLOCATE; + if(m != VM_PAGE_NULL) { + RELEASE_PAGE(m); + UNLOCK_AND_DEALLOCATE; + } else { + vm_object_deallocate(object); + } goto RetryFault; } @@ -2529,17 +2614,27 @@ FastPmapEnter: * has been copied while we left the map unlocked. */ prot &= retry_prot; - vm_object_unlock(m->object); + if(m != VM_PAGE_NULL) { + vm_object_unlock(m->object); + } else { + vm_object_unlock(object); + } + } + if(m != VM_PAGE_NULL) { + vm_object_lock(m->object); + } else { + vm_object_lock(object); } - vm_object_lock(m->object); /* * If the copy object changed while the top-level object * was unlocked, then we must take away write permission. */ - if (m->object->copy != old_copy_object) - prot &= ~VM_PROT_WRITE; + if(m != VM_PAGE_NULL) { + if (m->object->copy != old_copy_object) + prot &= ~VM_PROT_WRITE; + } /* * If we want to wire down this page, but no longer have @@ -2550,17 +2645,15 @@ FastPmapEnter: vm_map_verify_done(map, &version); if(pmap_map != map) vm_map_unlock(pmap_map); - RELEASE_PAGE(m); - UNLOCK_AND_DEALLOCATE; + if(m != VM_PAGE_NULL) { + RELEASE_PAGE(m); + UNLOCK_AND_DEALLOCATE; + } else { + vm_object_deallocate(object); + } goto RetryFault; } - /* - * It's critically important that a wired-down page be faulted - * only once in each map for which it is wired. - */ - vm_object_unlock(m->object); - /* * Put this page into the physical map. * We had to do the unlock above because pmap_enter @@ -2568,44 +2661,218 @@ FastPmapEnter: * the pageout queues. If the pageout daemon comes * across the page, it will remove it from the queues. */ - PMAP_ENTER(pmap, vaddr, m, prot, wired); + if (m != VM_PAGE_NULL) { + if (m->no_isync == TRUE) { + pmap_sync_caches_phys(m->phys_addr); + + m->no_isync = FALSE; + } + + cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK; + + if(caller_pmap) { + PMAP_ENTER(caller_pmap, + caller_pmap_addr, m, + prot, cache_attr, wired); + } else { + PMAP_ENTER(pmap, vaddr, m, + prot, cache_attr, wired); + } + { + tws_hash_line_t line; + task_t task; + kern_return_t kr; + + task = current_task(); + if((map != NULL) && + (task->dynamic_working_set != 0) + && (object->private)) { + vm_object_t base_object; + vm_object_offset_t base_offset; + base_object = m->object; + base_offset = m->offset; + while(base_object->shadow) { + base_offset += + base_object->shadow_offset; + base_object = + base_object->shadow; + } + kr = tws_lookup((tws_hash_t) + task->dynamic_working_set, + base_offset, base_object, &line); + if(kr == KERN_OPERATION_TIMED_OUT){ + write_startup_file = 1; + } else if (kr != KERN_SUCCESS) { + tws_insert((tws_hash_t) + task->dynamic_working_set, + base_offset, base_object, + vaddr, pmap_map); + kr = tws_insert((tws_hash_t) + task->dynamic_working_set, + base_offset, base_object, + vaddr, pmap_map); + if(kr == KERN_NO_SPACE) { + vm_object_unlock(m->object); + tws_expand_working_set( + task->dynamic_working_set, + TWS_HASH_LINE_COUNT, + FALSE); + vm_object_lock(m->object); + } + if(kr == KERN_OPERATION_TIMED_OUT) { + write_startup_file = 1; + } + } + } + } + } else { + +#ifndef i386 + int memattr; + struct phys_entry *pp; + vm_map_entry_t entry; + vm_offset_t laddr; + vm_offset_t ldelta, hdelta; + + /* + * do a pmap block mapping from the physical address + * in the object + */ + if(pp = pmap_find_physentry( + (vm_offset_t)object->shadow_offset)) { + memattr = ((pp->pte1 & 0x00000078) >> 3); + } else { + memattr = VM_WIMG_MASK & (int)object->wimg_bits; + } + + + /* While we do not worry about execution protection in */ + /* general, we may be able to read device memory and */ + /* still not be able to execute it. Here we check for */ + /* the guarded bit. If its set and we are attempting */ + /* to execute, we return with a protection failure. */ + + if((memattr & VM_MEM_GUARDED) && + (full_fault_type & VM_PROT_EXECUTE)) { + vm_map_verify_done(map, &version); + if(pmap_map != map) + vm_map_unlock(pmap_map); + vm_fault_cleanup(object, top_page); + vm_object_deallocate(object); + kr = KERN_PROTECTION_FAILURE; + goto done; + } - /* Sync I & D caches for new mapping*/ - pmap_attribute(pmap, - vaddr, - PAGE_SIZE, - MATTR_CACHE, - &mv_cache_sync); + + + if(pmap_map != map) { + vm_map_unlock(pmap_map); + } + if (original_map != map) { + vm_map_unlock_read(map); + vm_map_lock_read(original_map); + map = original_map; + } + pmap_map = map; + + laddr = vaddr; + hdelta = 0xFFFFF000; + ldelta = 0xFFFFF000; + + + while(vm_map_lookup_entry(map, laddr, &entry)) { + if(ldelta > (laddr - entry->vme_start)) + ldelta = laddr - entry->vme_start; + if(hdelta > (entry->vme_end - laddr)) + hdelta = entry->vme_end - laddr; + if(entry->is_sub_map) { + + laddr = (laddr - entry->vme_start) + + entry->offset; + vm_map_lock_read(entry->object.sub_map); + if(map != pmap_map) + vm_map_unlock_read(map); + if(entry->use_pmap) { + vm_map_unlock_read(pmap_map); + pmap_map = entry->object.sub_map; + } + map = entry->object.sub_map; + + } else { + break; + } + } + + if(vm_map_lookup_entry(map, laddr, &entry) && + (entry->object.vm_object != NULL) && + (entry->object.vm_object == object)) { + + + if(caller_pmap) { + pmap_map_block(caller_pmap, + caller_pmap_addr - ldelta, + ((vm_offset_t) + (entry->object.vm_object->shadow_offset)) + + entry->offset + + (laddr - entry->vme_start) - ldelta, + ldelta + hdelta, prot, + memattr, 0); /* Set up a block mapped area */ + } else { + pmap_map_block(pmap_map->pmap, vaddr - ldelta, + ((vm_offset_t) + (entry->object.vm_object->shadow_offset)) + + entry->offset + + (laddr - entry->vme_start) - ldelta, + ldelta + hdelta, prot, + memattr, 0); /* Set up a block mapped area */ + } + } +#else +#ifdef notyet + if(caller_pmap) { + pmap_enter(caller_pmap, caller_pmap_addr, + object->shadow_offset, prot, 0, TRUE); + } else { + pmap_enter(pmap, vaddr, + object->shadow_offset, prot, 0, TRUE); + } + /* Map it in */ +#endif +#endif + + } /* * If the page is not wired down and isn't already * on a pageout queue, then put it where the * pageout daemon can find it. */ - vm_object_lock(m->object); - vm_page_lock_queues(); - if (change_wiring) { - if (wired) - vm_page_wire(m); - else - vm_page_unwire(m); - } + if(m != VM_PAGE_NULL) { + vm_page_lock_queues(); + + if (change_wiring) { + if (wired) + vm_page_wire(m); + else + vm_page_unwire(m); + } #if VM_FAULT_STATIC_CONFIG - else { - if (!m->active && !m->inactive) - vm_page_activate(m); - m->reference = TRUE; - } + else { + if (!m->active && !m->inactive) + vm_page_activate(m); + m->reference = TRUE; + } #else - else if (software_reference_bits) { - if (!m->active && !m->inactive) + else if (software_reference_bits) { + if (!m->active && !m->inactive) + vm_page_activate(m); + m->reference = TRUE; + } else { vm_page_activate(m); - m->reference = TRUE; - } else { - vm_page_activate(m); - } + } #endif - vm_page_unlock_queues(); + vm_page_unlock_queues(); + } /* * Unlock everything, and return @@ -2614,26 +2881,34 @@ FastPmapEnter: vm_map_verify_done(map, &version); if(pmap_map != map) vm_map_unlock(pmap_map); - PAGE_WAKEUP_DONE(m); + if(m != VM_PAGE_NULL) { + PAGE_WAKEUP_DONE(m); + UNLOCK_AND_DEALLOCATE; + } else { + vm_fault_cleanup(object, top_page); + vm_object_deallocate(object); + } kr = KERN_SUCCESS; - UNLOCK_AND_DEALLOCATE; #undef UNLOCK_AND_DEALLOCATE #undef RELEASE_PAGE done: + if(write_startup_file) + tws_send_startup_info(current_task()); if (funnel_set) { thread_funnel_set( curflock, TRUE); funnel_set = FALSE; } - cur_thread->interruptible = interruptible_state; + thread_interrupt_level(interruptible_state); KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 0)) | DBG_FUNC_END, vaddr, - type_of_fault, + type_of_fault & 0xff, kr, - 0, + type_of_fault >> 8, 0); + return(kr); } @@ -2646,7 +2921,8 @@ kern_return_t vm_fault_wire( vm_map_t map, vm_map_entry_t entry, - pmap_t pmap) + pmap_t pmap, + vm_offset_t pmap_addr) { register vm_offset_t va; @@ -2655,13 +2931,20 @@ vm_fault_wire( assert(entry->in_transition); + if ((entry->object.vm_object != NULL) && + !entry->is_sub_map && + entry->object.vm_object->phys_contiguous) { + return KERN_SUCCESS; + } + /* * Inform the physical mapping system that the * range of addresses may not fault, so that * page tables and such can be locked down as well. */ - pmap_pageable(pmap, entry->vme_start, end_addr, FALSE); + pmap_pageable(pmap, pmap_addr, + pmap_addr + (end_addr - entry->vme_start), FALSE); /* * We simulate a fault to get the page and enter it @@ -2670,9 +2953,13 @@ vm_fault_wire( for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) { if ((rc = vm_fault_wire_fast( - map, va, entry, pmap)) != KERN_SUCCESS) { + map, va, entry, pmap, + pmap_addr + (va - entry->vme_start) + )) != KERN_SUCCESS) { rc = vm_fault(map, va, VM_PROT_NONE, TRUE, - (pmap == kernel_pmap) ? THREAD_UNINT : THREAD_ABORTSAFE); + (pmap == kernel_pmap) ? + THREAD_UNINT : THREAD_ABORTSAFE, + pmap, pmap_addr + (va - entry->vme_start)); } if (rc != KERN_SUCCESS) { @@ -2680,7 +2967,8 @@ vm_fault_wire( /* unwire wired pages */ tmp_entry.vme_end = va; - vm_fault_unwire(map, &tmp_entry, FALSE, pmap); + vm_fault_unwire(map, + &tmp_entry, FALSE, pmap, pmap_addr); return rc; } @@ -2698,7 +2986,8 @@ vm_fault_unwire( vm_map_t map, vm_map_entry_t entry, boolean_t deallocate, - pmap_t pmap) + pmap_t pmap, + vm_offset_t pmap_addr) { register vm_offset_t va; register vm_offset_t end_addr = entry->vme_end; @@ -2713,10 +3002,14 @@ vm_fault_unwire( */ for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) { - pmap_change_wiring(pmap, va, FALSE); + pmap_change_wiring(pmap, + pmap_addr + (va - entry->vme_start), FALSE); if (object == VM_OBJECT_NULL) { - (void) vm_fault(map, va, VM_PROT_NONE, TRUE, THREAD_UNINT); + (void) vm_fault(map, va, VM_PROT_NONE, + TRUE, THREAD_UNINT, pmap, pmap_addr); + } else if (object->phys_contiguous) { + continue; } else { vm_prot_t prot; vm_page_t result_page; @@ -2747,7 +3040,7 @@ vm_fault_unwire( &top_page, (int *)0, 0, map->no_zero_fill, - FALSE); + FALSE, NULL, 0); } while (result == VM_FAULT_RETRY); if (result != VM_FAULT_SUCCESS) @@ -2776,7 +3069,8 @@ vm_fault_unwire( * such may be unwired themselves. */ - pmap_pageable(pmap, entry->vme_start, end_addr, TRUE); + pmap_pageable(pmap, pmap_addr, + pmap_addr + (end_addr - entry->vme_start), TRUE); } @@ -2805,13 +3099,15 @@ vm_fault_wire_fast( vm_map_t map, vm_offset_t va, vm_map_entry_t entry, - pmap_t pmap) + pmap_t pmap, + vm_offset_t pmap_addr) { vm_object_t object; vm_object_offset_t offset; register vm_page_t m; vm_prot_t prot; thread_act_t thr_act; + unsigned int cache_attr; VM_STAT(faults++); @@ -2932,20 +3228,15 @@ vm_fault_wire_fast( * We have to unlock the object because pmap_enter * may cause other faults. */ - vm_object_unlock(object); + if (m->no_isync == TRUE) { + pmap_sync_caches_phys(m->phys_addr); - PMAP_ENTER(pmap, va, m, prot, TRUE); - /* Sync I & D caches for new mapping */ - pmap_attribute(pmap, - va, - PAGE_SIZE, - MATTR_CACHE, - &mv_cache_sync); + m->no_isync = FALSE; + } - /* - * Must relock object so that paging_in_progress can be cleared. - */ - vm_object_lock(object); + cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK; + + PMAP_ENTER(pmap, pmap_addr, m, prot, cache_attr, TRUE); /* * Unlock everything, and return @@ -3097,7 +3388,7 @@ vm_fault_copy( (int *)0, &error, dst_map->no_zero_fill, - FALSE)) { + FALSE, NULL, 0)) { case VM_FAULT_SUCCESS: break; case VM_FAULT_RETRY: @@ -3154,13 +3445,15 @@ vm_fault_copy( * zero-fill the page in dst_object. */ src_page = VM_PAGE_NULL; + result_page = VM_PAGE_NULL; } else { vm_object_lock(src_object); src_page = vm_page_lookup(src_object, trunc_page_64(src_offset)); - if (src_page == dst_page) + if (src_page == dst_page) { src_prot = dst_prot; - else { + result_page = VM_PAGE_NULL; + } else { src_prot = VM_PROT_READ; vm_object_paging_begin(src_object); @@ -3181,7 +3474,7 @@ vm_fault_copy( (int *)0, &error, FALSE, - FALSE)) { + FALSE, NULL, 0)) { case VM_FAULT_SUCCESS: break; @@ -3205,18 +3498,17 @@ vm_fault_copy( return(KERN_MEMORY_ERROR); } - src_page = result_page; assert((src_top_page == VM_PAGE_NULL) == - (src_page->object == src_object)); + (result_page->object == src_object)); } assert ((src_prot & VM_PROT_READ) != VM_PROT_NONE); - vm_object_unlock(src_page->object); + vm_object_unlock(result_page->object); } if (!vm_map_verify(dst_map, dst_version)) { - if (src_page != VM_PAGE_NULL && src_page != dst_page) - vm_fault_copy_cleanup(src_page, src_top_page); + if (result_page != VM_PAGE_NULL && src_page != dst_page) + vm_fault_copy_cleanup(result_page, src_top_page); vm_fault_copy_dst_cleanup(dst_page); break; } @@ -3226,8 +3518,8 @@ vm_fault_copy( if (dst_page->object->copy != old_copy_object) { vm_object_unlock(dst_page->object); vm_map_verify_done(dst_map, dst_version); - if (src_page != VM_PAGE_NULL && src_page != dst_page) - vm_fault_copy_cleanup(src_page, src_top_page); + if (result_page != VM_PAGE_NULL && src_page != dst_page) + vm_fault_copy_cleanup(result_page, src_top_page); vm_fault_copy_dst_cleanup(dst_page); break; } @@ -3257,11 +3549,11 @@ vm_fault_copy( part_size = amount_left; } - if (src_page == VM_PAGE_NULL) { + if (result_page == VM_PAGE_NULL) { vm_page_part_zero_fill(dst_page, dst_po, part_size); } else { - vm_page_part_copy(src_page, src_po, + vm_page_part_copy(result_page, src_po, dst_page, dst_po, part_size); if(!dst_page->dirty){ vm_object_lock(dst_object); @@ -3273,10 +3565,10 @@ vm_fault_copy( } else { part_size = PAGE_SIZE; - if (src_page == VM_PAGE_NULL) + if (result_page == VM_PAGE_NULL) vm_page_zero_fill(dst_page); else{ - vm_page_copy(src_page, dst_page); + vm_page_copy(result_page, dst_page); if(!dst_page->dirty){ vm_object_lock(dst_object); dst_page->dirty = TRUE; @@ -3292,8 +3584,8 @@ vm_fault_copy( vm_map_verify_done(dst_map, dst_version); - if (src_page != VM_PAGE_NULL && src_page != dst_page) - vm_fault_copy_cleanup(src_page, src_top_page); + if (result_page != VM_PAGE_NULL && src_page != dst_page) + vm_fault_copy_cleanup(result_page, src_top_page); vm_fault_copy_dst_cleanup(dst_page); amount_left -= part_size; @@ -3404,11 +3696,12 @@ vm_fault_page_overwrite( */ if (!dst_object->pager_ready) { - vm_object_assert_wait(dst_object, - VM_OBJECT_EVENT_PAGER_READY, - interruptible); + wait_result = vm_object_assert_wait(dst_object, + VM_OBJECT_EVENT_PAGER_READY, + interruptible); vm_object_unlock(dst_object); - wait_result = thread_block((void (*)(void))0); + if (wait_result == THREAD_WAITING) + wait_result = thread_block(THREAD_CONTINUE_NULL); if (wait_result != THREAD_AWAKENED) { DISCARD_PAGE; return(VM_FAULT_INTERRUPTED); @@ -3421,7 +3714,6 @@ vm_fault_page_overwrite( if ((rc = memory_object_data_unlock( dst_object->pager, - dst_object->pager_request, dst_offset + dst_object->paging_offset, PAGE_SIZE, u)) != KERN_SUCCESS) { @@ -3447,9 +3739,10 @@ vm_fault_page_overwrite( break; } - PAGE_ASSERT_WAIT(dst_page, interruptible); + wait_result = PAGE_ASSERT_WAIT(dst_page, interruptible); vm_object_unlock(dst_object); - wait_result = thread_block((void (*)(void))0); + if (wait_result == THREAD_WAITING) + wait_result = thread_block(THREAD_CONTINUE_NULL); if (wait_result != THREAD_AWAKENED) { DISCARD_PAGE; return(VM_FAULT_INTERRUPTED);