X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/d1ecb069dfe24481e4a83f44cb5217a2b06746d7..2dced7af2b695f87fe26496a3e73c219b7880cbc:/osfmk/vm/vm_fault.c diff --git a/osfmk/vm/vm_fault.c b/osfmk/vm/vm_fault.c index de5708199..663199f2a 100644 --- a/osfmk/vm/vm_fault.c +++ b/osfmk/vm/vm_fault.c @@ -64,7 +64,6 @@ #include #include -#include #include #include @@ -89,8 +88,8 @@ #include #include -#include - +#include +#include #include #include #include @@ -102,14 +101,17 @@ #include #include #include /* Needed by some vm_page.h macros */ +#include + +#include -#include +#include /* for struct timespec */ #define VM_FAULT_CLASSIFY 0 #define TRACEFAULTPAGE 0 /* (TEST/DEBUG) */ -int vm_object_pagein_throttle = 16; +unsigned int vm_object_pagein_throttle = 16; /* * We apply a hard throttle to the demand zero rate of tasks that we believe are running out of control which @@ -125,45 +127,51 @@ int vm_object_pagein_throttle = 16; * delay of HARD_THROTTLE_DELAY microseconds before being allowed to try the page fault again. */ -boolean_t thread_is_io_throttled(void); +extern void throttle_lowpri_io(int); uint64_t vm_hard_throttle_threshold; -extern unsigned int dp_pages_free, dp_pages_reserve; -#define NEED_TO_HARD_THROTTLE_THIS_TASK() (((dp_pages_free + dp_pages_reserve < 2000) && \ - (get_task_resident_size(current_task()) > vm_hard_throttle_threshold) && \ - (current_task() != kernel_task) && IP_VALID(memory_manager_default)) || \ - (vm_page_free_count < vm_page_throttle_limit && thread_is_io_throttled() && \ - (get_task_resident_size(current_task()) > vm_hard_throttle_threshold))) +#define NEED_TO_HARD_THROTTLE_THIS_TASK() (vm_wants_task_throttled(current_task()) || \ + (vm_page_free_count < vm_page_throttle_limit && \ + proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO) > THROTTLE_LEVEL_THROTTLED)) -#define HARD_THROTTLE_DELAY 10000 /* 10000 us == 10 ms */ +#define HARD_THROTTLE_DELAY 5000 /* 5000 us == 5 ms */ +#define SOFT_THROTTLE_DELAY 200 /* 200 us == .2 ms */ -extern int cs_debug; +#define VM_PAGE_CREATION_THROTTLE_PERIOD_SECS 6 +#define VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC 20000 -#if MACH_KDB -extern struct db_watchpoint *db_watchpoint_list; -#endif /* MACH_KDB */ boolean_t current_thread_aborted(void); /* Forward declarations of internal routines. */ -extern kern_return_t vm_fault_wire_fast( +static kern_return_t vm_fault_wire_fast( vm_map_t map, vm_map_offset_t va, + vm_prot_t prot, vm_map_entry_t entry, pmap_t pmap, - vm_map_offset_t pmap_addr); - -extern void vm_fault_continue(void); - -extern void vm_fault_copy_cleanup( + vm_map_offset_t pmap_addr, + ppnum_t *physpage_p); + +static kern_return_t vm_fault_internal( + vm_map_t map, + vm_map_offset_t vaddr, + vm_prot_t caller_prot, + boolean_t change_wiring, + int interruptible, + pmap_t pmap, + vm_map_offset_t pmap_addr, + ppnum_t *physpage_p); + +static void vm_fault_copy_cleanup( vm_page_t page, vm_page_t top_page); -extern void vm_fault_copy_dst_cleanup( +static void vm_fault_copy_dst_cleanup( vm_page_t page); #if VM_FAULT_CLASSIFY @@ -175,16 +183,20 @@ extern void vm_fault_classify_init(void); #endif unsigned long vm_pmap_enter_blocked = 0; +unsigned long vm_pmap_enter_retried = 0; unsigned long vm_cs_validates = 0; unsigned long vm_cs_revalidates = 0; unsigned long vm_cs_query_modified = 0; unsigned long vm_cs_validated_dirtied = 0; -#if CONFIG_ENFORCE_SIGNED_CODE -int cs_enforcement_disable=0; -#else -static const int cs_enforcement_disable=1; -#endif +unsigned long vm_cs_bitmap_validated = 0; + +void vm_pre_fault(vm_map_offset_t); + +extern int not_in_kdp; +extern char *kdp_compressor_decompressed_page; +extern addr64_t kdp_compressor_decompressed_page_paddr; +extern ppnum_t kdp_compressor_decompressed_page_ppnum; /* * Routine: vm_fault_init @@ -194,24 +206,54 @@ static const int cs_enforcement_disable=1; void vm_fault_init(void) { -#if !SECURE_KERNEL -#if CONFIG_ENFORCE_SIGNED_CODE - PE_parse_boot_argn("cs_enforcement_disable", &cs_enforcement_disable, - sizeof (cs_enforcement_disable)); -#endif - PE_parse_boot_argn("cs_debug", &cs_debug, sizeof (cs_debug)); -#endif - + int i, vm_compressor_temp; + boolean_t need_default_val = TRUE; /* * Choose a value for the hard throttle threshold based on the amount of ram. The threshold is * computed as a percentage of available memory, and the percentage used is scaled inversely with - * the amount of memory. The pertange runs between 10% and 35%. We use 35% for small memory systems + * the amount of memory. The percentage runs between 10% and 35%. We use 35% for small memory systems * and reduce the value down to 10% for very large memory configurations. This helps give us a * definition of a memory hog that makes more sense relative to the amount of ram in the machine. * The formula here simply uses the number of gigabytes of ram to adjust the percentage. */ vm_hard_throttle_threshold = sane_size * (35 - MIN((int)(sane_size / (1024*1024*1024)), 25)) / 100; + + /* + * Configure compressed pager behavior. A boot arg takes precedence over a device tree entry. + */ + + if (PE_parse_boot_argn("vm_compressor", &vm_compressor_temp, sizeof (vm_compressor_temp))) { + for ( i = 0; i < VM_PAGER_MAX_MODES; i++) { + if (vm_compressor_temp > 0 && + ((vm_compressor_temp & ( 1 << i)) == vm_compressor_temp)) { + need_default_val = FALSE; + vm_compressor_mode = vm_compressor_temp; + break; + } + } + if (need_default_val) + printf("Ignoring \"vm_compressor\" boot arg %d\n", vm_compressor_temp); + } + if (need_default_val) { + /* If no boot arg or incorrect boot arg, try device tree. */ + PE_get_default("kern.vm_compressor", &vm_compressor_mode, sizeof(vm_compressor_mode)); + } + PE_parse_boot_argn("vm_compressor_threads", &vm_compressor_thread_count, sizeof (vm_compressor_thread_count)); + + if (PE_parse_boot_argn("vm_compressor_immediate", &vm_compressor_temp, sizeof (vm_compressor_temp))) + vm_compressor_immediate_preferred_override = TRUE; + else { + if (PE_get_default("kern.vm_compressor_immediate", &vm_compressor_temp, sizeof(vm_compressor_temp))) + vm_compressor_immediate_preferred_override = TRUE; + } + if (vm_compressor_immediate_preferred_override == TRUE) { + if (vm_compressor_temp) + vm_compressor_immediate_preferred = TRUE; + else + vm_compressor_immediate_preferred = FALSE; + } + printf("\"vm_compressor_mode\" is %d\n", vm_compressor_mode); } /* @@ -234,7 +276,7 @@ vm_fault_cleanup( register vm_page_t top_page) { vm_object_paging_end(object); - vm_object_unlock(object); + vm_object_unlock(object); if (top_page != VM_PAGE_NULL) { object = top_page->object; @@ -494,9 +536,21 @@ vm_fault_deactivate_behind( for (n = 0; n < max_pages_in_run; n++) { m = vm_page_lookup(object, offset + run_offset + (n * pg_offset)); - if (m && !m->busy && !m->no_cache && !m->throttled && !m->fictitious && !m->absent) { + if (m && !m->laundry && !m->busy && !m->no_cache && !m->throttled && !m->fictitious && !m->absent) { page_run[pages_in_run++] = m; - pmap_clear_reference(m->phys_page); + + /* + * by not passing in a pmap_flush_context we will forgo any TLB flushing, local or otherwise... + * + * a TLB flush isn't really needed here since at worst we'll miss the reference bit being + * updated in the PTE if a remote processor still has this mapping cached in its TLB when the + * new reference happens. If no futher references happen on the page after that remote TLB flushes + * we'll see a clean, non-referenced page when it eventually gets pulled out of the inactive queue + * by pageout_scan, which is just fine since the last reference would have happened quite far + * in the past (TLB caches don't hang around for very long), and of course could just as easily + * have happened before we did the deactivate_behind. + */ + pmap_clear_refmod_options(m->phys_page, VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL); } } if (pages_in_run) { @@ -521,8 +575,14 @@ vm_fault_deactivate_behind( } -static boolean_t -vm_page_throttled(void) +#if (DEVELOPMENT || DEBUG) +uint32_t vm_page_creation_throttled_hard = 0; +uint32_t vm_page_creation_throttled_soft = 0; +uint64_t vm_page_creation_throttle_avoided = 0; +#endif /* DEVELOPMENT || DEBUG */ + +static int +vm_page_throttled(boolean_t page_kept) { clock_sec_t elapsed_sec; clock_sec_t tv_sec; @@ -531,23 +591,39 @@ vm_page_throttled(void) thread_t thread = current_thread(); if (thread->options & TH_OPT_VMPRIV) - return (FALSE); - - thread->t_page_creation_count++; + return (0); - if (NEED_TO_HARD_THROTTLE_THIS_TASK()) - return (TRUE); - - if (vm_page_free_count < vm_page_throttle_limit && - thread->t_page_creation_count > vm_page_creation_throttle) { + if (thread->t_page_creation_throttled) { + thread->t_page_creation_throttled = 0; + + if (page_kept == FALSE) + goto no_throttle; + } + if (NEED_TO_HARD_THROTTLE_THIS_TASK()) { +#if (DEVELOPMENT || DEBUG) + thread->t_page_creation_throttled_hard++; + OSAddAtomic(1, &vm_page_creation_throttled_hard); +#endif /* DEVELOPMENT || DEBUG */ + return (HARD_THROTTLE_DELAY); + } + if ((vm_page_free_count < vm_page_throttle_limit || ((COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) && SWAPPER_NEEDS_TO_UNTHROTTLE())) && + thread->t_page_creation_count > (VM_PAGE_CREATION_THROTTLE_PERIOD_SECS * VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC)) { + + if (vm_page_free_wanted == 0 && vm_page_free_wanted_privileged == 0) { +#if (DEVELOPMENT || DEBUG) + OSAddAtomic64(1, &vm_page_creation_throttle_avoided); +#endif + goto no_throttle; + } clock_get_system_microtime(&tv_sec, &tv_usec); elapsed_sec = tv_sec - thread->t_page_creation_time; - if (elapsed_sec <= 6 || (thread->t_page_creation_count / elapsed_sec) >= (vm_page_creation_throttle / 6)) { + if (elapsed_sec <= VM_PAGE_CREATION_THROTTLE_PERIOD_SECS || + (thread->t_page_creation_count / elapsed_sec) >= VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC) { - if (elapsed_sec >= 60) { + if (elapsed_sec >= (3 * VM_PAGE_CREATION_THROTTLE_PERIOD_SECS)) { /* * we'll reset our stats to give a well behaved app * that was unlucky enough to accumulate a bunch of pages @@ -558,16 +634,33 @@ vm_page_throttled(void) * will remain in the throttled state */ thread->t_page_creation_time = tv_sec; - thread->t_page_creation_count = (vm_page_creation_throttle / 6) * 5; + thread->t_page_creation_count = VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC * (VM_PAGE_CREATION_THROTTLE_PERIOD_SECS - 1); } ++vm_page_throttle_count; - return (TRUE); + thread->t_page_creation_throttled = 1; + + if ((COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) && HARD_THROTTLE_LIMIT_REACHED()) { +#if (DEVELOPMENT || DEBUG) + thread->t_page_creation_throttled_hard++; + OSAddAtomic(1, &vm_page_creation_throttled_hard); +#endif /* DEVELOPMENT || DEBUG */ + return (HARD_THROTTLE_DELAY); + } else { +#if (DEVELOPMENT || DEBUG) + thread->t_page_creation_throttled_soft++; + OSAddAtomic(1, &vm_page_creation_throttled_soft); +#endif /* DEVELOPMENT || DEBUG */ + return (SOFT_THROTTLE_DELAY); + } } thread->t_page_creation_time = tv_sec; thread->t_page_creation_count = 0; } - return (FALSE); +no_throttle: + thread->t_page_creation_count++; + + return (0); } @@ -580,8 +673,10 @@ vm_page_throttled(void) * object == m->object */ static vm_fault_return_t -vm_fault_check(vm_object_t object, vm_page_t m, vm_page_t first_m, boolean_t interruptible_state) +vm_fault_check(vm_object_t object, vm_page_t m, vm_page_t first_m, boolean_t interruptible_state, boolean_t page_throttle) { + int throttle_delay; + if (object->shadow_severed || VM_OBJECT_PURGEABLE_FAULT_ERROR(object)) { /* @@ -619,27 +714,28 @@ vm_fault_check(vm_object_t object, vm_page_t m, vm_page_t first_m, boolean_t int return (VM_FAULT_RETRY); } } - if (vm_page_throttled()) { - /* - * we're throttling zero-fills... - * treat this as if we couldn't grab a page - */ - if (m != VM_PAGE_NULL) - VM_PAGE_FREE(m); - vm_fault_cleanup(object, first_m); + if (page_throttle == TRUE) { + if ((throttle_delay = vm_page_throttled(FALSE))) { + /* + * we're throttling zero-fills... + * treat this as if we couldn't grab a page + */ + if (m != VM_PAGE_NULL) + VM_PAGE_FREE(m); + vm_fault_cleanup(object, first_m); + + VM_DEBUG_EVENT(vmf_check_zfdelay, VMF_CHECK_ZFDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0); - if (NEED_TO_HARD_THROTTLE_THIS_TASK()) { - delay(HARD_THROTTLE_DELAY); + delay(throttle_delay); if (current_thread_aborted()) { thread_interrupt_level(interruptible_state); return VM_FAULT_INTERRUPTED; } - } - - thread_interrupt_level(interruptible_state); + thread_interrupt_level(interruptible_state); - return (VM_FAULT_MEMORY_SHORTAGE); + return (VM_FAULT_MEMORY_SHORTAGE); + } } return (VM_FAULT_SUCCESS); } @@ -676,10 +772,14 @@ vm_fault_zero_page(vm_page_t m, boolean_t no_zero_fill) m->cs_validated = FALSE; m->cs_tainted = FALSE; + m->cs_nx = FALSE; - if (no_zero_fill == TRUE) - my_fault = DBG_NZF_PAGE_FAULT; - else { + if (no_zero_fill == TRUE) { + my_fault = DBG_NZF_PAGE_FAULT; + + if (m->absent && m->busy) + return (my_fault); + } else { vm_page_zero_fill(m); VM_STAT_INCR(zero_fill_count); @@ -689,22 +789,29 @@ vm_fault_zero_page(vm_page_t m, boolean_t no_zero_fill) assert(m->object != kernel_object); //assert(m->pageq.next == NULL && m->pageq.prev == NULL); - if (!IP_VALID(memory_manager_default) && + if (!VM_DYNAMIC_PAGING_ENABLED(memory_manager_default) && (m->object->purgable == VM_PURGABLE_DENY || m->object->purgable == VM_PURGABLE_NONVOLATILE || m->object->purgable == VM_PURGABLE_VOLATILE )) { + vm_page_lockspin_queues(); - queue_enter(&vm_page_queue_throttled, m, vm_page_t, pageq); - m->throttled = TRUE; - vm_page_throttled_count++; + if (!VM_DYNAMIC_PAGING_ENABLED(memory_manager_default)) { + assert(!VM_PAGE_WIRED(m)); - vm_page_unlock_queues(); - } else { - if (current_thread()->t_page_creation_count > vm_page_creation_throttle) { - m->zero_fill = TRUE; - VM_ZF_COUNT_INCR(); + /* + * can't be on the pageout queue since we don't + * have a pager to try and clean to + */ + assert(!m->pageout_queue); + + vm_page_queues_remove(m); + vm_page_check_pageable_safe(m); + queue_enter(&vm_page_queue_throttled, m, vm_page_t, pageq); + m->throttled = TRUE; + vm_page_throttled_count++; } + vm_page_unlock_queues(); } return (my_fault); } @@ -759,6 +866,7 @@ vm_fault_zero_page(vm_page_t m, boolean_t no_zero_fill) * paging_in_progress reference. */ unsigned int vm_fault_page_blocked_access = 0; +unsigned int vm_fault_page_forced_retry = 0; vm_fault_return_t vm_fault_page( @@ -767,10 +875,11 @@ vm_fault_page( vm_object_offset_t first_offset, /* Offset into object */ vm_prot_t fault_type, /* What access is requested */ boolean_t must_be_resident,/* Must page be resident? */ + boolean_t caller_lookup, /* caller looked up page */ /* Modifies in place: */ vm_prot_t *protection, /* Protection for mapping */ - /* Returns: */ vm_page_t *result_page, /* Page found, if successful */ + /* Returns: */ vm_page_t *top_page, /* Page in top object, if * not result_page. */ int *type_of_fault, /* if non-null, fill in with type of fault @@ -778,13 +887,9 @@ vm_fault_page( /* More arguments: */ kern_return_t *error_code, /* code if page is in error */ boolean_t no_zero_fill, /* don't zero fill absent pages */ -#if MACH_PAGEMAP boolean_t data_supply, /* treat as data_supply if * it is a write fault and a full * page is provided */ -#else - __unused boolean_t data_supply, -#endif vm_object_fault_info_t fault_info) { vm_page_t m; @@ -794,16 +899,21 @@ vm_fault_page( vm_object_t next_object; vm_object_t copy_object; boolean_t look_for_page; + boolean_t force_fault_retry = FALSE; vm_prot_t access_required = fault_type; vm_prot_t wants_copy_flag; CLUSTER_STAT(int pages_at_higher_offsets;) CLUSTER_STAT(int pages_at_lower_offsets;) kern_return_t wait_result; boolean_t interruptible_state; + boolean_t data_already_requested = FALSE; + vm_behavior_t orig_behavior; + vm_size_t orig_cluster_size; vm_fault_return_t error; int my_fault; uint32_t try_failed_count; int interruptible; /* how may fault be interrupted? */ + int external_state = VM_EXTERNAL_STATE_UNKNOWN; memory_object_t pager; vm_fault_return_t retval; @@ -834,14 +944,22 @@ vm_fault_page( * into a copy object in order to avoid a redundant page out operation. */ #if MACH_PAGEMAP -#define MUST_ASK_PAGER(o, f) (vm_external_state_get((o)->existence_map, (f)) \ - != VM_EXTERNAL_STATE_ABSENT) -#define PAGED_OUT(o, f) (vm_external_state_get((o)->existence_map, (f)) \ - == VM_EXTERNAL_STATE_EXISTS) -#else -#define MUST_ASK_PAGER(o, f) (TRUE) -#define PAGED_OUT(o, f) (FALSE) -#endif +#define MUST_ASK_PAGER(o, f, s) \ + ((vm_external_state_get((o)->existence_map, (f)) \ + != VM_EXTERNAL_STATE_ABSENT) && \ + (s = (VM_COMPRESSOR_PAGER_STATE_GET((o), (f)))) \ + != VM_EXTERNAL_STATE_ABSENT) +#define PAGED_OUT(o, f) \ + ((vm_external_state_get((o)->existence_map, (f)) \ + == VM_EXTERNAL_STATE_EXISTS) || \ + (VM_COMPRESSOR_PAGER_STATE_GET((o), (f)) \ + == VM_EXTERNAL_STATE_EXISTS)) +#else /* MACH_PAGEMAP */ +#define MUST_ASK_PAGER(o, f, s) \ + ((s = VM_COMPRESSOR_PAGER_STATE_GET((o), (f))) != VM_EXTERNAL_STATE_ABSENT) +#define PAGED_OUT(o, f) \ + (VM_COMPRESSOR_PAGER_STATE_GET((o), (f)) == VM_EXTERNAL_STATE_EXISTS) +#endif /* MACH_PAGEMAP */ /* * Recovery actions @@ -851,8 +969,12 @@ vm_fault_page( PAGE_WAKEUP_DONE(m); \ if (!m->active && !m->inactive && !m->throttled) { \ vm_page_lockspin_queues(); \ - if (!m->active && !m->inactive && !m->throttled) \ - vm_page_activate(m); \ + if (!m->active && !m->inactive && !m->throttled) { \ + if (COMPRESSED_PAGER_IS_ACTIVE) \ + vm_page_deactivate(m); \ + else \ + vm_page_activate(m); \ + } \ vm_page_unlock_queues(); \ } \ MACRO_END @@ -861,25 +983,6 @@ vm_fault_page( dbgTrace(0xBEEF0002, (unsigned int) first_object, (unsigned int) first_offset); /* (TEST/DEBUG) */ #endif - -#if MACH_KDB - /* - * If there are watchpoints set, then - * we don't want to give away write permission - * on a read fault. Make the task write fault, - * so that the watchpoint code notices the access. - */ - if (db_watchpoint_list) { - /* - * If we aren't asking for write permission, - * then don't give it away. We're using write - * faults to set the dirty bit. - */ - if (!(fault_type & VM_PROT_WRITE)) - *protection &= ~VM_PROT_WRITE; - } -#endif /* MACH_KDB */ - interruptible = fault_info->interruptible; interruptible_state = thread_interrupt_level(interruptible); @@ -945,6 +1048,7 @@ vm_fault_page( * must be a "large page" object. We do not deal * with VM pages for this object. */ + caller_lookup = FALSE; m = VM_PAGE_NULL; goto phys_contig_object; } @@ -956,6 +1060,7 @@ vm_fault_page( * a "activity_in_progress" reference and wait for * access to be unblocked. */ + caller_lookup = FALSE; /* no longer valid after sleep */ vm_object_activity_begin(object); vm_object_paging_end(object); while (object->blocked_access) { @@ -971,7 +1076,19 @@ vm_fault_page( /* * See whether the page at 'offset' is resident */ - m = vm_page_lookup(object, offset); + if (caller_lookup == TRUE) { + /* + * The caller has already looked up the page + * and gave us the result in "result_page". + * We can use this for the first lookup but + * it loses its validity as soon as we unlock + * the object. + */ + m = *result_page; + caller_lookup = FALSE; /* no longer valid after that */ + } else { + m = vm_page_lookup(object, offset); + } #if TRACEFAULTPAGE dbgTrace(0xBEEF0004, (unsigned int) m, (unsigned int) object); /* (TEST/DEBUG) */ #endif @@ -981,20 +1098,16 @@ vm_fault_page( /* * The page is being brought in, * wait for it and then retry. - * - * A possible optimization: if the page - * is known to be resident, we can ignore - * pages that are absent (regardless of - * whether they're busy). */ #if TRACEFAULTPAGE dbgTrace(0xBEEF0005, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */ #endif wait_result = PAGE_SLEEP(object, m, interruptible); + XPR(XPR_VM_FAULT, "vm_f_page: block busy obj 0x%X, offset 0x%X, page 0x%X\n", - object, offset, - m, 0, 0); + object, offset, + m, 0, 0); counter(c_vm_fault_page_block_busy_kernel++); if (wait_result != THREAD_AWAKENED) { @@ -1002,13 +1115,18 @@ vm_fault_page( thread_interrupt_level(interruptible_state); if (wait_result == THREAD_RESTART) - return (VM_FAULT_RETRY); + return (VM_FAULT_RETRY); else return (VM_FAULT_INTERRUPTED); } continue; } + if (m->laundry) { + m->pageout = FALSE; + if (!m->cleaning) + vm_pageout_steal_laundry(m, FALSE); + } if (m->phys_page == vm_page_guard_addr) { /* * Guard page: off limits ! @@ -1026,6 +1144,7 @@ vm_fault_page( *top_page = first_m; if (type_of_fault) *type_of_fault = DBG_GUARD_FAULT; + thread_interrupt_level(interruptible_state); return VM_FAULT_SUCCESS; } else { /* @@ -1101,7 +1220,7 @@ vm_fault_page( * fault cleanup in the case of an error condition * including resetting the thread_interrupt_level */ - error = vm_fault_check(object, m, first_m, interruptible_state); + error = vm_fault_check(object, m, first_m, interruptible_state, (type_of_fault == NULL) ? TRUE : FALSE); if (error != VM_FAULT_SUCCESS) return (error); @@ -1144,6 +1263,8 @@ vm_fault_page( m->absent = FALSE; m->busy = TRUE; } + if (fault_info->mark_zf_absent && no_zero_fill == TRUE) + m->absent = TRUE; /* * zero-fill the page and put it on * the correct paging queue @@ -1163,18 +1284,21 @@ vm_fault_page( m->busy = TRUE; vm_page_lockspin_queues(); - VM_PAGE_QUEUES_REMOVE(m); + + assert(!m->pageout_queue); + vm_page_queues_remove(m); + vm_page_unlock_queues(); } XPR(XPR_VM_FAULT, "vm_f_page: unavail obj 0x%X, off 0x%X, next_obj 0x%X, newoff 0x%X\n", object, offset, next_object, - offset+object->shadow_offset,0); + offset+object->vo_shadow_offset,0); - offset += object->shadow_offset; - fault_info->lo_offset += object->shadow_offset; - fault_info->hi_offset += object->shadow_offset; + offset += object->vo_shadow_offset; + fault_info->lo_offset += object->vo_shadow_offset; + fault_info->hi_offset += object->vo_shadow_offset; access_required = VM_PROT_READ; vm_object_lock(next_object); @@ -1258,7 +1382,8 @@ vm_fault_page( * the page in the speculative queue. */ vm_page_lockspin_queues(); - VM_PAGE_QUEUES_REMOVE(m); + if (m->speculative) + vm_page_queues_remove(m); vm_page_unlock_queues(); } @@ -1326,14 +1451,37 @@ vm_fault_page( * this object can provide the data or we're the top object... * object is locked; m == NULL */ - look_for_page = (object->pager_created && (MUST_ASK_PAGER(object, offset) == TRUE) && !data_supply); + if (must_be_resident) { + if (fault_type == VM_PROT_NONE && + object == kernel_object) { + /* + * We've been called from vm_fault_unwire() + * while removing a map entry that was allocated + * with KMA_KOBJECT and KMA_VAONLY. This page + * is not present and there's nothing more to + * do here (nothing to unwire). + */ + vm_fault_cleanup(object, first_m); + thread_interrupt_level(interruptible_state); + + return VM_FAULT_MEMORY_ERROR; + } + + goto dont_look_for_page; + } + +#if !MACH_PAGEMAP + data_supply = FALSE; +#endif /* !MACH_PAGEMAP */ + + look_for_page = (object->pager_created && (MUST_ASK_PAGER(object, offset, external_state) == TRUE) && !data_supply); #if TRACEFAULTPAGE dbgTrace(0xBEEF000C, (unsigned int) look_for_page, (unsigned int) object); /* (TEST/DEBUG) */ #endif - if ((look_for_page || (object == first_object)) && !must_be_resident && !object->phys_contiguous) { + if (!look_for_page && object == first_object && !object->phys_contiguous) { /* - * Allocate a new page for this object/offset pair + * Allocate a new page for this object/offset pair as a placeholder */ m = vm_page_grab(); #if TRACEFAULTPAGE @@ -1346,10 +1494,16 @@ vm_fault_page( return (VM_FAULT_MEMORY_SHORTAGE); } - vm_page_insert(m, object, offset); + + if (fault_info && fault_info->batch_pmap_op == TRUE) { + vm_page_insert_internal(m, object, offset, VM_KERN_MEMORY_NONE, FALSE, TRUE, TRUE, FALSE, NULL); + } else { + vm_page_insert(m, object, offset); + } } - if (look_for_page && !must_be_resident) { + if (look_for_page) { kern_return_t rc; + int my_fault_type; /* * If the memory manager is not ready, we @@ -1416,8 +1570,8 @@ vm_fault_page( vm_object_lock(object); assert(object->ref_count > 0); - if (object->paging_in_progress > vm_object_pagein_throttle) { - vm_object_assert_wait(object, VM_OBJECT_EVENT_PAGING_IN_PROGRESS, interruptible); + if (object->paging_in_progress >= vm_object_pagein_throttle) { + vm_object_assert_wait(object, VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS, interruptible); vm_object_unlock(object); wait_result = thread_block(THREAD_CONTINUE_NULL); @@ -1432,13 +1586,139 @@ vm_fault_page( return (VM_FAULT_RETRY); } } - if (m != VM_PAGE_NULL) { - /* - * Indicate that the page is waiting for data - * from the memory manager. - */ - m->list_req_pending = TRUE; + if (object->internal && + (COMPRESSED_PAGER_IS_ACTIVE + || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE)) { + int compressed_count_delta; + + if (m == VM_PAGE_NULL) { + /* + * Allocate a new page for this object/offset pair as a placeholder + */ + m = vm_page_grab(); +#if TRACEFAULTPAGE + dbgTrace(0xBEEF000D, (unsigned int) m, (unsigned int) object); /* (TEST/DEBUG) */ +#endif + if (m == VM_PAGE_NULL) { + + vm_fault_cleanup(object, first_m); + thread_interrupt_level(interruptible_state); + + return (VM_FAULT_MEMORY_SHORTAGE); + } + + m->absent = TRUE; + if (fault_info && fault_info->batch_pmap_op == TRUE) { + vm_page_insert_internal(m, object, offset, VM_KERN_MEMORY_NONE, FALSE, TRUE, TRUE, FALSE, NULL); + } else { + vm_page_insert(m, object, offset); + } + } + assert(m->busy); + m->absent = TRUE; + pager = object->pager; + + assert(object->paging_in_progress > 0); + vm_object_unlock(object); + + rc = vm_compressor_pager_get( + pager, + offset + object->paging_offset, + m->phys_page, + &my_fault_type, + 0, + &compressed_count_delta); + + if (type_of_fault == NULL) { + int throttle_delay; + + /* + * we weren't called from vm_fault, so we + * need to apply page creation throttling + * do it before we re-acquire any locks + */ + if (my_fault_type == DBG_COMPRESSOR_FAULT) { + if ((throttle_delay = vm_page_throttled(TRUE))) { + VM_DEBUG_EVENT(vmf_compressordelay, VMF_COMPRESSORDELAY, DBG_FUNC_NONE, throttle_delay, 0, 1, 0); + delay(throttle_delay); + } + } + } + vm_object_lock(object); + assert(object->paging_in_progress > 0); + + vm_compressor_pager_count( + pager, + compressed_count_delta, + FALSE, /* shared_lock */ + object); + + switch (rc) { + case KERN_SUCCESS: + m->absent = FALSE; + m->dirty = TRUE; + if ((m->object->wimg_bits & + VM_WIMG_MASK) != + VM_WIMG_USE_DEFAULT) { + /* + * If the page is not cacheable, + * we can't let its contents + * linger in the data cache + * after the decompression. + */ + pmap_sync_page_attributes_phys( + m->phys_page); + } else { + m->written_by_kernel = TRUE; + } + + /* + * If the object is purgeable, its + * owner's purgeable ledgers have been + * updated in vm_page_insert() but the + * page was also accounted for in a + * "compressed purgeable" ledger, so + * update that now. + */ + if ((object->purgable != + VM_PURGABLE_DENY) && + (object->vo_purgeable_owner != + NULL)) { + /* + * One less compressed + * purgeable page. + */ + vm_purgeable_compressed_update( + object, + -1); + } + + break; + case KERN_MEMORY_FAILURE: + m->unusual = TRUE; + m->error = TRUE; + m->absent = FALSE; + break; + case KERN_MEMORY_ERROR: + assert(m->absent); + break; + default: + panic("vm_fault_page(): unexpected " + "error %d from " + "vm_compressor_pager_get()\n", + rc); + } + PAGE_WAKEUP_DONE(m); + + rc = KERN_SUCCESS; + goto data_requested; + } + my_fault_type = DBG_PAGEIN_FAULT; + + if (m != VM_PAGE_NULL) { + VM_PAGE_FREE(m); + m = VM_PAGE_NULL; } #if TRACEFAULTPAGE @@ -1487,6 +1767,45 @@ vm_fault_page( object, offset, m, access_required | wants_copy_flag, 0); + if (object->copy == first_object) { + /* + * if we issue the memory_object_data_request in + * this state, we are subject to a deadlock with + * the underlying filesystem if it is trying to + * shrink the file resulting in a push of pages + * into the copy object... that push will stall + * on the placeholder page, and if the pushing thread + * is holding a lock that is required on the pagein + * path (such as a truncate lock), we'll deadlock... + * to avoid this potential deadlock, we throw away + * our placeholder page before calling memory_object_data_request + * and force this thread to retry the vm_fault_page after + * we have issued the I/O. the second time through this path + * we will find the page already in the cache (presumably still + * busy waiting for the I/O to complete) and then complete + * the fault w/o having to go through memory_object_data_request again + */ + assert(first_m != VM_PAGE_NULL); + assert(first_m->object == first_object); + + vm_object_lock(first_object); + VM_PAGE_FREE(first_m); + vm_object_paging_end(first_object); + vm_object_unlock(first_object); + + first_m = VM_PAGE_NULL; + force_fault_retry = TRUE; + + vm_fault_page_forced_retry++; + } + + if (data_already_requested == TRUE) { + orig_behavior = fault_info->behavior; + orig_cluster_size = fault_info->cluster_size; + + fault_info->behavior = VM_BEHAVIOR_RANDOM; + fault_info->cluster_size = PAGE_SIZE; + } /* * Call the memory manager to retrieve the data. */ @@ -1497,11 +1816,19 @@ vm_fault_page( access_required | wants_copy_flag, (memory_object_fault_info_t)fault_info); + if (data_already_requested == TRUE) { + fault_info->behavior = orig_behavior; + fault_info->cluster_size = orig_cluster_size; + } else + data_already_requested = TRUE; + + DTRACE_VM2(maj_fault, int, 1, (uint64_t *), NULL); #if TRACEFAULTPAGE dbgTrace(0xBEEF0013, (unsigned int) object, (unsigned int) rc); /* (TEST/DEBUG) */ #endif vm_object_lock(object); + data_requested: if (rc != KERN_SUCCESS) { vm_fault_cleanup(object, first_m); @@ -1513,18 +1840,27 @@ vm_fault_page( } else { clock_sec_t tv_sec; clock_usec_t tv_usec; - - clock_get_system_microtime(&tv_sec, &tv_usec); - current_thread()->t_page_creation_time = tv_sec; - current_thread()->t_page_creation_count = 0; + + if (my_fault_type == DBG_PAGEIN_FAULT) { + clock_get_system_microtime(&tv_sec, &tv_usec); + current_thread()->t_page_creation_time = tv_sec; + current_thread()->t_page_creation_count = 0; + } } - if ((interruptible != THREAD_UNINT) && (current_thread()->sched_mode & TH_MODE_ABORT)) { + if ((interruptible != THREAD_UNINT) && (current_thread()->sched_flags & TH_SFLAG_ABORT)) { vm_fault_cleanup(object, first_m); thread_interrupt_level(interruptible_state); return (VM_FAULT_INTERRUPTED); } + if (force_fault_retry == TRUE) { + + vm_fault_cleanup(object, first_m); + thread_interrupt_level(interruptible_state); + + return (VM_FAULT_RETRY); + } if (m == VM_PAGE_NULL && object->phys_contiguous) { /* * No page here means that the object we @@ -1547,7 +1883,7 @@ vm_fault_page( * if we make it through the state checks * above, than we'll count it as such */ - my_fault = DBG_PAGEIN_FAULT; + my_fault = my_fault_type; /* * Retry with same object/offset, since new data may @@ -1556,7 +1892,7 @@ vm_fault_page( */ continue; } - +dont_look_for_page: /* * We get here if the object has no pager, or an existence map * exists and indicates the page isn't present on the pager @@ -1605,7 +1941,7 @@ vm_fault_page( * fault cleanup in the case of an error condition * including resetting the thread_interrupt_level */ - error = vm_fault_check(object, m, first_m, interruptible_state); + error = vm_fault_check(object, m, first_m, interruptible_state, (type_of_fault == NULL) ? TRUE : FALSE); if (error != VM_FAULT_SUCCESS) return (error); @@ -1621,6 +1957,9 @@ vm_fault_page( } vm_page_insert(m, object, offset); } + if (fault_info->mark_zf_absent && no_zero_fill == TRUE) + m->absent = TRUE; + my_fault = vm_fault_zero_page(m, no_zero_fill); break; @@ -1633,9 +1972,9 @@ vm_fault_page( if ((object != first_object) || must_be_resident) vm_object_paging_end(object); - offset += object->shadow_offset; - fault_info->lo_offset += object->shadow_offset; - fault_info->hi_offset += object->shadow_offset; + offset += object->vo_shadow_offset; + fault_info->lo_offset += object->vo_shadow_offset; + fault_info->hi_offset += object->vo_shadow_offset; access_required = VM_PROT_READ; vm_object_lock(next_object); @@ -1776,12 +2115,17 @@ vm_fault_page( if (m->pmapped) pmap_disconnect(m->phys_page); + if (m->clustered) { + VM_PAGE_COUNT_AS_PAGEIN(m); + VM_PAGE_CONSUME_CLUSTERED(m); + } assert(!m->cleaning); /* * We no longer need the old page or object. */ - PAGE_WAKEUP_DONE(m); + RELEASE_PAGE(m); + vm_object_paging_end(object); vm_object_unlock(object); @@ -1807,7 +2151,7 @@ vm_fault_page( */ assert(copy_m->busy); vm_page_insert(copy_m, object, offset); - copy_m->dirty = TRUE; + SET_PAGE_DIRTY(copy_m, TRUE); m = copy_m; /* @@ -1880,9 +2224,9 @@ vm_fault_page( /* * Does the page exist in the copy? */ - copy_offset = first_offset - copy_object->shadow_offset; + copy_offset = first_offset - copy_object->vo_shadow_offset; - if (copy_object->size <= copy_offset) + if (copy_object->vo_size <= copy_offset) /* * Copy object doesn't cover this page -- do nothing. */ @@ -2003,15 +2347,20 @@ vm_fault_page( if (m->pmapped) pmap_disconnect(m->phys_page); + if (m->clustered) { + VM_PAGE_COUNT_AS_PAGEIN(m); + VM_PAGE_CONSUME_CLUSTERED(m); + } /* * If there's a pager, then immediately * page out this page, using the "initialize" * option. Else, we use the copy. */ - if ((!copy_object->pager_created) + if ((!copy_object->pager_ready) #if MACH_PAGEMAP || vm_external_state_get(copy_object->existence_map, copy_offset) == VM_EXTERNAL_STATE_ABSENT #endif + || VM_COMPRESSOR_PAGER_STATE_GET(copy_object, copy_offset) == VM_EXTERNAL_STATE_ABSENT ) { vm_page_lockspin_queues(); @@ -2019,17 +2368,81 @@ vm_fault_page( vm_page_activate(copy_m); vm_page_unlock_queues(); - copy_m->dirty = TRUE; + SET_PAGE_DIRTY(copy_m, TRUE); PAGE_WAKEUP_DONE(copy_m); - } - else { + + } else if (copy_object->internal && + (DEFAULT_PAGER_IS_ACTIVE || DEFAULT_FREEZER_IS_ACTIVE)) { + /* + * For internal objects check with the pager to see + * if the page already exists in the backing store. + * If yes, then we can drop the copy page. If not, + * then we'll activate it, mark it dirty and keep it + * around. + */ + + kern_return_t kr = KERN_SUCCESS; + + memory_object_t copy_pager = copy_object->pager; + assert(copy_pager != MEMORY_OBJECT_NULL); + vm_object_paging_begin(copy_object); + + vm_object_unlock(copy_object); + + kr = memory_object_data_request( + copy_pager, + copy_offset + copy_object->paging_offset, + 0, /* Only query the pager. */ + VM_PROT_READ, + NULL); + + vm_object_lock(copy_object); + + vm_object_paging_end(copy_object); + + /* + * Since we dropped the copy_object's lock, + * check whether we'll have to deallocate + * the hard way. + */ + if ((copy_object->shadow != object) || (copy_object->ref_count == 1)) { + vm_object_unlock(copy_object); + vm_object_deallocate(copy_object); + vm_object_lock(object); + + continue; + } + if (kr == KERN_SUCCESS) { + /* + * The pager has the page. We don't want to overwrite + * that page by sending this one out to the backing store. + * So we drop the copy page. + */ + VM_PAGE_FREE(copy_m); + + } else { + /* + * The pager doesn't have the page. We'll keep this one + * around in the copy object. It might get sent out to + * the backing store under memory pressure. + */ + vm_page_lockspin_queues(); + assert(!m->cleaning); + vm_page_activate(copy_m); + vm_page_unlock_queues(); + + SET_PAGE_DIRTY(copy_m, TRUE); + PAGE_WAKEUP_DONE(copy_m); + } + } else { + assert(copy_m->busy == TRUE); assert(!m->cleaning); /* * dirty is protected by the object lock */ - copy_m->dirty = TRUE; + SET_PAGE_DIRTY(copy_m, TRUE); /* * The page is already ready for pageout: @@ -2067,6 +2480,7 @@ vm_fault_page( */ vm_object_lock(object); } + /* * Because we're pushing a page upward * in the object tree, we must restart @@ -2108,20 +2522,15 @@ done: if (m != VM_PAGE_NULL) { retval = VM_FAULT_SUCCESS; + if (my_fault == DBG_PAGEIN_FAULT) { - VM_STAT_INCR(pageins); - DTRACE_VM2(pgin, int, 1, (uint64_t *), NULL); - DTRACE_VM2(maj_fault, int, 1, (uint64_t *), NULL); - current_task()->pageins++; + VM_PAGE_COUNT_AS_PAGEIN(m); - if (m->object->internal) { - DTRACE_VM2(anonpgin, int, 1, (uint64_t *), NULL); + if (m->object->internal) my_fault = DBG_PAGEIND_FAULT; - } else { - DTRACE_VM2(fspgin, int, 1, (uint64_t *), NULL); + else my_fault = DBG_PAGEINV_FAULT; - } /* * evaluate access pattern and update state @@ -2131,6 +2540,9 @@ done: vm_fault_is_sequential(object, offset, fault_info->behavior); vm_fault_deactivate_behind(object, offset, fault_info->behavior); + } else if (my_fault == DBG_COMPRESSOR_FAULT || my_fault == DBG_COMPRESSOR_SWAPIN_FAULT) { + + VM_STAT_INCR(decompressions); } if (type_of_fault) *type_of_fault = my_fault; @@ -2183,6 +2595,8 @@ backoff: * careful not to modify the VM object in any way that is not * legal under a shared lock... */ +extern int proc_selfpid(void); +extern char *proc_name_address(void *p); unsigned long cs_enter_tainted_rejected = 0; unsigned long cs_enter_tainted_accepted = 0; kern_return_t @@ -2190,17 +2604,25 @@ vm_fault_enter(vm_page_t m, pmap_t pmap, vm_map_offset_t vaddr, vm_prot_t prot, + vm_prot_t caller_prot, boolean_t wired, boolean_t change_wiring, boolean_t no_cache, + boolean_t cs_bypass, + __unused int user_tag, + int pmap_options, + boolean_t *need_retry, int *type_of_fault) { - unsigned int cache_attr; kern_return_t kr, pe_result; boolean_t previously_pmapped = m->pmapped; boolean_t must_disconnect = 0; boolean_t map_is_switched, map_is_switch_protected; + int cs_enforcement_enabled; + vm_prot_t fault_type; + fault_type = change_wiring ? VM_PROT_NONE : caller_prot; + vm_object_lock_assert_held(m->object); #if DEBUG lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED); @@ -2211,44 +2633,44 @@ vm_fault_enter(vm_page_t m, return KERN_SUCCESS; } - cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK; + if (*type_of_fault == DBG_ZERO_FILL_FAULT) { - if (m->pmapped == FALSE) { + vm_object_lock_assert_exclusive(m->object); + + } else if ((fault_type & VM_PROT_WRITE) == 0) { /* - * This is the first time this page is being - * mapped in an address space (pmapped == FALSE). - * - * Part of that page may still be in the data cache - * and not flushed to memory. In case we end up - * accessing that page via the instruction cache, - * we need to ensure that the 2 caches are in sync. + * This is not a "write" fault, so we + * might not have taken the object lock + * exclusively and we might not be able + * to update the "wpmapped" bit in + * vm_fault_enter(). + * Let's just grant read access to + * the page for now and we'll + * soft-fault again if we need write + * access later... */ - pmap_sync_page_data_phys(m->phys_page); - - if ((*type_of_fault == DBG_CACHE_HIT_FAULT) && m->clustered) { - /* - * found it in the cache, but this - * is the first fault-in of the page (m->pmapped == FALSE) - * so it must have come in as part of - * a cluster... account 1 pagein against it - */ - VM_STAT_INCR(pageins); - DTRACE_VM2(pgin, int, 1, (uint64_t *), NULL); + prot &= ~VM_PROT_WRITE; + } + if (m->pmapped == FALSE) { - if (m->object->internal) { - DTRACE_VM2(anonpgin, int, 1, (uint64_t *), NULL); - *type_of_fault = DBG_PAGEIND_FAULT; - } else { - DTRACE_VM2(fspgin, int, 1, (uint64_t *), NULL); - *type_of_fault = DBG_PAGEINV_FAULT; + if (m->clustered) { + if (*type_of_fault == DBG_CACHE_HIT_FAULT) { + /* + * found it in the cache, but this + * is the first fault-in of the page (m->pmapped == FALSE) + * so it must have come in as part of + * a cluster... account 1 pagein against it + */ + if (m->object->internal) + *type_of_fault = DBG_PAGEIND_FAULT; + else + *type_of_fault = DBG_PAGEINV_FAULT; + + VM_PAGE_COUNT_AS_PAGEIN(m); } - - current_task()->pageins++; + VM_PAGE_CONSUME_CLUSTERED(m); } - VM_PAGE_CONSUME_CLUSTERED(m); - - } else if (cache_attr != VM_WIMG_DEFAULT) - pmap_sync_page_attributes_phys(m->phys_page); + } if (*type_of_fault != DBG_COW_FAULT) { DTRACE_VM2(as_fault, int, 1, (uint64_t *), NULL); @@ -2272,6 +2694,7 @@ vm_fault_enter(vm_page_t m, } #define page_immutable(m,prot) ((m)->cs_validated /*&& ((prot) & VM_PROT_EXECUTE)*/) +#define page_nx(m) ((m)->cs_nx) map_is_switched = ((pmap != vm_map_pmap(current_task()->map)) && (pmap == vm_map_pmap(current_thread()->map))); @@ -2287,13 +2710,21 @@ vm_fault_enter(vm_page_t m, * from the current map. We do that below right before we do the * PMAP_ENTER. */ - if(!cs_enforcement_disable && map_is_switched && + cs_enforcement_enabled = cs_enforcement(NULL); + + if(cs_enforcement_enabled && map_is_switched && map_is_switch_protected && page_immutable(m, prot) && (prot & VM_PROT_WRITE)) { return KERN_CODESIGN_ERROR; } + if (cs_enforcement_enabled && page_nx(m) && (prot & VM_PROT_EXECUTE)) { + if (cs_debug) + printf("page marked to be NX, not letting it be mapped EXEC\n"); + return KERN_CODESIGN_ERROR; + } + /* A page could be tainted, or pose a risk of being tainted later. * Check whether the receiving process wants it, and make it feel * the consequences (that hapens in cs_invalid_page()). @@ -2304,11 +2735,12 @@ vm_fault_enter(vm_page_t m, * can be changed without the kernel noticing, therefore unsigned * code can be created */ - if (m->cs_tainted || - ( !cs_enforcement_disable && - (/* The page is unsigned and wants to be executable */ - (!m->cs_validated && (prot & VM_PROT_EXECUTE)) || - /* The page should be immutable, but is in danger of being modified + if (!cs_bypass && + (m->cs_tainted || + (cs_enforcement_enabled && + (/* The page is unsigned and wants to be executable */ + (!m->cs_validated && (prot & VM_PROT_EXECUTE)) || + /* The page should be immutable, but is in danger of being modified * This is the case where we want policy from the code directory - * is the page immutable or not? For now we have to assume that * code pages will be immutable, data pages not. @@ -2323,7 +2755,7 @@ vm_fault_enter(vm_page_t m, */ (page_immutable(m, prot) && ((prot & VM_PROT_WRITE) || m->wpmapped)) )) - ) + )) { /* We will have a tainted page. Have to handle the special case * of a switched map now. If the map is not switched, standard @@ -2338,27 +2770,167 @@ vm_fault_enter(vm_page_t m, assert(!(prot & VM_PROT_WRITE) || (map_is_switch_protected == FALSE)); reject_page = FALSE; } else { + if (cs_debug > 5) + printf("vm_fault: signed: %s validate: %s tainted: %s wpmapped: %s slid: %s prot: 0x%x\n", + m->object->code_signed ? "yes" : "no", + m->cs_validated ? "yes" : "no", + m->cs_tainted ? "yes" : "no", + m->wpmapped ? "yes" : "no", + m->slid ? "yes" : "no", + (int)prot); reject_page = cs_invalid_page((addr64_t) vaddr); } if (reject_page) { - /* reject the tainted page: abort the page fault */ + /* reject the invalid page: abort the page fault */ + int pid; + const char *procname; + task_t task; + vm_object_t file_object, shadow; + vm_object_offset_t file_offset; + char *pathname, *filename; + vm_size_t pathname_len, filename_len; + boolean_t truncated_path; +#define __PATH_MAX 1024 + struct timespec mtime, cs_mtime; + kr = KERN_CODESIGN_ERROR; cs_enter_tainted_rejected++; - } else { - /* proceed with the tainted page */ - kr = KERN_SUCCESS; - /* Page might have been tainted before or not; now it - * definitively is. If the page wasn't tainted, we must - * disconnect it from all pmaps later. */ - must_disconnect = !m->cs_tainted; - m->cs_tainted = TRUE; + + /* get process name and pid */ + procname = "?"; + task = current_task(); + pid = proc_selfpid(); + if (task->bsd_info != NULL) + procname = proc_name_address(task->bsd_info); + + /* get file's VM object */ + file_object = m->object; + file_offset = m->offset; + for (shadow = file_object->shadow; + shadow != VM_OBJECT_NULL; + shadow = file_object->shadow) { + vm_object_lock_shared(shadow); + if (file_object != m->object) { + vm_object_unlock(file_object); + } + file_offset += file_object->vo_shadow_offset; + file_object = shadow; + } + + mtime.tv_sec = 0; + mtime.tv_nsec = 0; + cs_mtime.tv_sec = 0; + cs_mtime.tv_nsec = 0; + + /* get file's pathname and/or filename */ + pathname = NULL; + filename = NULL; + pathname_len = 0; + filename_len = 0; + truncated_path = FALSE; + /* no pager -> no file -> no pathname, use "" in that case */ + if (file_object->pager != NULL) { + pathname = (char *)kalloc(__PATH_MAX * 2); + if (pathname) { + pathname[0] = '\0'; + pathname_len = __PATH_MAX; + filename = pathname + pathname_len; + filename_len = __PATH_MAX; + } + vnode_pager_get_object_name(file_object->pager, + pathname, + pathname_len, + filename, + filename_len, + &truncated_path); + if (pathname) { + /* safety first... */ + pathname[__PATH_MAX-1] = '\0'; + filename[__PATH_MAX-1] = '\0'; + } + vnode_pager_get_object_mtime(file_object->pager, + &mtime, + &cs_mtime); + } + printf("CODE SIGNING: process %d[%s]: " + "rejecting invalid page at address 0x%llx " + "from offset 0x%llx in file \"%s%s%s\" " + "(cs_mtime:%lu.%ld %s mtime:%lu.%ld) " + "(signed:%d validated:%d tainted:%d " + "wpmapped:%d slid:%d)\n", + pid, procname, (addr64_t) vaddr, + file_offset, + (pathname ? pathname : ""), + (truncated_path ? "/.../" : ""), + (truncated_path ? filename : ""), + cs_mtime.tv_sec, cs_mtime.tv_nsec, + ((cs_mtime.tv_sec == mtime.tv_sec && + cs_mtime.tv_nsec == mtime.tv_nsec) + ? "==" + : "!="), + mtime.tv_sec, mtime.tv_nsec, + m->object->code_signed, + m->cs_validated, + m->cs_tainted, + m->wpmapped, + m->slid); + if (file_object != m->object) { + vm_object_unlock(file_object); + } + if (pathname_len != 0) { + kfree(pathname, __PATH_MAX * 2); + pathname = NULL; + filename = NULL; + } + } else { + /* proceed with the invalid page */ + kr = KERN_SUCCESS; + if (!m->cs_validated) { + /* + * This page has not been validated, so it + * must not belong to a code-signed object + * and should not be forcefully considered + * as tainted. + * We're just concerned about it here because + * we've been asked to "execute" it but that + * does not mean that it should cause other + * accesses to fail. + * This happens when a debugger sets a + * breakpoint and we then execute code in + * that page. Marking the page as "tainted" + * would cause any inspection tool ("leaks", + * "vmmap", "CrashReporter", ...) to get killed + * due to code-signing violation on that page, + * even though they're just reading it and not + * executing from it. + */ + assert(!m->object->code_signed); + } else { + /* + * Page might have been tainted before or not; + * now it definitively is. If the page wasn't + * tainted, we must disconnect it from all + * pmaps later, to force existing mappings + * through that code path for re-consideration + * of the validity of that page. + */ + must_disconnect = !m->cs_tainted; + m->cs_tainted = TRUE; + } cs_enter_tainted_accepted++; } - if (cs_debug || kr != KERN_SUCCESS) { - printf("CODESIGNING: vm_fault_enter(0x%llx): " - "page %p obj %p off 0x%llx *** INVALID PAGE ***\n", - (long long)vaddr, m, m->object, m->offset); + if (kr != KERN_SUCCESS) { + if (cs_debug) { + printf("CODESIGNING: vm_fault_enter(0x%llx): " + "*** INVALID PAGE ***\n", + (long long)vaddr); + } +#if !SECURE_KERNEL + if (cs_enforcement_panic) { + panic("CODESIGNING: panicking on invalid page\n"); + } +#endif } } else { @@ -2366,6 +2938,175 @@ vm_fault_enter(vm_page_t m, kr = KERN_SUCCESS; } + boolean_t page_queues_locked = FALSE; +#define __VM_PAGE_LOCKSPIN_QUEUES_IF_NEEDED() \ +MACRO_BEGIN \ + if (! page_queues_locked) { \ + page_queues_locked = TRUE; \ + vm_page_lockspin_queues(); \ + } \ +MACRO_END +#define __VM_PAGE_UNLOCK_QUEUES_IF_NEEDED() \ +MACRO_BEGIN \ + if (page_queues_locked) { \ + page_queues_locked = FALSE; \ + vm_page_unlock_queues(); \ + } \ +MACRO_END + + /* + * Hold queues lock to manipulate + * the page queues. Change wiring + * case is obvious. + */ + assert(m->compressor || m->object != compressor_object); + if (m->compressor) { + /* + * Compressor pages are neither wired + * nor pageable and should never change. + */ + assert(m->object == compressor_object); + } else if (change_wiring) { + __VM_PAGE_LOCKSPIN_QUEUES_IF_NEEDED(); + + if (wired) { + if (kr == KERN_SUCCESS) { + vm_page_wire(m, VM_PROT_MEMORY_TAG(caller_prot), TRUE); + } + } else { + vm_page_unwire(m, TRUE); + } + /* we keep the page queues lock, if we need it later */ + + } else { + if (kr != KERN_SUCCESS) { + __VM_PAGE_LOCKSPIN_QUEUES_IF_NEEDED(); + vm_page_deactivate(m); + /* we keep the page queues lock, if we need it later */ + } else if (((!m->active && !m->inactive) || + m->clean_queue || + no_cache) && + !VM_PAGE_WIRED(m) && !m->throttled) { + + if (vm_page_local_q && + !no_cache && + (*type_of_fault == DBG_COW_FAULT || + *type_of_fault == DBG_ZERO_FILL_FAULT) ) { + struct vpl *lq; + uint32_t lid; + + __VM_PAGE_UNLOCK_QUEUES_IF_NEEDED(); + vm_object_lock_assert_exclusive(m->object); + + /* + * we got a local queue to stuff this + * new page on... + * its safe to manipulate local and + * local_id at this point since we're + * behind an exclusive object lock and + * the page is not on any global queue. + * + * we'll use the current cpu number to + * select the queue note that we don't + * need to disable preemption... we're + * going to behind the local queue's + * lock to do the real work + */ + lid = cpu_number(); + + lq = &vm_page_local_q[lid].vpl_un.vpl; + + VPL_LOCK(&lq->vpl_lock); + + vm_page_check_pageable_safe(m); + queue_enter(&lq->vpl_queue, m, + vm_page_t, pageq); + m->local = TRUE; + m->local_id = lid; + lq->vpl_count++; + + if (m->object->internal) + lq->vpl_internal_count++; + else + lq->vpl_external_count++; + + VPL_UNLOCK(&lq->vpl_lock); + + if (lq->vpl_count > vm_page_local_q_soft_limit) + { + /* + * we're beyond the soft limit + * for the local queue + * vm_page_reactivate_local will + * 'try' to take the global page + * queue lock... if it can't + * that's ok... we'll let the + * queue continue to grow up + * to the hard limit... at that + * point we'll wait for the + * lock... once we've got the + * lock, we'll transfer all of + * the pages from the local + * queue to the global active + * queue + */ + vm_page_reactivate_local(lid, FALSE, FALSE); + } + } else { + + __VM_PAGE_LOCKSPIN_QUEUES_IF_NEEDED(); + + /* + * test again now that we hold the + * page queue lock + */ + if (!VM_PAGE_WIRED(m)) { + if (m->clean_queue) { + vm_page_queues_remove(m); + + vm_pageout_cleaned_reactivated++; + vm_pageout_cleaned_fault_reactivated++; + } + + if ((!m->active && + !m->inactive) || + no_cache) { + /* + * If this is a no_cache mapping + * and the page has never been + * mapped before or was + * previously a no_cache page, + * then we want to leave pages + * in the speculative state so + * that they can be readily + * recycled if free memory runs + * low. Otherwise the page is + * activated as normal. + */ + + if (no_cache && + (!previously_pmapped || + m->no_cache)) { + m->no_cache = TRUE; + + if (!m->speculative) + vm_page_speculate(m, FALSE); + + } else if (!m->active && + !m->inactive) { + + vm_page_activate(m); + } + } + } + /* we keep the page queues lock, if we need it later */ + } + } + } + /* we're done with the page queues lock, if we ever took it */ + __VM_PAGE_UNLOCK_QUEUES_IF_NEEDED(); + + /* If we have a KERN_SUCCESS from the previous checks, we either have * a good page, or a tainted page that has been accepted by the process. * In both cases the page will be entered into the pmap. @@ -2373,51 +3114,151 @@ vm_fault_enter(vm_page_t m, * now so those processes can take note. */ if (kr == KERN_SUCCESS) { + /* * NOTE: we may only hold the vm_object lock SHARED - * at this point, but the update of pmapped is ok - * since this is the ONLY bit updated behind the SHARED - * lock... however, we need to figure out how to do an atomic - * update on a bit field to make this less fragile... right - * now I don't know how to coerce 'C' to give me the offset info - * that's needed for an AtomicCompareAndSwap + * at this point, so we need the phys_page lock to + * properly serialize updating the pmapped and + * xpmapped bits */ - m->pmapped = TRUE; - if (prot & VM_PROT_WRITE) { + if ((prot & VM_PROT_EXECUTE) && !m->xpmapped) { + + pmap_lock_phys_page(m->phys_page); + /* + * go ahead and take the opportunity + * to set 'pmapped' here so that we don't + * need to grab this lock a 2nd time + * just below + */ + m->pmapped = TRUE; + + if (!m->xpmapped) { + + m->xpmapped = TRUE; + + pmap_unlock_phys_page(m->phys_page); + + if (!m->object->internal) + OSAddAtomic(1, &vm_page_xpmapped_external_count); + + if ((COMPRESSED_PAGER_IS_ACTIVE) && + m->object->internal && + m->object->pager != NULL) { + /* + * This page could have been + * uncompressed by the + * compressor pager and its + * contents might be only in + * the data cache. + * Since it's being mapped for + * "execute" for the fist time, + * make sure the icache is in + * sync. + */ + pmap_sync_page_data_phys(m->phys_page); + } + } else + pmap_unlock_phys_page(m->phys_page); + } else { + if (m->pmapped == FALSE) { + pmap_lock_phys_page(m->phys_page); + m->pmapped = TRUE; + pmap_unlock_phys_page(m->phys_page); + } + } + if (vm_page_is_slideable(m)) { + boolean_t was_busy = m->busy; + vm_object_lock_assert_exclusive(m->object); - m->wpmapped = TRUE; - if(must_disconnect) { - /* We can only get here - * because of the CSE logic */ - assert(cs_enforcement_disable == FALSE); + + m->busy = TRUE; + kr = vm_page_slide(m, 0); + assert(m->busy); + if(!was_busy) { + PAGE_WAKEUP_DONE(m); + } + if (kr != KERN_SUCCESS) { + /* + * This page has not been slid correctly, + * do not do the pmap_enter() ! + * Let vm_fault_enter() return the error + * so the caller can fail the fault. + */ + goto after_the_pmap_enter; + } + } + + if (fault_type & VM_PROT_WRITE) { + + if (m->wpmapped == FALSE) { + vm_object_lock_assert_exclusive(m->object); + + m->wpmapped = TRUE; + } + if (must_disconnect) { + /* + * We can only get here + * because of the CSE logic + */ + assert(cs_enforcement_enabled); pmap_disconnect(m->phys_page); - /* If we are faulting for a write, we can clear + /* + * If we are faulting for a write, we can clear * the execute bit - that will ensure the page is * checked again before being executable, which * protects against a map switch. * This only happens the first time the page * gets tainted, so we won't get stuck here - * to make an already writeable page executable. */ - prot &= ~VM_PROT_EXECUTE; + * to make an already writeable page executable. + */ + if (!cs_bypass){ + prot &= ~VM_PROT_EXECUTE; + } } } /* Prevent a deadlock by not * holding the object lock if we need to wait for a page in * pmap_enter() - */ - PMAP_ENTER_OPTIONS(pmap, vaddr, m, prot, cache_attr, - wired, PMAP_OPTIONS_NOWAIT, pe_result); + PMAP_ENTER_OPTIONS(pmap, vaddr, m, prot, fault_type, 0, + wired, + pmap_options | PMAP_OPTIONS_NOWAIT, + pe_result); if(pe_result == KERN_RESOURCE_SHORTAGE) { + + if (need_retry) { + /* + * this will be non-null in the case where we hold the lock + * on the top-object in this chain... we can't just drop + * the lock on the object we're inserting the page into + * and recall the PMAP_ENTER since we can still cause + * a deadlock if one of the critical paths tries to + * acquire the lock on the top-object and we're blocked + * in PMAP_ENTER waiting for memory... our only recourse + * is to deal with it at a higher level where we can + * drop both locks. + */ + *need_retry = TRUE; + vm_pmap_enter_retried++; + goto after_the_pmap_enter; + } /* The nonblocking version of pmap_enter did not succeed. - * Use the blocking version instead. Requires marking + * and we don't need to drop other locks and retry + * at the level above us, so + * use the blocking version instead. Requires marking * the page busy and unlocking the object */ boolean_t was_busy = m->busy; + + vm_object_lock_assert_exclusive(m->object); + m->busy = TRUE; vm_object_unlock(m->object); - PMAP_ENTER(pmap, vaddr, m, prot, cache_attr, wired); - + PMAP_ENTER_OPTIONS(pmap, vaddr, m, prot, fault_type, + 0, wired, + pmap_options, pe_result); + /* Take the object lock again. */ vm_object_lock(m->object); @@ -2431,108 +3272,23 @@ vm_fault_enter(vm_page_t m, } } - /* - * Hold queues lock to manipulate - * the page queues. Change wiring - * case is obvious. - */ - if (change_wiring) { - vm_page_lockspin_queues(); - - if (wired) { - if (kr == KERN_SUCCESS) { - vm_page_wire(m); - } - } else { - vm_page_unwire(m); - } - vm_page_unlock_queues(); - - } else { - if (kr != KERN_SUCCESS) { - vm_page_lockspin_queues(); - vm_page_deactivate(m); - vm_page_unlock_queues(); - } else { - if (((!m->active && !m->inactive) || no_cache) && !VM_PAGE_WIRED(m) && !m->throttled) { - - if ( vm_page_local_q && !no_cache && (*type_of_fault == DBG_COW_FAULT || *type_of_fault == DBG_ZERO_FILL_FAULT) ) { - struct vpl *lq; - uint32_t lid; - - /* - * we got a local queue to stuff this new page on... - * its safe to manipulate local and local_id at this point - * since we're behind an exclusive object lock and the - * page is not on any global queue. - * - * we'll use the current cpu number to select the queue - * note that we don't need to disable preemption... we're - * going to behind the local queue's lock to do the real - * work - */ - lid = cpu_number(); - - lq = &vm_page_local_q[lid].vpl_un.vpl; - - VPL_LOCK(&lq->vpl_lock); - - queue_enter(&lq->vpl_queue, m, vm_page_t, pageq); - m->local = TRUE; - m->local_id = lid; - lq->vpl_count++; - - VPL_UNLOCK(&lq->vpl_lock); - - if (lq->vpl_count > vm_page_local_q_soft_limit) { - /* - * we're beyond the soft limit for the local queue - * vm_page_reactivate_local will 'try' to take - * the global page queue lock... if it can't that's - * ok... we'll let the queue continue to grow up - * to the hard limit... at that point we'll wait - * for the lock... once we've got the lock, we'll - * transfer all of the pages from the local queue - * to the global active queue - */ - vm_page_reactivate_local(lid, FALSE, FALSE); - } - return kr; - } - - vm_page_lockspin_queues(); - /* - * test again now that we hold the page queue lock - */ - if (((!m->active && !m->inactive) || no_cache) && !VM_PAGE_WIRED(m)) { - - /* - * If this is a no_cache mapping and the page has never been - * mapped before or was previously a no_cache page, then we - * want to leave pages in the speculative state so that they - * can be readily recycled if free memory runs low. Otherwise - * the page is activated as normal. - */ - - if (no_cache && (!previously_pmapped || m->no_cache)) { - m->no_cache = TRUE; - - if (m->active || m->inactive) - VM_PAGE_QUEUES_REMOVE(m); - - if (!m->speculative) - vm_page_speculate(m, TRUE); - - } else if (!m->active && !m->inactive) - vm_page_activate(m); - - } +after_the_pmap_enter: + return kr; +} - vm_page_unlock_queues(); - } - } +void +vm_pre_fault(vm_map_offset_t vaddr) +{ + if (pmap_find_phys(current_map()->pmap, vaddr) == 0) { + + vm_fault(current_map(), /* map */ + vaddr, /* vaddr */ + VM_PROT_READ, /* fault_type */ + FALSE, /* change_wiring */ + THREAD_UNINT, /* interruptible */ + NULL, /* caller_pmap */ + 0 /* caller_pmap_addr */); } - return kr; } @@ -2555,6 +3311,7 @@ extern int _map_enter_debug; unsigned long vm_fault_collapse_total = 0; unsigned long vm_fault_collapse_skipped = 0; + kern_return_t vm_fault( vm_map_t map, @@ -2564,6 +3321,23 @@ vm_fault( int interruptible, pmap_t caller_pmap, vm_map_offset_t caller_pmap_addr) +{ + return vm_fault_internal(map, vaddr, fault_type, change_wiring, + interruptible, caller_pmap, caller_pmap_addr, + NULL); +} + + +kern_return_t +vm_fault_internal( + vm_map_t map, + vm_map_offset_t vaddr, + vm_prot_t caller_prot, + boolean_t change_wiring, + int interruptible, + pmap_t caller_pmap, + vm_map_offset_t caller_pmap_addr, + ppnum_t *physpage_p) { vm_map_version_t version; /* Map version for verificiation */ boolean_t wired; /* Should mapping be wired down? */ @@ -2586,25 +3360,32 @@ vm_fault( boolean_t interruptible_state; vm_map_t real_map = map; vm_map_t original_map = map; + vm_prot_t fault_type; vm_prot_t original_fault_type; struct vm_object_fault_info fault_info; boolean_t need_collapse = FALSE; + boolean_t need_retry = FALSE; + boolean_t *need_retry_ptr = NULL; int object_lock_type = 0; int cur_object_lock_type; vm_object_t top_object = VM_OBJECT_NULL; + int throttle_delay; + int compressed_count_delta; - KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_START, - (int)((uint64_t)vaddr >> 32), - (int)vaddr, - 0, + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + (MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_START, + ((uint64_t)vaddr >> 32), + vaddr, + (map == kernel_map), 0, 0); if (get_preemption_level() != 0) { - KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_END, - (int)((uint64_t)vaddr >> 32), - (int)vaddr, + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + (MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_END, + ((uint64_t)vaddr >> 32), + vaddr, KERN_FAILURE, 0, 0); @@ -2614,6 +3395,8 @@ vm_fault( interruptible_state = thread_interrupt_level(interruptible); + fault_type = (change_wiring ? VM_PROT_NONE : caller_prot); + VM_STAT_INCR(faults); current_task()->faults++; original_fault_type = fault_type; @@ -2647,6 +3430,7 @@ RetryFault: &fault_info, &real_map); + if (kr != KERN_SUCCESS) { vm_map_unlock_read(map); goto done; @@ -2654,6 +3438,9 @@ RetryFault: pmap = real_map->pmap; fault_info.interruptible = interruptible; fault_info.stealth = FALSE; + fault_info.io_sync = FALSE; + fault_info.mark_zf_absent = FALSE; + fault_info.batch_pmap_op = FALSE; /* * If the page is wired, we must fault for the current protection @@ -2754,7 +3541,6 @@ RetryFault: * have object that 'm' belongs to locked exclusively */ if (object != cur_object) { - vm_object_unlock(object); if (cur_object_lock_type == OBJECT_LOCK_SHARED) { @@ -2763,11 +3549,13 @@ RetryFault: if (vm_object_lock_upgrade(cur_object) == FALSE) { /* * couldn't upgrade so go do a full retry - * immediately since we've already dropped - * the top object lock associated with this page - * and the current one got dropped due to the - * failed upgrade... the state is no longer valid + * immediately since we can no longer be + * certain about cur_object (since we + * don't hold a reference on it)... + * first drop the top object lock */ + vm_object_unlock(object); + vm_map_unlock_read(map); if (real_map != map) vm_map_unlock(real_map); @@ -2794,6 +3582,30 @@ RetryFault: continue; } } + if (m->pageout_queue && m->object->internal && COMPRESSED_PAGER_IS_ACTIVE) { + /* + * m->busy == TRUE and the object is locked exclusively + * if m->pageout_queue == TRUE after we acquire the + * queues lock, we are guaranteed that it is stable on + * the pageout queue and therefore reclaimable + * + * NOTE: this is only true for the internal pageout queue + * in the compressor world + */ + vm_page_lock_queues(); + + if (m->pageout_queue) { + vm_pageout_throttle_up(m); + vm_page_unlock_queues(); + + PAGE_WAKEUP_DONE(m); + goto reclaimed_from_pageout; + } + vm_page_unlock_queues(); + } + if (object != cur_object) + vm_object_unlock(object); + vm_map_unlock_read(map); if (real_map != map) vm_map_unlock(real_map); @@ -2813,6 +3625,46 @@ RetryFault: kr = KERN_ABORTED; goto done; } +reclaimed_from_pageout: + if (m->laundry) { + if (object != cur_object) { + if (cur_object_lock_type == OBJECT_LOCK_SHARED) { + cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE; + + vm_object_unlock(object); + vm_object_unlock(cur_object); + + vm_map_unlock_read(map); + if (real_map != map) + vm_map_unlock(real_map); + + goto RetryFault; + } + + } else if (object_lock_type == OBJECT_LOCK_SHARED) { + + object_lock_type = OBJECT_LOCK_EXCLUSIVE; + + if (vm_object_lock_upgrade(object) == FALSE) { + /* + * couldn't upgrade, so explictly take the lock + * exclusively and go relookup the page since we + * will have dropped the object lock and + * a different thread could have inserted + * a page at this offset + * no need for a full retry since we're + * at the top level of the object chain + */ + vm_object_lock(object); + + continue; + } + } + m->pageout = FALSE; + + vm_pageout_steal_laundry(m, FALSE); + } + if (m->phys_page == vm_page_guard_addr) { /* * Guard page: let the slow path deal with it @@ -2910,7 +3762,36 @@ RetryFault: } ASSERT_PAGE_DECRYPTED(m); - if (VM_FAULT_NEED_CS_VALIDATION(map->pmap, m)) { + if(vm_page_is_slideable(m)) { + /* + * We might need to slide this page, and so, + * we want to hold the VM object exclusively. + */ + if (object != cur_object) { + if (cur_object_lock_type == OBJECT_LOCK_SHARED) { + vm_object_unlock(object); + vm_object_unlock(cur_object); + + cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE; + + vm_map_unlock_read(map); + if (real_map != map) + vm_map_unlock(real_map); + + goto RetryFault; + } + } else if (object_lock_type == OBJECT_LOCK_SHARED) { + + vm_object_unlock(object); + object_lock_type = OBJECT_LOCK_EXCLUSIVE; + vm_map_unlock_read(map); + goto RetryFault; + } + } + + if (VM_FAULT_NEED_CS_VALIDATION(map->pmap, m) || + (physpage_p != NULL && (prot & VM_PROT_WRITE))) { +upgrade_for_validation: /* * We might need to validate this page * against its code signature, so we @@ -2958,27 +3839,12 @@ RetryFault: */ if (object == cur_object && object->copy == VM_OBJECT_NULL) { - if ((fault_type & VM_PROT_WRITE) == 0) { - /* - * This is not a "write" fault, so we - * might not have taken the object lock - * exclusively and we might not be able - * to update the "wpmapped" bit in - * vm_fault_enter(). - * Let's just grant read access to - * the page for now and we'll - * soft-fault again if we need write - * access later... - */ - prot &= ~VM_PROT_WRITE; - } + goto FastPmapEnter; } if ((fault_type & VM_PROT_WRITE) == 0) { - prot &= ~VM_PROT_WRITE; - if (object != cur_object) { /* * We still need to hold the top object @@ -3015,30 +3881,52 @@ FastPmapEnter: * cur_object == NULL or it's been unlocked * no paging references on either object or cur_object */ -#if MACH_KDB - if (db_watchpoint_list && (fault_type & VM_PROT_WRITE) == 0) - prot &= ~VM_PROT_WRITE; -#endif + if (top_object != VM_OBJECT_NULL || object_lock_type != OBJECT_LOCK_EXCLUSIVE) + need_retry_ptr = &need_retry; + else + need_retry_ptr = NULL; + if (caller_pmap) { kr = vm_fault_enter(m, caller_pmap, caller_pmap_addr, prot, + caller_prot, wired, change_wiring, fault_info.no_cache, + fault_info.cs_bypass, + fault_info.user_tag, + fault_info.pmap_options, + need_retry_ptr, &type_of_fault); } else { kr = vm_fault_enter(m, pmap, vaddr, prot, + caller_prot, wired, change_wiring, fault_info.no_cache, + fault_info.cs_bypass, + fault_info.user_tag, + fault_info.pmap_options, + need_retry_ptr, &type_of_fault); } + if (kr == KERN_SUCCESS && + physpage_p != NULL) { + /* for vm_map_wire_and_extract() */ + *physpage_p = m->phys_page; + if (prot & VM_PROT_WRITE) { + vm_object_lock_assert_exclusive( + m->object); + m->dirty = TRUE; + } + } + if (top_object != VM_OBJECT_NULL) { /* * It's safe to drop the top object @@ -3055,8 +3943,9 @@ FastPmapEnter: if (need_collapse == TRUE) vm_object_collapse(object, offset, TRUE); - - if (type_of_fault == DBG_PAGEIND_FAULT || type_of_fault == DBG_PAGEINV_FAULT || type_of_fault == DBG_CACHE_HIT_FAULT) { + + if (need_retry == FALSE && + (type_of_fault == DBG_PAGEIND_FAULT || type_of_fault == DBG_PAGEINV_FAULT || type_of_fault == DBG_CACHE_HIT_FAULT)) { /* * evaluate access pattern and update state * vm_fault_deactivate_behind depends on the @@ -3078,6 +3967,22 @@ FastPmapEnter: if (real_map != map) vm_map_unlock(real_map); + if (need_retry == TRUE) { + /* + * vm_fault_enter couldn't complete the PMAP_ENTER... + * at this point we don't hold any locks so it's safe + * to ask the pmap layer to expand the page table to + * accommodate this mapping... once expanded, we'll + * re-drive the fault which should result in vm_fault_enter + * being able to successfully enter the mapping this time around + */ + (void)pmap_enter_options( + pmap, vaddr, 0, 0, 0, 0, 0, + PMAP_OPTIONS_NOENTER, NULL); + + need_retry = FALSE; + goto RetryFault; + } goto done; } /* @@ -3085,30 +3990,6 @@ FastPmapEnter: */ assert(object_lock_type == OBJECT_LOCK_EXCLUSIVE); - if (vm_page_throttled()) { - /* - * drop all of our locks... - * wait until the free queue is - * pumped back up and then - * redrive the fault - */ - if (object != cur_object) - vm_object_unlock(cur_object); - vm_object_unlock(object); - vm_map_unlock_read(map); - if (real_map != map) - vm_map_unlock(real_map); - - if (NEED_TO_HARD_THROTTLE_THIS_TASK()) - delay(HARD_THROTTLE_DELAY); - - if (!current_thread_aborted() && vm_page_wait((change_wiring) ? - THREAD_UNINT : - THREAD_ABORTSAFE)) - goto RetryFault; - kr = KERN_ABORTED; - goto done; - } /* * If objects match, then * object->copy must not be NULL (else control @@ -3123,11 +4004,19 @@ FastPmapEnter: */ break; } + /* * This is now a shadow based copy on write * fault -- it requires a copy up the shadow * chain. - * + */ + + if ((cur_object_lock_type == OBJECT_LOCK_SHARED) && + VM_FAULT_NEED_CS_VALIDATION(NULL, m)) { + goto upgrade_for_validation; + } + + /* * Allocate a page in the original top level * object. Give up if allocate fails. Also * need to remember current page, as it's the @@ -3157,14 +4046,18 @@ FastPmapEnter: */ vm_page_copy(cur_m, m); vm_page_insert(m, object, offset); - m->dirty = TRUE; + SET_PAGE_DIRTY(m, FALSE); /* * Now cope with the source page and object */ if (object->ref_count > 1 && cur_m->pmapped) pmap_disconnect(cur_m->phys_page); - + + if (cur_m->clustered) { + VM_PAGE_COUNT_AS_PAGEIN(cur_m); + VM_PAGE_CONSUME_CLUSTERED(cur_m); + } need_collapse = TRUE; if (!cur_object->internal && @@ -3210,12 +4103,226 @@ FastPmapEnter: * No page at cur_object, cur_offset... m == NULL */ if (cur_object->pager_created) { - if (MUST_ASK_PAGER(cur_object, cur_offset) == TRUE) { + int compressor_external_state = VM_EXTERNAL_STATE_UNKNOWN; + + if (MUST_ASK_PAGER(cur_object, cur_offset, compressor_external_state) == TRUE) { + int my_fault_type; + int c_flags = C_DONT_BLOCK; + boolean_t insert_cur_object = FALSE; + /* * May have to talk to a pager... - * take the slow path. + * if so, take the slow path by + * doing a 'break' from the while (TRUE) loop + * + * external_state will only be set to VM_EXTERNAL_STATE_EXISTS + * if the compressor is active and the page exists there + */ + if (compressor_external_state != VM_EXTERNAL_STATE_EXISTS) + break; + + if (map == kernel_map || real_map == kernel_map) { + /* + * can't call into the compressor with the kernel_map + * lock held, since the compressor may try to operate + * on the kernel map in order to return an empty c_segment + */ + break; + } + if (object != cur_object) { + if (fault_type & VM_PROT_WRITE) + c_flags |= C_KEEP; + else + insert_cur_object = TRUE; + } + if (insert_cur_object == TRUE) { + + if (cur_object_lock_type == OBJECT_LOCK_SHARED) { + + cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE; + + if (vm_object_lock_upgrade(cur_object) == FALSE) { + /* + * couldn't upgrade so go do a full retry + * immediately since we can no longer be + * certain about cur_object (since we + * don't hold a reference on it)... + * first drop the top object lock + */ + vm_object_unlock(object); + + vm_map_unlock_read(map); + if (real_map != map) + vm_map_unlock(real_map); + + goto RetryFault; + } + } + } else if (object_lock_type == OBJECT_LOCK_SHARED) { + + object_lock_type = OBJECT_LOCK_EXCLUSIVE; + + if (object != cur_object) { + /* + * we can't go for the upgrade on the top + * lock since the upgrade may block waiting + * for readers to drain... since we hold + * cur_object locked at this point, waiting + * for the readers to drain would represent + * a lock order inversion since the lock order + * for objects is the reference order in the + * shadown chain + */ + vm_object_unlock(object); + vm_object_unlock(cur_object); + + vm_map_unlock_read(map); + if (real_map != map) + vm_map_unlock(real_map); + + goto RetryFault; + } + if (vm_object_lock_upgrade(object) == FALSE) { + /* + * couldn't upgrade, so explictly take the lock + * exclusively and go relookup the page since we + * will have dropped the object lock and + * a different thread could have inserted + * a page at this offset + * no need for a full retry since we're + * at the top level of the object chain + */ + vm_object_lock(object); + + continue; + } + } + m = vm_page_grab(); + + if (m == VM_PAGE_NULL) { + /* + * no free page currently available... + * must take the slow path + */ + break; + } + + /* + * The object is and remains locked + * so no need to take a + * "paging_in_progress" reference. + */ + boolean_t shared_lock; + if ((object == cur_object && + object_lock_type == OBJECT_LOCK_EXCLUSIVE) || + (object != cur_object && + cur_object_lock_type == OBJECT_LOCK_EXCLUSIVE)) { + shared_lock = FALSE; + } else { + shared_lock = TRUE; + } + + kr = vm_compressor_pager_get( + cur_object->pager, + (cur_offset + + cur_object->paging_offset), + m->phys_page, + &my_fault_type, + c_flags, + &compressed_count_delta); + + vm_compressor_pager_count( + cur_object->pager, + compressed_count_delta, + shared_lock, + cur_object); + + if (kr != KERN_SUCCESS) { + vm_page_release(m); + break; + } + m->dirty = TRUE; + + /* + * If the object is purgeable, its + * owner's purgeable ledgers will be + * updated in vm_page_insert() but the + * page was also accounted for in a + * "compressed purgeable" ledger, so + * update that now. */ - break; + if (object != cur_object && + !insert_cur_object) { + /* + * We're not going to insert + * the decompressed page into + * the object it came from. + * + * We're dealing with a + * copy-on-write fault on + * "object". + * We're going to decompress + * the page directly into the + * target "object" while + * keepin the compressed + * page for "cur_object", so + * no ledger update in that + * case. + */ + } else if ((cur_object->purgable == + VM_PURGABLE_DENY) || + (cur_object->vo_purgeable_owner == + NULL)) { + /* + * "cur_object" is not purgeable + * or is not owned, so no + * purgeable ledgers to update. + */ + } else { + /* + * One less compressed + * purgeable page for + * cur_object's owner. + */ + vm_purgeable_compressed_update( + cur_object, + -1); + } + + if (insert_cur_object) { + vm_page_insert(m, cur_object, cur_offset); + } else { + vm_page_insert(m, object, offset); + } + + if ((m->object->wimg_bits & VM_WIMG_MASK) != VM_WIMG_USE_DEFAULT) { + /* + * If the page is not cacheable, + * we can't let its contents + * linger in the data cache + * after the decompression. + */ + pmap_sync_page_attributes_phys(m->phys_page); + } + + type_of_fault = my_fault_type; + + VM_STAT_INCR(decompressions); + + if (cur_object != object) { + if (insert_cur_object) { + top_object = object; + /* + * switch to the object that has the new page + */ + object = cur_object; + object_lock_type = cur_object_lock_type; + } else { + vm_object_unlock(cur_object); + cur_object = object; + } + } + goto FastPmapEnter; } /* * existence map present and indicates @@ -3241,30 +4348,6 @@ FastPmapEnter: kr = KERN_MEMORY_ERROR; goto done; } - if (vm_page_throttled()) { - /* - * drop all of our locks... - * wait until the free queue is - * pumped back up and then - * redrive the fault - */ - if (object != cur_object) - vm_object_unlock(cur_object); - vm_object_unlock(object); - vm_map_unlock_read(map); - if (real_map != map) - vm_map_unlock(real_map); - - if (NEED_TO_HARD_THROTTLE_THIS_TASK()) - delay(HARD_THROTTLE_DELAY); - - if (!current_thread_aborted() && vm_page_wait((change_wiring) ? - THREAD_UNINT : - THREAD_ABORTSAFE)) - goto RetryFault; - kr = KERN_ABORTED; - goto done; - } if (vm_backing_store_low) { /* * we are protecting the system from @@ -3324,7 +4407,7 @@ FastPmapEnter: /* * On to the next level in the shadow chain */ - cur_offset += cur_object->shadow_offset; + cur_offset += cur_object->vo_shadow_offset; new_object = cur_object->shadow; /* @@ -3387,8 +4470,10 @@ handle_copy_delay: error_code = 0; + result_page = VM_PAGE_NULL; kr = vm_fault_page(object, offset, fault_type, (change_wiring && !wired), + FALSE, /* page not looked up */ &prot, &result_page, &top_page, &type_of_fault, &error_code, map->no_zero_fill, @@ -3623,18 +4708,28 @@ handle_copy_delay: caller_pmap, caller_pmap_addr, prot, + caller_prot, wired, change_wiring, fault_info.no_cache, + fault_info.cs_bypass, + fault_info.user_tag, + fault_info.pmap_options, + NULL, &type_of_fault); } else { kr = vm_fault_enter(m, pmap, vaddr, prot, + caller_prot, wired, change_wiring, fault_info.no_cache, + fault_info.cs_bypass, + fault_info.user_tag, + fault_info.pmap_options, + NULL, &type_of_fault); } if (kr != KERN_SUCCESS) { @@ -3647,6 +4742,14 @@ handle_copy_delay: vm_object_deallocate(object); goto done; } + if (physpage_p != NULL) { + /* for vm_map_wire_and_extract() */ + *physpage_p = m->phys_page; + if (prot & VM_PROT_WRITE) { + vm_object_lock_assert_exclusive(m->object); + m->dirty = TRUE; + } + } } else { vm_map_entry_t entry; @@ -3665,7 +4768,7 @@ handle_copy_delay: /* to execute, we return with a protection failure. */ if ((fault_type & VM_PROT_EXECUTE) && - (!pmap_eligible_for_execute((ppnum_t)(object->shadow_offset >> 12)))) { + (!pmap_eligible_for_execute((ppnum_t)(object->vo_shadow_offset >> 12)))) { vm_map_verify_done(map, &version); @@ -3701,17 +4804,17 @@ handle_copy_delay: hdelta = entry->vme_end - laddr; if (entry->is_sub_map) { - laddr = (laddr - entry->vme_start) - + entry->offset; - vm_map_lock_read(entry->object.sub_map); + laddr = ((laddr - entry->vme_start) + + VME_OFFSET(entry)); + vm_map_lock_read(VME_SUBMAP(entry)); if (map != real_map) vm_map_unlock_read(map); if (entry->use_pmap) { vm_map_unlock_read(real_map); - real_map = entry->object.sub_map; + real_map = VME_SUBMAP(entry); } - map = entry->object.sub_map; + map = VME_SUBMAP(entry); } else { break; @@ -3719,31 +4822,48 @@ handle_copy_delay: } if (vm_map_lookup_entry(map, laddr, &entry) && - (entry->object.vm_object != NULL) && - (entry->object.vm_object == object)) { + (VME_OBJECT(entry) != NULL) && + (VME_OBJECT(entry) == object)) { + int superpage; + + if (!object->pager_created && + object->phys_contiguous) { + superpage = VM_MEM_SUPERPAGE; + } else { + superpage = 0; + } + + if (superpage && physpage_p) { + /* for vm_map_wire_and_extract() */ + *physpage_p = (ppnum_t) + ((((vm_map_offset_t) + object->vo_shadow_offset) + + VME_OFFSET(entry) + + (laddr - entry->vme_start)) + >> PAGE_SHIFT); + } - int superpage = (!object->pager_created && object->phys_contiguous)? VM_MEM_SUPERPAGE : 0; if (caller_pmap) { /* * Set up a block mapped area */ - assert((uint32_t)((ldelta + hdelta) >> 12) == ((ldelta + hdelta) >> 12)); + assert((uint32_t)((ldelta + hdelta) >> PAGE_SHIFT) == ((ldelta + hdelta) >> PAGE_SHIFT)); pmap_map_block(caller_pmap, (addr64_t)(caller_pmap_addr - ldelta), - (ppnum_t)((((vm_map_offset_t) (entry->object.vm_object->shadow_offset)) + - entry->offset + (laddr - entry->vme_start) - ldelta) >> 12), - (uint32_t)((ldelta + hdelta) >> 12), prot, + (ppnum_t)((((vm_map_offset_t) (VME_OBJECT(entry)->vo_shadow_offset)) + + VME_OFFSET(entry) + (laddr - entry->vme_start) - ldelta) >> PAGE_SHIFT), + (uint32_t)((ldelta + hdelta) >> PAGE_SHIFT), prot, (VM_WIMG_MASK & (int)object->wimg_bits) | superpage, 0); } else { /* * Set up a block mapped area */ - assert((uint32_t)((ldelta + hdelta) >> 12) == ((ldelta + hdelta) >> 12)); + assert((uint32_t)((ldelta + hdelta) >> PAGE_SHIFT) == ((ldelta + hdelta) >> PAGE_SHIFT)); pmap_map_block(real_map->pmap, (addr64_t)(vaddr - ldelta), - (ppnum_t)((((vm_map_offset_t)(entry->object.vm_object->shadow_offset)) + - entry->offset + (laddr - entry->vme_start) - ldelta) >> 12), - (uint32_t)((ldelta + hdelta) >> 12), prot, + (ppnum_t)((((vm_map_offset_t)(VME_OBJECT(entry)->vo_shadow_offset)) + + VME_OFFSET(entry) + (laddr - entry->vme_start) - ldelta) >> PAGE_SHIFT), + (uint32_t)((ldelta + hdelta) >> PAGE_SHIFT), prot, (VM_WIMG_MASK & (int)object->wimg_bits) | superpage, 0); } } @@ -3771,9 +4891,32 @@ handle_copy_delay: done: thread_interrupt_level(interruptible_state); - KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_END, - (int)((uint64_t)vaddr >> 32), - (int)vaddr, + /* + * Only I/O throttle on faults which cause a pagein/swapin. + */ + if ((type_of_fault == DBG_PAGEIND_FAULT) || (type_of_fault == DBG_PAGEINV_FAULT) || (type_of_fault == DBG_COMPRESSOR_SWAPIN_FAULT)) { + throttle_lowpri_io(1); + } else { + if (kr == KERN_SUCCESS && type_of_fault != DBG_CACHE_HIT_FAULT && type_of_fault != DBG_GUARD_FAULT) { + + if ((throttle_delay = vm_page_throttled(TRUE))) { + + if (vm_debug_events) { + if (type_of_fault == DBG_COMPRESSOR_FAULT) + VM_DEBUG_EVENT(vmf_compressordelay, VMF_COMPRESSORDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0); + else if (type_of_fault == DBG_COW_FAULT) + VM_DEBUG_EVENT(vmf_cowdelay, VMF_COWDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0); + else + VM_DEBUG_EVENT(vmf_zfdelay, VMF_ZFDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0); + } + delay(throttle_delay); + } + } + } + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + (MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_END, + ((uint64_t)vaddr >> 32), + vaddr, kr, type_of_fault, 0); @@ -3790,8 +4933,10 @@ kern_return_t vm_fault_wire( vm_map_t map, vm_map_entry_t entry, + vm_prot_t prot, pmap_t pmap, - vm_map_offset_t pmap_addr) + vm_map_offset_t pmap_addr, + ppnum_t *physpage_p) { register vm_map_offset_t va; @@ -3800,9 +4945,9 @@ vm_fault_wire( assert(entry->in_transition); - if ((entry->object.vm_object != NULL) && - !entry->is_sub_map && - entry->object.vm_object->phys_contiguous) { + if ((VME_OBJECT(entry) != NULL) && + !entry->is_sub_map && + VME_OBJECT(entry)->phys_contiguous) { return KERN_SUCCESS; } @@ -3821,14 +4966,18 @@ vm_fault_wire( */ for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) { - if ((rc = vm_fault_wire_fast( - map, va, entry, pmap, - pmap_addr + (va - entry->vme_start) - )) != KERN_SUCCESS) { - rc = vm_fault(map, va, VM_PROT_NONE, TRUE, - (pmap == kernel_pmap) ? - THREAD_UNINT : THREAD_ABORTSAFE, - pmap, pmap_addr + (va - entry->vme_start)); + rc = vm_fault_wire_fast(map, va, prot, entry, pmap, + pmap_addr + (va - entry->vme_start), + physpage_p); + if (rc != KERN_SUCCESS) { + rc = vm_fault_internal(map, va, prot, TRUE, + ((pmap == kernel_pmap) + ? THREAD_UNINT + : THREAD_ABORTSAFE), + pmap, + (pmap_addr + + (va - entry->vme_start)), + physpage_p); DTRACE_VM2(softlock, int, 1, (uint64_t *), NULL); } @@ -3864,8 +5013,7 @@ vm_fault_unwire( vm_object_t object; struct vm_object_fault_info fault_info; - object = (entry->is_sub_map) - ? VM_OBJECT_NULL : entry->object.vm_object; + object = (entry->is_sub_map) ? VM_OBJECT_NULL : VME_OBJECT(entry); /* * If it's marked phys_contiguous, then vm_fault_wire() didn't actually @@ -3878,11 +5026,20 @@ vm_fault_unwire( fault_info.interruptible = THREAD_UNINT; fault_info.behavior = entry->behavior; - fault_info.user_tag = entry->alias; - fault_info.lo_offset = entry->offset; - fault_info.hi_offset = (entry->vme_end - entry->vme_start) + entry->offset; + fault_info.user_tag = VME_ALIAS(entry); + fault_info.pmap_options = 0; + if (entry->iokit_acct || + (!entry->is_sub_map && !entry->use_pmap)) { + fault_info.pmap_options |= PMAP_OPTIONS_ALT_ACCT; + } + fault_info.lo_offset = VME_OFFSET(entry); + fault_info.hi_offset = (entry->vme_end - entry->vme_start) + VME_OFFSET(entry); fault_info.no_cache = entry->no_cache; fault_info.stealth = TRUE; + fault_info.io_sync = FALSE; + fault_info.cs_bypass = FALSE; + fault_info.mark_zf_absent = FALSE; + fault_info.batch_pmap_op = FALSE; /* * Since the pages are wired down, we must be able to @@ -3921,10 +5078,13 @@ vm_fault_unwire( XPR(XPR_VM_FAULT, "vm_fault_unwire -> vm_fault_page\n", 0,0,0,0,0); + result_page = VM_PAGE_NULL; result = vm_fault_page( object, - entry->offset + (va - entry->vme_start), + (VME_OFFSET(entry) + + (va - entry->vme_start)), VM_PROT_NONE, TRUE, + FALSE, /* page not looked up */ &prot, &result_page, &top_page, (int *)0, NULL, map->no_zero_fill, @@ -3944,24 +5104,37 @@ vm_fault_unwire( if (result == VM_FAULT_MEMORY_ERROR && !object->alive) continue; + if (result == VM_FAULT_MEMORY_ERROR && + object == kernel_object) { + /* + * This must have been allocated with + * KMA_KOBJECT and KMA_VAONLY and there's + * no physical page at this offset. + * We're done (no page to free). + */ + assert(deallocate); + continue; + } + if (result != VM_FAULT_SUCCESS) panic("vm_fault_unwire: failure"); result_object = result_page->object; - if ((pmap) && (result_page->phys_page != vm_page_guard_addr)) { - pmap_change_wiring(pmap, - pmap_addr + (va - entry->vme_start), FALSE); - } if (deallocate) { assert(result_page->phys_page != vm_page_fictitious_addr); pmap_disconnect(result_page->phys_page); VM_PAGE_FREE(result_page); } else { + if ((pmap) && (result_page->phys_page != vm_page_guard_addr)) + pmap_change_wiring(pmap, + pmap_addr + (va - entry->vme_start), FALSE); + + if (VM_PAGE_WIRED(result_page)) { vm_page_lockspin_queues(); - vm_page_unwire(result_page); + vm_page_unwire(result_page, TRUE); vm_page_unlock_queues(); } if(entry->zero_wired_pages) { @@ -4006,13 +5179,15 @@ vm_fault_unwire( * other than the common case will return KERN_FAILURE, and the caller * is expected to call vm_fault(). */ -kern_return_t +static kern_return_t vm_fault_wire_fast( __unused vm_map_t map, vm_map_offset_t va, + vm_prot_t caller_prot, vm_map_entry_t entry, - pmap_t pmap, - vm_map_offset_t pmap_addr) + pmap_t pmap, + vm_map_offset_t pmap_addr, + ppnum_t *physpage_p) { vm_object_t object; vm_object_offset_t offset; @@ -4035,7 +5210,7 @@ vm_fault_wire_fast( #define RELEASE_PAGE(m) { \ PAGE_WAKEUP_DONE(m); \ vm_page_lockspin_queues(); \ - vm_page_unwire(m); \ + vm_page_unwire(m, TRUE); \ vm_page_unlock_queues(); \ } @@ -4064,15 +5239,17 @@ vm_fault_wire_fast( /* * If this entry is not directly to a vm_object, bail out. */ - if (entry->is_sub_map) + if (entry->is_sub_map) { + assert(physpage_p == NULL); return(KERN_FAILURE); + } /* * Find the backing store object and offset into it. */ - object = entry->object.vm_object; - offset = (va - entry->vme_start) + entry->offset; + object = VME_OBJECT(entry); + offset = (va - entry->vme_start) + VME_OFFSET(entry); prot = entry->protection; /* @@ -4128,7 +5305,7 @@ vm_fault_wire_fast( */ vm_page_lockspin_queues(); - vm_page_wire(m); + vm_page_wire(m, VM_PROT_MEMORY_TAG(caller_prot), TRUE); vm_page_unlock_queues(); /* @@ -4154,9 +5331,17 @@ vm_fault_wire_fast( pmap, pmap_addr, prot, + prot, TRUE, FALSE, FALSE, + FALSE, + VME_ALIAS(entry), + ((entry->iokit_acct || + (!entry->is_sub_map && !entry->use_pmap)) + ? PMAP_OPTIONS_ALT_ACCT + : 0), + NULL, &type_of_fault); done: @@ -4164,6 +5349,19 @@ done: * Unlock everything, and return */ + if (physpage_p) { + /* for vm_map_wire_and_extract() */ + if (kr == KERN_SUCCESS) { + *physpage_p = m->phys_page; + if (prot & VM_PROT_WRITE) { + vm_object_lock_assert_exclusive(m->object); + m->dirty = TRUE; + } + } else { + *physpage_p = 0; + } + } + PAGE_WAKEUP_DONE(m); UNLOCK_AND_DEALLOCATE; @@ -4177,7 +5375,7 @@ done: * Release a page used by vm_fault_copy. */ -void +static void vm_fault_copy_cleanup( vm_page_t page, vm_page_t top_page) @@ -4195,7 +5393,7 @@ vm_fault_copy_cleanup( vm_fault_cleanup(object, top_page); } -void +static void vm_fault_copy_dst_cleanup( vm_page_t page) { @@ -4205,7 +5403,7 @@ vm_fault_copy_dst_cleanup( object = page->object; vm_object_lock(object); vm_page_lockspin_queues(); - vm_page_unwire(page); + vm_page_unwire(page, TRUE); vm_page_unlock_queues(); vm_object_paging_end(object); vm_object_unlock(object); @@ -4285,18 +5483,28 @@ vm_fault_copy( fault_info_src.interruptible = interruptible; fault_info_src.behavior = VM_BEHAVIOR_SEQUENTIAL; fault_info_src.user_tag = 0; + fault_info_src.pmap_options = 0; fault_info_src.lo_offset = vm_object_trunc_page(src_offset); fault_info_src.hi_offset = fault_info_src.lo_offset + amount_left; fault_info_src.no_cache = FALSE; fault_info_src.stealth = TRUE; + fault_info_src.io_sync = FALSE; + fault_info_src.cs_bypass = FALSE; + fault_info_src.mark_zf_absent = FALSE; + fault_info_src.batch_pmap_op = FALSE; fault_info_dst.interruptible = interruptible; fault_info_dst.behavior = VM_BEHAVIOR_SEQUENTIAL; fault_info_dst.user_tag = 0; + fault_info_dst.pmap_options = 0; fault_info_dst.lo_offset = vm_object_trunc_page(dst_offset); fault_info_dst.hi_offset = fault_info_dst.lo_offset + amount_left; fault_info_dst.no_cache = FALSE; fault_info_dst.stealth = TRUE; + fault_info_dst.io_sync = FALSE; + fault_info_dst.cs_bypass = FALSE; + fault_info_dst.mark_zf_absent = FALSE; + fault_info_dst.batch_pmap_op = FALSE; do { /* while (amount_left > 0) */ /* @@ -4322,10 +5530,12 @@ vm_fault_copy( } XPR(XPR_VM_FAULT,"vm_fault_copy -> vm_fault_page\n",0,0,0,0,0); + dst_page = VM_PAGE_NULL; result = vm_fault_page(dst_object, vm_object_trunc_page(dst_offset), VM_PROT_WRITE|VM_PROT_READ, FALSE, + FALSE, /* page not looked up */ &dst_prot, &dst_page, &dst_top_page, (int *)0, &error, @@ -4373,7 +5583,7 @@ vm_fault_copy( */ vm_page_lockspin_queues(); - vm_page_wire(dst_page); + vm_page_wire(dst_page, VM_KERN_MEMORY_OSFMK, TRUE); vm_page_unlock_queues(); PAGE_WAKEUP_DONE(dst_page); vm_object_unlock(dst_page->object); @@ -4416,10 +5626,12 @@ vm_fault_copy( XPR(XPR_VM_FAULT, "vm_fault_copy(2) -> vm_fault_page\n", 0,0,0,0,0); + result_page = VM_PAGE_NULL; result = vm_fault_page( src_object, vm_object_trunc_page(src_offset), VM_PROT_READ, FALSE, + FALSE, /* page not looked up */ &src_prot, &result_page, &src_top_page, (int *)0, &error, FALSE, @@ -4522,7 +5734,7 @@ vm_fault_copy( (vm_size_t)part_size); if(!dst_page->dirty){ vm_object_lock(dst_object); - dst_page->dirty = TRUE; + SET_PAGE_DIRTY(dst_page, TRUE); vm_object_unlock(dst_page->object); } @@ -4533,10 +5745,13 @@ vm_fault_copy( if (result_page == VM_PAGE_NULL) vm_page_zero_fill(dst_page); else{ + vm_object_lock(result_page->object); vm_page_copy(result_page, dst_page); + vm_object_unlock(result_page->object); + if(!dst_page->dirty){ vm_object_lock(dst_object); - dst_page->dirty = TRUE; + SET_PAGE_DIRTY(dst_page, TRUE); vm_object_unlock(dst_page->object); } } @@ -4617,7 +5832,7 @@ vm_fault_classify(vm_object_t object, break; } - offset += object->shadow_offset; + offset += object->vo_shadow_offset; object = object->shadow; level++; continue; @@ -4649,28 +5864,154 @@ vm_fault_classify_init(void) } #endif /* VM_FAULT_CLASSIFY */ +vm_offset_t +kdp_lightweight_fault(vm_map_t map, vm_offset_t cur_target_addr, uint32_t *fault_results) +{ +#pragma unused(map, cur_target_addr, fault_results) + + return 0; +#if 0 + vm_map_entry_t entry; + vm_object_t object; + vm_offset_t object_offset; + vm_page_t m; + int compressor_external_state, compressed_count_delta; + int compressor_flags = (C_DONT_BLOCK | C_KEEP | C_KDP); + int my_fault_type = VM_PROT_READ; + kern_return_t kr; + + + if (not_in_kdp) { + panic("kdp_lightweight_fault called from outside of debugger context"); + } + + assert(map != VM_MAP_NULL); + + assert((cur_target_addr & PAGE_MASK) == 0); + if ((cur_target_addr & PAGE_MASK) != 0) { + return 0; + } + + if (kdp_lck_rw_lock_is_acquired_exclusive(&map->lock)) { + return 0; + } + + if (!vm_map_lookup_entry(map, cur_target_addr, &entry)) { + return 0; + } + + if (entry->is_sub_map) { + return 0; + } + + object = VME_OBJECT(entry); + if (object == VM_OBJECT_NULL) { + return 0; + } + + object_offset = cur_target_addr - entry->vme_start + VME_OFFSET(entry); + + while (TRUE) { + if (kdp_lck_rw_lock_is_acquired_exclusive(&object->Lock)) { + return 0; + } + + if (object->pager_created && (object->paging_in_progress || + object->activity_in_progress)) { + return 0; + } + + m = kdp_vm_page_lookup(object, object_offset); + + if (m != VM_PAGE_NULL) { + + if ((object->wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT) { + return 0; + } + + if (m->laundry || m->busy || m->pageout || m->absent || m->error || m->cleaning || + m->overwriting || m->restart || m->unusual) { + return 0; + } + + assert(!m->private); + if (m->private) { + return 0; + } + + assert(!m->fictitious); + if (m->fictitious) { + return 0; + } + + assert(!m->encrypted); + if (m->encrypted) { + return 0; + } + + assert(!m->encrypted_cleaning); + if (m->encrypted_cleaning) { + return 0; + } + + assert(!m->compressor); + if (m->compressor) { + return 0; + } + + if (fault_results) { + *fault_results |= kThreadFaultedBT; + } + return ptoa(m->phys_page); + } + + compressor_external_state = VM_EXTERNAL_STATE_UNKNOWN; + + if (object->pager_created && MUST_ASK_PAGER(object, object_offset, compressor_external_state)) { + if (compressor_external_state == VM_EXTERNAL_STATE_EXISTS) { + kr = vm_compressor_pager_get(object->pager, (object_offset + object->paging_offset), + kdp_compressor_decompressed_page_ppnum, &my_fault_type, + compressor_flags, &compressed_count_delta); + if (kr == KERN_SUCCESS) { + if (fault_results) { + *fault_results |= kThreadDecompressedBT; + } + return kdp_compressor_decompressed_page_paddr; + } else { + return 0; + } + } + } + + if (object->shadow == VM_OBJECT_NULL) { + return 0; + } + + object_offset += object->vo_shadow_offset; + object = object->shadow; + } +#endif /* 0 */ +} -extern int cs_validation; +#define CODE_SIGNING_CHUNK_SIZE 4096 void vm_page_validate_cs_mapped( vm_page_t page, const void *kaddr) { vm_object_t object; - vm_object_offset_t offset; + vm_object_offset_t offset, offset_in_page; kern_return_t kr; memory_object_t pager; void *blobs; - boolean_t validated, tainted; + boolean_t validated; + unsigned tainted; + int num_chunks, num_chunks_validated; assert(page->busy); vm_object_lock_assert_exclusive(page->object); - if (!cs_validation) { - return; - } - if (page->wpmapped && !page->cs_tainted) { /* * This page was mapped for "write" access sometime in the @@ -4690,7 +6031,7 @@ vm_page_validate_cs_mapped( vm_cs_validated_dirtied++; } - if (page->cs_validated) { + if (page->cs_validated || page->cs_tainted) { return; } @@ -4724,14 +6065,32 @@ vm_page_validate_cs_mapped( } /* verify the SHA1 hash for this page */ - validated = cs_validate_page(blobs, - offset + object->paging_offset, - (const void *)kaddr, - &tainted); - - page->cs_validated = validated; - if (validated) { - page->cs_tainted = tainted; + num_chunks_validated = 0; + for (offset_in_page = 0, num_chunks = 0; + offset_in_page < PAGE_SIZE_64; + offset_in_page += CODE_SIGNING_CHUNK_SIZE, num_chunks++) { + tainted = 0; + validated = cs_validate_page(blobs, + pager, + (object->paging_offset + + offset + + offset_in_page), + (const void *)((const char *)kaddr + + offset_in_page), + &tainted); + if (validated) { + num_chunks_validated++; + } + if (tainted & CS_VALIDATE_TAINTED) { + page->cs_tainted = TRUE; + } + if (tainted & CS_VALIDATE_NX) { + page->cs_nx = TRUE; + } + } + /* page is validated only if all its chunks are */ + if (num_chunks_validated == num_chunks) { + page->cs_validated = TRUE; } } @@ -4746,13 +6105,10 @@ vm_page_validate_cs( vm_offset_t kaddr; kern_return_t kr; boolean_t busy_page; + boolean_t need_unmap; vm_object_lock_assert_held(page->object); - if (!cs_validation) { - return; - } - if (page->wpmapped && !page->cs_tainted) { vm_object_lock_assert_exclusive(page->object); @@ -4774,10 +6130,23 @@ vm_page_validate_cs( vm_cs_validated_dirtied++; } - if (page->cs_validated) { + if (page->cs_validated || page->cs_tainted) { return; } + if (page->slid) { + panic("vm_page_validate_cs(%p): page is slid\n", page); + } + assert(!page->slid); + +#if CHECK_CS_VALIDATION_BITMAP + if ( vnode_pager_cs_check_validation_bitmap( page->object->pager, trunc_page(page->offset + page->object->paging_offset), CS_BITMAP_CHECK ) == KERN_SUCCESS) { + page->cs_validated = TRUE; + page->cs_tainted = FALSE; + vm_cs_bitmap_validated++; + return; + } +#endif vm_object_lock_assert_exclusive(page->object); object = page->object; @@ -4798,15 +6167,17 @@ vm_page_validate_cs( vm_object_paging_begin(object); /* map the page in the kernel address space */ - koffset = 0; ksize = PAGE_SIZE_64; - kr = vm_paging_map_object(&koffset, - page, + koffset = 0; + need_unmap = FALSE; + kr = vm_paging_map_object(page, object, offset, - &ksize, VM_PROT_READ, - FALSE); /* can't unlock object ! */ + FALSE, /* can't unlock object ! */ + &ksize, + &koffset, + &need_unmap); if (kr != KERN_SUCCESS) { panic("vm_page_validate_cs: could not map page: 0x%x\n", kr); } @@ -4815,6 +6186,11 @@ vm_page_validate_cs( /* validate the mapped page */ vm_page_validate_cs_mapped(page, (const void *) kaddr); +#if CHECK_CS_VALIDATION_BITMAP + if ( page->cs_validated == TRUE && page->cs_tainted == FALSE ) { + vnode_pager_cs_check_validation_bitmap( object->pager, trunc_page( offset + object->paging_offset), CS_BITMAP_SET ); + } +#endif assert(page->busy); assert(object == page->object); vm_object_lock_assert_exclusive(object); @@ -4822,7 +6198,7 @@ vm_page_validate_cs( if (!busy_page) { PAGE_WAKEUP_DONE(page); } - if (koffset != 0) { + if (need_unmap) { /* unmap the map from the kernel address space */ vm_paging_unmap_object(object, koffset, koffset + ksize); koffset = 0; @@ -4831,3 +6207,74 @@ vm_page_validate_cs( } vm_object_paging_end(object); } + +void +vm_page_validate_cs_mapped_chunk( + vm_page_t page, + const void *kaddr, + vm_offset_t chunk_offset, + boolean_t *validated_p, + unsigned *tainted_p) +{ + vm_object_t object; + vm_object_offset_t offset, offset_in_page; + kern_return_t kr; + memory_object_t pager; + void *blobs; + boolean_t validated; + unsigned tainted; + + *validated_p = FALSE; + *tainted_p = 0; + + assert(page->busy); + vm_object_lock_assert_exclusive(page->object); + + object = page->object; + assert(object->code_signed); + offset = page->offset; + + if (!object->alive || object->terminating || object->pager == NULL) { + /* + * The object is terminating and we don't have its pager + * so we can't validate the data... + */ + return; + } + /* + * Since we get here to validate a page that was brought in by + * the pager, we know that this pager is all setup and ready + * by now. + */ + assert(!object->internal); + assert(object->pager != NULL); + assert(object->pager_ready); + + pager = object->pager; + assert(object->paging_in_progress); + kr = vnode_pager_get_object_cs_blobs(pager, &blobs); + if (kr != KERN_SUCCESS) { + blobs = NULL; + } + + /* verify the signature for this chunk */ + offset_in_page = chunk_offset; + assert(offset_in_page < PAGE_SIZE); + assert((offset_in_page & (CODE_SIGNING_CHUNK_SIZE-1)) == 0); + + tainted = 0; + validated = cs_validate_page(blobs, + pager, + (object->paging_offset + + offset + + offset_in_page), + (const void *)((const char *)kaddr + + offset_in_page), + &tainted); + if (validated) { + *validated_p = TRUE; + } + if (tainted) { + *tainted_p = tainted; + } +}