+ vm_page_deactivate_behind_count++;
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF0019, (unsigned int) object, (unsigned int) m); /* (TEST/DEBUG) */
+#endif
+ }
+ vm_page_unlock_queues();
+
+ return TRUE;
+ }
+ return FALSE;
+}
+
+
+#if (DEVELOPMENT || DEBUG)
+uint32_t vm_page_creation_throttled_hard = 0;
+uint32_t vm_page_creation_throttled_soft = 0;
+uint64_t vm_page_creation_throttle_avoided = 0;
+#endif /* DEVELOPMENT || DEBUG */
+
+static int
+vm_page_throttled(boolean_t page_kept)
+{
+ clock_sec_t elapsed_sec;
+ clock_sec_t tv_sec;
+ clock_usec_t tv_usec;
+
+ thread_t thread = current_thread();
+
+ if (thread->options & TH_OPT_VMPRIV) {
+ return 0;
+ }
+
+ if (thread->t_page_creation_throttled) {
+ thread->t_page_creation_throttled = 0;
+
+ if (page_kept == FALSE) {
+ goto no_throttle;
+ }
+ }
+ if (NEED_TO_HARD_THROTTLE_THIS_TASK()) {
+#if (DEVELOPMENT || DEBUG)
+ thread->t_page_creation_throttled_hard++;
+ OSAddAtomic(1, &vm_page_creation_throttled_hard);
+#endif /* DEVELOPMENT || DEBUG */
+ return HARD_THROTTLE_DELAY;
+ }
+
+ if ((vm_page_free_count < vm_page_throttle_limit || (VM_CONFIG_COMPRESSOR_IS_PRESENT && SWAPPER_NEEDS_TO_UNTHROTTLE())) &&
+ thread->t_page_creation_count > (VM_PAGE_CREATION_THROTTLE_PERIOD_SECS * VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC)) {
+ if (vm_page_free_wanted == 0 && vm_page_free_wanted_privileged == 0) {
+#if (DEVELOPMENT || DEBUG)
+ OSAddAtomic64(1, &vm_page_creation_throttle_avoided);
+#endif
+ goto no_throttle;
+ }
+ clock_get_system_microtime(&tv_sec, &tv_usec);
+
+ elapsed_sec = tv_sec - thread->t_page_creation_time;
+
+ if (elapsed_sec <= VM_PAGE_CREATION_THROTTLE_PERIOD_SECS ||
+ (thread->t_page_creation_count / elapsed_sec) >= VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC) {
+ if (elapsed_sec >= (3 * VM_PAGE_CREATION_THROTTLE_PERIOD_SECS)) {
+ /*
+ * we'll reset our stats to give a well behaved app
+ * that was unlucky enough to accumulate a bunch of pages
+ * over a long period of time a chance to get out of
+ * the throttled state... we reset the counter and timestamp
+ * so that if it stays under the rate limit for the next second
+ * it will be back in our good graces... if it exceeds it, it
+ * will remain in the throttled state
+ */
+ thread->t_page_creation_time = tv_sec;
+ thread->t_page_creation_count = VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC * (VM_PAGE_CREATION_THROTTLE_PERIOD_SECS - 1);
+ }
+ VM_PAGEOUT_DEBUG(vm_page_throttle_count, 1);
+
+ thread->t_page_creation_throttled = 1;
+
+ if (VM_CONFIG_COMPRESSOR_IS_PRESENT && HARD_THROTTLE_LIMIT_REACHED()) {
+#if (DEVELOPMENT || DEBUG)
+ thread->t_page_creation_throttled_hard++;
+ OSAddAtomic(1, &vm_page_creation_throttled_hard);
+#endif /* DEVELOPMENT || DEBUG */
+ return HARD_THROTTLE_DELAY;
+ } else {
+#if (DEVELOPMENT || DEBUG)
+ thread->t_page_creation_throttled_soft++;
+ OSAddAtomic(1, &vm_page_creation_throttled_soft);
+#endif /* DEVELOPMENT || DEBUG */
+ return SOFT_THROTTLE_DELAY;
+ }
+ }
+ thread->t_page_creation_time = tv_sec;
+ thread->t_page_creation_count = 0;
+ }
+no_throttle:
+ thread->t_page_creation_count++;
+
+ return 0;
+}
+
+
+/*
+ * check for various conditions that would
+ * prevent us from creating a ZF page...
+ * cleanup is based on being called from vm_fault_page
+ *
+ * object must be locked
+ * object == m->vmp_object
+ */
+static vm_fault_return_t
+vm_fault_check(vm_object_t object, vm_page_t m, vm_page_t first_m, wait_interrupt_t interruptible_state, boolean_t page_throttle)
+{
+ int throttle_delay;
+
+ if (object->shadow_severed ||
+ VM_OBJECT_PURGEABLE_FAULT_ERROR(object)) {
+ /*
+ * Either:
+ * 1. the shadow chain was severed,
+ * 2. the purgeable object is volatile or empty and is marked
+ * to fault on access while volatile.
+ * Just have to return an error at this point
+ */
+ if (m != VM_PAGE_NULL) {
+ VM_PAGE_FREE(m);
+ }
+ vm_fault_cleanup(object, first_m);
+
+ thread_interrupt_level(interruptible_state);
+
+ return VM_FAULT_MEMORY_ERROR;
+ }
+ if (page_throttle == TRUE) {
+ if ((throttle_delay = vm_page_throttled(FALSE))) {
+ /*
+ * we're throttling zero-fills...
+ * treat this as if we couldn't grab a page
+ */
+ if (m != VM_PAGE_NULL) {
+ VM_PAGE_FREE(m);
+ }
+ vm_fault_cleanup(object, first_m);
+
+ VM_DEBUG_EVENT(vmf_check_zfdelay, VMF_CHECK_ZFDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0);
+
+ delay(throttle_delay);
+
+ if (current_thread_aborted()) {
+ thread_interrupt_level(interruptible_state);
+ return VM_FAULT_INTERRUPTED;
+ }
+ thread_interrupt_level(interruptible_state);
+
+ return VM_FAULT_MEMORY_SHORTAGE;
+ }
+ }
+ return VM_FAULT_SUCCESS;
+}
+
+/*
+ * Clear the code signing bits on the given page_t
+ */
+static void
+vm_fault_cs_clear(vm_page_t m)
+{
+ m->vmp_cs_validated = VMP_CS_ALL_FALSE;
+ m->vmp_cs_tainted = VMP_CS_ALL_FALSE;
+ m->vmp_cs_nx = VMP_CS_ALL_FALSE;
+}
+
+/*
+ * Enqueues the given page on the throttled queue.
+ * The caller must hold the vm_page_queue_lock and it will be held on return.
+ */
+static void
+vm_fault_enqueue_throttled_locked(vm_page_t m)
+{
+ LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+ assert(!VM_PAGE_WIRED(m));
+
+ /*
+ * can't be on the pageout queue since we don't
+ * have a pager to try and clean to
+ */
+ vm_page_queues_remove(m, TRUE);
+ vm_page_check_pageable_safe(m);
+ vm_page_queue_enter(&vm_page_queue_throttled, m, vmp_pageq);
+ m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;
+ vm_page_throttled_count++;
+}
+
+/*
+ * do the work to zero fill a page and
+ * inject it into the correct paging queue
+ *
+ * m->vmp_object must be locked
+ * page queue lock must NOT be held
+ */
+static int
+vm_fault_zero_page(vm_page_t m, boolean_t no_zero_fill)
+{
+ int my_fault = DBG_ZERO_FILL_FAULT;
+ vm_object_t object;
+
+ object = VM_PAGE_OBJECT(m);
+
+ /*
+ * This is is a zero-fill page fault...
+ *
+ * Checking the page lock is a waste of
+ * time; this page was absent, so
+ * it can't be page locked by a pager.
+ *
+ * we also consider it undefined
+ * with respect to instruction
+ * execution. i.e. it is the responsibility
+ * of higher layers to call for an instruction
+ * sync after changing the contents and before
+ * sending a program into this area. We
+ * choose this approach for performance
+ */
+ vm_fault_cs_clear(m);
+ m->vmp_pmapped = TRUE;
+
+ if (no_zero_fill == TRUE) {
+ my_fault = DBG_NZF_PAGE_FAULT;
+
+ if (m->vmp_absent && m->vmp_busy) {
+ return my_fault;
+ }
+ } else {
+ vm_page_zero_fill(m);
+
+ VM_STAT_INCR(zero_fill_count);
+ DTRACE_VM2(zfod, int, 1, (uint64_t *), NULL);
+ }
+ assert(!m->vmp_laundry);
+ assert(object != kernel_object);
+ //assert(m->vmp_pageq.next == 0 && m->vmp_pageq.prev == 0);
+ if (!VM_DYNAMIC_PAGING_ENABLED() &&
+ (object->purgable == VM_PURGABLE_DENY ||
+ object->purgable == VM_PURGABLE_NONVOLATILE ||
+ object->purgable == VM_PURGABLE_VOLATILE)) {
+ vm_page_lockspin_queues();
+ if (!VM_DYNAMIC_PAGING_ENABLED()) {
+ vm_fault_enqueue_throttled_locked(m);
+ }
+ vm_page_unlock_queues();
+ }
+ return my_fault;
+}
+
+
+/*
+ * Routine: vm_fault_page
+ * Purpose:
+ * Find the resident page for the virtual memory
+ * specified by the given virtual memory object
+ * and offset.
+ * Additional arguments:
+ * The required permissions for the page is given
+ * in "fault_type". Desired permissions are included
+ * in "protection".
+ * fault_info is passed along to determine pagein cluster
+ * limits... it contains the expected reference pattern,
+ * cluster size if available, etc...
+ *
+ * If the desired page is known to be resident (for
+ * example, because it was previously wired down), asserting
+ * the "unwiring" parameter will speed the search.
+ *
+ * If the operation can be interrupted (by thread_abort
+ * or thread_terminate), then the "interruptible"
+ * parameter should be asserted.
+ *
+ * Results:
+ * The page containing the proper data is returned
+ * in "result_page".
+ *
+ * In/out conditions:
+ * The source object must be locked and referenced,
+ * and must donate one paging reference. The reference
+ * is not affected. The paging reference and lock are
+ * consumed.
+ *
+ * If the call succeeds, the object in which "result_page"
+ * resides is left locked and holding a paging reference.
+ * If this is not the original object, a busy page in the
+ * original object is returned in "top_page", to prevent other
+ * callers from pursuing this same data, along with a paging
+ * reference for the original object. The "top_page" should
+ * be destroyed when this guarantee is no longer required.
+ * The "result_page" is also left busy. It is not removed
+ * from the pageout queues.
+ * Special Case:
+ * A return value of VM_FAULT_SUCCESS_NO_PAGE means that the
+ * fault succeeded but there's no VM page (i.e. the VM object
+ * does not actually hold VM pages, but device memory or
+ * large pages). The object is still locked and we still hold a
+ * paging_in_progress reference.
+ */
+unsigned int vm_fault_page_blocked_access = 0;
+unsigned int vm_fault_page_forced_retry = 0;
+
+vm_fault_return_t
+vm_fault_page(
+ /* Arguments: */
+ vm_object_t first_object, /* Object to begin search */
+ vm_object_offset_t first_offset, /* Offset into object */
+ vm_prot_t fault_type, /* What access is requested */
+ boolean_t must_be_resident,/* Must page be resident? */
+ boolean_t caller_lookup, /* caller looked up page */
+ /* Modifies in place: */
+ vm_prot_t *protection, /* Protection for mapping */
+ vm_page_t *result_page, /* Page found, if successful */
+ /* Returns: */
+ vm_page_t *top_page, /* Page in top object, if
+ * not result_page. */
+ int *type_of_fault, /* if non-null, fill in with type of fault
+ * COW, zero-fill, etc... returned in trace point */
+ /* More arguments: */
+ kern_return_t *error_code, /* code if page is in error */
+ boolean_t no_zero_fill, /* don't zero fill absent pages */
+ boolean_t data_supply, /* treat as data_supply if
+ * it is a write fault and a full
+ * page is provided */
+ vm_object_fault_info_t fault_info)
+{
+ vm_page_t m;
+ vm_object_t object;
+ vm_object_offset_t offset;
+ vm_page_t first_m;
+ vm_object_t next_object;
+ vm_object_t copy_object;
+ boolean_t look_for_page;
+ boolean_t force_fault_retry = FALSE;
+ vm_prot_t access_required = fault_type;
+ vm_prot_t wants_copy_flag;
+ kern_return_t wait_result;
+ wait_interrupt_t interruptible_state;
+ boolean_t data_already_requested = FALSE;
+ vm_behavior_t orig_behavior;
+ vm_size_t orig_cluster_size;
+ vm_fault_return_t error;
+ int my_fault;
+ uint32_t try_failed_count;
+ int interruptible; /* how may fault be interrupted? */
+ int external_state = VM_EXTERNAL_STATE_UNKNOWN;
+ memory_object_t pager;
+ vm_fault_return_t retval;
+ int grab_options;
+
+/*
+ * MUST_ASK_PAGER() evaluates to TRUE if the page specified by object/offset is
+ * marked as paged out in the compressor pager or the pager doesn't exist.
+ * Note also that if the pager for an internal object
+ * has not been created, the pager is not invoked regardless of the value
+ * of MUST_ASK_PAGER().
+ *
+ * PAGED_OUT() evaluates to TRUE if the page specified by the object/offset
+ * is marked as paged out in the compressor pager.
+ * PAGED_OUT() is used to determine if a page has already been pushed
+ * into a copy object in order to avoid a redundant page out operation.
+ */
+#define MUST_ASK_PAGER(o, f, s) \
+ ((s = VM_COMPRESSOR_PAGER_STATE_GET((o), (f))) != VM_EXTERNAL_STATE_ABSENT)
+
+#define PAGED_OUT(o, f) \
+ (VM_COMPRESSOR_PAGER_STATE_GET((o), (f)) == VM_EXTERNAL_STATE_EXISTS)
+
+/*
+ * Recovery actions
+ */
+#define RELEASE_PAGE(m) \
+ MACRO_BEGIN \
+ PAGE_WAKEUP_DONE(m); \
+ if ( !VM_PAGE_PAGEABLE(m)) { \
+ vm_page_lockspin_queues(); \
+ if ( !VM_PAGE_PAGEABLE(m)) { \
+ if (VM_CONFIG_COMPRESSOR_IS_ACTIVE) \
+ vm_page_deactivate(m); \
+ else \
+ vm_page_activate(m); \
+ } \
+ vm_page_unlock_queues(); \
+ } \
+ MACRO_END
+
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF0002, (unsigned int) first_object, (unsigned int) first_offset); /* (TEST/DEBUG) */
+#endif
+
+ interruptible = fault_info->interruptible;
+ interruptible_state = thread_interrupt_level(interruptible);
+
+ /*
+ * INVARIANTS (through entire routine):
+ *
+ * 1) At all times, we must either have the object
+ * lock or a busy page in some object to prevent
+ * some other thread from trying to bring in
+ * the same page.
+ *
+ * Note that we cannot hold any locks during the
+ * pager access or when waiting for memory, so
+ * we use a busy page then.
+ *
+ * 2) To prevent another thread from racing us down the
+ * shadow chain and entering a new page in the top
+ * object before we do, we must keep a busy page in
+ * the top object while following the shadow chain.
+ *
+ * 3) We must increment paging_in_progress on any object
+ * for which we have a busy page before dropping
+ * the object lock
+ *
+ * 4) We leave busy pages on the pageout queues.
+ * If the pageout daemon comes across a busy page,
+ * it will remove the page from the pageout queues.
+ */
+
+ object = first_object;
+ offset = first_offset;
+ first_m = VM_PAGE_NULL;
+ access_required = fault_type;
+
+ /*
+ * default type of fault
+ */
+ my_fault = DBG_CACHE_HIT_FAULT;
+
+ while (TRUE) {
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF0003, (unsigned int) 0, (unsigned int) 0); /* (TEST/DEBUG) */
+#endif
+
+ grab_options = 0;
+#if CONFIG_SECLUDED_MEMORY
+ if (object->can_grab_secluded) {
+ grab_options |= VM_PAGE_GRAB_SECLUDED;
+ }
+#endif /* CONFIG_SECLUDED_MEMORY */
+
+ if (!object->alive) {
+ /*
+ * object is no longer valid
+ * clean up and return error
+ */
+ vm_fault_cleanup(object, first_m);
+ thread_interrupt_level(interruptible_state);
+
+ return VM_FAULT_MEMORY_ERROR;
+ }
+
+ if (!object->pager_created && object->phys_contiguous) {
+ /*
+ * A physically-contiguous object without a pager:
+ * must be a "large page" object. We do not deal
+ * with VM pages for this object.
+ */
+ caller_lookup = FALSE;
+ m = VM_PAGE_NULL;
+ goto phys_contig_object;
+ }
+
+ if (object->blocked_access) {
+ /*
+ * Access to this VM object has been blocked.
+ * Replace our "paging_in_progress" reference with
+ * a "activity_in_progress" reference and wait for
+ * access to be unblocked.
+ */
+ caller_lookup = FALSE; /* no longer valid after sleep */
+ vm_object_activity_begin(object);
+ vm_object_paging_end(object);
+ while (object->blocked_access) {
+ vm_object_sleep(object,
+ VM_OBJECT_EVENT_UNBLOCKED,
+ THREAD_UNINT);
+ }
+ vm_fault_page_blocked_access++;
+ vm_object_paging_begin(object);
+ vm_object_activity_end(object);
+ }
+
+ /*
+ * See whether the page at 'offset' is resident
+ */
+ if (caller_lookup == TRUE) {
+ /*
+ * The caller has already looked up the page
+ * and gave us the result in "result_page".
+ * We can use this for the first lookup but
+ * it loses its validity as soon as we unlock
+ * the object.
+ */
+ m = *result_page;
+ caller_lookup = FALSE; /* no longer valid after that */
+ } else {
+ m = vm_page_lookup(object, vm_object_trunc_page(offset));
+ }
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF0004, (unsigned int) m, (unsigned int) object); /* (TEST/DEBUG) */
+#endif
+ if (m != VM_PAGE_NULL) {
+ if (m->vmp_busy) {
+ /*
+ * The page is being brought in,
+ * wait for it and then retry.
+ */
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF0005, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */
+#endif
+ wait_result = PAGE_SLEEP(object, m, interruptible);
+
+ counter(c_vm_fault_page_block_busy_kernel++);
+
+ if (wait_result != THREAD_AWAKENED) {
+ vm_fault_cleanup(object, first_m);
+ thread_interrupt_level(interruptible_state);
+
+ if (wait_result == THREAD_RESTART) {
+ return VM_FAULT_RETRY;
+ } else {
+ return VM_FAULT_INTERRUPTED;
+ }
+ }
+ continue;
+ }
+ if (m->vmp_laundry) {
+ m->vmp_free_when_done = FALSE;
+
+ if (!m->vmp_cleaning) {
+ vm_pageout_steal_laundry(m, FALSE);
+ }
+ }
+ if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) {
+ /*
+ * Guard page: off limits !
+ */
+ if (fault_type == VM_PROT_NONE) {
+ /*
+ * The fault is not requesting any
+ * access to the guard page, so it must
+ * be just to wire or unwire it.
+ * Let's pretend it succeeded...
+ */
+ m->vmp_busy = TRUE;
+ *result_page = m;
+ assert(first_m == VM_PAGE_NULL);
+ *top_page = first_m;
+ if (type_of_fault) {
+ *type_of_fault = DBG_GUARD_FAULT;
+ }
+ thread_interrupt_level(interruptible_state);
+ return VM_FAULT_SUCCESS;
+ } else {
+ /*
+ * The fault requests access to the
+ * guard page: let's deny that !
+ */
+ vm_fault_cleanup(object, first_m);
+ thread_interrupt_level(interruptible_state);
+ return VM_FAULT_MEMORY_ERROR;
+ }
+ }
+
+ if (m->vmp_error) {
+ /*
+ * The page is in error, give up now.
+ */
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF0006, (unsigned int) m, (unsigned int) error_code); /* (TEST/DEBUG) */
+#endif
+ if (error_code) {
+ *error_code = KERN_MEMORY_ERROR;
+ }
+ VM_PAGE_FREE(m);
+
+ vm_fault_cleanup(object, first_m);
+ thread_interrupt_level(interruptible_state);
+
+ return VM_FAULT_MEMORY_ERROR;
+ }
+ if (m->vmp_restart) {
+ /*
+ * The pager wants us to restart
+ * at the top of the chain,
+ * typically because it has moved the
+ * page to another pager, then do so.
+ */
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF0007, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */
+#endif
+ VM_PAGE_FREE(m);
+
+ vm_fault_cleanup(object, first_m);
+ thread_interrupt_level(interruptible_state);
+
+ return VM_FAULT_RETRY;
+ }
+ if (m->vmp_absent) {
+ /*
+ * The page isn't busy, but is absent,
+ * therefore it's deemed "unavailable".
+ *
+ * Remove the non-existent page (unless it's
+ * in the top object) and move on down to the
+ * next object (if there is one).
+ */
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF0008, (unsigned int) m, (unsigned int) object->shadow); /* (TEST/DEBUG) */
+#endif
+ next_object = object->shadow;
+
+ if (next_object == VM_OBJECT_NULL) {
+ /*
+ * Absent page at bottom of shadow
+ * chain; zero fill the page we left
+ * busy in the first object, and free
+ * the absent page.
+ */
+ assert(!must_be_resident);
+
+ /*
+ * check for any conditions that prevent
+ * us from creating a new zero-fill page
+ * vm_fault_check will do all of the
+ * fault cleanup in the case of an error condition
+ * including resetting the thread_interrupt_level
+ */
+ error = vm_fault_check(object, m, first_m, interruptible_state, (type_of_fault == NULL) ? TRUE : FALSE);
+
+ if (error != VM_FAULT_SUCCESS) {
+ return error;
+ }
+
+ if (object != first_object) {
+ /*
+ * free the absent page we just found
+ */
+ VM_PAGE_FREE(m);
+
+ /*
+ * drop reference and lock on current object
+ */
+ vm_object_paging_end(object);
+ vm_object_unlock(object);
+
+ /*
+ * grab the original page we
+ * 'soldered' in place and
+ * retake lock on 'first_object'
+ */
+ m = first_m;
+ first_m = VM_PAGE_NULL;
+
+ object = first_object;
+ offset = first_offset;
+
+ vm_object_lock(object);
+ } else {
+ /*
+ * we're going to use the absent page we just found
+ * so convert it to a 'busy' page
+ */
+ m->vmp_absent = FALSE;
+ m->vmp_busy = TRUE;
+ }
+ if (fault_info->mark_zf_absent && no_zero_fill == TRUE) {
+ m->vmp_absent = TRUE;
+ }
+ /*
+ * zero-fill the page and put it on
+ * the correct paging queue
+ */
+ my_fault = vm_fault_zero_page(m, no_zero_fill);
+
+ break;
+ } else {
+ if (must_be_resident) {
+ vm_object_paging_end(object);
+ } else if (object != first_object) {
+ vm_object_paging_end(object);
+ VM_PAGE_FREE(m);
+ } else {
+ first_m = m;
+ m->vmp_absent = FALSE;
+ m->vmp_busy = TRUE;
+
+ vm_page_lockspin_queues();
+ vm_page_queues_remove(m, FALSE);
+ vm_page_unlock_queues();
+ }
+
+ offset += object->vo_shadow_offset;
+ fault_info->lo_offset += object->vo_shadow_offset;
+ fault_info->hi_offset += object->vo_shadow_offset;
+ access_required = VM_PROT_READ;
+
+ vm_object_lock(next_object);
+ vm_object_unlock(object);
+ object = next_object;
+ vm_object_paging_begin(object);
+
+ /*
+ * reset to default type of fault
+ */
+ my_fault = DBG_CACHE_HIT_FAULT;
+
+ continue;
+ }
+ }
+ if ((m->vmp_cleaning)
+ && ((object != first_object) || (object->copy != VM_OBJECT_NULL))
+ && (fault_type & VM_PROT_WRITE)) {
+ /*
+ * This is a copy-on-write fault that will
+ * cause us to revoke access to this page, but
+ * this page is in the process of being cleaned
+ * in a clustered pageout. We must wait until
+ * the cleaning operation completes before
+ * revoking access to the original page,
+ * otherwise we might attempt to remove a
+ * wired mapping.
+ */
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF0009, (unsigned int) m, (unsigned int) offset); /* (TEST/DEBUG) */
+#endif
+ /*
+ * take an extra ref so that object won't die
+ */
+ vm_object_reference_locked(object);
+
+ vm_fault_cleanup(object, first_m);
+
+ counter(c_vm_fault_page_block_backoff_kernel++);
+ vm_object_lock(object);
+ assert(object->ref_count > 0);
+
+ m = vm_page_lookup(object, vm_object_trunc_page(offset));
+
+ if (m != VM_PAGE_NULL && m->vmp_cleaning) {
+ PAGE_ASSERT_WAIT(m, interruptible);
+
+ vm_object_unlock(object);
+ wait_result = thread_block(THREAD_CONTINUE_NULL);
+ vm_object_deallocate(object);
+
+ goto backoff;
+ } else {
+ vm_object_unlock(object);
+
+ vm_object_deallocate(object);
+ thread_interrupt_level(interruptible_state);
+
+ return VM_FAULT_RETRY;
+ }
+ }
+ if (type_of_fault == NULL && (m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) &&
+ !(fault_info != NULL && fault_info->stealth)) {
+ /*
+ * If we were passed a non-NULL pointer for
+ * "type_of_fault", than we came from
+ * vm_fault... we'll let it deal with
+ * this condition, since it
+ * needs to see m->vmp_speculative to correctly
+ * account the pageins, otherwise...
+ * take it off the speculative queue, we'll
+ * let the caller of vm_fault_page deal
+ * with getting it onto the correct queue
+ *
+ * If the caller specified in fault_info that
+ * it wants a "stealth" fault, we also leave
+ * the page in the speculative queue.
+ */
+ vm_page_lockspin_queues();
+ if (m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) {
+ vm_page_queues_remove(m, FALSE);
+ }
+ vm_page_unlock_queues();
+ }
+ assert(object == VM_PAGE_OBJECT(m));
+
+ if (object->code_signed) {
+ /*
+ * CODE SIGNING:
+ * We just paged in a page from a signed
+ * memory object but we don't need to
+ * validate it now. We'll validate it if
+ * when it gets mapped into a user address
+ * space for the first time or when the page
+ * gets copied to another object as a result
+ * of a copy-on-write.
+ */
+ }
+
+ /*
+ * We mark the page busy and leave it on
+ * the pageout queues. If the pageout
+ * deamon comes across it, then it will
+ * remove the page from the queue, but not the object
+ */
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF000B, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */
+#endif
+ assert(!m->vmp_busy);
+ assert(!m->vmp_absent);
+
+ m->vmp_busy = TRUE;
+ break;
+ }
+
+
+ /*
+ * we get here when there is no page present in the object at
+ * the offset we're interested in... we'll allocate a page
+ * at this point if the pager associated with
+ * this object can provide the data or we're the top object...
+ * object is locked; m == NULL
+ */
+
+ if (must_be_resident) {
+ if (fault_type == VM_PROT_NONE &&
+ object == kernel_object) {
+ /*
+ * We've been called from vm_fault_unwire()
+ * while removing a map entry that was allocated
+ * with KMA_KOBJECT and KMA_VAONLY. This page
+ * is not present and there's nothing more to
+ * do here (nothing to unwire).
+ */
+ vm_fault_cleanup(object, first_m);
+ thread_interrupt_level(interruptible_state);
+
+ return VM_FAULT_MEMORY_ERROR;
+ }
+
+ goto dont_look_for_page;
+ }
+
+ /* Don't expect to fault pages into the kernel object. */
+ assert(object != kernel_object);
+
+ data_supply = FALSE;
+
+ look_for_page = (object->pager_created && (MUST_ASK_PAGER(object, offset, external_state) == TRUE) && !data_supply);
+
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF000C, (unsigned int) look_for_page, (unsigned int) object); /* (TEST/DEBUG) */
+#endif
+ if (!look_for_page && object == first_object && !object->phys_contiguous) {
+ /*
+ * Allocate a new page for this object/offset pair as a placeholder
+ */
+ m = vm_page_grab_options(grab_options);
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF000D, (unsigned int) m, (unsigned int) object); /* (TEST/DEBUG) */
+#endif
+ if (m == VM_PAGE_NULL) {
+ vm_fault_cleanup(object, first_m);
+ thread_interrupt_level(interruptible_state);
+
+ return VM_FAULT_MEMORY_SHORTAGE;
+ }
+
+ if (fault_info && fault_info->batch_pmap_op == TRUE) {
+ vm_page_insert_internal(m, object,
+ vm_object_trunc_page(offset),
+ VM_KERN_MEMORY_NONE, FALSE, TRUE, TRUE, FALSE, NULL);
+ } else {
+ vm_page_insert(m, object, vm_object_trunc_page(offset));
+ }
+ }
+ if (look_for_page) {
+ kern_return_t rc;
+ int my_fault_type;
+
+ /*
+ * If the memory manager is not ready, we
+ * cannot make requests.
+ */
+ if (!object->pager_ready) {
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF000E, (unsigned int) 0, (unsigned int) 0); /* (TEST/DEBUG) */
+#endif
+ if (m != VM_PAGE_NULL) {
+ VM_PAGE_FREE(m);
+ }
+
+ /*
+ * take an extra ref so object won't die
+ */
+ vm_object_reference_locked(object);
+ vm_fault_cleanup(object, first_m);
+ counter(c_vm_fault_page_block_backoff_kernel++);
+
+ vm_object_lock(object);
+ assert(object->ref_count > 0);
+
+ if (!object->pager_ready) {
+ wait_result = vm_object_assert_wait(object, VM_OBJECT_EVENT_PAGER_READY, interruptible);
+
+ vm_object_unlock(object);
+ if (wait_result == THREAD_WAITING) {
+ wait_result = thread_block(THREAD_CONTINUE_NULL);
+ }
+ vm_object_deallocate(object);
+
+ goto backoff;
+ } else {
+ vm_object_unlock(object);
+ vm_object_deallocate(object);
+ thread_interrupt_level(interruptible_state);
+
+ return VM_FAULT_RETRY;
+ }
+ }
+ if (!object->internal && !object->phys_contiguous && object->paging_in_progress > vm_object_pagein_throttle) {
+ /*
+ * If there are too many outstanding page
+ * requests pending on this external object, we
+ * wait for them to be resolved now.
+ */
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF0010, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */
+#endif
+ if (m != VM_PAGE_NULL) {
+ VM_PAGE_FREE(m);
+ }
+ /*
+ * take an extra ref so object won't die
+ */
+ vm_object_reference_locked(object);
+
+ vm_fault_cleanup(object, first_m);
+
+ counter(c_vm_fault_page_block_backoff_kernel++);
+
+ vm_object_lock(object);
+ assert(object->ref_count > 0);
+
+ if (object->paging_in_progress >= vm_object_pagein_throttle) {
+ vm_object_assert_wait(object, VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS, interruptible);
+
+ vm_object_unlock(object);
+ wait_result = thread_block(THREAD_CONTINUE_NULL);
+ vm_object_deallocate(object);
+
+ goto backoff;
+ } else {
+ vm_object_unlock(object);
+ vm_object_deallocate(object);
+ thread_interrupt_level(interruptible_state);
+
+ return VM_FAULT_RETRY;
+ }
+ }
+ if (object->internal) {
+ int compressed_count_delta;
+
+ assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
+
+ if (m == VM_PAGE_NULL) {
+ /*
+ * Allocate a new page for this object/offset pair as a placeholder
+ */
+ m = vm_page_grab_options(grab_options);
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF000D, (unsigned int) m, (unsigned int) object); /* (TEST/DEBUG) */
+#endif
+ if (m == VM_PAGE_NULL) {
+ vm_fault_cleanup(object, first_m);
+ thread_interrupt_level(interruptible_state);
+
+ return VM_FAULT_MEMORY_SHORTAGE;
+ }
+
+ m->vmp_absent = TRUE;
+ if (fault_info && fault_info->batch_pmap_op == TRUE) {
+ vm_page_insert_internal(m, object, vm_object_trunc_page(offset), VM_KERN_MEMORY_NONE, FALSE, TRUE, TRUE, FALSE, NULL);
+ } else {
+ vm_page_insert(m, object, vm_object_trunc_page(offset));
+ }
+ }
+ assert(m->vmp_busy);
+
+ m->vmp_absent = TRUE;
+ pager = object->pager;
+
+ assert(object->paging_in_progress > 0);
+ vm_object_unlock(object);
+
+ rc = vm_compressor_pager_get(
+ pager,
+ offset + object->paging_offset,
+ VM_PAGE_GET_PHYS_PAGE(m),
+ &my_fault_type,
+ 0,
+ &compressed_count_delta);
+
+ if (type_of_fault == NULL) {
+ int throttle_delay;
+
+ /*
+ * we weren't called from vm_fault, so we
+ * need to apply page creation throttling
+ * do it before we re-acquire any locks
+ */
+ if (my_fault_type == DBG_COMPRESSOR_FAULT) {
+ if ((throttle_delay = vm_page_throttled(TRUE))) {
+ VM_DEBUG_EVENT(vmf_compressordelay, VMF_COMPRESSORDELAY, DBG_FUNC_NONE, throttle_delay, 0, 1, 0);
+ delay(throttle_delay);
+ }
+ }
+ }
+ vm_object_lock(object);
+ assert(object->paging_in_progress > 0);
+
+ vm_compressor_pager_count(
+ pager,
+ compressed_count_delta,
+ FALSE, /* shared_lock */
+ object);
+
+ switch (rc) {
+ case KERN_SUCCESS:
+ m->vmp_absent = FALSE;
+ m->vmp_dirty = TRUE;
+ if ((object->wimg_bits &
+ VM_WIMG_MASK) !=
+ VM_WIMG_USE_DEFAULT) {
+ /*
+ * If the page is not cacheable,
+ * we can't let its contents
+ * linger in the data cache
+ * after the decompression.
+ */
+ pmap_sync_page_attributes_phys(
+ VM_PAGE_GET_PHYS_PAGE(m));
+ } else {
+ m->vmp_written_by_kernel = TRUE;
+ }
+
+ /*
+ * If the object is purgeable, its
+ * owner's purgeable ledgers have been
+ * updated in vm_page_insert() but the
+ * page was also accounted for in a
+ * "compressed purgeable" ledger, so
+ * update that now.
+ */
+ if (((object->purgable !=
+ VM_PURGABLE_DENY) ||
+ object->vo_ledger_tag) &&
+ (object->vo_owner !=
+ NULL)) {
+ /*
+ * One less compressed
+ * purgeable/tagged page.
+ */
+ vm_object_owner_compressed_update(
+ object,
+ -1);
+ }
+
+ break;
+ case KERN_MEMORY_FAILURE:
+ m->vmp_unusual = TRUE;
+ m->vmp_error = TRUE;
+ m->vmp_absent = FALSE;
+ break;
+ case KERN_MEMORY_ERROR:
+ assert(m->vmp_absent);
+ break;
+ default:
+ panic("vm_fault_page(): unexpected "
+ "error %d from "
+ "vm_compressor_pager_get()\n",
+ rc);
+ }
+ PAGE_WAKEUP_DONE(m);
+
+ rc = KERN_SUCCESS;
+ goto data_requested;
+ }
+ my_fault_type = DBG_PAGEIN_FAULT;
+
+ if (m != VM_PAGE_NULL) {
+ VM_PAGE_FREE(m);
+ m = VM_PAGE_NULL;
+ }
+
+#if TRACEFAULTPAGE
+ dbgTrace(0xBEEF0012, (unsigned int) object, (unsigned int) 0); /* (TEST/DEBUG) */
+#endif
+
+ /*
+ * It's possible someone called vm_object_destroy while we weren't
+ * holding the object lock. If that has happened, then bail out
+ * here.
+ */
+
+ pager = object->pager;
+
+ if (pager == MEMORY_OBJECT_NULL) {
+ vm_fault_cleanup(object, first_m);
+ thread_interrupt_level(interruptible_state);
+ return VM_FAULT_MEMORY_ERROR;
+ }
+
+ /*
+ * We have an absent page in place for the faulting offset,
+ * so we can release the object lock.
+ */
+
+ if (object->object_is_shared_cache) {
+ set_thread_rwlock_boost();
+ }
+
+ vm_object_unlock(object);
+
+ /*
+ * If this object uses a copy_call strategy,
+ * and we are interested in a copy of this object
+ * (having gotten here only by following a
+ * shadow chain), then tell the memory manager
+ * via a flag added to the desired_access
+ * parameter, so that it can detect a race
+ * between our walking down the shadow chain
+ * and its pushing pages up into a copy of
+ * the object that it manages.
+ */
+ if (object->copy_strategy == MEMORY_OBJECT_COPY_CALL && object != first_object) {
+ wants_copy_flag = VM_PROT_WANTS_COPY;
+ } else {
+ wants_copy_flag = VM_PROT_NONE;
+ }
projects / apple / xnu.git / blobdiff