+ inactive_burst_count = 0;
+
+ lock_yield_check = TRUE;
+ continue;
+ }
+ if (inactive_throttled == TRUE) {
+ vps_deal_with_throttled_queues(m, &object, &vm_pageout_inactive_external_forced_reactivate_limit,
+ &delayed_unlock, &force_anonymous, page_from_bg_q);
+
+ inactive_burst_count = 0;
+
+ if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
+ VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reactivated, 1);
+ }
+
+ lock_yield_check = TRUE;
+ continue;
+ }
+
+ /*
+ * we've got a page that we can steal...
+ * eliminate all mappings and make sure
+ * we have the up-to-date modified state
+ *
+ * if we need to do a pmap_disconnect then we
+ * need to re-evaluate m->vmp_dirty since the pmap_disconnect
+ * provides the true state atomically... the
+ * page was still mapped up to the pmap_disconnect
+ * and may have been dirtied at the last microsecond
+ *
+ * Note that if 'pmapped' is FALSE then the page is not
+ * and has not been in any map, so there is no point calling
+ * pmap_disconnect(). m->vmp_dirty could have been set in anticipation
+ * of likely usage of the page.
+ */
+ if (m->vmp_pmapped == TRUE) {
+ int pmap_options;
+
+ /*
+ * Don't count this page as going into the compressor
+ * if any of these are true:
+ * 1) compressed pager isn't enabled
+ * 2) Freezer enabled device with compressed pager
+ * backend (exclusive use) i.e. most of the VM system
+ * (including vm_pageout_scan) has no knowledge of
+ * the compressor
+ * 3) This page belongs to a file and hence will not be
+ * sent into the compressor
+ */
+ if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE ||
+ object->internal == FALSE) {
+ pmap_options = 0;
+ } else if (m->vmp_dirty || m->vmp_precious) {
+ /*
+ * VM knows that this page is dirty (or
+ * precious) and needs to be compressed
+ * rather than freed.
+ * Tell the pmap layer to count this page
+ * as "compressed".
+ */
+ pmap_options = PMAP_OPTIONS_COMPRESSOR;
+ } else {
+ /*
+ * VM does not know if the page needs to
+ * be preserved but the pmap layer might tell
+ * us if any mapping has "modified" it.
+ * Let's the pmap layer to count this page
+ * as compressed if and only if it has been
+ * modified.
+ */
+ pmap_options =
+ PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED;
+ }
+ refmod_state = pmap_disconnect_options(VM_PAGE_GET_PHYS_PAGE(m),
+ pmap_options,
+ NULL);
+ if (refmod_state & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(m, FALSE);
+ }
+ }
+
+ /*
+ * reset our count of pages that have been reclaimed
+ * since the last page was 'stolen'
+ */
+ inactive_reclaim_run = 0;
+
+ /*
+ * If it's clean and not precious, we can free the page.
+ */
+ if (!m->vmp_dirty && !m->vmp_precious) {
+ vm_pageout_state.vm_pageout_inactive_clean++;
+
+ /*
+ * OK, at this point we have found a page we are going to free.
+ */
+#if CONFIG_PHANTOM_CACHE
+ if (!object->internal) {
+ vm_phantom_cache_add_ghost(m);
+ }
+#endif
+ goto reclaim_page;
+ }
+
+ /*
+ * The page may have been dirtied since the last check
+ * for a throttled target queue (which may have been skipped
+ * if the page was clean then). With the dirty page
+ * disconnected here, we can make one final check.
+ */
+ if (object->internal) {
+ if (VM_PAGE_Q_THROTTLED(iq)) {
+ inactive_throttled = TRUE;
+ }
+ } else if (VM_PAGE_Q_THROTTLED(eq)) {
+ inactive_throttled = TRUE;
+ }
+
+ if (inactive_throttled == TRUE) {
+ goto throttle_inactive;
+ }
+
+#if VM_PRESSURE_EVENTS
+#if CONFIG_JETSAM
+
+ /*
+ * If Jetsam is enabled, then the sending
+ * of memory pressure notifications is handled
+ * from the same thread that takes care of high-water
+ * and other jetsams i.e. the memorystatus_thread.
+ */
+
+#else /* CONFIG_JETSAM */
+
+ vm_pressure_response();
+
+#endif /* CONFIG_JETSAM */
+#endif /* VM_PRESSURE_EVENTS */
+
+ if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) {
+ VM_PAGEOUT_DEBUG(vm_pageout_speculative_dirty, 1);
+ }
+
+ if (object->internal) {
+ vm_pageout_vminfo.vm_pageout_inactive_dirty_internal++;
+ } else {
+ vm_pageout_vminfo.vm_pageout_inactive_dirty_external++;
+ }
+
+ /*
+ * internal pages will go to the compressor...
+ * external pages will go to the appropriate pager to be cleaned
+ * and upon completion will end up on 'vm_page_queue_cleaned' which
+ * is a preferred queue to steal from
+ */
+ vm_pageout_cluster(m);
+ inactive_burst_count = 0;
+
+ /*
+ * back to top of pageout scan loop
+ */
+ }
+}
+
+
+void
+vm_page_free_reserve(
+ int pages)
+{
+ int free_after_reserve;
+
+ if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {
+ if ((vm_page_free_reserved + pages + COMPRESSOR_FREE_RESERVED_LIMIT) >= (VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT)) {
+ vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT;
+ } else {
+ vm_page_free_reserved += (pages + COMPRESSOR_FREE_RESERVED_LIMIT);
+ }
+ } else {
+ if ((vm_page_free_reserved + pages) >= VM_PAGE_FREE_RESERVED_LIMIT) {
+ vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT;
+ } else {
+ vm_page_free_reserved += pages;
+ }
+ }
+ free_after_reserve = vm_pageout_state.vm_page_free_count_init - vm_page_free_reserved;
+
+ vm_page_free_min = vm_page_free_reserved +
+ VM_PAGE_FREE_MIN(free_after_reserve);
+
+ if (vm_page_free_min > VM_PAGE_FREE_MIN_LIMIT) {
+ vm_page_free_min = VM_PAGE_FREE_MIN_LIMIT;
+ }
+
+ vm_page_free_target = vm_page_free_reserved +
+ VM_PAGE_FREE_TARGET(free_after_reserve);
+
+ if (vm_page_free_target > VM_PAGE_FREE_TARGET_LIMIT) {
+ vm_page_free_target = VM_PAGE_FREE_TARGET_LIMIT;
+ }
+
+ if (vm_page_free_target < vm_page_free_min + 5) {
+ vm_page_free_target = vm_page_free_min + 5;
+ }
+
+ vm_page_throttle_limit = vm_page_free_target - (vm_page_free_target / 2);
+}
+
+/*
+ * vm_pageout is the high level pageout daemon.
+ */
+
+void
+vm_pageout_continue(void)
+{
+ DTRACE_VM2(pgrrun, int, 1, (uint64_t *), NULL);
+ VM_PAGEOUT_DEBUG(vm_pageout_scan_event_counter, 1);
+
+ lck_mtx_lock(&vm_page_queue_free_lock);
+ vm_pageout_running = TRUE;
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+
+ vm_pageout_scan();
+ /*
+ * we hold both the vm_page_queue_free_lock
+ * and the vm_page_queues_lock at this point
+ */
+ assert(vm_page_free_wanted == 0);
+ assert(vm_page_free_wanted_privileged == 0);
+ assert_wait((event_t) &vm_page_free_wanted, THREAD_UNINT);
+
+ vm_pageout_running = FALSE;
+#if !CONFIG_EMBEDDED
+ if (vm_pageout_waiter) {
+ vm_pageout_waiter = FALSE;
+ thread_wakeup((event_t)&vm_pageout_waiter);
+ }
+#endif /* !CONFIG_EMBEDDED */
+
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+ vm_page_unlock_queues();
+
+ counter(c_vm_pageout_block++);
+ thread_block((thread_continue_t)vm_pageout_continue);
+ /*NOTREACHED*/
+}
+
+#if !CONFIG_EMBEDDED
+kern_return_t
+vm_pageout_wait(uint64_t deadline)
+{
+ kern_return_t kr;
+
+ lck_mtx_lock(&vm_page_queue_free_lock);
+ for (kr = KERN_SUCCESS; vm_pageout_running && (KERN_SUCCESS == kr);) {
+ vm_pageout_waiter = TRUE;
+ if (THREAD_AWAKENED != lck_mtx_sleep_deadline(
+ &vm_page_queue_free_lock, LCK_SLEEP_DEFAULT,
+ (event_t) &vm_pageout_waiter, THREAD_UNINT, deadline)) {
+ kr = KERN_OPERATION_TIMED_OUT;
+ }
+ }
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+
+ return kr;
+}
+#endif /* !CONFIG_EMBEDDED */
+
+
+static void
+vm_pageout_iothread_external_continue(struct vm_pageout_queue *q)
+{
+ vm_page_t m = NULL;
+ vm_object_t object;
+ vm_object_offset_t offset;
+ memory_object_t pager;
+
+ /* On systems with a compressor, the external IO thread clears its
+ * VM privileged bit to accommodate large allocations (e.g. bulk UPL
+ * creation)
+ */
+ if (vm_pageout_state.vm_pageout_internal_iothread != THREAD_NULL) {
+ current_thread()->options &= ~TH_OPT_VMPRIV;
+ }
+
+ vm_page_lockspin_queues();
+
+ while (!vm_page_queue_empty(&q->pgo_pending)) {
+ q->pgo_busy = TRUE;
+ vm_page_queue_remove_first(&q->pgo_pending, m, vmp_pageq);
+
+ assert(m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q);
+ VM_PAGE_CHECK(m);
+ /*
+ * grab a snapshot of the object and offset this
+ * page is tabled in so that we can relookup this
+ * page after we've taken the object lock - these
+ * fields are stable while we hold the page queues lock
+ * but as soon as we drop it, there is nothing to keep
+ * this page in this object... we hold an activity_in_progress
+ * on this object which will keep it from terminating
+ */
+ object = VM_PAGE_OBJECT(m);
+ offset = m->vmp_offset;
+
+ m->vmp_q_state = VM_PAGE_NOT_ON_Q;
+ VM_PAGE_ZERO_PAGEQ_ENTRY(m);
+
+ vm_page_unlock_queues();
+
+ vm_object_lock(object);
+
+ m = vm_page_lookup(object, offset);
+
+ if (m == NULL || m->vmp_busy || m->vmp_cleaning ||
+ !m->vmp_laundry || (m->vmp_q_state != VM_PAGE_NOT_ON_Q)) {
+ /*
+ * it's either the same page that someone else has
+ * started cleaning (or it's finished cleaning or
+ * been put back on the pageout queue), or
+ * the page has been freed or we have found a
+ * new page at this offset... in all of these cases
+ * we merely need to release the activity_in_progress
+ * we took when we put the page on the pageout queue
+ */
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+
+ vm_page_lockspin_queues();
+ continue;
+ }
+ pager = object->pager;
+
+ if (pager == MEMORY_OBJECT_NULL) {
+ /*
+ * This pager has been destroyed by either
+ * memory_object_destroy or vm_object_destroy, and
+ * so there is nowhere for the page to go.
+ */
+ if (m->vmp_free_when_done) {
+ /*
+ * Just free the page... VM_PAGE_FREE takes
+ * care of cleaning up all the state...
+ * including doing the vm_pageout_throttle_up
+ */
+ VM_PAGE_FREE(m);
+ } else {
+ vm_page_lockspin_queues();
+
+ vm_pageout_throttle_up(m);
+ vm_page_activate(m);
+
+ vm_page_unlock_queues();
+
+ /*
+ * And we are done with it.
+ */
+ }
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+
+ vm_page_lockspin_queues();
+ continue;
+ }
+#if 0
+ /*
+ * we don't hold the page queue lock
+ * so this check isn't safe to make
+ */
+ VM_PAGE_CHECK(m);
+#endif
+ /*
+ * give back the activity_in_progress reference we
+ * took when we queued up this page and replace it
+ * it with a paging_in_progress reference that will
+ * also hold the paging offset from changing and
+ * prevent the object from terminating
+ */
+ vm_object_activity_end(object);
+ vm_object_paging_begin(object);
+ vm_object_unlock(object);
+
+ /*
+ * Send the data to the pager.
+ * any pageout clustering happens there
+ */
+ memory_object_data_return(pager,
+ m->vmp_offset + object->paging_offset,
+ PAGE_SIZE,
+ NULL,
+ NULL,
+ FALSE,
+ FALSE,
+ 0);
+
+ vm_object_lock(object);
+ vm_object_paging_end(object);
+ vm_object_unlock(object);
+
+ vm_pageout_io_throttle();
+
+ vm_page_lockspin_queues();
+ }
+ q->pgo_busy = FALSE;
+ q->pgo_idle = TRUE;
+
+ assert_wait((event_t) &q->pgo_pending, THREAD_UNINT);
+ vm_page_unlock_queues();
+
+ thread_block_parameter((thread_continue_t)vm_pageout_iothread_external_continue, (void *) q);
+ /*NOTREACHED*/
+}
+
+
+#define MAX_FREE_BATCH 32
+uint32_t vm_compressor_time_thread; /* Set via sysctl to record time accrued by
+ * this thread.
+ */
+
+
+void
+vm_pageout_iothread_internal_continue(struct cq *);
+void
+vm_pageout_iothread_internal_continue(struct cq *cq)
+{
+ struct vm_pageout_queue *q;
+ vm_page_t m = NULL;
+ boolean_t pgo_draining;
+ vm_page_t local_q;
+ int local_cnt;
+ vm_page_t local_freeq = NULL;
+ int local_freed = 0;
+ int local_batch_size;
+#if DEVELOPMENT || DEBUG
+ int ncomps = 0;
+ boolean_t marked_active = FALSE;
+#endif
+ KERNEL_DEBUG(0xe040000c | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+ q = cq->q;
+#if __AMP__
+ if (vm_compressor_ebound && (vm_pageout_state.vm_compressor_thread_count > 1)) {
+ local_batch_size = (q->pgo_maxlaundry >> 3);
+ local_batch_size = MAX(local_batch_size, 16);
+ } else {
+ local_batch_size = q->pgo_maxlaundry / (vm_pageout_state.vm_compressor_thread_count * 2);
+ }
+#else
+ local_batch_size = q->pgo_maxlaundry / (vm_pageout_state.vm_compressor_thread_count * 2);
+#endif
+
+#if RECORD_THE_COMPRESSED_DATA
+ if (q->pgo_laundry) {
+ c_compressed_record_init();
+ }
+#endif
+ while (TRUE) {
+ int pages_left_on_q = 0;
+
+ local_cnt = 0;
+ local_q = NULL;
+
+ KERNEL_DEBUG(0xe0400014 | DBG_FUNC_START, 0, 0, 0, 0, 0);
+
+ vm_page_lock_queues();
+#if DEVELOPMENT || DEBUG
+ if (marked_active == FALSE) {
+ vmct_active++;
+ vmct_state[cq->id] = VMCT_ACTIVE;
+ marked_active = TRUE;
+ if (vmct_active == 1) {
+ vm_compressor_epoch_start = mach_absolute_time();
+ }
+ }
+#endif
+ KERNEL_DEBUG(0xe0400014 | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+ KERNEL_DEBUG(0xe0400018 | DBG_FUNC_START, q->pgo_laundry, 0, 0, 0, 0);
+
+ while (!vm_page_queue_empty(&q->pgo_pending) && local_cnt < local_batch_size) {
+ vm_page_queue_remove_first(&q->pgo_pending, m, vmp_pageq);
+ assert(m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q);
+ VM_PAGE_CHECK(m);
+
+ m->vmp_q_state = VM_PAGE_NOT_ON_Q;
+ VM_PAGE_ZERO_PAGEQ_ENTRY(m);
+ m->vmp_laundry = FALSE;
+
+ m->vmp_snext = local_q;
+ local_q = m;
+ local_cnt++;
+ }
+ if (local_q == NULL) {
+ break;
+ }
+
+ q->pgo_busy = TRUE;
+
+ if ((pgo_draining = q->pgo_draining) == FALSE) {
+ vm_pageout_throttle_up_batch(q, local_cnt);
+ pages_left_on_q = q->pgo_laundry;
+ } else {
+ pages_left_on_q = q->pgo_laundry - local_cnt;
+ }
+
+ vm_page_unlock_queues();
+
+#if !RECORD_THE_COMPRESSED_DATA
+ if (pages_left_on_q >= local_batch_size && cq->id < (vm_pageout_state.vm_compressor_thread_count - 1)) {
+ thread_wakeup((event_t) ((uintptr_t)&q->pgo_pending + cq->id + 1));
+ }
+#endif
+ KERNEL_DEBUG(0xe0400018 | DBG_FUNC_END, q->pgo_laundry, 0, 0, 0, 0);
+
+ while (local_q) {
+ KERNEL_DEBUG(0xe0400024 | DBG_FUNC_START, local_cnt, 0, 0, 0, 0);
+
+ m = local_q;
+ local_q = m->vmp_snext;
+ m->vmp_snext = NULL;
+
+ if (vm_pageout_compress_page(&cq->current_chead, cq->scratch_buf, m) == KERN_SUCCESS) {
+#if DEVELOPMENT || DEBUG
+ ncomps++;
+#endif
+ KERNEL_DEBUG(0xe0400024 | DBG_FUNC_END, local_cnt, 0, 0, 0, 0);
+
+ m->vmp_snext = local_freeq;
+ local_freeq = m;
+ local_freed++;
+
+ if (local_freed >= MAX_FREE_BATCH) {
+ OSAddAtomic64(local_freed, &vm_pageout_vminfo.vm_pageout_compressions);
+
+ vm_page_free_list(local_freeq, TRUE);
+
+ local_freeq = NULL;
+ local_freed = 0;
+ }
+ }
+#if !CONFIG_JETSAM
+ while (vm_page_free_count < COMPRESSOR_FREE_RESERVED_LIMIT) {
+ kern_return_t wait_result;
+ int need_wakeup = 0;
+
+ if (local_freeq) {
+ OSAddAtomic64(local_freed, &vm_pageout_vminfo.vm_pageout_compressions);
+
+ vm_page_free_list(local_freeq, TRUE);
+ local_freeq = NULL;
+ local_freed = 0;
+
+ continue;
+ }
+ lck_mtx_lock_spin(&vm_page_queue_free_lock);
+
+ if (vm_page_free_count < COMPRESSOR_FREE_RESERVED_LIMIT) {
+ if (vm_page_free_wanted_privileged++ == 0) {
+ need_wakeup = 1;
+ }
+ wait_result = assert_wait((event_t)&vm_page_free_wanted_privileged, THREAD_UNINT);
+
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+
+ if (need_wakeup) {
+ thread_wakeup((event_t)&vm_page_free_wanted);
+ }
+
+ if (wait_result == THREAD_WAITING) {
+ thread_block(THREAD_CONTINUE_NULL);
+ }
+ } else {
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+ }
+ }
+#endif
+ }
+ if (local_freeq) {
+ OSAddAtomic64(local_freed, &vm_pageout_vminfo.vm_pageout_compressions);
+
+ vm_page_free_list(local_freeq, TRUE);
+ local_freeq = NULL;
+ local_freed = 0;
+ }
+ if (pgo_draining == TRUE) {
+ vm_page_lockspin_queues();
+ vm_pageout_throttle_up_batch(q, local_cnt);
+ vm_page_unlock_queues();
+ }
+ }
+ KERNEL_DEBUG(0xe040000c | DBG_FUNC_START, 0, 0, 0, 0, 0);
+
+ /*
+ * queue lock is held and our q is empty
+ */
+ q->pgo_busy = FALSE;
+ q->pgo_idle = TRUE;
+
+ assert_wait((event_t) ((uintptr_t)&q->pgo_pending + cq->id), THREAD_UNINT);
+#if DEVELOPMENT || DEBUG
+ if (marked_active == TRUE) {
+ vmct_active--;
+ vmct_state[cq->id] = VMCT_IDLE;
+
+ if (vmct_active == 0) {
+ vm_compressor_epoch_stop = mach_absolute_time();
+ assertf(vm_compressor_epoch_stop >= vm_compressor_epoch_start,
+ "Compressor epoch non-monotonic: 0x%llx -> 0x%llx",
+ vm_compressor_epoch_start, vm_compressor_epoch_stop);
+ /* This interval includes intervals where one or more
+ * compressor threads were pre-empted
+ */
+ vmct_stats.vmct_cthreads_total += vm_compressor_epoch_stop - vm_compressor_epoch_start;
+ }
+ }
+#endif
+ vm_page_unlock_queues();
+#if DEVELOPMENT || DEBUG
+ if (__improbable(vm_compressor_time_thread)) {
+ vmct_stats.vmct_runtimes[cq->id] = thread_get_runtime_self();
+ vmct_stats.vmct_pages[cq->id] += ncomps;
+ vmct_stats.vmct_iterations[cq->id]++;
+ if (ncomps > vmct_stats.vmct_maxpages[cq->id]) {
+ vmct_stats.vmct_maxpages[cq->id] = ncomps;
+ }
+ if (ncomps < vmct_stats.vmct_minpages[cq->id]) {
+ vmct_stats.vmct_minpages[cq->id] = ncomps;
+ }
+ }
+#endif
+
+ KERNEL_DEBUG(0xe0400018 | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+ thread_block_parameter((thread_continue_t)vm_pageout_iothread_internal_continue, (void *) cq);
+ /*NOTREACHED*/
+}
+
+
+kern_return_t
+vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m)
+{
+ vm_object_t object;
+ memory_object_t pager;
+ int compressed_count_delta;
+ kern_return_t retval;
+
+ object = VM_PAGE_OBJECT(m);
+
+ assert(!m->vmp_free_when_done);
+ assert(!m->vmp_laundry);
+
+ pager = object->pager;
+
+ if (!object->pager_initialized || pager == MEMORY_OBJECT_NULL) {
+ KERNEL_DEBUG(0xe0400010 | DBG_FUNC_START, object, pager, 0, 0, 0);
+
+ vm_object_lock(object);
+
+ /*
+ * If there is no memory object for the page, create
+ * one and hand it to the compression pager.
+ */
+
+ if (!object->pager_initialized) {
+ vm_object_collapse(object, (vm_object_offset_t) 0, TRUE);
+ }
+ if (!object->pager_initialized) {
+ vm_object_compressor_pager_create(object);
+ }
+
+ pager = object->pager;
+
+ if (!object->pager_initialized || pager == MEMORY_OBJECT_NULL) {
+ /*
+ * Still no pager for the object,
+ * or the pager has been destroyed.
+ * Reactivate the page.
+ *
+ * Should only happen if there is no
+ * compression pager
+ */
+ PAGE_WAKEUP_DONE(m);
+
+ vm_page_lockspin_queues();
+ vm_page_activate(m);
+ VM_PAGEOUT_DEBUG(vm_pageout_dirty_no_pager, 1);
+ vm_page_unlock_queues();
+
+ /*
+ * And we are done with it.
+ */
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+
+ return KERN_FAILURE;
+ }
+ vm_object_unlock(object);
+
+ KERNEL_DEBUG(0xe0400010 | DBG_FUNC_END, object, pager, 0, 0, 0);
+ }
+ assert(object->pager_initialized && pager != MEMORY_OBJECT_NULL);
+ assert(object->activity_in_progress > 0);
+
+ retval = vm_compressor_pager_put(
+ pager,
+ m->vmp_offset + object->paging_offset,
+ VM_PAGE_GET_PHYS_PAGE(m),
+ current_chead,
+ scratch_buf,
+ &compressed_count_delta);
+
+ vm_object_lock(object);
+
+ assert(object->activity_in_progress > 0);
+ assert(VM_PAGE_OBJECT(m) == object);
+ assert( !VM_PAGE_WIRED(m));
+
+ vm_compressor_pager_count(pager,
+ compressed_count_delta,
+ FALSE, /* shared_lock */
+ object);
+
+ if (retval == KERN_SUCCESS) {
+ /*
+ * If the object is purgeable, its owner's
+ * purgeable ledgers will be updated in
+ * vm_page_remove() but the page still
+ * contributes to the owner's memory footprint,
+ * so account for it as such.
+ */
+ if ((object->purgable != VM_PURGABLE_DENY ||
+ object->vo_ledger_tag) &&
+ object->vo_owner != NULL) {
+ /* one more compressed purgeable/tagged page */
+ vm_object_owner_compressed_update(object,
+ +1);
+ }
+ VM_STAT_INCR(compressions);
+
+ if (m->vmp_tabled) {
+ vm_page_remove(m, TRUE);
+ }
+ } else {
+ PAGE_WAKEUP_DONE(m);
+
+ vm_page_lockspin_queues();
+
+ vm_page_activate(m);
+ vm_pageout_vminfo.vm_compressor_failed++;
+
+ vm_page_unlock_queues();
+ }
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+
+ return retval;
+}
+
+
+static void
+vm_pageout_adjust_eq_iothrottle(struct vm_pageout_queue *eq, boolean_t req_lowpriority)
+{
+ uint32_t policy;
+
+ if (hibernate_cleaning_in_progress == TRUE) {
+ req_lowpriority = FALSE;
+ }
+
+ if (eq->pgo_inited == TRUE && eq->pgo_lowpriority != req_lowpriority) {
+ vm_page_unlock_queues();
+
+ if (req_lowpriority == TRUE) {
+ policy = THROTTLE_LEVEL_PAGEOUT_THROTTLED;
+ DTRACE_VM(laundrythrottle);
+ } else {
+ policy = THROTTLE_LEVEL_PAGEOUT_UNTHROTTLED;
+ DTRACE_VM(laundryunthrottle);
+ }
+ proc_set_thread_policy_with_tid(kernel_task, eq->pgo_tid,
+ TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy);
+
+ vm_page_lock_queues();
+ eq->pgo_lowpriority = req_lowpriority;
+ }
+}
+
+
+static void
+vm_pageout_iothread_external(void)
+{
+ thread_t self = current_thread();
+
+ self->options |= TH_OPT_VMPRIV;
+
+ DTRACE_VM2(laundrythrottle, int, 1, (uint64_t *), NULL);
+
+ proc_set_thread_policy(self, TASK_POLICY_EXTERNAL,
+ TASK_POLICY_IO, THROTTLE_LEVEL_PAGEOUT_THROTTLED);
+
+ vm_page_lock_queues();
+
+ vm_pageout_queue_external.pgo_tid = self->thread_id;
+ vm_pageout_queue_external.pgo_lowpriority = TRUE;
+ vm_pageout_queue_external.pgo_inited = TRUE;
+
+ vm_page_unlock_queues();
+
+ vm_pageout_iothread_external_continue(&vm_pageout_queue_external);
+
+ /*NOTREACHED*/
+}
+
+
+static void
+vm_pageout_iothread_internal(struct cq *cq)
+{
+ thread_t self = current_thread();
+
+ self->options |= TH_OPT_VMPRIV;
+
+ vm_page_lock_queues();
+
+ vm_pageout_queue_internal.pgo_tid = self->thread_id;
+ vm_pageout_queue_internal.pgo_lowpriority = TRUE;
+ vm_pageout_queue_internal.pgo_inited = TRUE;
+
+ vm_page_unlock_queues();
+
+ if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) {
+ thread_vm_bind_group_add();
+ }
+
+#if CONFIG_THREAD_GROUPS
+ thread_group_vm_add();
+#endif /* CONFIG_THREAD_GROUPS */
+
+#if __AMP__
+ if (vm_compressor_ebound) {
+ /*
+ * Use the soft bound option for vm_compressor to allow it to run on
+ * P-cores if E-cluster is unavailable.
+ */
+ thread_bind_cluster_type(self, 'E', true);
+ }
+#endif /* __AMP__ */
+
+ thread_set_thread_name(current_thread(), "VM_compressor");
+#if DEVELOPMENT || DEBUG
+ vmct_stats.vmct_minpages[cq->id] = INT32_MAX;
+#endif
+ vm_pageout_iothread_internal_continue(cq);
+
+ /*NOTREACHED*/
+}
+
+kern_return_t
+vm_set_buffer_cleanup_callout(boolean_t (*func)(int))
+{
+ if (OSCompareAndSwapPtr(NULL, ptrauth_nop_cast(void *, func), (void * volatile *) &consider_buffer_cache_collect)) {
+ return KERN_SUCCESS;
+ } else {
+ return KERN_FAILURE; /* Already set */
+ }
+}
+
+extern boolean_t memorystatus_manual_testing_on;
+extern unsigned int memorystatus_level;
+
+
+#if VM_PRESSURE_EVENTS
+
+boolean_t vm_pressure_events_enabled = FALSE;
+
+void
+vm_pressure_response(void)
+{
+ vm_pressure_level_t old_level = kVMPressureNormal;
+ int new_level = -1;
+ unsigned int total_pages;
+ uint64_t available_memory = 0;
+
+ if (vm_pressure_events_enabled == FALSE) {
+ return;
+ }
+
+#if CONFIG_EMBEDDED
+
+ available_memory = (uint64_t) memorystatus_available_pages;
+
+#else /* CONFIG_EMBEDDED */
+
+ available_memory = (uint64_t) AVAILABLE_NON_COMPRESSED_MEMORY;
+ memorystatus_available_pages = (uint64_t) AVAILABLE_NON_COMPRESSED_MEMORY;
+
+#endif /* CONFIG_EMBEDDED */
+
+ total_pages = (unsigned int) atop_64(max_mem);
+#if CONFIG_SECLUDED_MEMORY
+ total_pages -= vm_page_secluded_count;
+#endif /* CONFIG_SECLUDED_MEMORY */
+ memorystatus_level = (unsigned int) ((available_memory * 100) / total_pages);
+
+ if (memorystatus_manual_testing_on) {
+ return;
+ }
+
+ old_level = memorystatus_vm_pressure_level;
+
+ switch (memorystatus_vm_pressure_level) {
+ case kVMPressureNormal:
+ {
+ if (VM_PRESSURE_WARNING_TO_CRITICAL()) {
+ new_level = kVMPressureCritical;
+ } else if (VM_PRESSURE_NORMAL_TO_WARNING()) {
+ new_level = kVMPressureWarning;
+ }
+ break;
+ }
+
+ case kVMPressureWarning:
+ case kVMPressureUrgent:
+ {
+ if (VM_PRESSURE_WARNING_TO_NORMAL()) {
+ new_level = kVMPressureNormal;
+ } else if (VM_PRESSURE_WARNING_TO_CRITICAL()) {
+ new_level = kVMPressureCritical;
+ }
+ break;
+ }
+
+ case kVMPressureCritical:
+ {
+ if (VM_PRESSURE_WARNING_TO_NORMAL()) {
+ new_level = kVMPressureNormal;
+ } else if (VM_PRESSURE_CRITICAL_TO_WARNING()) {
+ new_level = kVMPressureWarning;
+ }
+ break;
+ }
+
+ default:
+ return;
+ }
+
+ if (new_level != -1) {
+ memorystatus_vm_pressure_level = (vm_pressure_level_t) new_level;
+
+ if (new_level != (int) old_level) {
+ VM_DEBUG_CONSTANT_EVENT(vm_pressure_level_change, VM_PRESSURE_LEVEL_CHANGE, DBG_FUNC_NONE,
+ new_level, old_level, 0, 0);
+ }
+
+ if ((memorystatus_vm_pressure_level != kVMPressureNormal) || (old_level != memorystatus_vm_pressure_level)) {
+ if (vm_pageout_state.vm_pressure_thread_running == FALSE) {
+ thread_wakeup(&vm_pressure_thread);
+ }
+
+ if (old_level != memorystatus_vm_pressure_level) {
+ thread_wakeup(&vm_pageout_state.vm_pressure_changed);
+ }
+ }
+ }
+}
+#endif /* VM_PRESSURE_EVENTS */
+
+/*
+ * Function called by a kernel thread to either get the current pressure level or
+ * wait until memory pressure changes from a given level.
+ */
+kern_return_t
+mach_vm_pressure_level_monitor(__unused boolean_t wait_for_pressure, __unused unsigned int *pressure_level)
+{
+#if !VM_PRESSURE_EVENTS
+
+ return KERN_FAILURE;
+
+#else /* VM_PRESSURE_EVENTS */
+
+ wait_result_t wr = 0;
+ vm_pressure_level_t old_level = memorystatus_vm_pressure_level;
+
+ if (pressure_level == NULL) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ if (*pressure_level == kVMPressureJetsam) {
+ if (!wait_for_pressure) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ lck_mtx_lock(&memorystatus_jetsam_fg_band_lock);
+ wr = assert_wait((event_t)&memorystatus_jetsam_fg_band_waiters,
+ THREAD_INTERRUPTIBLE);
+ if (wr == THREAD_WAITING) {
+ ++memorystatus_jetsam_fg_band_waiters;
+ lck_mtx_unlock(&memorystatus_jetsam_fg_band_lock);
+ wr = thread_block(THREAD_CONTINUE_NULL);
+ } else {
+ lck_mtx_unlock(&memorystatus_jetsam_fg_band_lock);
+ }
+ if (wr != THREAD_AWAKENED) {
+ return KERN_ABORTED;
+ }
+ *pressure_level = kVMPressureJetsam;
+ return KERN_SUCCESS;
+ }
+
+ if (wait_for_pressure == TRUE) {
+ while (old_level == *pressure_level) {
+ wr = assert_wait((event_t) &vm_pageout_state.vm_pressure_changed,
+ THREAD_INTERRUPTIBLE);
+ if (wr == THREAD_WAITING) {
+ wr = thread_block(THREAD_CONTINUE_NULL);
+ }
+ if (wr == THREAD_INTERRUPTED) {
+ return KERN_ABORTED;
+ }
+
+ if (wr == THREAD_AWAKENED) {
+ old_level = memorystatus_vm_pressure_level;
+ }
+ }
+ }
+
+ *pressure_level = old_level;
+ return KERN_SUCCESS;
+#endif /* VM_PRESSURE_EVENTS */
+}
+
+#if VM_PRESSURE_EVENTS
+void
+vm_pressure_thread(void)
+{
+ static boolean_t thread_initialized = FALSE;
+
+ if (thread_initialized == TRUE) {
+ vm_pageout_state.vm_pressure_thread_running = TRUE;
+ consider_vm_pressure_events();
+ vm_pageout_state.vm_pressure_thread_running = FALSE;
+ }
+
+ thread_set_thread_name(current_thread(), "VM_pressure");
+ thread_initialized = TRUE;
+ assert_wait((event_t) &vm_pressure_thread, THREAD_UNINT);
+ thread_block((thread_continue_t)vm_pressure_thread);
+}
+#endif /* VM_PRESSURE_EVENTS */
+
+
+/*
+ * called once per-second via "compute_averages"
+ */
+void
+compute_pageout_gc_throttle(__unused void *arg)
+{
+ if (vm_pageout_vminfo.vm_pageout_considered_page != vm_pageout_state.vm_pageout_considered_page_last) {
+ vm_pageout_state.vm_pageout_considered_page_last = vm_pageout_vminfo.vm_pageout_considered_page;
+
+ thread_wakeup((event_t) &vm_pageout_garbage_collect);
+ }
+}
+
+/*
+ * vm_pageout_garbage_collect can also be called when the zone allocator needs
+ * to call zone_gc on a different thread in order to trigger zone-map-exhaustion
+ * jetsams. We need to check if the zone map size is above its jetsam limit to
+ * decide if this was indeed the case.
+ *
+ * We need to do this on a different thread because of the following reasons:
+ *
+ * 1. In the case of synchronous jetsams, the leaking process can try to jetsam
+ * itself causing the system to hang. We perform synchronous jetsams if we're
+ * leaking in the VM map entries zone, so the leaking process could be doing a
+ * zalloc for a VM map entry while holding its vm_map lock, when it decides to
+ * jetsam itself. We also need the vm_map lock on the process termination path,
+ * which would now lead the dying process to deadlock against itself.
+ *
+ * 2. The jetsam path might need to allocate zone memory itself. We could try
+ * using the non-blocking variant of zalloc for this path, but we can still
+ * end up trying to do a kernel_memory_allocate when the zone maps are almost
+ * full.
+ */
+
+void
+vm_pageout_garbage_collect(int collect)
+{
+ if (collect) {
+ if (is_zone_map_nearing_exhaustion()) {
+ /*
+ * Woken up by the zone allocator for zone-map-exhaustion jetsams.
+ *
+ * Bail out after calling zone_gc (which triggers the
+ * zone-map-exhaustion jetsams). If we fall through, the subsequent
+ * operations that clear out a bunch of caches might allocate zone
+ * memory themselves (for eg. vm_map operations would need VM map
+ * entries). Since the zone map is almost full at this point, we
+ * could end up with a panic. We just need to quickly jetsam a
+ * process and exit here.
+ *
+ * It could so happen that we were woken up to relieve memory
+ * pressure and the zone map also happened to be near its limit at
+ * the time, in which case we'll skip out early. But that should be
+ * ok; if memory pressure persists, the thread will simply be woken
+ * up again.
+ */
+ consider_zone_gc(TRUE);
+ } else {
+ /* Woken up by vm_pageout_scan or compute_pageout_gc_throttle. */
+ boolean_t buf_large_zfree = FALSE;
+ boolean_t first_try = TRUE;
+
+ stack_collect();
+
+ consider_machine_collect();
+ mbuf_drain(FALSE);
+
+ do {
+ if (consider_buffer_cache_collect != NULL) {
+ buf_large_zfree = (*consider_buffer_cache_collect)(0);
+ }
+ if (first_try == TRUE || buf_large_zfree == TRUE) {
+ /*
+ * consider_zone_gc should be last, because the other operations
+ * might return memory to zones.
+ */
+ consider_zone_gc(FALSE);
+ }
+ first_try = FALSE;
+ } while (buf_large_zfree == TRUE && vm_page_free_count < vm_page_free_target);
+
+ consider_machine_adjust();
+ }
+ }
+
+ assert_wait((event_t) &vm_pageout_garbage_collect, THREAD_UNINT);
+
+ thread_block_parameter((thread_continue_t) vm_pageout_garbage_collect, (void *)1);
+ /*NOTREACHED*/
+}
+
+
+#if VM_PAGE_BUCKETS_CHECK
+#if VM_PAGE_FAKE_BUCKETS
+extern vm_map_offset_t vm_page_fake_buckets_start, vm_page_fake_buckets_end;
+#endif /* VM_PAGE_FAKE_BUCKETS */
+#endif /* VM_PAGE_BUCKETS_CHECK */
+
+
+
+void
+vm_set_restrictions(unsigned int num_cpus)
+{
+ int vm_restricted_to_single_processor = 0;
+
+ if (PE_parse_boot_argn("vm_restricted_to_single_processor", &vm_restricted_to_single_processor, sizeof(vm_restricted_to_single_processor))) {
+ kprintf("Overriding vm_restricted_to_single_processor to %d\n", vm_restricted_to_single_processor);
+ vm_pageout_state.vm_restricted_to_single_processor = (vm_restricted_to_single_processor ? TRUE : FALSE);
+ } else {
+ assert(num_cpus > 0);
+
+ if (num_cpus <= 3) {
+ /*
+ * on systems with a limited number of CPUS, bind the
+ * 4 major threads that can free memory and that tend to use
+ * a fair bit of CPU under pressured conditions to a single processor.
+ * This insures that these threads don't hog all of the available CPUs
+ * (important for camera launch), while allowing them to run independently
+ * w/r to locks... the 4 threads are
+ * vm_pageout_scan, vm_pageout_iothread_internal (compressor),
+ * vm_compressor_swap_trigger_thread (minor and major compactions),
+ * memorystatus_thread (jetsams).
+ *
+ * the first time the thread is run, it is responsible for checking the
+ * state of vm_restricted_to_single_processor, and if TRUE it calls
+ * thread_bind_master... someday this should be replaced with a group
+ * scheduling mechanism and KPI.
+ */
+ vm_pageout_state.vm_restricted_to_single_processor = TRUE;
+ } else {
+ vm_pageout_state.vm_restricted_to_single_processor = FALSE;
+ }
+ }
+}
+
+void
+vm_pageout(void)
+{
+ thread_t self = current_thread();
+ thread_t thread;
+ kern_return_t result;
+ spl_t s;
+
+ /*
+ * Set thread privileges.
+ */
+ s = splsched();
+
+ vm_pageout_scan_thread = self;
+
+#if CONFIG_VPS_DYNAMIC_PRIO
+
+ int vps_dynprio_bootarg = 0;
+
+ if (PE_parse_boot_argn("vps_dynamic_priority_enabled", &vps_dynprio_bootarg, sizeof(vps_dynprio_bootarg))) {
+ vps_dynamic_priority_enabled = (vps_dynprio_bootarg ? TRUE : FALSE);
+ kprintf("Overriding vps_dynamic_priority_enabled to %d\n", vps_dynamic_priority_enabled);
+ } else {
+ if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) {
+ vps_dynamic_priority_enabled = TRUE;
+ } else {
+ vps_dynamic_priority_enabled = FALSE;
+ }
+ }
+
+ if (vps_dynamic_priority_enabled) {
+ sched_set_kernel_thread_priority(self, MAXPRI_THROTTLE);
+ thread_set_eager_preempt(self);
+ } else {
+ sched_set_kernel_thread_priority(self, BASEPRI_VM);
+ }
+
+#else /* CONFIG_VPS_DYNAMIC_PRIO */
+
+ vps_dynamic_priority_enabled = FALSE;
+ sched_set_kernel_thread_priority(self, BASEPRI_VM);
+
+#endif /* CONFIG_VPS_DYNAMIC_PRIO */
+
+ thread_lock(self);
+ self->options |= TH_OPT_VMPRIV;
+ thread_unlock(self);
+
+ if (!self->reserved_stack) {
+ self->reserved_stack = self->kernel_stack;
+ }
+
+ if (vm_pageout_state.vm_restricted_to_single_processor == TRUE &&
+ vps_dynamic_priority_enabled == FALSE) {
+ thread_vm_bind_group_add();
+ }
+
+
+#if CONFIG_THREAD_GROUPS
+ thread_group_vm_add();
+#endif /* CONFIG_THREAD_GROUPS */
+
+#if __AMP__
+ PE_parse_boot_argn("vmpgo_pcluster", &vm_pgo_pbound, sizeof(vm_pgo_pbound));
+ if (vm_pgo_pbound) {
+ /*
+ * Use the soft bound option for vm pageout to allow it to run on
+ * E-cores if P-cluster is unavailable.
+ */
+ thread_bind_cluster_type(self, 'P', true);
+ }
+#endif /* __AMP__ */
+
+ splx(s);
+
+ thread_set_thread_name(current_thread(), "VM_pageout_scan");
+
+ /*
+ * Initialize some paging parameters.
+ */
+
+ vm_pageout_state.vm_pressure_thread_running = FALSE;
+ vm_pageout_state.vm_pressure_changed = FALSE;
+ vm_pageout_state.memorystatus_purge_on_warning = 2;
+ vm_pageout_state.memorystatus_purge_on_urgent = 5;
+ vm_pageout_state.memorystatus_purge_on_critical = 8;
+ vm_pageout_state.vm_page_speculative_q_age_ms = VM_PAGE_SPECULATIVE_Q_AGE_MS;
+ vm_pageout_state.vm_page_speculative_percentage = 5;
+ vm_pageout_state.vm_page_speculative_target = 0;
+
+ vm_pageout_state.vm_pageout_external_iothread = THREAD_NULL;
+ vm_pageout_state.vm_pageout_internal_iothread = THREAD_NULL;
+
+ vm_pageout_state.vm_pageout_swap_wait = 0;
+ vm_pageout_state.vm_pageout_idle_wait = 0;
+ vm_pageout_state.vm_pageout_empty_wait = 0;
+ vm_pageout_state.vm_pageout_burst_wait = 0;
+ vm_pageout_state.vm_pageout_deadlock_wait = 0;
+ vm_pageout_state.vm_pageout_deadlock_relief = 0;
+ vm_pageout_state.vm_pageout_burst_inactive_throttle = 0;
+
+ vm_pageout_state.vm_pageout_inactive = 0;
+ vm_pageout_state.vm_pageout_inactive_used = 0;
+ vm_pageout_state.vm_pageout_inactive_clean = 0;
+
+ vm_pageout_state.vm_memory_pressure = 0;
+ vm_pageout_state.vm_page_filecache_min = 0;
+#if CONFIG_JETSAM
+ vm_pageout_state.vm_page_filecache_min_divisor = 70;
+ vm_pageout_state.vm_page_xpmapped_min_divisor = 40;
+#else
+ vm_pageout_state.vm_page_filecache_min_divisor = 27;
+ vm_pageout_state.vm_page_xpmapped_min_divisor = 36;
+#endif
+ vm_pageout_state.vm_page_free_count_init = vm_page_free_count;
+
+ vm_pageout_state.vm_pageout_considered_page_last = 0;
+
+ if (vm_pageout_state.vm_pageout_swap_wait == 0) {
+ vm_pageout_state.vm_pageout_swap_wait = VM_PAGEOUT_SWAP_WAIT;
+ }
+
+ if (vm_pageout_state.vm_pageout_idle_wait == 0) {
+ vm_pageout_state.vm_pageout_idle_wait = VM_PAGEOUT_IDLE_WAIT;
+ }
+
+ if (vm_pageout_state.vm_pageout_burst_wait == 0) {
+ vm_pageout_state.vm_pageout_burst_wait = VM_PAGEOUT_BURST_WAIT;
+ }
+
+ if (vm_pageout_state.vm_pageout_empty_wait == 0) {
+ vm_pageout_state.vm_pageout_empty_wait = VM_PAGEOUT_EMPTY_WAIT;
+ }
+
+ if (vm_pageout_state.vm_pageout_deadlock_wait == 0) {
+ vm_pageout_state.vm_pageout_deadlock_wait = VM_PAGEOUT_DEADLOCK_WAIT;
+ }
+
+ if (vm_pageout_state.vm_pageout_deadlock_relief == 0) {
+ vm_pageout_state.vm_pageout_deadlock_relief = VM_PAGEOUT_DEADLOCK_RELIEF;
+ }
+
+ if (vm_pageout_state.vm_pageout_burst_inactive_throttle == 0) {
+ vm_pageout_state.vm_pageout_burst_inactive_throttle = VM_PAGEOUT_BURST_INACTIVE_THROTTLE;
+ }
+ /*
+ * even if we've already called vm_page_free_reserve
+ * call it again here to insure that the targets are
+ * accurately calculated (it uses vm_page_free_count_init)
+ * calling it with an arg of 0 will not change the reserve
+ * but will re-calculate free_min and free_target
+ */
+ if (vm_page_free_reserved < VM_PAGE_FREE_RESERVED(processor_count)) {
+ vm_page_free_reserve((VM_PAGE_FREE_RESERVED(processor_count)) - vm_page_free_reserved);
+ } else {
+ vm_page_free_reserve(0);
+ }
+
+
+ vm_page_queue_init(&vm_pageout_queue_external.pgo_pending);
+ vm_pageout_queue_external.pgo_maxlaundry = VM_PAGE_LAUNDRY_MAX;
+ vm_pageout_queue_external.pgo_laundry = 0;
+ vm_pageout_queue_external.pgo_idle = FALSE;
+ vm_pageout_queue_external.pgo_busy = FALSE;
+ vm_pageout_queue_external.pgo_throttled = FALSE;
+ vm_pageout_queue_external.pgo_draining = FALSE;
+ vm_pageout_queue_external.pgo_lowpriority = FALSE;
+ vm_pageout_queue_external.pgo_tid = -1;
+ vm_pageout_queue_external.pgo_inited = FALSE;
+
+ vm_page_queue_init(&vm_pageout_queue_internal.pgo_pending);
+ vm_pageout_queue_internal.pgo_maxlaundry = 0;
+ vm_pageout_queue_internal.pgo_laundry = 0;
+ vm_pageout_queue_internal.pgo_idle = FALSE;
+ vm_pageout_queue_internal.pgo_busy = FALSE;
+ vm_pageout_queue_internal.pgo_throttled = FALSE;
+ vm_pageout_queue_internal.pgo_draining = FALSE;
+ vm_pageout_queue_internal.pgo_lowpriority = FALSE;
+ vm_pageout_queue_internal.pgo_tid = -1;
+ vm_pageout_queue_internal.pgo_inited = FALSE;
+
+ /* internal pageout thread started when default pager registered first time */
+ /* external pageout and garbage collection threads started here */
+
+ result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_external, NULL,
+ BASEPRI_VM,
+ &vm_pageout_state.vm_pageout_external_iothread);
+ if (result != KERN_SUCCESS) {
+ panic("vm_pageout_iothread_external: create failed");
+ }
+ thread_set_thread_name(vm_pageout_state.vm_pageout_external_iothread, "VM_pageout_external_iothread");
+ thread_deallocate(vm_pageout_state.vm_pageout_external_iothread);
+
+ result = kernel_thread_start_priority((thread_continue_t)vm_pageout_garbage_collect, NULL,
+ BASEPRI_DEFAULT,
+ &thread);
+ if (result != KERN_SUCCESS) {
+ panic("vm_pageout_garbage_collect: create failed");
+ }
+ thread_set_thread_name(thread, "VM_pageout_garbage_collect");
+ thread_deallocate(thread);
+
+#if VM_PRESSURE_EVENTS
+ result = kernel_thread_start_priority((thread_continue_t)vm_pressure_thread, NULL,
+ BASEPRI_DEFAULT,
+ &thread);
+
+ if (result != KERN_SUCCESS) {
+ panic("vm_pressure_thread: create failed");
+ }
+
+ thread_deallocate(thread);
+#endif
+
+ vm_object_reaper_init();
+
+
+ bzero(&vm_config, sizeof(vm_config));
+
+ switch (vm_compressor_mode) {
+ case VM_PAGER_DEFAULT:
+ printf("mapping deprecated VM_PAGER_DEFAULT to VM_PAGER_COMPRESSOR_WITH_SWAP\n");
+ OS_FALLTHROUGH;
+
+ case VM_PAGER_COMPRESSOR_WITH_SWAP:
+ vm_config.compressor_is_present = TRUE;
+ vm_config.swap_is_present = TRUE;
+ vm_config.compressor_is_active = TRUE;
+ vm_config.swap_is_active = TRUE;
+ break;
+
+ case VM_PAGER_COMPRESSOR_NO_SWAP:
+ vm_config.compressor_is_present = TRUE;
+ vm_config.swap_is_present = TRUE;
+ vm_config.compressor_is_active = TRUE;
+ break;
+
+ case VM_PAGER_FREEZER_DEFAULT:
+ printf("mapping deprecated VM_PAGER_FREEZER_DEFAULT to VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP\n");
+ OS_FALLTHROUGH;
+
+ case VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP:
+ vm_config.compressor_is_present = TRUE;
+ vm_config.swap_is_present = TRUE;
+ break;
+
+ case VM_PAGER_COMPRESSOR_NO_SWAP_PLUS_FREEZER_COMPRESSOR_WITH_SWAP:
+ vm_config.compressor_is_present = TRUE;
+ vm_config.swap_is_present = TRUE;
+ vm_config.compressor_is_active = TRUE;
+ vm_config.freezer_swap_is_active = TRUE;
+ break;
+
+ case VM_PAGER_NOT_CONFIGURED:
+ break;
+
+ default:
+ printf("unknown compressor mode - %x\n", vm_compressor_mode);
+ break;
+ }
+ if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {
+ vm_compressor_pager_init();
+ }
+
+#if VM_PRESSURE_EVENTS
+ vm_pressure_events_enabled = TRUE;
+#endif /* VM_PRESSURE_EVENTS */
+
+#if CONFIG_PHANTOM_CACHE
+ vm_phantom_cache_init();
+#endif
+#if VM_PAGE_BUCKETS_CHECK
+#if VM_PAGE_FAKE_BUCKETS
+ printf("**** DEBUG: protecting fake buckets [0x%llx:0x%llx]\n",
+ (uint64_t) vm_page_fake_buckets_start,
+ (uint64_t) vm_page_fake_buckets_end);
+ pmap_protect(kernel_pmap,
+ vm_page_fake_buckets_start,
+ vm_page_fake_buckets_end,
+ VM_PROT_READ);
+// *(char *) vm_page_fake_buckets_start = 'x'; /* panic! */
+#endif /* VM_PAGE_FAKE_BUCKETS */
+#endif /* VM_PAGE_BUCKETS_CHECK */
+
+#if VM_OBJECT_TRACKING
+ vm_object_tracking_init();
+#endif /* VM_OBJECT_TRACKING */
+
+ vm_pageout_continue();
+
+ /*
+ * Unreached code!
+ *
+ * The vm_pageout_continue() call above never returns, so the code below is never
+ * executed. We take advantage of this to declare several DTrace VM related probe
+ * points that our kernel doesn't have an analog for. These are probe points that
+ * exist in Solaris and are in the DTrace documentation, so people may have written
+ * scripts that use them. Declaring the probe points here means their scripts will
+ * compile and execute which we want for portability of the scripts, but since this
+ * section of code is never reached, the probe points will simply never fire. Yes,
+ * this is basically a hack. The problem is the DTrace probe points were chosen with
+ * Solaris specific VM events in mind, not portability to different VM implementations.
+ */
+
+ DTRACE_VM2(execfree, int, 1, (uint64_t *), NULL);
+ DTRACE_VM2(execpgin, int, 1, (uint64_t *), NULL);
+ DTRACE_VM2(execpgout, int, 1, (uint64_t *), NULL);
+ DTRACE_VM2(pgswapin, int, 1, (uint64_t *), NULL);
+ DTRACE_VM2(pgswapout, int, 1, (uint64_t *), NULL);
+ DTRACE_VM2(swapin, int, 1, (uint64_t *), NULL);
+ DTRACE_VM2(swapout, int, 1, (uint64_t *), NULL);
+ /*NOTREACHED*/
+}
+
+
+
+kern_return_t
+vm_pageout_internal_start(void)
+{
+ kern_return_t result;
+ host_basic_info_data_t hinfo;
+ vm_offset_t buf, bufsize;
+
+ assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
+
+ mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
+#define BSD_HOST 1
+ host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
+
+ assert(hinfo.max_cpus > 0);
+
+#if CONFIG_EMBEDDED
+ vm_pageout_state.vm_compressor_thread_count = 1;
+#else
+ if (hinfo.max_cpus > 4) {
+ vm_pageout_state.vm_compressor_thread_count = 2;
+ } else {
+ vm_pageout_state.vm_compressor_thread_count = 1;
+ }
+#endif
+ PE_parse_boot_argn("vmcomp_threads", &vm_pageout_state.vm_compressor_thread_count,
+ sizeof(vm_pageout_state.vm_compressor_thread_count));
+
+#if __AMP__
+ PE_parse_boot_argn("vmcomp_ecluster", &vm_compressor_ebound, sizeof(vm_compressor_ebound));
+ if (vm_compressor_ebound) {
+ vm_pageout_state.vm_compressor_thread_count = 2;
+ }
+#endif
+ if (vm_pageout_state.vm_compressor_thread_count >= hinfo.max_cpus) {
+ vm_pageout_state.vm_compressor_thread_count = hinfo.max_cpus - 1;
+ }
+ if (vm_pageout_state.vm_compressor_thread_count <= 0) {
+ vm_pageout_state.vm_compressor_thread_count = 1;
+ } else if (vm_pageout_state.vm_compressor_thread_count > MAX_COMPRESSOR_THREAD_COUNT) {
+ vm_pageout_state.vm_compressor_thread_count = MAX_COMPRESSOR_THREAD_COUNT;
+ }
+
+ vm_pageout_queue_internal.pgo_maxlaundry =
+ (vm_pageout_state.vm_compressor_thread_count * 4) * VM_PAGE_LAUNDRY_MAX;
+
+ PE_parse_boot_argn("vmpgoi_maxlaundry",
+ &vm_pageout_queue_internal.pgo_maxlaundry,
+ sizeof(vm_pageout_queue_internal.pgo_maxlaundry));
+
+ bufsize = COMPRESSOR_SCRATCH_BUF_SIZE;
+ if (kernel_memory_allocate(kernel_map, &buf,
+ bufsize * vm_pageout_state.vm_compressor_thread_count,
+ 0, KMA_KOBJECT | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR)) {
+ panic("vm_pageout_internal_start: Unable to allocate %zd bytes",
+ (size_t)(bufsize * vm_pageout_state.vm_compressor_thread_count));
+ }
+
+ for (int i = 0; i < vm_pageout_state.vm_compressor_thread_count; i++) {
+ ciq[i].id = i;
+ ciq[i].q = &vm_pageout_queue_internal;
+ ciq[i].current_chead = NULL;
+ ciq[i].scratch_buf = (char *)(buf + i * bufsize);
+
+ result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal,
+ (void *)&ciq[i], BASEPRI_VM,
+ &vm_pageout_state.vm_pageout_internal_iothread);
+
+ if (result == KERN_SUCCESS) {
+ thread_deallocate(vm_pageout_state.vm_pageout_internal_iothread);
+ } else {
+ break;
+ }
+ }
+ return result;
+}
+
+#if CONFIG_IOSCHED
+/*
+ * To support I/O Expedite for compressed files we mark the upls with special flags.
+ * The way decmpfs works is that we create a big upl which marks all the pages needed to
+ * represent the compressed file as busy. We tag this upl with the flag UPL_DECMP_REQ. Decmpfs
+ * then issues smaller I/Os for compressed I/Os, deflates them and puts the data into the pages
+ * being held in the big original UPL. We mark each of these smaller UPLs with the flag
+ * UPL_DECMP_REAL_IO. Any outstanding real I/O UPL is tracked by the big req upl using the
+ * decmp_io_upl field (in the upl structure). This link is protected in the forward direction
+ * by the req upl lock (the reverse link doesnt need synch. since we never inspect this link
+ * unless the real I/O upl is being destroyed).
+ */
+
+
+static void
+upl_set_decmp_info(upl_t upl, upl_t src_upl)
+{
+ assert((src_upl->flags & UPL_DECMP_REQ) != 0);
+
+ upl_lock(src_upl);
+ if (src_upl->decmp_io_upl) {
+ /*
+ * If there is already an alive real I/O UPL, ignore this new UPL.
+ * This case should rarely happen and even if it does, it just means
+ * that we might issue a spurious expedite which the driver is expected
+ * to handle.
+ */
+ upl_unlock(src_upl);
+ return;
+ }
+ src_upl->decmp_io_upl = (void *)upl;
+ src_upl->ref_count++;
+
+ upl->flags |= UPL_DECMP_REAL_IO;
+ upl->decmp_io_upl = (void *)src_upl;
+ upl_unlock(src_upl);
+}
+#endif /* CONFIG_IOSCHED */
+
+#if UPL_DEBUG
+int upl_debug_enabled = 1;
+#else
+int upl_debug_enabled = 0;
+#endif
+
+static upl_t
+upl_create(int type, int flags, upl_size_t size)
+{
+ upl_t upl;
+ vm_size_t page_field_size = 0;
+ int upl_flags = 0;
+ vm_size_t upl_size = sizeof(struct upl);
+
+ assert(page_aligned(size));
+
+ size = round_page_32(size);
+
+ if (type & UPL_CREATE_LITE) {
+ page_field_size = (atop(size) + 7) >> 3;
+ page_field_size = (page_field_size + 3) & 0xFFFFFFFC;
+
+ upl_flags |= UPL_LITE;
+ }
+ if (type & UPL_CREATE_INTERNAL) {
+ upl_size += sizeof(struct upl_page_info) * atop(size);
+
+ upl_flags |= UPL_INTERNAL;
+ }
+ upl = (upl_t)kalloc(upl_size + page_field_size);
+
+ if (page_field_size) {
+ bzero((char *)upl + upl_size, page_field_size);
+ }
+
+ upl->flags = upl_flags | flags;
+ upl->kaddr = (vm_offset_t)0;
+ upl->u_offset = 0;
+ upl->u_size = 0;
+ upl->map_object = NULL;
+ upl->ref_count = 1;
+ upl->ext_ref_count = 0;
+ upl->highest_page = 0;
+ upl_lock_init(upl);
+ upl->vector_upl = NULL;
+ upl->associated_upl = NULL;
+ upl->upl_iodone = NULL;
+#if CONFIG_IOSCHED
+ if (type & UPL_CREATE_IO_TRACKING) {
+ upl->upl_priority = proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO);
+ }
+
+ upl->upl_reprio_info = 0;
+ upl->decmp_io_upl = 0;
+ if ((type & UPL_CREATE_INTERNAL) && (type & UPL_CREATE_EXPEDITE_SUP)) {
+ /* Only support expedite on internal UPLs */
+ thread_t curthread = current_thread();
+ upl->upl_reprio_info = (uint64_t *)kalloc(sizeof(uint64_t) * atop(size));
+ bzero(upl->upl_reprio_info, (sizeof(uint64_t) * atop(size)));
+ upl->flags |= UPL_EXPEDITE_SUPPORTED;
+ if (curthread->decmp_upl != NULL) {
+ upl_set_decmp_info(upl, curthread->decmp_upl);
+ }
+ }
+#endif
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if ((type & UPL_CREATE_IO_TRACKING) || upl_debug_enabled) {
+ upl->upl_creator = current_thread();
+ upl->uplq.next = 0;
+ upl->uplq.prev = 0;
+ upl->flags |= UPL_TRACKED_BY_OBJECT;
+ }
+#endif
+
+#if UPL_DEBUG
+ upl->ubc_alias1 = 0;
+ upl->ubc_alias2 = 0;
+
+ upl->upl_state = 0;
+ upl->upl_commit_index = 0;
+ bzero(&upl->upl_commit_records[0], sizeof(upl->upl_commit_records));
+
+ (void) OSBacktrace(&upl->upl_create_retaddr[0], UPL_DEBUG_STACK_FRAMES);
+#endif /* UPL_DEBUG */
+
+ return upl;
+}
+
+static void
+upl_destroy(upl_t upl)
+{
+ int page_field_size; /* bit field in word size buf */
+ int size;
+
+// DEBUG4K_UPL("upl %p (u_offset 0x%llx u_size 0x%llx) object %p\n", upl, (uint64_t)upl->u_offset, (uint64_t)upl->u_size, upl->map_object);
+
+ if (upl->ext_ref_count) {
+ panic("upl(%p) ext_ref_count", upl);
+ }
+
+#if CONFIG_IOSCHED
+ if ((upl->flags & UPL_DECMP_REAL_IO) && upl->decmp_io_upl) {
+ upl_t src_upl;
+ src_upl = upl->decmp_io_upl;
+ assert((src_upl->flags & UPL_DECMP_REQ) != 0);
+ upl_lock(src_upl);
+ src_upl->decmp_io_upl = NULL;
+ upl_unlock(src_upl);
+ upl_deallocate(src_upl);
+ }
+#endif /* CONFIG_IOSCHED */
+
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if (((upl->flags & UPL_TRACKED_BY_OBJECT) || upl_debug_enabled) &&
+ !(upl->flags & UPL_VECTOR)) {
+ vm_object_t object;
+
+ if (upl->flags & UPL_SHADOWED) {
+ object = upl->map_object->shadow;
+ } else {
+ object = upl->map_object;
+ }
+
+ vm_object_lock(object);
+ queue_remove(&object->uplq, upl, upl_t, uplq);
+ vm_object_activity_end(object);
+ vm_object_collapse(object, 0, TRUE);
+ vm_object_unlock(object);
+ }
+#endif
+ /*
+ * drop a reference on the map_object whether or
+ * not a pageout object is inserted
+ */
+ if (upl->flags & UPL_SHADOWED) {
+ vm_object_deallocate(upl->map_object);
+ }
+
+ if (upl->flags & UPL_DEVICE_MEMORY) {
+ size = PAGE_SIZE;
+ } else {
+ size = upl_adjusted_size(upl, PAGE_MASK);
+ }
+ page_field_size = 0;
+
+ if (upl->flags & UPL_LITE) {
+ page_field_size = ((size / PAGE_SIZE) + 7) >> 3;
+ page_field_size = (page_field_size + 3) & 0xFFFFFFFC;
+ }
+ upl_lock_destroy(upl);
+ upl->vector_upl = (vector_upl_t) 0xfeedbeef;
+
+#if CONFIG_IOSCHED
+ if (upl->flags & UPL_EXPEDITE_SUPPORTED) {
+ kfree(upl->upl_reprio_info, sizeof(uint64_t) * (size / PAGE_SIZE));
+ }
+#endif
+
+ if (upl->flags & UPL_INTERNAL) {
+ kfree(upl,
+ sizeof(struct upl) +
+ (sizeof(struct upl_page_info) * (size / PAGE_SIZE))
+ + page_field_size);
+ } else {
+ kfree(upl, sizeof(struct upl) + page_field_size);
+ }
+}
+
+void
+upl_deallocate(upl_t upl)
+{
+ upl_lock(upl);
+
+ if (--upl->ref_count == 0) {
+ if (vector_upl_is_valid(upl)) {
+ vector_upl_deallocate(upl);
+ }
+ upl_unlock(upl);
+
+ if (upl->upl_iodone) {
+ upl_callout_iodone(upl);
+ }
+
+ upl_destroy(upl);
+ } else {
+ upl_unlock(upl);
+ }
+}
+
+#if CONFIG_IOSCHED
+void
+upl_mark_decmp(upl_t upl)
+{
+ if (upl->flags & UPL_TRACKED_BY_OBJECT) {
+ upl->flags |= UPL_DECMP_REQ;
+ upl->upl_creator->decmp_upl = (void *)upl;
+ }
+}
+
+void
+upl_unmark_decmp(upl_t upl)
+{
+ if (upl && (upl->flags & UPL_DECMP_REQ)) {
+ upl->upl_creator->decmp_upl = NULL;
+ }
+}
+
+#endif /* CONFIG_IOSCHED */
+
+#define VM_PAGE_Q_BACKING_UP(q) \
+ ((q)->pgo_laundry >= (((q)->pgo_maxlaundry * 8) / 10))
+
+boolean_t must_throttle_writes(void);
+
+boolean_t
+must_throttle_writes()
+{
+ if (VM_PAGE_Q_BACKING_UP(&vm_pageout_queue_external) &&
+ vm_page_pageable_external_count > (AVAILABLE_NON_COMPRESSED_MEMORY * 6) / 10) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+#define MIN_DELAYED_WORK_CTX_ALLOCATED (16)
+#define MAX_DELAYED_WORK_CTX_ALLOCATED (512)
+
+int vm_page_delayed_work_ctx_needed = 0;
+zone_t dw_ctx_zone = ZONE_NULL;
+
+void
+vm_page_delayed_work_init_ctx(void)
+{
+ int nelems = 0, elem_size = 0;
+
+ elem_size = sizeof(struct vm_page_delayed_work_ctx);
+
+ dw_ctx_zone = zone_create_ext("delayed-work-ctx", elem_size,
+ ZC_NOGC, ZONE_ID_ANY, ^(zone_t z) {
+ zone_set_exhaustible(z, MAX_DELAYED_WORK_CTX_ALLOCATED * elem_size);
+ });
+
+ nelems = zfill(dw_ctx_zone, MIN_DELAYED_WORK_CTX_ALLOCATED);
+ if (nelems < MIN_DELAYED_WORK_CTX_ALLOCATED) {
+ printf("vm_page_delayed_work_init_ctx: Failed to preallocate minimum delayed work contexts (%d vs %d).\n", nelems, MIN_DELAYED_WORK_CTX_ALLOCATED);
+#if DEVELOPMENT || DEBUG
+ panic("Failed to preallocate minimum delayed work contexts (%d vs %d).\n", nelems, MIN_DELAYED_WORK_CTX_ALLOCATED);
+#endif /* DEVELOPMENT || DEBUG */
+ }
+}
+
+struct vm_page_delayed_work*
+vm_page_delayed_work_get_ctx(void)
+{
+ struct vm_page_delayed_work_ctx * dw_ctx = NULL;
+
+ dw_ctx = (struct vm_page_delayed_work_ctx*) zalloc_noblock(dw_ctx_zone);
+
+ if (dw_ctx) {
+ dw_ctx->delayed_owner = current_thread();
+ } else {
+ vm_page_delayed_work_ctx_needed++;
+ }
+ return dw_ctx ? dw_ctx->dwp : NULL;
+}
+
+void
+vm_page_delayed_work_finish_ctx(struct vm_page_delayed_work* dwp)
+{
+ struct vm_page_delayed_work_ctx *ldw_ctx;
+
+ ldw_ctx = (struct vm_page_delayed_work_ctx *)dwp;
+ ldw_ctx->delayed_owner = NULL;
+
+ zfree(dw_ctx_zone, ldw_ctx);
+}
+
+/*
+ * Routine: vm_object_upl_request
+ * Purpose:
+ * Cause the population of a portion of a vm_object.
+ * Depending on the nature of the request, the pages
+ * returned may be contain valid data or be uninitialized.
+ * A page list structure, listing the physical pages
+ * will be returned upon request.
+ * This function is called by the file system or any other
+ * supplier of backing store to a pager.
+ * IMPORTANT NOTE: The caller must still respect the relationship
+ * between the vm_object and its backing memory object. The
+ * caller MUST NOT substitute changes in the backing file
+ * without first doing a memory_object_lock_request on the
+ * target range unless it is know that the pages are not
+ * shared with another entity at the pager level.
+ * Copy_in_to:
+ * if a page list structure is present
+ * return the mapped physical pages, where a
+ * page is not present, return a non-initialized
+ * one. If the no_sync bit is turned on, don't
+ * call the pager unlock to synchronize with other
+ * possible copies of the page. Leave pages busy
+ * in the original object, if a page list structure
+ * was specified. When a commit of the page list
+ * pages is done, the dirty bit will be set for each one.
+ * Copy_out_from:
+ * If a page list structure is present, return
+ * all mapped pages. Where a page does not exist
+ * map a zero filled one. Leave pages busy in
+ * the original object. If a page list structure
+ * is not specified, this call is a no-op.
+ *
+ * Note: access of default pager objects has a rather interesting
+ * twist. The caller of this routine, presumably the file system
+ * page cache handling code, will never actually make a request
+ * against a default pager backed object. Only the default
+ * pager will make requests on backing store related vm_objects
+ * In this way the default pager can maintain the relationship
+ * between backing store files (abstract memory objects) and
+ * the vm_objects (cache objects), they support.
+ *
+ */
+
+__private_extern__ kern_return_t
+vm_object_upl_request(
+ vm_object_t object,
+ vm_object_offset_t offset,
+ upl_size_t size,
+ upl_t *upl_ptr,
+ upl_page_info_array_t user_page_list,
+ unsigned int *page_list_count,
+ upl_control_flags_t cntrl_flags,
+ vm_tag_t tag)
+{
+ vm_page_t dst_page = VM_PAGE_NULL;
+ vm_object_offset_t dst_offset;
+ upl_size_t xfer_size;
+ unsigned int size_in_pages;
+ boolean_t dirty;
+ boolean_t hw_dirty;
+ upl_t upl = NULL;
+ unsigned int entry;
+ vm_page_t alias_page = NULL;
+ int refmod_state = 0;
+ wpl_array_t lite_list = NULL;
+ vm_object_t last_copy_object;
+ struct vm_page_delayed_work dw_array;
+ struct vm_page_delayed_work *dwp, *dwp_start;
+ bool dwp_finish_ctx = TRUE;
+ int dw_count;
+ int dw_limit;
+ int io_tracking_flag = 0;
+ int grab_options;
+ int page_grab_count = 0;
+ ppnum_t phys_page;
+ pmap_flush_context pmap_flush_context_storage;
+ boolean_t pmap_flushes_delayed = FALSE;
+#if DEVELOPMENT || DEBUG
+ task_t task = current_task();
+#endif /* DEVELOPMENT || DEBUG */
+
+ dwp_start = dwp = NULL;
+
+ if (cntrl_flags & ~UPL_VALID_FLAGS) {
+ /*
+ * For forward compatibility's sake,
+ * reject any unknown flag.
+ */
+ return KERN_INVALID_VALUE;
+ }
+ if ((!object->internal) && (object->paging_offset != 0)) {
+ panic("vm_object_upl_request: external object with non-zero paging offset\n");
+ }
+ if (object->phys_contiguous) {
+ panic("vm_object_upl_request: contiguous object specified\n");
+ }
+
+ assertf(page_aligned(offset) && page_aligned(size),
+ "offset 0x%llx size 0x%x",
+ offset, size);
+
+ VM_DEBUG_CONSTANT_EVENT(vm_object_upl_request, VM_UPL_REQUEST, DBG_FUNC_START, size, cntrl_flags, 0, 0);
+
+ dw_count = 0;
+ dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
+ dwp_start = vm_page_delayed_work_get_ctx();
+ if (dwp_start == NULL) {
+ dwp_start = &dw_array;
+ dw_limit = 1;
+ dwp_finish_ctx = FALSE;
+ }
+
+ dwp = dwp_start;
+
+ if (size > MAX_UPL_SIZE_BYTES) {
+ size = MAX_UPL_SIZE_BYTES;
+ }
+
+ if ((cntrl_flags & UPL_SET_INTERNAL) && page_list_count != NULL) {
+ *page_list_count = MAX_UPL_SIZE_BYTES >> PAGE_SHIFT;
+ }
+
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if (object->io_tracking || upl_debug_enabled) {
+ io_tracking_flag |= UPL_CREATE_IO_TRACKING;
+ }
+#endif
+#if CONFIG_IOSCHED
+ if (object->io_tracking) {
+ io_tracking_flag |= UPL_CREATE_EXPEDITE_SUP;
+ }
+#endif
+
+ if (cntrl_flags & UPL_SET_INTERNAL) {
+ if (cntrl_flags & UPL_SET_LITE) {
+ upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE | io_tracking_flag, 0, size);
+
+ user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
+ lite_list = (wpl_array_t)
+ (((uintptr_t)user_page_list) +
+ ((size / PAGE_SIZE) * sizeof(upl_page_info_t)));
+ if (size == 0) {
+ user_page_list = NULL;
+ lite_list = NULL;
+ }
+ } else {
+ upl = upl_create(UPL_CREATE_INTERNAL | io_tracking_flag, 0, size);
+
+ user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
+ if (size == 0) {
+ user_page_list = NULL;
+ }
+ }
+ } else {
+ if (cntrl_flags & UPL_SET_LITE) {
+ upl = upl_create(UPL_CREATE_EXTERNAL | UPL_CREATE_LITE | io_tracking_flag, 0, size);
+
+ lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));
+ if (size == 0) {
+ lite_list = NULL;
+ }
+ } else {
+ upl = upl_create(UPL_CREATE_EXTERNAL | io_tracking_flag, 0, size);
+ }
+ }
+ *upl_ptr = upl;
+
+ if (user_page_list) {
+ user_page_list[0].device = FALSE;
+ }
+
+ if (cntrl_flags & UPL_SET_LITE) {
+ upl->map_object = object;
+ } else {
+ upl->map_object = vm_object_allocate(size);
+ /*
+ * No neeed to lock the new object: nobody else knows
+ * about it yet, so it's all ours so far.
+ */
+ upl->map_object->shadow = object;
+ upl->map_object->pageout = TRUE;
+ upl->map_object->can_persist = FALSE;
+ upl->map_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
+ upl->map_object->vo_shadow_offset = offset;
+ upl->map_object->wimg_bits = object->wimg_bits;
+ assertf(page_aligned(upl->map_object->vo_shadow_offset),
+ "object %p shadow_offset 0x%llx",
+ upl->map_object, upl->map_object->vo_shadow_offset);
+
+ VM_PAGE_GRAB_FICTITIOUS(alias_page);
+
+ upl->flags |= UPL_SHADOWED;
+ }
+ if (cntrl_flags & UPL_FOR_PAGEOUT) {
+ upl->flags |= UPL_PAGEOUT;
+ }
+
+ vm_object_lock(object);
+ vm_object_activity_begin(object);
+
+ grab_options = 0;
+#if CONFIG_SECLUDED_MEMORY
+ if (object->can_grab_secluded) {
+ grab_options |= VM_PAGE_GRAB_SECLUDED;
+ }
+#endif /* CONFIG_SECLUDED_MEMORY */
+
+ /*
+ * we can lock in the paging_offset once paging_in_progress is set
+ */
+ upl->u_size = size;
+ upl->u_offset = offset + object->paging_offset;
+
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if (object->io_tracking || upl_debug_enabled) {
+ vm_object_activity_begin(object);
+ queue_enter(&object->uplq, upl, upl_t, uplq);
+ }
+#endif
+ if ((cntrl_flags & UPL_WILL_MODIFY) && object->copy != VM_OBJECT_NULL) {
+ /*
+ * Honor copy-on-write obligations
+ *
+ * The caller is gathering these pages and
+ * might modify their contents. We need to
+ * make sure that the copy object has its own
+ * private copies of these pages before we let
+ * the caller modify them.
+ */
+ vm_object_update(object,
+ offset,
+ size,
+ NULL,
+ NULL,
+ FALSE, /* should_return */
+ MEMORY_OBJECT_COPY_SYNC,
+ VM_PROT_NO_CHANGE);
+
+ VM_PAGEOUT_DEBUG(upl_cow, 1);
+ VM_PAGEOUT_DEBUG(upl_cow_pages, (size >> PAGE_SHIFT));
+ }
+ /*
+ * remember which copy object we synchronized with
+ */
+ last_copy_object = object->copy;
+ entry = 0;
+
+ xfer_size = size;
+ dst_offset = offset;
+ size_in_pages = size / PAGE_SIZE;
+
+ if (vm_page_free_count > (vm_page_free_target + size_in_pages) ||
+ object->resident_page_count < ((MAX_UPL_SIZE_BYTES * 2) >> PAGE_SHIFT)) {
+ object->scan_collisions = 0;
+ }
+
+ if ((cntrl_flags & UPL_WILL_MODIFY) && must_throttle_writes() == TRUE) {
+ boolean_t isSSD = FALSE;
+
+#if CONFIG_EMBEDDED
+ isSSD = TRUE;
+#else
+ vnode_pager_get_isSSD(object->pager, &isSSD);
+#endif
+ vm_object_unlock(object);
+
+ OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages);
+
+ if (isSSD == TRUE) {
+ delay(1000 * size_in_pages);
+ } else {
+ delay(5000 * size_in_pages);
+ }
+ OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
+
+ vm_object_lock(object);
+ }
+
+ while (xfer_size) {
+ dwp->dw_mask = 0;
+
+ if ((alias_page == NULL) && !(cntrl_flags & UPL_SET_LITE)) {
+ vm_object_unlock(object);
+ VM_PAGE_GRAB_FICTITIOUS(alias_page);
+ vm_object_lock(object);
+ }
+ if (cntrl_flags & UPL_COPYOUT_FROM) {
+ upl->flags |= UPL_PAGE_SYNC_DONE;
+
+ if (((dst_page = vm_page_lookup(object, dst_offset)) == VM_PAGE_NULL) ||
+ dst_page->vmp_fictitious ||
+ dst_page->vmp_absent ||
+ dst_page->vmp_error ||
+ dst_page->vmp_cleaning ||
+ (VM_PAGE_WIRED(dst_page))) {
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = 0;
+ }
+
+ goto try_next_page;
+ }
+ phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
+
+ /*
+ * grab this up front...
+ * a high percentange of the time we're going to
+ * need the hardware modification state a bit later
+ * anyway... so we can eliminate an extra call into
+ * the pmap layer by grabbing it here and recording it
+ */
+ if (dst_page->vmp_pmapped) {
+ refmod_state = pmap_get_refmod(phys_page);
+ } else {
+ refmod_state = 0;
+ }
+
+ if ((refmod_state & VM_MEM_REFERENCED) && VM_PAGE_INACTIVE(dst_page)) {
+ /*
+ * page is on inactive list and referenced...
+ * reactivate it now... this gets it out of the
+ * way of vm_pageout_scan which would have to
+ * reactivate it upon tripping over it
+ */
+ dwp->dw_mask |= DW_vm_page_activate;
+ }
+ if (cntrl_flags & UPL_RET_ONLY_DIRTY) {
+ /*
+ * we're only asking for DIRTY pages to be returned
+ */
+ if (dst_page->vmp_laundry || !(cntrl_flags & UPL_FOR_PAGEOUT)) {
+ /*
+ * if we were the page stolen by vm_pageout_scan to be
+ * cleaned (as opposed to a buddy being clustered in
+ * or this request is not being driven by a PAGEOUT cluster
+ * then we only need to check for the page being dirty or
+ * precious to decide whether to return it
+ */
+ if (dst_page->vmp_dirty || dst_page->vmp_precious || (refmod_state & VM_MEM_MODIFIED)) {
+ goto check_busy;
+ }
+ goto dont_return;
+ }
+ /*
+ * this is a request for a PAGEOUT cluster and this page
+ * is merely along for the ride as a 'buddy'... not only
+ * does it have to be dirty to be returned, but it also
+ * can't have been referenced recently...
+ */
+ if ((hibernate_cleaning_in_progress == TRUE ||
+ (!((refmod_state & VM_MEM_REFERENCED) || dst_page->vmp_reference) ||
+ (dst_page->vmp_q_state == VM_PAGE_ON_THROTTLED_Q))) &&
+ ((refmod_state & VM_MEM_MODIFIED) || dst_page->vmp_dirty || dst_page->vmp_precious)) {
+ goto check_busy;
+ }
+dont_return:
+ /*
+ * if we reach here, we're not to return
+ * the page... go on to the next one
+ */
+ if (dst_page->vmp_laundry == TRUE) {
+ /*
+ * if we get here, the page is not 'cleaning' (filtered out above).
+ * since it has been referenced, remove it from the laundry
+ * so we don't pay the cost of an I/O to clean a page
+ * we're just going to take back
+ */
+ vm_page_lockspin_queues();
+
+ vm_pageout_steal_laundry(dst_page, TRUE);
+ vm_page_activate(dst_page);
+
+ vm_page_unlock_queues();
+ }
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = 0;
+ }
+
+ goto try_next_page;
+ }
+check_busy:
+ if (dst_page->vmp_busy) {
+ if (cntrl_flags & UPL_NOBLOCK) {
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = 0;
+ }
+ dwp->dw_mask = 0;
+
+ goto try_next_page;
+ }
+ /*
+ * someone else is playing with the
+ * page. We will have to wait.
+ */
+ PAGE_SLEEP(object, dst_page, THREAD_UNINT);
+
+ continue;
+ }
+ if (dst_page->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {
+ vm_page_lockspin_queues();
+
+ if (dst_page->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {
+ /*
+ * we've buddied up a page for a clustered pageout
+ * that has already been moved to the pageout
+ * queue by pageout_scan... we need to remove
+ * it from the queue and drop the laundry count
+ * on that queue
+ */
+ vm_pageout_throttle_up(dst_page);
+ }
+ vm_page_unlock_queues();
+ }
+ hw_dirty = refmod_state & VM_MEM_MODIFIED;
+ dirty = hw_dirty ? TRUE : dst_page->vmp_dirty;
+
+ if (phys_page > upl->highest_page) {
+ upl->highest_page = phys_page;
+ }
+
+ assert(!pmap_is_noencrypt(phys_page));
+
+ if (cntrl_flags & UPL_SET_LITE) {
+ unsigned int pg_num;
+
+ pg_num = (unsigned int) ((dst_offset - offset) / PAGE_SIZE);
+ assert(pg_num == (dst_offset - offset) / PAGE_SIZE);
+ lite_list[pg_num >> 5] |= 1U << (pg_num & 31);
+
+ if (hw_dirty) {
+ if (pmap_flushes_delayed == FALSE) {
+ pmap_flush_context_init(&pmap_flush_context_storage);
+ pmap_flushes_delayed = TRUE;
+ }
+ pmap_clear_refmod_options(phys_page,
+ VM_MEM_MODIFIED,
+ PMAP_OPTIONS_NOFLUSH | PMAP_OPTIONS_CLEAR_WRITE,
+ &pmap_flush_context_storage);
+ }
+
+ /*
+ * Mark original page as cleaning
+ * in place.
+ */
+ dst_page->vmp_cleaning = TRUE;
+ dst_page->vmp_precious = FALSE;
+ } else {
+ /*
+ * use pageclean setup, it is more
+ * convenient even for the pageout
+ * cases here
+ */
+ vm_object_lock(upl->map_object);
+ vm_pageclean_setup(dst_page, alias_page, upl->map_object, size - xfer_size);
+ vm_object_unlock(upl->map_object);
+
+ alias_page->vmp_absent = FALSE;
+ alias_page = NULL;
+ }
+ if (dirty) {
+ SET_PAGE_DIRTY(dst_page, FALSE);
+ } else {
+ dst_page->vmp_dirty = FALSE;
+ }
+
+ if (!dirty) {
+ dst_page->vmp_precious = TRUE;
+ }
+
+ if (!(cntrl_flags & UPL_CLEAN_IN_PLACE)) {
+ if (!VM_PAGE_WIRED(dst_page)) {
+ dst_page->vmp_free_when_done = TRUE;
+ }
+ }
+ } else {
+ if ((cntrl_flags & UPL_WILL_MODIFY) && object->copy != last_copy_object) {
+ /*
+ * Honor copy-on-write obligations
+ *
+ * The copy object has changed since we
+ * last synchronized for copy-on-write.
+ * Another copy object might have been
+ * inserted while we released the object's
+ * lock. Since someone could have seen the
+ * original contents of the remaining pages
+ * through that new object, we have to
+ * synchronize with it again for the remaining
+ * pages only. The previous pages are "busy"
+ * so they can not be seen through the new
+ * mapping. The new mapping will see our
+ * upcoming changes for those previous pages,
+ * but that's OK since they couldn't see what
+ * was there before. It's just a race anyway
+ * and there's no guarantee of consistency or
+ * atomicity. We just don't want new mappings
+ * to see both the *before* and *after* pages.
+ */
+ if (object->copy != VM_OBJECT_NULL) {
+ vm_object_update(
+ object,
+ dst_offset,/* current offset */
+ xfer_size, /* remaining size */
+ NULL,
+ NULL,
+ FALSE, /* should_return */
+ MEMORY_OBJECT_COPY_SYNC,
+ VM_PROT_NO_CHANGE);
+
+ VM_PAGEOUT_DEBUG(upl_cow_again, 1);
+ VM_PAGEOUT_DEBUG(upl_cow_again_pages, (xfer_size >> PAGE_SHIFT));
+ }
+ /*
+ * remember the copy object we synced with
+ */
+ last_copy_object = object->copy;
+ }
+ dst_page = vm_page_lookup(object, dst_offset);
+
+ if (dst_page != VM_PAGE_NULL) {
+ if ((cntrl_flags & UPL_RET_ONLY_ABSENT)) {
+ /*
+ * skip over pages already present in the cache
+ */
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = 0;
+ }
+
+ goto try_next_page;
+ }
+ if (dst_page->vmp_fictitious) {
+ panic("need corner case for fictitious page");
+ }
+
+ if (dst_page->vmp_busy || dst_page->vmp_cleaning) {
+ /*
+ * someone else is playing with the
+ * page. We will have to wait.
+ */
+ PAGE_SLEEP(object, dst_page, THREAD_UNINT);
+
+ continue;
+ }
+ if (dst_page->vmp_laundry) {
+ vm_pageout_steal_laundry(dst_page, FALSE);
+ }
+ } else {
+ if (object->private) {
+ /*
+ * This is a nasty wrinkle for users
+ * of upl who encounter device or
+ * private memory however, it is
+ * unavoidable, only a fault can
+ * resolve the actual backing
+ * physical page by asking the
+ * backing device.
+ */
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = 0;
+ }
+
+ goto try_next_page;
+ }
+ if (object->scan_collisions) {
+ /*
+ * the pageout_scan thread is trying to steal
+ * pages from this object, but has run into our
+ * lock... grab 2 pages from the head of the object...
+ * the first is freed on behalf of pageout_scan, the
+ * 2nd is for our own use... we use vm_object_page_grab
+ * in both cases to avoid taking pages from the free
+ * list since we are under memory pressure and our
+ * lock on this object is getting in the way of
+ * relieving it
+ */
+ dst_page = vm_object_page_grab(object);
+
+ if (dst_page != VM_PAGE_NULL) {
+ vm_page_release(dst_page,
+ FALSE);
+ }
+
+ dst_page = vm_object_page_grab(object);
+ }
+ if (dst_page == VM_PAGE_NULL) {
+ /*
+ * need to allocate a page
+ */
+ dst_page = vm_page_grab_options(grab_options);
+ if (dst_page != VM_PAGE_NULL) {
+ page_grab_count++;
+ }
+ }
+ if (dst_page == VM_PAGE_NULL) {
+ if ((cntrl_flags & (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) == (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) {
+ /*
+ * we don't want to stall waiting for pages to come onto the free list
+ * while we're already holding absent pages in this UPL
+ * the caller will deal with the empty slots
+ */
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = 0;
+ }
+
+ goto try_next_page;
+ }
+ /*
+ * no pages available... wait
+ * then try again for the same
+ * offset...
+ */
+ vm_object_unlock(object);
+
+ OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages);
+
+ VM_DEBUG_EVENT(vm_upl_page_wait, VM_UPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);
+
+ VM_PAGE_WAIT();
+ OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
+
+ VM_DEBUG_EVENT(vm_upl_page_wait, VM_UPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
+
+ vm_object_lock(object);
+
+ continue;
+ }
+ vm_page_insert(dst_page, object, dst_offset);
+
+ dst_page->vmp_absent = TRUE;
+ dst_page->vmp_busy = FALSE;
+
+ if (cntrl_flags & UPL_RET_ONLY_ABSENT) {
+ /*
+ * if UPL_RET_ONLY_ABSENT was specified,
+ * than we're definitely setting up a
+ * upl for a clustered read/pagein
+ * operation... mark the pages as clustered
+ * so upl_commit_range can put them on the
+ * speculative list
+ */
+ dst_page->vmp_clustered = TRUE;
+
+ if (!(cntrl_flags & UPL_FILE_IO)) {
+ VM_STAT_INCR(pageins);
+ }
+ }
+ }
+ phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
+
+ dst_page->vmp_overwriting = TRUE;
+
+ if (dst_page->vmp_pmapped) {
+ if (!(cntrl_flags & UPL_FILE_IO)) {
+ /*
+ * eliminate all mappings from the
+ * original object and its prodigy
+ */
+ refmod_state = pmap_disconnect(phys_page);
+ } else {
+ refmod_state = pmap_get_refmod(phys_page);
+ }
+ } else {
+ refmod_state = 0;
+ }
+
+ hw_dirty = refmod_state & VM_MEM_MODIFIED;
+ dirty = hw_dirty ? TRUE : dst_page->vmp_dirty;
+
+ if (cntrl_flags & UPL_SET_LITE) {
+ unsigned int pg_num;
+
+ pg_num = (unsigned int) ((dst_offset - offset) / PAGE_SIZE);
+ assert(pg_num == (dst_offset - offset) / PAGE_SIZE);
+ lite_list[pg_num >> 5] |= 1U << (pg_num & 31);
+
+ if (hw_dirty) {
+ pmap_clear_modify(phys_page);
+ }
+
+ /*
+ * Mark original page as cleaning
+ * in place.
+ */
+ dst_page->vmp_cleaning = TRUE;
+ dst_page->vmp_precious = FALSE;
+ } else {
+ /*
+ * use pageclean setup, it is more
+ * convenient even for the pageout
+ * cases here
+ */
+ vm_object_lock(upl->map_object);
+ vm_pageclean_setup(dst_page, alias_page, upl->map_object, size - xfer_size);
+ vm_object_unlock(upl->map_object);
+
+ alias_page->vmp_absent = FALSE;
+ alias_page = NULL;
+ }
+
+ if (cntrl_flags & UPL_REQUEST_SET_DIRTY) {
+ upl->flags &= ~UPL_CLEAR_DIRTY;
+ upl->flags |= UPL_SET_DIRTY;
+ dirty = TRUE;
+ /*
+ * Page belonging to a code-signed object is about to
+ * be written. Mark it tainted and disconnect it from
+ * all pmaps so processes have to fault it back in and
+ * deal with the tainted bit.
+ */
+ if (object->code_signed && dst_page->vmp_cs_tainted != VMP_CS_ALL_TRUE) {
+ dst_page->vmp_cs_tainted = VMP_CS_ALL_TRUE;
+ vm_page_upl_tainted++;
+ if (dst_page->vmp_pmapped) {
+ refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(dst_page));
+ if (refmod_state & VM_MEM_REFERENCED) {
+ dst_page->vmp_reference = TRUE;
+ }
+ }
+ }
+ } else if (cntrl_flags & UPL_CLEAN_IN_PLACE) {
+ /*
+ * clean in place for read implies
+ * that a write will be done on all
+ * the pages that are dirty before
+ * a upl commit is done. The caller
+ * is obligated to preserve the
+ * contents of all pages marked dirty
+ */
+ upl->flags |= UPL_CLEAR_DIRTY;
+ }
+ dst_page->vmp_dirty = dirty;
+
+ if (!dirty) {
+ dst_page->vmp_precious = TRUE;
+ }
+
+ if (!VM_PAGE_WIRED(dst_page)) {
+ /*
+ * deny access to the target page while
+ * it is being worked on
+ */
+ dst_page->vmp_busy = TRUE;
+ } else {
+ dwp->dw_mask |= DW_vm_page_wire;
+ }
+
+ /*
+ * We might be about to satisfy a fault which has been
+ * requested. So no need for the "restart" bit.
+ */
+ dst_page->vmp_restart = FALSE;
+ if (!dst_page->vmp_absent && !(cntrl_flags & UPL_WILL_MODIFY)) {
+ /*
+ * expect the page to be used
+ */
+ dwp->dw_mask |= DW_set_reference;
+ }
+ if (cntrl_flags & UPL_PRECIOUS) {
+ if (object->internal) {
+ SET_PAGE_DIRTY(dst_page, FALSE);
+ dst_page->vmp_precious = FALSE;
+ } else {
+ dst_page->vmp_precious = TRUE;
+ }
+ } else {
+ dst_page->vmp_precious = FALSE;
+ }
+ }
+ if (dst_page->vmp_busy) {
+ upl->flags |= UPL_HAS_BUSY;
+ }
+
+ if (phys_page > upl->highest_page) {
+ upl->highest_page = phys_page;
+ }
+ assert(!pmap_is_noencrypt(phys_page));
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = phys_page;
+ user_page_list[entry].free_when_done = dst_page->vmp_free_when_done;
+ user_page_list[entry].absent = dst_page->vmp_absent;
+ user_page_list[entry].dirty = dst_page->vmp_dirty;
+ user_page_list[entry].precious = dst_page->vmp_precious;
+ user_page_list[entry].device = FALSE;
+ user_page_list[entry].needed = FALSE;
+ if (dst_page->vmp_clustered == TRUE) {
+ user_page_list[entry].speculative = (dst_page->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE;
+ } else {
+ user_page_list[entry].speculative = FALSE;
+ }
+ user_page_list[entry].cs_validated = dst_page->vmp_cs_validated;
+ user_page_list[entry].cs_tainted = dst_page->vmp_cs_tainted;
+ user_page_list[entry].cs_nx = dst_page->vmp_cs_nx;
+ user_page_list[entry].mark = FALSE;
+ }
+ /*
+ * if UPL_RET_ONLY_ABSENT is set, then
+ * we are working with a fresh page and we've
+ * just set the clustered flag on it to
+ * indicate that it was drug in as part of a
+ * speculative cluster... so leave it alone
+ */
+ if (!(cntrl_flags & UPL_RET_ONLY_ABSENT)) {
+ /*
+ * someone is explicitly grabbing this page...
+ * update clustered and speculative state
+ *
+ */
+ if (dst_page->vmp_clustered) {
+ VM_PAGE_CONSUME_CLUSTERED(dst_page);
+ }
+ }
+try_next_page:
+ if (dwp->dw_mask) {
+ if (dwp->dw_mask & DW_vm_page_activate) {
+ VM_STAT_INCR(reactivations);
+ }
+
+ VM_PAGE_ADD_DELAYED_WORK(dwp, dst_page, dw_count);
+
+ if (dw_count >= dw_limit) {
+ vm_page_do_delayed_work(object, tag, dwp_start, dw_count);
+
+ dwp = dwp_start;
+ dw_count = 0;
+ }
+ }
+ entry++;
+ dst_offset += PAGE_SIZE_64;
+ xfer_size -= PAGE_SIZE;
+ }
+ if (dw_count) {
+ vm_page_do_delayed_work(object, tag, dwp_start, dw_count);
+ dwp = dwp_start;
+ dw_count = 0;
+ }
+
+ if (alias_page != NULL) {
+ VM_PAGE_FREE(alias_page);
+ }
+ if (pmap_flushes_delayed == TRUE) {
+ pmap_flush(&pmap_flush_context_storage);
+ }
+
+ if (page_list_count != NULL) {
+ if (upl->flags & UPL_INTERNAL) {
+ *page_list_count = 0;
+ } else if (*page_list_count > entry) {
+ *page_list_count = entry;
+ }
+ }
+#if UPL_DEBUG
+ upl->upl_state = 1;
+#endif
+ vm_object_unlock(object);
+
+ VM_DEBUG_CONSTANT_EVENT(vm_object_upl_request, VM_UPL_REQUEST, DBG_FUNC_END, page_grab_count, 0, 0, 0);
+#if DEVELOPMENT || DEBUG
+ if (task != NULL) {
+ ledger_credit(task->ledger, task_ledgers.pages_grabbed_upl, page_grab_count);
+ }
+#endif /* DEVELOPMENT || DEBUG */
+
+ if (dwp_start && dwp_finish_ctx) {
+ vm_page_delayed_work_finish_ctx(dwp_start);
+ dwp_start = dwp = NULL;
+ }
+
+ return KERN_SUCCESS;
+}
+
+/*
+ * Routine: vm_object_super_upl_request
+ * Purpose:
+ * Cause the population of a portion of a vm_object
+ * in much the same way as memory_object_upl_request.
+ * Depending on the nature of the request, the pages
+ * returned may be contain valid data or be uninitialized.
+ * However, the region may be expanded up to the super
+ * cluster size provided.
+ */
+
+__private_extern__ kern_return_t
+vm_object_super_upl_request(
+ vm_object_t object,
+ vm_object_offset_t offset,
+ upl_size_t size,
+ upl_size_t super_cluster,
+ upl_t *upl,
+ upl_page_info_t *user_page_list,
+ unsigned int *page_list_count,
+ upl_control_flags_t cntrl_flags,
+ vm_tag_t tag)
+{
+ if (object->paging_offset > offset || ((cntrl_flags & UPL_VECTOR) == UPL_VECTOR)) {
+ return KERN_FAILURE;
+ }
+
+ assert(object->paging_in_progress);
+ offset = offset - object->paging_offset;
+
+ if (super_cluster > size) {
+ vm_object_offset_t base_offset;
+ upl_size_t super_size;
+ vm_object_size_t super_size_64;
+
+ base_offset = (offset & ~((vm_object_offset_t) super_cluster - 1));
+ super_size = (offset + size) > (base_offset + super_cluster) ? super_cluster << 1 : super_cluster;
+ super_size_64 = ((base_offset + super_size) > object->vo_size) ? (object->vo_size - base_offset) : super_size;
+ super_size = (upl_size_t) super_size_64;
+ assert(super_size == super_size_64);
+
+ if (offset > (base_offset + super_size)) {
+ panic("vm_object_super_upl_request: Missed target pageout"
+ " %#llx,%#llx, %#x, %#x, %#x, %#llx\n",
+ offset, base_offset, super_size, super_cluster,
+ size, object->paging_offset);
+ }
+ /*
+ * apparently there is a case where the vm requests a
+ * page to be written out who's offset is beyond the
+ * object size
+ */
+ if ((offset + size) > (base_offset + super_size)) {
+ super_size_64 = (offset + size) - base_offset;
+ super_size = (upl_size_t) super_size_64;
+ assert(super_size == super_size_64);
+ }
+
+ offset = base_offset;
+ size = super_size;
+ }
+ return vm_object_upl_request(object, offset, size, upl, user_page_list, page_list_count, cntrl_flags, tag);
+}
+
+int cs_executable_create_upl = 0;
+extern int proc_selfpid(void);
+extern char *proc_name_address(void *p);
+
+kern_return_t
+vm_map_create_upl(
+ vm_map_t map,
+ vm_map_address_t offset,
+ upl_size_t *upl_size,
+ upl_t *upl,
+ upl_page_info_array_t page_list,
+ unsigned int *count,
+ upl_control_flags_t *flags,
+ vm_tag_t tag)
+{
+ vm_map_entry_t entry;
+ upl_control_flags_t caller_flags;
+ int force_data_sync;
+ int sync_cow_data;
+ vm_object_t local_object;
+ vm_map_offset_t local_offset;
+ vm_map_offset_t local_start;
+ kern_return_t ret;
+ vm_map_address_t original_offset;
+ vm_map_size_t original_size, adjusted_size;
+ vm_map_offset_t local_entry_start;
+ vm_object_offset_t local_entry_offset;
+ vm_object_offset_t offset_in_mapped_page;
+ boolean_t release_map = FALSE;
+
+start_with_map:
+
+ original_offset = offset;
+ original_size = *upl_size;
+ adjusted_size = original_size;
+
+ caller_flags = *flags;
+
+ if (caller_flags & ~UPL_VALID_FLAGS) {
+ /*
+ * For forward compatibility's sake,
+ * reject any unknown flag.
+ */
+ ret = KERN_INVALID_VALUE;
+ goto done;
+ }
+ force_data_sync = (caller_flags & UPL_FORCE_DATA_SYNC);
+ sync_cow_data = !(caller_flags & UPL_COPYOUT_FROM);
+
+ if (upl == NULL) {
+ ret = KERN_INVALID_ARGUMENT;
+ goto done;
+ }
+
+REDISCOVER_ENTRY:
+ vm_map_lock_read(map);
+
+ if (!vm_map_lookup_entry(map, offset, &entry)) {
+ vm_map_unlock_read(map);
+ ret = KERN_FAILURE;
+ goto done;
+ }
+
+ local_entry_start = entry->vme_start;
+ local_entry_offset = VME_OFFSET(entry);
+
+ if (VM_MAP_PAGE_SHIFT(map) < PAGE_SHIFT) {
+ DEBUG4K_UPL("map %p (%d) offset 0x%llx size 0x%x flags 0x%llx\n", map, VM_MAP_PAGE_SHIFT(map), (uint64_t)offset, *upl_size, *flags);
+ }
+
+ if (entry->vme_end - original_offset < adjusted_size) {
+ adjusted_size = entry->vme_end - original_offset;
+ assert(adjusted_size > 0);
+ *upl_size = (upl_size_t) adjusted_size;
+ assert(*upl_size == adjusted_size);
+ }
+
+ if (caller_flags & UPL_QUERY_OBJECT_TYPE) {
+ *flags = 0;
+
+ if (!entry->is_sub_map &&
+ VME_OBJECT(entry) != VM_OBJECT_NULL) {
+ if (VME_OBJECT(entry)->private) {
+ *flags = UPL_DEV_MEMORY;
+ }
+
+ if (VME_OBJECT(entry)->phys_contiguous) {
+ *flags |= UPL_PHYS_CONTIG;
+ }
+ }
+ vm_map_unlock_read(map);
+ ret = KERN_SUCCESS;
+ goto done;
+ }
+
+ offset_in_mapped_page = 0;
+ if (VM_MAP_PAGE_SIZE(map) < PAGE_SIZE) {
+ offset = vm_map_trunc_page(original_offset, VM_MAP_PAGE_MASK(map));
+ *upl_size = (upl_size_t)
+ (vm_map_round_page(original_offset + adjusted_size,
+ VM_MAP_PAGE_MASK(map))
+ - offset);
+
+ offset_in_mapped_page = original_offset - offset;
+ assert(offset_in_mapped_page < VM_MAP_PAGE_SIZE(map));
+
+ DEBUG4K_UPL("map %p (%d) offset 0x%llx size 0x%llx flags 0x%llx -> offset 0x%llx adjusted_size 0x%llx *upl_size 0x%x offset_in_mapped_page 0x%llx\n", map, VM_MAP_PAGE_SHIFT(map), (uint64_t)original_offset, (uint64_t)original_size, *flags, (uint64_t)offset, (uint64_t)adjusted_size, *upl_size, offset_in_mapped_page);
+ }
+
+ if (VME_OBJECT(entry) == VM_OBJECT_NULL ||
+ !VME_OBJECT(entry)->phys_contiguous) {
+ if (*upl_size > MAX_UPL_SIZE_BYTES) {
+ *upl_size = MAX_UPL_SIZE_BYTES;
+ }
+ }
+
+ /*
+ * Create an object if necessary.
+ */
+ if (VME_OBJECT(entry) == VM_OBJECT_NULL) {
+ if (vm_map_lock_read_to_write(map)) {
+ goto REDISCOVER_ENTRY;
+ }
+
+ VME_OBJECT_SET(entry,
+ vm_object_allocate((vm_size_t)
+ vm_object_round_page((entry->vme_end - entry->vme_start))));
+ VME_OFFSET_SET(entry, 0);
+ assert(entry->use_pmap);
+
+ vm_map_lock_write_to_read(map);
+ }
+
+ if (!(caller_flags & UPL_COPYOUT_FROM) &&
+ !entry->is_sub_map &&
+ !(entry->protection & VM_PROT_WRITE)) {
+ vm_map_unlock_read(map);
+ ret = KERN_PROTECTION_FAILURE;
+ goto done;
+ }
+
+#if CONFIG_EMBEDDED
+ if (map->pmap != kernel_pmap &&
+ (caller_flags & UPL_COPYOUT_FROM) &&
+ (entry->protection & VM_PROT_EXECUTE) &&
+ !(entry->protection & VM_PROT_WRITE)) {
+ vm_offset_t kaddr;
+ vm_size_t ksize;
+
+ /*
+ * We're about to create a read-only UPL backed by
+ * memory from an executable mapping.
+ * Wiring the pages would result in the pages being copied
+ * (due to the "MAP_PRIVATE" mapping) and no longer
+ * code-signed, so no longer eligible for execution.
+ * Instead, let's copy the data into a kernel buffer and
+ * create the UPL from this kernel buffer.
+ * The kernel buffer is then freed, leaving the UPL holding
+ * the last reference on the VM object, so the memory will
+ * be released when the UPL is committed.
+ */
+
+ vm_map_unlock_read(map);
+ entry = VM_MAP_ENTRY_NULL;
+ /* allocate kernel buffer */
+ ksize = round_page(*upl_size);
+ kaddr = 0;
+ ret = kmem_alloc_pageable(kernel_map,
+ &kaddr,
+ ksize,
+ tag);
+ if (ret == KERN_SUCCESS) {
+ /* copyin the user data */
+ ret = copyinmap(map, offset, (void *)kaddr, *upl_size);
+ }
+ if (ret == KERN_SUCCESS) {
+ if (ksize > *upl_size) {
+ /* zero out the extra space in kernel buffer */
+ memset((void *)(kaddr + *upl_size),
+ 0,
+ ksize - *upl_size);
+ }
+ /* create the UPL from the kernel buffer */
+ vm_object_offset_t offset_in_object;
+ vm_object_offset_t offset_in_object_page;
+
+ offset_in_object = offset - local_entry_start + local_entry_offset;
+ offset_in_object_page = offset_in_object - vm_object_trunc_page(offset_in_object);
+ assert(offset_in_object_page < PAGE_SIZE);
+ assert(offset_in_object_page + offset_in_mapped_page < PAGE_SIZE);
+ *upl_size -= offset_in_object_page + offset_in_mapped_page;
+ ret = vm_map_create_upl(kernel_map,
+ (vm_map_address_t)(kaddr + offset_in_object_page + offset_in_mapped_page),
+ upl_size, upl, page_list, count, flags, tag);
+ }
+ if (kaddr != 0) {
+ /* free the kernel buffer */
+ kmem_free(kernel_map, kaddr, ksize);
+ kaddr = 0;
+ ksize = 0;
+ }
+#if DEVELOPMENT || DEBUG
+ DTRACE_VM4(create_upl_from_executable,
+ vm_map_t, map,
+ vm_map_address_t, offset,
+ upl_size_t, *upl_size,
+ kern_return_t, ret);
+#endif /* DEVELOPMENT || DEBUG */
+ goto done;
+ }
+#endif /* CONFIG_EMBEDDED */
+
+ local_object = VME_OBJECT(entry);
+ assert(local_object != VM_OBJECT_NULL);
+
+ if (!entry->is_sub_map &&
+ !entry->needs_copy &&
+ *upl_size != 0 &&
+ local_object->vo_size > *upl_size && /* partial UPL */
+ entry->wired_count == 0 && /* No COW for entries that are wired */
+ (map->pmap != kernel_pmap) && /* alias checks */
+ (vm_map_entry_should_cow_for_true_share(entry) /* case 1 */
+ ||
+ ( /* case 2 */
+ local_object->internal &&
+ (local_object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) &&
+ local_object->ref_count > 1))) {
+ vm_prot_t prot;
+
+ /*
+ * Case 1:
+ * Set up the targeted range for copy-on-write to avoid
+ * applying true_share/copy_delay to the entire object.
+ *
+ * Case 2:
+ * This map entry covers only part of an internal
+ * object. There could be other map entries covering
+ * other areas of this object and some of these map
+ * entries could be marked as "needs_copy", which
+ * assumes that the object is COPY_SYMMETRIC.
+ * To avoid marking this object as COPY_DELAY and
+ * "true_share", let's shadow it and mark the new
+ * (smaller) object as "true_share" and COPY_DELAY.
+ */
+
+ if (vm_map_lock_read_to_write(map)) {
+ goto REDISCOVER_ENTRY;
+ }
+ vm_map_lock_assert_exclusive(map);
+ assert(VME_OBJECT(entry) == local_object);
+
+ vm_map_clip_start(map,
+ entry,
+ vm_map_trunc_page(offset,
+ VM_MAP_PAGE_MASK(map)));
+ vm_map_clip_end(map,
+ entry,
+ vm_map_round_page(offset + *upl_size,
+ VM_MAP_PAGE_MASK(map)));
+ if ((entry->vme_end - offset) < *upl_size) {
+ *upl_size = (upl_size_t) (entry->vme_end - offset);
+ assert(*upl_size == entry->vme_end - offset);
+ }
+
+ prot = entry->protection & ~VM_PROT_WRITE;
+ if (override_nx(map, VME_ALIAS(entry)) && prot) {
+ prot |= VM_PROT_EXECUTE;
+ }
+ vm_object_pmap_protect(local_object,
+ VME_OFFSET(entry),
+ entry->vme_end - entry->vme_start,
+ ((entry->is_shared ||
+ map->mapped_in_other_pmaps)
+ ? PMAP_NULL
+ : map->pmap),
+ VM_MAP_PAGE_SIZE(map),
+ entry->vme_start,
+ prot);
+
+ assert(entry->wired_count == 0);
+
+ /*
+ * Lock the VM object and re-check its status: if it's mapped
+ * in another address space, we could still be racing with
+ * another thread holding that other VM map exclusively.
+ */
+ vm_object_lock(local_object);
+ if (local_object->true_share) {
+ /* object is already in proper state: no COW needed */
+ assert(local_object->copy_strategy !=
+ MEMORY_OBJECT_COPY_SYMMETRIC);
+ } else {
+ /* not true_share: ask for copy-on-write below */
+ assert(local_object->copy_strategy ==
+ MEMORY_OBJECT_COPY_SYMMETRIC);
+ entry->needs_copy = TRUE;
+ }
+ vm_object_unlock(local_object);
+
+ vm_map_lock_write_to_read(map);
+ }
+
+ if (entry->needs_copy) {
+ /*
+ * Honor copy-on-write for COPY_SYMMETRIC
+ * strategy.
+ */
+ vm_map_t local_map;
+ vm_object_t object;
+ vm_object_offset_t new_offset;
+ vm_prot_t prot;
+ boolean_t wired;
+ vm_map_version_t version;
+ vm_map_t real_map;
+ vm_prot_t fault_type;
+
+ if (entry->vme_start < VM_MAP_TRUNC_PAGE(offset, VM_MAP_PAGE_MASK(map)) ||
+ entry->vme_end > VM_MAP_ROUND_PAGE(offset + *upl_size, VM_MAP_PAGE_MASK(map))) {
+ /*
+ * Clip the requested range first to minimize the
+ * amount of potential copying...
+ */
+ if (vm_map_lock_read_to_write(map)) {
+ goto REDISCOVER_ENTRY;
+ }
+ vm_map_lock_assert_exclusive(map);
+ assert(VME_OBJECT(entry) == local_object);
+ vm_map_clip_start(map, entry,
+ VM_MAP_TRUNC_PAGE(offset, VM_MAP_PAGE_MASK(map)));
+ vm_map_clip_end(map, entry,
+ VM_MAP_ROUND_PAGE(offset + *upl_size, VM_MAP_PAGE_MASK(map)));
+ vm_map_lock_write_to_read(map);
+ }
+
+ local_map = map;
+
+ if (caller_flags & UPL_COPYOUT_FROM) {
+ fault_type = VM_PROT_READ | VM_PROT_COPY;
+ vm_counters.create_upl_extra_cow++;
+ vm_counters.create_upl_extra_cow_pages +=
+ (entry->vme_end - entry->vme_start) / PAGE_SIZE;
+ } else {
+ fault_type = VM_PROT_WRITE;
+ }
+ if (vm_map_lookup_locked(&local_map,
+ offset, fault_type,
+ OBJECT_LOCK_EXCLUSIVE,
+ &version, &object,
+ &new_offset, &prot, &wired,
+ NULL,
+ &real_map, NULL) != KERN_SUCCESS) {
+ if (fault_type == VM_PROT_WRITE) {
+ vm_counters.create_upl_lookup_failure_write++;
+ } else {
+ vm_counters.create_upl_lookup_failure_copy++;
+ }
+ vm_map_unlock_read(local_map);
+ ret = KERN_FAILURE;
+ goto done;
+ }
+ if (real_map != local_map) {
+ vm_map_unlock(real_map);
+ }
+ vm_map_unlock_read(local_map);
+
+ vm_object_unlock(object);
+
+ goto REDISCOVER_ENTRY;
+ }
+
+ if (entry->is_sub_map) {
+ vm_map_t submap;
+
+ submap = VME_SUBMAP(entry);
+ local_start = entry->vme_start;
+ local_offset = (vm_map_offset_t)VME_OFFSET(entry);
+
+ vm_map_reference(submap);
+ vm_map_unlock_read(map);
+
+ DEBUG4K_UPL("map %p offset 0x%llx (0x%llx) size 0x%x (adjusted 0x%llx original 0x%llx) offset_in_mapped_page 0x%llx submap %p\n", map, (uint64_t)offset, (uint64_t)original_offset, *upl_size, (uint64_t)adjusted_size, (uint64_t)original_size, offset_in_mapped_page, submap);
+ offset += offset_in_mapped_page;
+ *upl_size -= offset_in_mapped_page;
+
+ if (release_map) {
+ vm_map_deallocate(map);
+ }
+ map = submap;
+ release_map = TRUE;
+ offset = local_offset + (offset - local_start);
+ goto start_with_map;
+ }
+
+ if (sync_cow_data &&
+ (VME_OBJECT(entry)->shadow ||
+ VME_OBJECT(entry)->copy)) {
+ local_object = VME_OBJECT(entry);
+ local_start = entry->vme_start;
+ local_offset = (vm_map_offset_t)VME_OFFSET(entry);
+
+ vm_object_reference(local_object);
+ vm_map_unlock_read(map);
+
+ if (local_object->shadow && local_object->copy) {
+ vm_object_lock_request(local_object->shadow,
+ ((vm_object_offset_t)
+ ((offset - local_start) +
+ local_offset) +
+ local_object->vo_shadow_offset),
+ *upl_size, FALSE,
+ MEMORY_OBJECT_DATA_SYNC,
+ VM_PROT_NO_CHANGE);
+ }
+ sync_cow_data = FALSE;
+ vm_object_deallocate(local_object);
+
+ goto REDISCOVER_ENTRY;
+ }
+ if (force_data_sync) {
+ local_object = VME_OBJECT(entry);
+ local_start = entry->vme_start;
+ local_offset = (vm_map_offset_t)VME_OFFSET(entry);
+
+ vm_object_reference(local_object);
+ vm_map_unlock_read(map);
+
+ vm_object_lock_request(local_object,
+ ((vm_object_offset_t)
+ ((offset - local_start) +
+ local_offset)),
+ (vm_object_size_t)*upl_size,
+ FALSE,
+ MEMORY_OBJECT_DATA_SYNC,
+ VM_PROT_NO_CHANGE);
+
+ force_data_sync = FALSE;
+ vm_object_deallocate(local_object);
+
+ goto REDISCOVER_ENTRY;
+ }
+ if (VME_OBJECT(entry)->private) {
+ *flags = UPL_DEV_MEMORY;
+ } else {
+ *flags = 0;
+ }
+
+ if (VME_OBJECT(entry)->phys_contiguous) {
+ *flags |= UPL_PHYS_CONTIG;
+ }
+
+ local_object = VME_OBJECT(entry);
+ local_offset = (vm_map_offset_t)VME_OFFSET(entry);
+ local_start = entry->vme_start;
+
+ /*
+ * Wiring will copy the pages to the shadow object.
+ * The shadow object will not be code-signed so
+ * attempting to execute code from these copied pages
+ * would trigger a code-signing violation.
+ */
+ if (entry->protection & VM_PROT_EXECUTE) {
+#if MACH_ASSERT
+ printf("pid %d[%s] create_upl out of executable range from "
+ "0x%llx to 0x%llx: side effects may include "
+ "code-signing violations later on\n",
+ proc_selfpid(),
+ (current_task()->bsd_info
+ ? proc_name_address(current_task()->bsd_info)
+ : "?"),
+ (uint64_t) entry->vme_start,
+ (uint64_t) entry->vme_end);
+#endif /* MACH_ASSERT */
+ DTRACE_VM2(cs_executable_create_upl,
+ uint64_t, (uint64_t)entry->vme_start,
+ uint64_t, (uint64_t)entry->vme_end);
+ cs_executable_create_upl++;
+ }
+
+ vm_object_lock(local_object);
+
+ /*
+ * Ensure that this object is "true_share" and "copy_delay" now,
+ * while we're still holding the VM map lock. After we unlock the map,
+ * anything could happen to that mapping, including some copy-on-write
+ * activity. We need to make sure that the IOPL will point at the
+ * same memory as the mapping.
+ */
+ if (local_object->true_share) {
+ assert(local_object->copy_strategy !=
+ MEMORY_OBJECT_COPY_SYMMETRIC);
+ } else if (local_object != kernel_object &&
+ local_object != compressor_object &&
+ !local_object->phys_contiguous) {
+#if VM_OBJECT_TRACKING_OP_TRUESHARE
+ if (!local_object->true_share &&
+ vm_object_tracking_inited) {
+ void *bt[VM_OBJECT_TRACKING_BTDEPTH];
+ int num = 0;
+ num = OSBacktrace(bt,
+ VM_OBJECT_TRACKING_BTDEPTH);
+ btlog_add_entry(vm_object_tracking_btlog,
+ local_object,
+ VM_OBJECT_TRACKING_OP_TRUESHARE,
+ bt,
+ num);
+ }
+#endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
+ local_object->true_share = TRUE;
+ if (local_object->copy_strategy ==
+ MEMORY_OBJECT_COPY_SYMMETRIC) {
+ local_object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
+ }
+ }
+
+ vm_object_reference_locked(local_object);
+ vm_object_unlock(local_object);
+
+ vm_map_unlock_read(map);
+
+ offset += offset_in_mapped_page;
+ assert(*upl_size > offset_in_mapped_page);
+ *upl_size -= offset_in_mapped_page;
+
+ ret = vm_object_iopl_request(local_object,
+ ((vm_object_offset_t)
+ ((offset - local_start) + local_offset)),
+ *upl_size,
+ upl,
+ page_list,
+ count,
+ caller_flags,
+ tag);
+ vm_object_deallocate(local_object);
+
+done:
+ if (release_map) {
+ vm_map_deallocate(map);
+ }
+
+ return ret;
+}
+
+/*
+ * Internal routine to enter a UPL into a VM map.
+ *
+ * JMM - This should just be doable through the standard
+ * vm_map_enter() API.
+ */
+kern_return_t
+vm_map_enter_upl(
+ vm_map_t map,
+ upl_t upl,
+ vm_map_offset_t *dst_addr)
+{
+ vm_map_size_t size;
+ vm_object_offset_t offset;
+ vm_map_offset_t addr;
+ vm_page_t m;
+ kern_return_t kr;
+ int isVectorUPL = 0, curr_upl = 0;
+ upl_t vector_upl = NULL;
+ vm_offset_t vector_upl_dst_addr = 0;
+ vm_map_t vector_upl_submap = NULL;
+ upl_offset_t subupl_offset = 0;
+ upl_size_t subupl_size = 0;
+
+ if (upl == UPL_NULL) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ DEBUG4K_UPL("map %p upl %p flags 0x%x object %p offset 0x%llx size 0x%x \n", map, upl, upl->flags, upl->map_object, upl->u_offset, upl->u_size);
+ assert(map == kernel_map);
+
+ if ((isVectorUPL = vector_upl_is_valid(upl))) {
+ int mapped = 0, valid_upls = 0;
+ vector_upl = upl;
+
+ upl_lock(vector_upl);
+ for (curr_upl = 0; curr_upl < MAX_VECTOR_UPL_ELEMENTS; curr_upl++) {
+ upl = vector_upl_subupl_byindex(vector_upl, curr_upl );
+ if (upl == NULL) {
+ continue;
+ }
+ valid_upls++;
+ if (UPL_PAGE_LIST_MAPPED & upl->flags) {
+ mapped++;
+ }
+ }
+
+ if (mapped) {
+ if (mapped != valid_upls) {
+ panic("Only %d of the %d sub-upls within the Vector UPL are alread mapped\n", mapped, valid_upls);
+ } else {
+ upl_unlock(vector_upl);
+ return KERN_FAILURE;
+ }
+ }
+
+ if (VM_MAP_PAGE_MASK(map) < PAGE_MASK) {
+ panic("TODO4K: vector UPL not implemented");
+ }
+
+ kr = kmem_suballoc(map, &vector_upl_dst_addr,
+ vector_upl->u_size,
+ FALSE,
+ VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_NONE,
+ &vector_upl_submap);
+ if (kr != KERN_SUCCESS) {
+ panic("Vector UPL submap allocation failed\n");
+ }
+ map = vector_upl_submap;
+ vector_upl_set_submap(vector_upl, vector_upl_submap, vector_upl_dst_addr);
+ curr_upl = 0;
+ } else {
+ upl_lock(upl);
+ }
+
+process_upl_to_enter:
+ if (isVectorUPL) {
+ if (curr_upl == MAX_VECTOR_UPL_ELEMENTS) {
+ *dst_addr = vector_upl_dst_addr;
+ upl_unlock(vector_upl);
+ return KERN_SUCCESS;
+ }
+ upl = vector_upl_subupl_byindex(vector_upl, curr_upl++ );
+ if (upl == NULL) {
+ goto process_upl_to_enter;
+ }
+
+ vector_upl_get_iostate(vector_upl, upl, &subupl_offset, &subupl_size);
+ *dst_addr = (vm_map_offset_t)(vector_upl_dst_addr + (vm_map_offset_t)subupl_offset);
+ } else {
+ /*
+ * check to see if already mapped
+ */
+ if (UPL_PAGE_LIST_MAPPED & upl->flags) {
+ upl_unlock(upl);
+ return KERN_FAILURE;
+ }
+ }
+
+ size = upl_adjusted_size(upl, VM_MAP_PAGE_MASK(map));
+
+ if ((!(upl->flags & UPL_SHADOWED)) &&
+ ((upl->flags & UPL_HAS_BUSY) ||
+ !((upl->flags & (UPL_DEVICE_MEMORY | UPL_IO_WIRE)) || (upl->map_object->phys_contiguous)))) {
+ vm_object_t object;
+ vm_page_t alias_page;
+ vm_object_offset_t new_offset;
+ unsigned int pg_num;
+ wpl_array_t lite_list;
+
+ if (upl->flags & UPL_INTERNAL) {
+ lite_list = (wpl_array_t)
+ ((((uintptr_t)upl) + sizeof(struct upl))
+ + ((size / PAGE_SIZE) * sizeof(upl_page_info_t)));
+ } else {
+ lite_list = (wpl_array_t)(((uintptr_t)upl) + sizeof(struct upl));
+ }
+ object = upl->map_object;
+ upl->map_object = vm_object_allocate(vm_object_round_page(size));
+
+ vm_object_lock(upl->map_object);
+
+ upl->map_object->shadow = object;
+ upl->map_object->pageout = TRUE;
+ upl->map_object->can_persist = FALSE;
+ upl->map_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
+ upl->map_object->vo_shadow_offset = upl_adjusted_offset(upl, PAGE_MASK) - object->paging_offset;
+ assertf(page_aligned(upl->map_object->vo_shadow_offset),
+ "object %p shadow_offset 0x%llx",
+ upl->map_object,
+ (uint64_t)upl->map_object->vo_shadow_offset);
+ upl->map_object->wimg_bits = object->wimg_bits;
+ assertf(page_aligned(upl->map_object->vo_shadow_offset),
+ "object %p shadow_offset 0x%llx",
+ upl->map_object, upl->map_object->vo_shadow_offset);
+ offset = upl->map_object->vo_shadow_offset;
+ new_offset = 0;
+ size = upl_adjusted_size(upl, VM_MAP_PAGE_MASK(map));
+
+ upl->flags |= UPL_SHADOWED;
+
+ while (size) {
+ pg_num = (unsigned int) (new_offset / PAGE_SIZE);
+ assert(pg_num == new_offset / PAGE_SIZE);
+
+ if (lite_list[pg_num >> 5] & (1U << (pg_num & 31))) {
+ VM_PAGE_GRAB_FICTITIOUS(alias_page);
+
+ vm_object_lock(object);
+
+ m = vm_page_lookup(object, offset);
+ if (m == VM_PAGE_NULL) {
+ panic("vm_upl_map: page missing\n");
+ }
+
+ /*
+ * Convert the fictitious page to a private
+ * shadow of the real page.
+ */
+ assert(alias_page->vmp_fictitious);
+ alias_page->vmp_fictitious = FALSE;
+ alias_page->vmp_private = TRUE;
+ alias_page->vmp_free_when_done = TRUE;
+ /*
+ * since m is a page in the upl it must
+ * already be wired or BUSY, so it's
+ * safe to assign the underlying physical
+ * page to the alias
+ */
+ VM_PAGE_SET_PHYS_PAGE(alias_page, VM_PAGE_GET_PHYS_PAGE(m));
+
+ vm_object_unlock(object);
+
+ vm_page_lockspin_queues();
+ vm_page_wire(alias_page, VM_KERN_MEMORY_NONE, TRUE);
+ vm_page_unlock_queues();
+
+ vm_page_insert_wired(alias_page, upl->map_object, new_offset, VM_KERN_MEMORY_NONE);
+
+ assert(!alias_page->vmp_wanted);
+ alias_page->vmp_busy = FALSE;
+ alias_page->vmp_absent = FALSE;
+ }
+ size -= PAGE_SIZE;
+ offset += PAGE_SIZE_64;
+ new_offset += PAGE_SIZE_64;
+ }
+ vm_object_unlock(upl->map_object);
+ }
+ if (upl->flags & UPL_SHADOWED) {
+ offset = 0;
+ } else {
+ offset = upl_adjusted_offset(upl, VM_MAP_PAGE_MASK(map)) - upl->map_object->paging_offset;
+ }
+
+ size = upl_adjusted_size(upl, VM_MAP_PAGE_MASK(map));
+
+ vm_object_reference(upl->map_object);
+
+ if (!isVectorUPL) {
+ *dst_addr = 0;
+ /*
+ * NEED A UPL_MAP ALIAS
+ */
+ kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,
+ VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_OSFMK,
+ upl->map_object, offset, FALSE,
+ VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
+
+ if (kr != KERN_SUCCESS) {
+ vm_object_deallocate(upl->map_object);
+ upl_unlock(upl);
+ return kr;
+ }
+ } else {
+ kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,
+ VM_FLAGS_FIXED, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_OSFMK,
+ upl->map_object, offset, FALSE,
+ VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
+ if (kr) {
+ panic("vm_map_enter failed for a Vector UPL\n");
+ }
+ }
+ vm_object_lock(upl->map_object);
+
+ for (addr = *dst_addr; size > 0; size -= PAGE_SIZE, addr += PAGE_SIZE) {
+ m = vm_page_lookup(upl->map_object, offset);
+
+ if (m) {
+ m->vmp_pmapped = TRUE;
+
+ /* CODE SIGNING ENFORCEMENT: page has been wpmapped,
+ * but only in kernel space. If this was on a user map,
+ * we'd have to set the wpmapped bit. */
+ /* m->vmp_wpmapped = TRUE; */
+ assert(map->pmap == kernel_pmap);
+
+ PMAP_ENTER(map->pmap, addr, m, VM_PROT_DEFAULT, VM_PROT_NONE, 0, TRUE, kr);
+
+ assert(kr == KERN_SUCCESS);
+#if KASAN
+ kasan_notify_address(addr, PAGE_SIZE_64);
+#endif
+ }
+ offset += PAGE_SIZE_64;
+ }
+ vm_object_unlock(upl->map_object);
+
+ /*
+ * hold a reference for the mapping
+ */
+ upl->ref_count++;
+ upl->flags |= UPL_PAGE_LIST_MAPPED;
+ upl->kaddr = (vm_offset_t) *dst_addr;
+ assert(upl->kaddr == *dst_addr);
+
+ if (isVectorUPL) {
+ goto process_upl_to_enter;
+ }
+
+ if (!isVectorUPL) {
+ vm_map_offset_t addr_adjustment;
+
+ addr_adjustment = (vm_map_offset_t)(upl->u_offset - upl_adjusted_offset(upl, VM_MAP_PAGE_MASK(map)));
+ if (addr_adjustment) {
+ assert(VM_MAP_PAGE_MASK(map) != PAGE_MASK);
+ DEBUG4K_UPL("dst_addr 0x%llx (+ 0x%llx) -> 0x%llx\n", (uint64_t)*dst_addr, (uint64_t)addr_adjustment, (uint64_t)(*dst_addr + addr_adjustment));
+ *dst_addr += addr_adjustment;
+ }
+ }
+
+ upl_unlock(upl);
+
+ return KERN_SUCCESS;
+}
+
+/*
+ * Internal routine to remove a UPL mapping from a VM map.
+ *
+ * XXX - This should just be doable through a standard
+ * vm_map_remove() operation. Otherwise, implicit clean-up
+ * of the target map won't be able to correctly remove
+ * these (and release the reference on the UPL). Having
+ * to do this means we can't map these into user-space
+ * maps yet.
+ */
+kern_return_t
+vm_map_remove_upl(
+ vm_map_t map,
+ upl_t upl)
+{
+ vm_address_t addr;
+ upl_size_t size;
+ int isVectorUPL = 0, curr_upl = 0;
+ upl_t vector_upl = NULL;
+
+ if (upl == UPL_NULL) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ if ((isVectorUPL = vector_upl_is_valid(upl))) {
+ int unmapped = 0, valid_upls = 0;
+ vector_upl = upl;
+ upl_lock(vector_upl);
+ for (curr_upl = 0; curr_upl < MAX_VECTOR_UPL_ELEMENTS; curr_upl++) {
+ upl = vector_upl_subupl_byindex(vector_upl, curr_upl );
+ if (upl == NULL) {
+ continue;
+ }
+ valid_upls++;
+ if (!(UPL_PAGE_LIST_MAPPED & upl->flags)) {
+ unmapped++;
+ }
+ }
+
+ if (unmapped) {
+ if (unmapped != valid_upls) {
+ panic("%d of the %d sub-upls within the Vector UPL is/are not mapped\n", unmapped, valid_upls);
+ } else {
+ upl_unlock(vector_upl);
+ return KERN_FAILURE;
+ }
+ }
+ curr_upl = 0;
+ } else {
+ upl_lock(upl);
+ }
+
+process_upl_to_remove:
+ if (isVectorUPL) {
+ if (curr_upl == MAX_VECTOR_UPL_ELEMENTS) {
+ vm_map_t v_upl_submap;
+ vm_offset_t v_upl_submap_dst_addr;
+ vector_upl_get_submap(vector_upl, &v_upl_submap, &v_upl_submap_dst_addr);
+
+ vm_map_remove(map, v_upl_submap_dst_addr,
+ v_upl_submap_dst_addr + vector_upl->u_size,
+ VM_MAP_REMOVE_NO_FLAGS);
+ vm_map_deallocate(v_upl_submap);
+ upl_unlock(vector_upl);
+ return KERN_SUCCESS;
+ }
+
+ upl = vector_upl_subupl_byindex(vector_upl, curr_upl++ );
+ if (upl == NULL) {
+ goto process_upl_to_remove;
+ }
+ }
+
+ if (upl->flags & UPL_PAGE_LIST_MAPPED) {
+ addr = upl->kaddr;
+ size = upl_adjusted_size(upl, VM_MAP_PAGE_MASK(map));
+
+ assert(upl->ref_count > 1);
+ upl->ref_count--; /* removing mapping ref */
+
+ upl->flags &= ~UPL_PAGE_LIST_MAPPED;
+ upl->kaddr = (vm_offset_t) 0;
+
+ if (!isVectorUPL) {
+ upl_unlock(upl);
+
+ vm_map_remove(
+ map,
+ vm_map_trunc_page(addr,
+ VM_MAP_PAGE_MASK(map)),
+ vm_map_round_page(addr + size,
+ VM_MAP_PAGE_MASK(map)),
+ VM_MAP_REMOVE_NO_FLAGS);
+ return KERN_SUCCESS;
+ } else {
+ /*
+ * If it's a Vectored UPL, we'll be removing the entire
+ * submap anyways, so no need to remove individual UPL
+ * element mappings from within the submap
+ */
+ goto process_upl_to_remove;
+ }
+ }
+ upl_unlock(upl);
+
+ return KERN_FAILURE;
+}
+
+
+kern_return_t
+upl_commit_range(
+ upl_t upl,
+ upl_offset_t offset,
+ upl_size_t size,
+ int flags,
+ upl_page_info_t *page_list,
+ mach_msg_type_number_t count,
+ boolean_t *empty)
+{
+ upl_size_t xfer_size, subupl_size;
+ vm_object_t shadow_object;
+ vm_object_t object;
+ vm_object_t m_object;
+ vm_object_offset_t target_offset;
+ upl_offset_t subupl_offset = offset;
+ int entry;
+ wpl_array_t lite_list;
+ int occupied;
+ int clear_refmod = 0;
+ int pgpgout_count = 0;
+ struct vm_page_delayed_work dw_array;
+ struct vm_page_delayed_work *dwp, *dwp_start;
+ bool dwp_finish_ctx = TRUE;
+ int dw_count;
+ int dw_limit;
+ int isVectorUPL = 0;
+ upl_t vector_upl = NULL;
+ boolean_t should_be_throttled = FALSE;
+
+ vm_page_t nxt_page = VM_PAGE_NULL;
+ int fast_path_possible = 0;
+ int fast_path_full_commit = 0;
+ int throttle_page = 0;
+ int unwired_count = 0;
+ int local_queue_count = 0;
+ vm_page_t first_local, last_local;
+ vm_object_offset_t obj_start, obj_end, obj_offset;
+ kern_return_t kr = KERN_SUCCESS;
+
+// DEBUG4K_UPL("upl %p (u_offset 0x%llx u_size 0x%llx) object %p offset 0x%llx size 0x%llx flags 0x%x\n", upl, (uint64_t)upl->u_offset, (uint64_t)upl->u_size, upl->map_object, (uint64_t)offset, (uint64_t)size, flags);
+
+ dwp_start = dwp = NULL;
+
+ subupl_size = size;
+ *empty = FALSE;
+
+ if (upl == UPL_NULL) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ dw_count = 0;
+ dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
+ dwp_start = vm_page_delayed_work_get_ctx();
+ if (dwp_start == NULL) {
+ dwp_start = &dw_array;
+ dw_limit = 1;
+ dwp_finish_ctx = FALSE;
+ }
+
+ dwp = dwp_start;
+
+ if (count == 0) {
+ page_list = NULL;
+ }
+
+ if ((isVectorUPL = vector_upl_is_valid(upl))) {
+ vector_upl = upl;
+ upl_lock(vector_upl);
+ } else {
+ upl_lock(upl);
+ }
+
+process_upl_to_commit:
+
+ if (isVectorUPL) {
+ size = subupl_size;
+ offset = subupl_offset;
+ if (size == 0) {
+ upl_unlock(vector_upl);
+ kr = KERN_SUCCESS;
+ goto done;
+ }
+ upl = vector_upl_subupl_byoffset(vector_upl, &offset, &size);
+ if (upl == NULL) {
+ upl_unlock(vector_upl);
+ kr = KERN_FAILURE;
+ goto done;
+ }
+ page_list = UPL_GET_INTERNAL_PAGE_LIST_SIMPLE(upl);
+ subupl_size -= size;
+ subupl_offset += size;
+ }
+
+#if UPL_DEBUG
+ if (upl->upl_commit_index < UPL_DEBUG_COMMIT_RECORDS) {
+ (void) OSBacktrace(&upl->upl_commit_records[upl->upl_commit_index].c_retaddr[0], UPL_DEBUG_STACK_FRAMES);
+
+ upl->upl_commit_records[upl->upl_commit_index].c_beg = offset;
+ upl->upl_commit_records[upl->upl_commit_index].c_end = (offset + size);
+
+ upl->upl_commit_index++;
+ }
+#endif
+ if (upl->flags & UPL_DEVICE_MEMORY) {
+ xfer_size = 0;
+ } else if ((offset + size) <= upl_adjusted_size(upl, PAGE_MASK)) {
+ xfer_size = size;
+ } else {
+ if (!isVectorUPL) {
+ upl_unlock(upl);
+ } else {
+ upl_unlock(vector_upl);
+ }
+ DEBUG4K_ERROR("upl %p (u_offset 0x%llx u_size 0x%x) offset 0x%x size 0x%x\n", upl, upl->u_offset, upl->u_size, offset, size);
+ kr = KERN_FAILURE;
+ goto done;
+ }
+ if (upl->flags & UPL_SET_DIRTY) {
+ flags |= UPL_COMMIT_SET_DIRTY;
+ }
+ if (upl->flags & UPL_CLEAR_DIRTY) {
+ flags |= UPL_COMMIT_CLEAR_DIRTY;
+ }
+
+ if (upl->flags & UPL_INTERNAL) {
+ lite_list = (wpl_array_t) ((((uintptr_t)upl) + sizeof(struct upl))
+ + ((upl_adjusted_size(upl, PAGE_MASK) / PAGE_SIZE) * sizeof(upl_page_info_t)));
+ } else {
+ lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));
+ }
+
+ object = upl->map_object;
+
+ if (upl->flags & UPL_SHADOWED) {
+ vm_object_lock(object);
+ shadow_object = object->shadow;
+ } else {
+ shadow_object = object;
+ }
+ entry = offset / PAGE_SIZE;
+ target_offset = (vm_object_offset_t)offset;
+
+ if (upl->flags & UPL_KERNEL_OBJECT) {
+ vm_object_lock_shared(shadow_object);
+ } else {
+ vm_object_lock(shadow_object);
+ }
+
+ VM_OBJECT_WIRED_PAGE_UPDATE_START(shadow_object);
+
+ if (upl->flags & UPL_ACCESS_BLOCKED) {
+ assert(shadow_object->blocked_access);
+ shadow_object->blocked_access = FALSE;
+ vm_object_wakeup(object, VM_OBJECT_EVENT_UNBLOCKED);
+ }
+
+ if (shadow_object->code_signed) {
+ /*
+ * CODE SIGNING:
+ * If the object is code-signed, do not let this UPL tell
+ * us if the pages are valid or not. Let the pages be
+ * validated by VM the normal way (when they get mapped or
+ * copied).
+ */
+ flags &= ~UPL_COMMIT_CS_VALIDATED;
+ }
+ if (!page_list) {
+ /*
+ * No page list to get the code-signing info from !?
+ */
+ flags &= ~UPL_COMMIT_CS_VALIDATED;
+ }
+ if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal) {
+ should_be_throttled = TRUE;
+ }
+
+ if ((upl->flags & UPL_IO_WIRE) &&
+ !(flags & UPL_COMMIT_FREE_ABSENT) &&
+ !isVectorUPL &&
+ shadow_object->purgable != VM_PURGABLE_VOLATILE &&
+ shadow_object->purgable != VM_PURGABLE_EMPTY) {
+ if (!vm_page_queue_empty(&shadow_object->memq)) {
+ if (size == shadow_object->vo_size) {
+ nxt_page = (vm_page_t)vm_page_queue_first(&shadow_object->memq);
+ fast_path_full_commit = 1;
+ }
+ fast_path_possible = 1;
+
+ if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal &&
+ (shadow_object->purgable == VM_PURGABLE_DENY ||
+ shadow_object->purgable == VM_PURGABLE_NONVOLATILE ||
+ shadow_object->purgable == VM_PURGABLE_VOLATILE)) {
+ throttle_page = 1;
+ }
+ }
+ }
+ first_local = VM_PAGE_NULL;
+ last_local = VM_PAGE_NULL;
+
+ obj_start = target_offset + upl->u_offset - shadow_object->paging_offset;
+ obj_end = obj_start + xfer_size;
+ obj_start = vm_object_trunc_page(obj_start);
+ obj_end = vm_object_round_page(obj_end);
+ for (obj_offset = obj_start;
+ obj_offset < obj_end;
+ obj_offset += PAGE_SIZE) {
+ vm_page_t t, m;
+
+ dwp->dw_mask = 0;
+ clear_refmod = 0;
+
+ m = VM_PAGE_NULL;
+
+ if (upl->flags & UPL_LITE) {
+ unsigned int pg_num;
+
+ if (nxt_page != VM_PAGE_NULL) {
+ m = nxt_page;
+ nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->vmp_listq);
+ target_offset = m->vmp_offset;
+ }
+ pg_num = (unsigned int) (target_offset / PAGE_SIZE);
+ assert(pg_num == target_offset / PAGE_SIZE);
+
+ if (lite_list[pg_num >> 5] & (1U << (pg_num & 31))) {
+ lite_list[pg_num >> 5] &= ~(1U << (pg_num & 31));
+
+ if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL) {
+ m = vm_page_lookup(shadow_object, obj_offset);
+ }
+ } else {
+ m = NULL;
+ }
+ }
+ if (upl->flags & UPL_SHADOWED) {
+ if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) {
+ t->vmp_free_when_done = FALSE;
+
+ VM_PAGE_FREE(t);
+
+ if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL) {
+ m = vm_page_lookup(shadow_object, target_offset + object->vo_shadow_offset);
+ }
+ }
+ }
+ if (m == VM_PAGE_NULL) {
+ goto commit_next_page;
+ }
+
+ m_object = VM_PAGE_OBJECT(m);
+
+ if (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
+ assert(m->vmp_busy);
+
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
+ goto commit_next_page;
+ }
+
+ if (flags & UPL_COMMIT_CS_VALIDATED) {
+ /*
+ * CODE SIGNING:
+ * Set the code signing bits according to
+ * what the UPL says they should be.
+ */
+ m->vmp_cs_validated |= page_list[entry].cs_validated;
+ m->vmp_cs_tainted |= page_list[entry].cs_tainted;
+ m->vmp_cs_nx |= page_list[entry].cs_nx;
+ }
+ if (flags & UPL_COMMIT_WRITTEN_BY_KERNEL) {
+ m->vmp_written_by_kernel = TRUE;
+ }
+
+ if (upl->flags & UPL_IO_WIRE) {
+ if (page_list) {
+ page_list[entry].phys_addr = 0;
+ }
+
+ if (flags & UPL_COMMIT_SET_DIRTY) {
+ SET_PAGE_DIRTY(m, FALSE);
+ } else if (flags & UPL_COMMIT_CLEAR_DIRTY) {
+ m->vmp_dirty = FALSE;
+
+ if (!(flags & UPL_COMMIT_CS_VALIDATED) &&
+ m->vmp_cs_validated &&
+ m->vmp_cs_tainted != VMP_CS_ALL_TRUE) {
+ /*
+ * CODE SIGNING:
+ * This page is no longer dirty
+ * but could have been modified,
+ * so it will need to be
+ * re-validated.
+ */
+ m->vmp_cs_validated = VMP_CS_ALL_FALSE;
+
+ VM_PAGEOUT_DEBUG(vm_cs_validated_resets, 1);
+
+ pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
+ }
+ clear_refmod |= VM_MEM_MODIFIED;
+ }
+ if (upl->flags & UPL_ACCESS_BLOCKED) {
+ /*
+ * We blocked access to the pages in this UPL.
+ * Clear the "busy" bit and wake up any waiter
+ * for this page.
+ */
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
+ }
+ if (fast_path_possible) {
+ assert(m_object->purgable != VM_PURGABLE_EMPTY);
+ assert(m_object->purgable != VM_PURGABLE_VOLATILE);
+ if (m->vmp_absent) {
+ assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
+ assert(m->vmp_wire_count == 0);
+ assert(m->vmp_busy);
+
+ m->vmp_absent = FALSE;
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
+ } else {
+ if (m->vmp_wire_count == 0) {
+ panic("wire_count == 0, m = %p, obj = %p\n", m, shadow_object);
+ }
+ assert(m->vmp_q_state == VM_PAGE_IS_WIRED);
+
+ /*
+ * XXX FBDP need to update some other
+ * counters here (purgeable_wired_count)
+ * (ledgers), ...
+ */
+ assert(m->vmp_wire_count > 0);
+ m->vmp_wire_count--;
+
+ if (m->vmp_wire_count == 0) {
+ m->vmp_q_state = VM_PAGE_NOT_ON_Q;
+ unwired_count++;
+ }
+ }
+ if (m->vmp_wire_count == 0) {
+ assert(m->vmp_pageq.next == 0 && m->vmp_pageq.prev == 0);
+
+ if (last_local == VM_PAGE_NULL) {
+ assert(first_local == VM_PAGE_NULL);
+
+ last_local = m;
+ first_local = m;
+ } else {
+ assert(first_local != VM_PAGE_NULL);
+
+ m->vmp_pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local);
+ first_local->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(m);
+ first_local = m;
+ }
+ local_queue_count++;
+
+ if (throttle_page) {
+ m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;
+ } else {
+ if (flags & UPL_COMMIT_INACTIVATE) {
+ if (shadow_object->internal) {
+ m->vmp_q_state = VM_PAGE_ON_INACTIVE_INTERNAL_Q;
+ } else {
+ m->vmp_q_state = VM_PAGE_ON_INACTIVE_EXTERNAL_Q;
+ }
+ } else {
+ m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
+ }
+ }
+ }
+ } else {
+ if (flags & UPL_COMMIT_INACTIVATE) {
+ dwp->dw_mask |= DW_vm_page_deactivate_internal;
+ clear_refmod |= VM_MEM_REFERENCED;
+ }
+ if (m->vmp_absent) {
+ if (flags & UPL_COMMIT_FREE_ABSENT) {
+ dwp->dw_mask |= DW_vm_page_free;
+ } else {
+ m->vmp_absent = FALSE;
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
+
+ if (!(dwp->dw_mask & DW_vm_page_deactivate_internal)) {
+ dwp->dw_mask |= DW_vm_page_activate;
+ }
+ }
+ } else {
+ dwp->dw_mask |= DW_vm_page_unwire;
+ }
+ }
+ goto commit_next_page;
+ }
+ assert(m->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR);
+
+ if (page_list) {
+ page_list[entry].phys_addr = 0;
+ }
+
+ /*
+ * make sure to clear the hardware
+ * modify or reference bits before
+ * releasing the BUSY bit on this page
+ * otherwise we risk losing a legitimate
+ * change of state
+ */
+ if (flags & UPL_COMMIT_CLEAR_DIRTY) {
+ m->vmp_dirty = FALSE;
+
+ clear_refmod |= VM_MEM_MODIFIED;
+ }
+ if (m->vmp_laundry) {
+ dwp->dw_mask |= DW_vm_pageout_throttle_up;
+ }
+
+ if (VM_PAGE_WIRED(m)) {
+ m->vmp_free_when_done = FALSE;
+ }
+
+ if (!(flags & UPL_COMMIT_CS_VALIDATED) &&
+ m->vmp_cs_validated &&
+ m->vmp_cs_tainted != VMP_CS_ALL_TRUE) {
+ /*
+ * CODE SIGNING:
+ * This page is no longer dirty
+ * but could have been modified,
+ * so it will need to be
+ * re-validated.
+ */
+ m->vmp_cs_validated = VMP_CS_ALL_FALSE;
+
+ VM_PAGEOUT_DEBUG(vm_cs_validated_resets, 1);
+
+ pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
+ }
+ if (m->vmp_overwriting) {
+ /*
+ * the (COPY_OUT_FROM == FALSE) request_page_list case
+ */
+ if (m->vmp_busy) {
+#if CONFIG_PHANTOM_CACHE
+ if (m->vmp_absent && !m_object->internal) {
+ dwp->dw_mask |= DW_vm_phantom_cache_update;
+ }
+#endif
+ m->vmp_absent = FALSE;
+
+ dwp->dw_mask |= DW_clear_busy;
+ } else {
+ /*
+ * alternate (COPY_OUT_FROM == FALSE) page_list case
+ * Occurs when the original page was wired
+ * at the time of the list request
+ */
+ assert(VM_PAGE_WIRED(m));
+
+ dwp->dw_mask |= DW_vm_page_unwire; /* reactivates */
+ }
+ m->vmp_overwriting = FALSE;
+ }
+ m->vmp_cleaning = FALSE;
+
+ if (m->vmp_free_when_done) {
+ /*
+ * With the clean queue enabled, UPL_PAGEOUT should
+ * no longer set the pageout bit. Its pages now go
+ * to the clean queue.
+ *
+ * We don't use the cleaned Q anymore and so this
+ * assert isn't correct. The code for the clean Q
+ * still exists and might be used in the future. If we
+ * go back to the cleaned Q, we will re-enable this
+ * assert.
+ *
+ * assert(!(upl->flags & UPL_PAGEOUT));
+ */
+ assert(!m_object->internal);
+
+ m->vmp_free_when_done = FALSE;
+
+ if ((flags & UPL_COMMIT_SET_DIRTY) ||
+ (m->vmp_pmapped && (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED))) {
+ /*
+ * page was re-dirtied after we started
+ * the pageout... reactivate it since
+ * we don't know whether the on-disk
+ * copy matches what is now in memory
+ */
+ SET_PAGE_DIRTY(m, FALSE);
+
+ dwp->dw_mask |= DW_vm_page_activate | DW_PAGE_WAKEUP;
+
+ if (upl->flags & UPL_PAGEOUT) {
+ VM_STAT_INCR(reactivations);
+ DTRACE_VM2(pgrec, int, 1, (uint64_t *), NULL);
+ }
+ } else {
+ /*
+ * page has been successfully cleaned
+ * go ahead and free it for other use
+ */
+ if (m_object->internal) {
+ DTRACE_VM2(anonpgout, int, 1, (uint64_t *), NULL);
+ } else {
+ DTRACE_VM2(fspgout, int, 1, (uint64_t *), NULL);
+ }
+ m->vmp_dirty = FALSE;
+ m->vmp_busy = TRUE;
+
+ dwp->dw_mask |= DW_vm_page_free;
+ }
+ goto commit_next_page;
+ }
+ /*
+ * It is a part of the semantic of COPYOUT_FROM
+ * UPLs that a commit implies cache sync
+ * between the vm page and the backing store
+ * this can be used to strip the precious bit
+ * as well as clean
+ */
+ if ((upl->flags & UPL_PAGE_SYNC_DONE) || (flags & UPL_COMMIT_CLEAR_PRECIOUS)) {
+ m->vmp_precious = FALSE;
+ }
+
+ if (flags & UPL_COMMIT_SET_DIRTY) {
+ SET_PAGE_DIRTY(m, FALSE);
+ } else {
+ m->vmp_dirty = FALSE;
+ }
+
+ /* with the clean queue on, move *all* cleaned pages to the clean queue */
+ if (hibernate_cleaning_in_progress == FALSE && !m->vmp_dirty && (upl->flags & UPL_PAGEOUT)) {
+ pgpgout_count++;
+
+ VM_STAT_INCR(pageouts);
+ DTRACE_VM2(pgout, int, 1, (uint64_t *), NULL);
+
+ dwp->dw_mask |= DW_enqueue_cleaned;
+ } else if (should_be_throttled == TRUE && (m->vmp_q_state == VM_PAGE_NOT_ON_Q)) {
+ /*
+ * page coming back in from being 'frozen'...
+ * it was dirty before it was frozen, so keep it so
+ * the vm_page_activate will notice that it really belongs
+ * on the throttle queue and put it there
+ */
+ SET_PAGE_DIRTY(m, FALSE);
+ dwp->dw_mask |= DW_vm_page_activate;
+ } else {
+ if ((flags & UPL_COMMIT_INACTIVATE) && !m->vmp_clustered && (m->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q)) {
+ dwp->dw_mask |= DW_vm_page_deactivate_internal;
+ clear_refmod |= VM_MEM_REFERENCED;
+ } else if (!VM_PAGE_PAGEABLE(m)) {
+ if (m->vmp_clustered || (flags & UPL_COMMIT_SPECULATE)) {
+ dwp->dw_mask |= DW_vm_page_speculate;
+ } else if (m->vmp_reference) {
+ dwp->dw_mask |= DW_vm_page_activate;
+ } else {
+ dwp->dw_mask |= DW_vm_page_deactivate_internal;
+ clear_refmod |= VM_MEM_REFERENCED;
+ }
+ }
+ }
+ if (upl->flags & UPL_ACCESS_BLOCKED) {
+ /*
+ * We blocked access to the pages in this URL.
+ * Clear the "busy" bit on this page before we
+ * wake up any waiter.
+ */
+ dwp->dw_mask |= DW_clear_busy;
+ }
+ /*
+ * Wakeup any thread waiting for the page to be un-cleaning.
+ */
+ dwp->dw_mask |= DW_PAGE_WAKEUP;
projects / apple / xnu.git / blobdiff