/*
- * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2014 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#include <debug.h>
#include <mach_pagemap.h>
#include <mach_cluster_stats.h>
-#include <mach_kdb.h>
-#include <advisory_pageout.h>
#include <mach/mach_types.h>
#include <mach/memory_object.h>
#include <machine/vm_tuning.h>
#include <machine/commpage.h>
-#if CONFIG_EMBEDDED
-#include <sys/kern_memorystatus.h>
-#endif
-
#include <vm/pmap.h>
+#include <vm/vm_compressor_pager.h>
#include <vm/vm_fault.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/vm_protos.h> /* must be last */
#include <vm/memory_object.h>
#include <vm/vm_purgeable_internal.h>
+#include <vm/vm_shared_region.h>
+#include <vm/vm_compressor.h>
+#if CONFIG_PHANTOM_CACHE
+#include <vm/vm_phantom_cache.h>
+#endif
/*
* ENCRYPTED SWAP:
*/
-#include <../bsd/crypto/aes/aes.h>
+#include <libkern/crypto/aes.h>
extern u_int32_t random(void); /* from <libkern/libkern.h> */
+extern int cs_debug;
+
#if UPL_DEBUG
#include <libkern/OSDebug.h>
#endif
+extern void m_drain(void);
+
+#if VM_PRESSURE_EVENTS
+extern unsigned int memorystatus_available_pages;
+extern unsigned int memorystatus_available_pages_pressure;
+extern unsigned int memorystatus_available_pages_critical;
+extern unsigned int memorystatus_frozen_count;
+extern unsigned int memorystatus_suspended_count;
+
+extern vm_pressure_level_t memorystatus_vm_pressure_level;
+int memorystatus_purge_on_warning = 2;
+int memorystatus_purge_on_urgent = 5;
+int memorystatus_purge_on_critical = 8;
+
+void vm_pressure_response(void);
+boolean_t vm_pressure_thread_running = FALSE;
+extern void consider_vm_pressure_events(void);
+
+#define MEMORYSTATUS_SUSPENDED_THRESHOLD 4
+#endif /* VM_PRESSURE_EVENTS */
+
+boolean_t vm_pressure_changed = FALSE;
+
#ifndef VM_PAGEOUT_BURST_ACTIVE_THROTTLE /* maximum iterations of the active queue to move pages to inactive */
#define VM_PAGEOUT_BURST_ACTIVE_THROTTLE 100
#endif
#ifndef VM_PAGEOUT_BURST_INACTIVE_THROTTLE /* maximum iterations of the inactive queue w/o stealing/cleaning a page */
-#ifdef CONFIG_EMBEDDED
-#define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 1024
-#else
#define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 4096
#endif
-#endif
#ifndef VM_PAGEOUT_DEADLOCK_RELIEF
#define VM_PAGEOUT_DEADLOCK_RELIEF 100 /* number of pages to move to break deadlock */
#endif
#ifndef VM_PAGE_LAUNDRY_MAX
-#define VM_PAGE_LAUNDRY_MAX 16UL /* maximum pageouts on a given pageout queue */
+#define VM_PAGE_LAUNDRY_MAX 128UL /* maximum pageouts on a given pageout queue */
#endif /* VM_PAGEOUT_LAUNDRY_MAX */
#ifndef VM_PAGEOUT_BURST_WAIT
-#define VM_PAGEOUT_BURST_WAIT 30 /* milliseconds per page */
+#define VM_PAGEOUT_BURST_WAIT 10 /* milliseconds */
#endif /* VM_PAGEOUT_BURST_WAIT */
#ifndef VM_PAGEOUT_EMPTY_WAIT
#define VM_PAGEOUT_IDLE_WAIT 10 /* milliseconds */
#endif /* VM_PAGEOUT_IDLE_WAIT */
+#ifndef VM_PAGEOUT_SWAP_WAIT
+#define VM_PAGEOUT_SWAP_WAIT 50 /* milliseconds */
+#endif /* VM_PAGEOUT_SWAP_WAIT */
+
+#ifndef VM_PAGEOUT_PRESSURE_PAGES_CONSIDERED
+#define VM_PAGEOUT_PRESSURE_PAGES_CONSIDERED 1000 /* maximum pages considered before we issue a pressure event */
+#endif /* VM_PAGEOUT_PRESSURE_PAGES_CONSIDERED */
+
+#ifndef VM_PAGEOUT_PRESSURE_EVENT_MONITOR_SECS
+#define VM_PAGEOUT_PRESSURE_EVENT_MONITOR_SECS 5 /* seconds */
+#endif /* VM_PAGEOUT_PRESSURE_EVENT_MONITOR_SECS */
+
+unsigned int vm_page_speculative_q_age_ms = VM_PAGE_SPECULATIVE_Q_AGE_MS;
+unsigned int vm_page_speculative_percentage = 5;
+
#ifndef VM_PAGE_SPECULATIVE_TARGET
-#define VM_PAGE_SPECULATIVE_TARGET(total) ((total) * 1 / 20)
+#define VM_PAGE_SPECULATIVE_TARGET(total) ((total) * 1 / (100 / vm_page_speculative_percentage))
#endif /* VM_PAGE_SPECULATIVE_TARGET */
+
#ifndef VM_PAGE_INACTIVE_HEALTHY_LIMIT
#define VM_PAGE_INACTIVE_HEALTHY_LIMIT(total) ((total) * 1 / 200)
#endif /* VM_PAGE_INACTIVE_HEALTHY_LIMIT */
*/
#ifndef VM_PAGE_INACTIVE_TARGET
-#define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 1 / 3)
+#define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 1 / 2)
#endif /* VM_PAGE_INACTIVE_TARGET */
/*
*/
#ifndef VM_PAGE_FREE_TARGET
-#ifdef CONFIG_EMBEDDED
-#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 100)
-#else
#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80)
-#endif
#endif /* VM_PAGE_FREE_TARGET */
+
/*
* The pageout daemon always starts running once vm_page_free_count
* falls below vm_page_free_min.
*/
#ifndef VM_PAGE_FREE_MIN
-#ifdef CONFIG_EMBEDDED
-#define VM_PAGE_FREE_MIN(free) (10 + (free) / 200)
-#else
#define VM_PAGE_FREE_MIN(free) (10 + (free) / 100)
-#endif
#endif /* VM_PAGE_FREE_MIN */
-#define VM_PAGE_FREE_MIN_LIMIT 1500
-#define VM_PAGE_FREE_TARGET_LIMIT 2000
-
+#define VM_PAGE_FREE_RESERVED_LIMIT 1700
+#define VM_PAGE_FREE_MIN_LIMIT 3500
+#define VM_PAGE_FREE_TARGET_LIMIT 4000
/*
* When vm_page_free_count falls below vm_page_free_reserved,
*/
#define VM_PAGE_REACTIVATE_LIMIT_MAX 20000
#ifndef VM_PAGE_REACTIVATE_LIMIT
-#ifdef CONFIG_EMBEDDED
-#define VM_PAGE_REACTIVATE_LIMIT(avail) (VM_PAGE_INACTIVE_TARGET(avail) / 2)
-#else
#define VM_PAGE_REACTIVATE_LIMIT(avail) (MAX((avail) * 1 / 20,VM_PAGE_REACTIVATE_LIMIT_MAX))
-#endif
#endif /* VM_PAGE_REACTIVATE_LIMIT */
#define VM_PAGEOUT_INACTIVE_FORCE_RECLAIM 100
+extern boolean_t hibernate_cleaning_in_progress;
+
/*
* Exported variable used to broadcast the activation of the pageout scan
* Working Set uses this to throttle its use of pmap removes. In this
/*
* Forward declarations for internal routines.
*/
+struct cq {
+ struct vm_pageout_queue *q;
+ void *current_chead;
+ char *scratch_buf;
+};
+
+#if VM_PRESSURE_EVENTS
+void vm_pressure_thread(void);
+
+boolean_t VM_PRESSURE_NORMAL_TO_WARNING(void);
+boolean_t VM_PRESSURE_WARNING_TO_CRITICAL(void);
+
+boolean_t VM_PRESSURE_WARNING_TO_NORMAL(void);
+boolean_t VM_PRESSURE_CRITICAL_TO_WARNING(void);
+#endif
static void vm_pageout_garbage_collect(int);
static void vm_pageout_iothread_continue(struct vm_pageout_queue *);
static void vm_pageout_iothread_external(void);
-static void vm_pageout_iothread_internal(void);
+static void vm_pageout_iothread_internal(struct cq *cq);
+static void vm_pageout_adjust_io_throttles(struct vm_pageout_queue *, struct vm_pageout_queue *, boolean_t);
extern void vm_pageout_continue(void);
extern void vm_pageout_scan(void);
unsigned int vm_pageout_reserved_internal = 0;
unsigned int vm_pageout_reserved_really = 0;
+unsigned int vm_pageout_swap_wait = 0;
unsigned int vm_pageout_idle_wait = 0; /* milliseconds */
unsigned int vm_pageout_empty_wait = 0; /* milliseconds */
unsigned int vm_pageout_burst_wait = 0; /* milliseconds */
unsigned int vm_pageout_burst_active_throttle = 0;
unsigned int vm_pageout_burst_inactive_throttle = 0;
-/*
- * Protection against zero fill flushing live working sets derived
- * from existing backing store and files
- */
-unsigned int vm_accellerate_zf_pageout_trigger = 400;
-unsigned int zf_queue_min_count = 100;
-unsigned int vm_zf_queue_count = 0;
+int vm_upl_wait_for_pages = 0;
-#if defined(__ppc__) /* On ppc, vm statistics are still 32-bit */
-unsigned int vm_zf_count = 0;
-#else
-uint64_t vm_zf_count __attribute__((aligned(8))) = 0;
-#endif
/*
* These variables record the pageout daemon's actions:
*/
unsigned int vm_pageout_active = 0; /* debugging */
+unsigned int vm_pageout_active_busy = 0; /* debugging */
unsigned int vm_pageout_inactive = 0; /* debugging */
unsigned int vm_pageout_inactive_throttled = 0; /* debugging */
unsigned int vm_pageout_inactive_forced = 0; /* debugging */
unsigned int vm_pageout_inactive_nolock = 0; /* debugging */
unsigned int vm_pageout_inactive_avoid = 0; /* debugging */
unsigned int vm_pageout_inactive_busy = 0; /* debugging */
+unsigned int vm_pageout_inactive_error = 0; /* debugging */
unsigned int vm_pageout_inactive_absent = 0; /* debugging */
+unsigned int vm_pageout_inactive_notalive = 0; /* debugging */
unsigned int vm_pageout_inactive_used = 0; /* debugging */
+unsigned int vm_pageout_cache_evicted = 0; /* debugging */
unsigned int vm_pageout_inactive_clean = 0; /* debugging */
-unsigned int vm_pageout_inactive_dirty = 0; /* debugging */
+unsigned int vm_pageout_speculative_clean = 0; /* debugging */
+
+unsigned int vm_pageout_freed_from_cleaned = 0;
+unsigned int vm_pageout_freed_from_speculative = 0;
+unsigned int vm_pageout_freed_from_inactive_clean = 0;
+
+unsigned int vm_pageout_enqueued_cleaned_from_inactive_clean = 0;
+unsigned int vm_pageout_enqueued_cleaned_from_inactive_dirty = 0;
+
+unsigned int vm_pageout_cleaned_reclaimed = 0; /* debugging; how many cleaned pages are reclaimed by the pageout scan */
+unsigned int vm_pageout_cleaned_reactivated = 0; /* debugging; how many cleaned pages are found to be referenced on pageout (and are therefore reactivated) */
+unsigned int vm_pageout_cleaned_reference_reactivated = 0;
+unsigned int vm_pageout_cleaned_volatile_reactivated = 0;
+unsigned int vm_pageout_cleaned_fault_reactivated = 0;
+unsigned int vm_pageout_cleaned_commit_reactivated = 0; /* debugging; how many cleaned pages are found to be referenced on commit (and are therefore reactivated) */
+unsigned int vm_pageout_cleaned_busy = 0;
+unsigned int vm_pageout_cleaned_nolock = 0;
+
+unsigned int vm_pageout_inactive_dirty_internal = 0; /* debugging */
+unsigned int vm_pageout_inactive_dirty_external = 0; /* debugging */
unsigned int vm_pageout_inactive_deactivated = 0; /* debugging */
-unsigned int vm_pageout_inactive_zf = 0; /* debugging */
+unsigned int vm_pageout_inactive_anonymous = 0; /* debugging */
unsigned int vm_pageout_dirty_no_pager = 0; /* debugging */
unsigned int vm_pageout_purged_objects = 0; /* debugging */
unsigned int vm_stat_discard = 0; /* debugging */
unsigned int vm_pageout_catch_ups = 0; /* debugging */
unsigned int vm_pageout_inactive_force_reclaim = 0; /* debugging */
+unsigned int vm_pageout_scan_reclaimed_throttled = 0;
unsigned int vm_pageout_scan_active_throttled = 0;
-unsigned int vm_pageout_scan_inactive_throttled = 0;
+unsigned int vm_pageout_scan_inactive_throttled_internal = 0;
+unsigned int vm_pageout_scan_inactive_throttled_external = 0;
unsigned int vm_pageout_scan_throttle = 0; /* debugging */
-unsigned int vm_pageout_scan_throttle_aborted = 0; /* debugging */
unsigned int vm_pageout_scan_burst_throttle = 0; /* debugging */
unsigned int vm_pageout_scan_empty_throttle = 0; /* debugging */
+unsigned int vm_pageout_scan_swap_throttle = 0; /* debugging */
unsigned int vm_pageout_scan_deadlock_detected = 0; /* debugging */
unsigned int vm_pageout_scan_active_throttle_success = 0; /* debugging */
unsigned int vm_pageout_scan_inactive_throttle_success = 0; /* debugging */
-
+unsigned int vm_pageout_inactive_external_forced_jetsam_count = 0; /* debugging */
unsigned int vm_page_speculative_count_drifts = 0;
unsigned int vm_page_speculative_count_drift_max = 0;
+
/*
* Backing store throttle when BS is exhausted
*/
unsigned long vm_cs_validated_resets = 0;
#endif
+int vm_debug_events = 0;
+
+#if CONFIG_MEMORYSTATUS
+#if !CONFIG_JETSAM
+extern boolean_t memorystatus_idle_exit_from_VM(void);
+#endif
+extern boolean_t memorystatus_kill_on_VM_page_shortage(boolean_t async);
+extern void memorystatus_on_pageout_scan_end(void);
+#endif
+
+boolean_t vm_page_compressions_failing = FALSE;
+
/*
* Routine: vm_backing_store_disable
* Purpose:
assert(p->pageout);
p->pageout = FALSE;
assert(!p->cleaning);
+ assert(!p->laundry);
offset = p->offset;
VM_PAGE_FREE(p);
p = VM_PAGE_NULL;
m = vm_page_lookup(shadow_object,
- offset + object->shadow_offset);
+ offset + object->vo_shadow_offset);
if(m == VM_PAGE_NULL)
continue;
- assert(m->cleaning);
- /* used as a trigger on upl_commit etc to recognize the */
- /* pageout daemon's subseqent desire to pageout a cleaning */
- /* page. When the bit is on the upl commit code will */
- /* respect the pageout bit in the target page over the */
- /* caller's page list indication */
- m->dump_cleaning = FALSE;
assert((m->dirty) || (m->precious) ||
(m->busy && m->cleaning));
* Also decrement the burst throttle (if external).
*/
vm_page_lock_queues();
- if (m->laundry) {
+ if (m->pageout_queue)
vm_pageout_throttle_up(m);
- }
/*
* Handle the "target" page(s). These pages are to be freed if
* can detect whether the page was redirtied during
* pageout by checking the modify state.
*/
- if (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED)
- m->dirty = TRUE;
- else
- m->dirty = FALSE;
+ if (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(m, FALSE);
+ } else {
+ m->dirty = FALSE;
+ }
if (m->dirty) {
CLUSTER_STAT(vm_pageout_target_page_dirtied++;)
else
vm_page_deactivate(m);
}
- if((m->busy) && (m->cleaning)) {
-
- /* the request_page_list case, (COPY_OUT_FROM FALSE) */
- m->busy = FALSE;
-
- /* We do not re-set m->dirty ! */
- /* The page was busy so no extraneous activity */
- /* could have occurred. COPY_INTO is a read into the */
- /* new pages. CLEAN_IN_PLACE does actually write */
- /* out the pages but handling outside of this code */
- /* will take care of resetting dirty. We clear the */
- /* modify however for the Programmed I/O case. */
- pmap_clear_modify(m->phys_page);
+ if (m->overwriting) {
+ /*
+ * the (COPY_OUT_FROM == FALSE) request_page_list case
+ */
+ if (m->busy) {
+ /*
+ * We do not re-set m->dirty !
+ * The page was busy so no extraneous activity
+ * could have occurred. COPY_INTO is a read into the
+ * new pages. CLEAN_IN_PLACE does actually write
+ * out the pages but handling outside of this code
+ * will take care of resetting dirty. We clear the
+ * modify however for the Programmed I/O case.
+ */
+ pmap_clear_modify(m->phys_page);
- m->absent = FALSE;
- m->overwriting = FALSE;
- } else if (m->overwriting) {
- /* alternate request page list, write to page_list */
- /* case. Occurs when the original page was wired */
- /* at the time of the list request */
- assert(VM_PAGE_WIRED(m));
- vm_page_unwire(m, TRUE); /* reactivates */
+ m->busy = FALSE;
+ m->absent = FALSE;
+ } else {
+ /*
+ * alternate (COPY_OUT_FROM == FALSE) request_page_list case
+ * Occurs when the original page was wired
+ * at the time of the list request
+ */
+ assert(VM_PAGE_WIRED(m));
+ vm_page_unwire(m, TRUE); /* reactivates */
+ }
m->overwriting = FALSE;
} else {
- /*
- * Set the dirty state according to whether or not the page was
- * modified during the pageout. Note that we purposefully do
- * NOT call pmap_clear_modify since the page is still mapped.
- * If the page were to be dirtied between the 2 calls, this
- * this fact would be lost. This code is only necessary to
- * maintain statistics, since the pmap module is always
- * consulted if m->dirty is false.
- */
+ /*
+ * Set the dirty state according to whether or not the page was
+ * modified during the pageout. Note that we purposefully do
+ * NOT call pmap_clear_modify since the page is still mapped.
+ * If the page were to be dirtied between the 2 calls, this
+ * this fact would be lost. This code is only necessary to
+ * maintain statistics, since the pmap module is always
+ * consulted if m->dirty is false.
+ */
#if MACH_CLUSTER_STATS
m->dirty = pmap_is_modified(m->phys_page);
else vm_pageout_cluster_cleaned++;
if (m->wanted) vm_pageout_cluster_collisions++;
#else
- m->dirty = 0;
+ m->dirty = FALSE;
#endif
}
+ if (m->encrypted_cleaning == TRUE) {
+ m->encrypted_cleaning = FALSE;
+ m->busy = FALSE;
+ }
m->cleaning = FALSE;
- m->encrypted_cleaning = FALSE;
/*
* Wakeup any thread waiting for the page to be un-cleaning.
* Mark original page as cleaning in place.
*/
m->cleaning = TRUE;
- m->dirty = TRUE;
+ SET_PAGE_DIRTY(m, FALSE);
m->precious = FALSE;
/*
{
vm_object_t object;
vm_object_offset_t paging_offset;
- vm_page_t holding_page;
memory_object_t pager;
XPR(XPR_VM_PAGEOUT,
return;
}
- /* set the page for future call to vm_fault_list_request */
- vm_object_paging_begin(object);
- holding_page = NULL;
-
+ /*
+ * set the page for future call to vm_fault_list_request
+ */
pmap_clear_modify(m->phys_page);
- m->dirty = TRUE;
- m->busy = TRUE;
- m->list_req_pending = TRUE;
- m->cleaning = TRUE;
+ SET_PAGE_DIRTY(m, FALSE);
m->pageout = TRUE;
- vm_page_lockspin_queues();
- vm_page_wire(m);
- vm_page_unlock_queues();
-
+ /*
+ * keep the object from collapsing or terminating
+ */
+ vm_object_paging_begin(object);
vm_object_unlock(object);
/*
* which will page it out and attempt to clean adjacent pages
* in the same operation.
*
- * The page must be busy, and the object and queues locked. We will take a
+ * The object and queues must be locked. We will take a
* paging reference to prevent deallocation or collapse when we
* release the object lock back at the call site. The I/O thread
* is responsible for consuming this reference
*/
void
-vm_pageout_cluster(vm_page_t m)
+vm_pageout_cluster(vm_page_t m, boolean_t pageout)
{
vm_object_t object = m->object;
struct vm_pageout_queue *q;
object, m->offset, m, 0, 0);
VM_PAGE_CHECK(m);
+#if DEBUG
+ lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+#endif
+ vm_object_lock_assert_exclusive(object);
/*
* Only a certain kind of page is appreciated here.
*/
- assert(m->busy && (m->dirty || m->precious) && (!VM_PAGE_WIRED(m)));
- assert(!m->cleaning && !m->pageout && !m->inactive && !m->active);
+ assert((m->dirty || m->precious) && (!VM_PAGE_WIRED(m)));
+ assert(!m->cleaning && !m->pageout && !m->laundry);
+#ifndef CONFIG_FREEZE
+ assert(!m->inactive && !m->active);
assert(!m->throttled);
+#endif
/*
- * protect the object from collapse -
- * locking in the object's paging_offset.
+ * protect the object from collapse or termination
*/
- vm_object_paging_begin(object);
+ vm_object_activity_begin(object);
- /*
- * set the page for future call to vm_fault_list_request
- * page should already be marked busy
- */
- vm_page_wire(m);
- m->list_req_pending = TRUE;
- m->cleaning = TRUE;
- m->pageout = TRUE;
+ m->pageout = pageout;
+
+ if (object->internal == TRUE) {
+ if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE)
+ m->busy = TRUE;
- if (object->internal == TRUE)
q = &vm_pageout_queue_internal;
- else
+ } else
q = &vm_pageout_queue_external;
- /*
+ /*
* pgo_laundry count is tied to the laundry bit
*/
- m->laundry = TRUE;
+ m->laundry = TRUE;
q->pgo_laundry++;
m->pageout_queue = TRUE;
queue_enter(&q->pgo_pending, m, vm_page_t, pageq);
if (q->pgo_idle == TRUE) {
- q->pgo_idle = FALSE;
- thread_wakeup((event_t) &q->pgo_pending);
+ q->pgo_idle = FALSE;
+ thread_wakeup((event_t) &q->pgo_pending);
}
-
VM_PAGE_CHECK(m);
}
*/
void
vm_pageout_throttle_up(
- vm_page_t m)
+ vm_page_t m)
{
- struct vm_pageout_queue *q;
+ struct vm_pageout_queue *q;
- assert(m->object != VM_OBJECT_NULL);
- assert(m->object != kernel_object);
+ assert(m->object != VM_OBJECT_NULL);
+ assert(m->object != kernel_object);
- vm_pageout_throttle_up_count++;
+#if DEBUG
+ lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+ vm_object_lock_assert_exclusive(m->object);
+#endif
- if (m->object->internal == TRUE)
- q = &vm_pageout_queue_internal;
- else
- q = &vm_pageout_queue_external;
+ vm_pageout_throttle_up_count++;
- if (m->pageout_queue == TRUE) {
+ if (m->object->internal == TRUE)
+ q = &vm_pageout_queue_internal;
+ else
+ q = &vm_pageout_queue_external;
- queue_remove(&q->pgo_pending, m, vm_page_t, pageq);
- m->pageout_queue = FALSE;
+ if (m->pageout_queue == TRUE) {
- m->pageq.next = NULL;
- m->pageq.prev = NULL;
+ queue_remove(&q->pgo_pending, m, vm_page_t, pageq);
+ m->pageout_queue = FALSE;
- vm_object_paging_end(m->object);
- }
- if (m->laundry == TRUE) {
- m->laundry = FALSE;
- q->pgo_laundry--;
+ m->pageq.next = NULL;
+ m->pageq.prev = NULL;
- if (q->pgo_throttled == TRUE) {
- q->pgo_throttled = FALSE;
- thread_wakeup((event_t) &q->pgo_laundry);
- }
- if (q->pgo_draining == TRUE && q->pgo_laundry == 0) {
- q->pgo_draining = FALSE;
- thread_wakeup((event_t) (&q->pgo_laundry+1));
- }
+ vm_object_activity_end(m->object);
+ }
+ if (m->laundry == TRUE) {
+
+ m->laundry = FALSE;
+ q->pgo_laundry--;
+
+ if (q->pgo_throttled == TRUE) {
+ q->pgo_throttled = FALSE;
+ thread_wakeup((event_t) &q->pgo_laundry);
+ }
+ if (q->pgo_draining == TRUE && q->pgo_laundry == 0) {
+ q->pgo_draining = FALSE;
+ thread_wakeup((event_t) (&q->pgo_laundry+1));
+ }
}
}
-/*
- * vm_pageout_scan does the dirty work for the pageout daemon.
- * It returns with vm_page_queue_free_lock held and
- * vm_page_free_wanted == 0.
- */
+static void
+vm_pageout_throttle_up_batch(
+ struct vm_pageout_queue *q,
+ int batch_cnt)
+{
+#if DEBUG
+ lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+#endif
-#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT (3 * MAX_UPL_TRANSFER)
+ vm_pageout_throttle_up_count += batch_cnt;
-#define FCS_IDLE 0
-#define FCS_DELAYED 1
-#define FCS_DEADLOCK_DETECTED 2
+ q->pgo_laundry -= batch_cnt;
+
+ if (q->pgo_throttled == TRUE) {
+ q->pgo_throttled = FALSE;
+ thread_wakeup((event_t) &q->pgo_laundry);
+ }
+ if (q->pgo_draining == TRUE && q->pgo_laundry == 0) {
+ q->pgo_draining = FALSE;
+ thread_wakeup((event_t) (&q->pgo_laundry+1));
+ }
+}
-struct flow_control {
- int state;
- mach_timespec_t ts;
-};
/*
#define VM_PAGEOUT_STAT_AFTER(i) \
(((i) == VM_PAGEOUT_STAT_SIZE - 1) ? 0 : (i) + 1)
+#if VM_PAGE_BUCKETS_CHECK
+int vm_page_buckets_check_interval = 10; /* in seconds */
+#endif /* VM_PAGE_BUCKETS_CHECK */
+
/*
* Called from compute_averages().
*/
{
unsigned int vm_pageout_next;
+#if VM_PAGE_BUCKETS_CHECK
+ /* check the consistency of VM page buckets at regular interval */
+ static int counter = 0;
+ if ((++counter % vm_page_buckets_check_interval) == 0) {
+ vm_page_buckets_check();
+ }
+#endif /* VM_PAGE_BUCKETS_CHECK */
+
vm_memory_pressure =
vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].reclaimed;
vm_pageout_stat_now = vm_pageout_next;
}
+
+/*
+ * IMPORTANT
+ * mach_vm_ctl_page_free_wanted() is called indirectly, via
+ * mach_vm_pressure_monitor(), when taking a stackshot. Therefore,
+ * it must be safe in the restricted stackshot context. Locks and/or
+ * blocking are not allowable.
+ */
unsigned int
mach_vm_ctl_page_free_wanted(void)
{
return page_free_wanted;
}
+
+/*
+ * IMPORTANT:
+ * mach_vm_pressure_monitor() is called when taking a stackshot, with
+ * wait_for_pressure FALSE, so that code path must remain safe in the
+ * restricted stackshot context. No blocking or locks are allowable.
+ * on that code path.
+ */
+
kern_return_t
mach_vm_pressure_monitor(
boolean_t wait_for_pressure,
return KERN_SUCCESS;
}
-/* Page States: Used below to maintain the page state
- before it's removed from it's Q. This saved state
- helps us do the right accounting in certain cases
-*/
-
-#define PAGE_STATE_SPECULATIVE 1
-#define PAGE_STATE_THROTTLED 2
-#define PAGE_STATE_ZEROFILL 3
-#define PAGE_STATE_INACTIVE 4
-
-#define VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m) \
- MACRO_BEGIN \
- /* \
- * If a "reusable" page somehow made it back into \
- * the active queue, it's been re-used and is not \
- * quite re-usable. \
- * If the VM object was "all_reusable", consider it \
- * as "all re-used" instead of converting it to \
- * "partially re-used", which could be expensive. \
- */ \
- if ((m)->reusable || \
- (m)->object->all_reusable) { \
- vm_object_reuse_pages((m)->object, \
- (m)->offset, \
- (m)->offset + PAGE_SIZE_64, \
- FALSE); \
- } \
- MACRO_END
+
+/*
+ * function in BSD to apply I/O throttle to the pageout thread
+ */
+extern void vm_pageout_io_throttle(void);
+
+/*
+ * Page States: Used below to maintain the page state
+ * before it's removed from it's Q. This saved state
+ * helps us do the right accounting in certain cases
+ */
+#define PAGE_STATE_SPECULATIVE 1
+#define PAGE_STATE_ANONYMOUS 2
+#define PAGE_STATE_INACTIVE 3
+#define PAGE_STATE_INACTIVE_FIRST 4
+#define PAGE_STATE_CLEAN 5
+
+
+#define VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m) \
+ MACRO_BEGIN \
+ /* \
+ * If a "reusable" page somehow made it back into \
+ * the active queue, it's been re-used and is not \
+ * quite re-usable. \
+ * If the VM object was "all_reusable", consider it \
+ * as "all re-used" instead of converting it to \
+ * "partially re-used", which could be expensive. \
+ */ \
+ if ((m)->reusable || \
+ (m)->object->all_reusable) { \
+ vm_object_reuse_pages((m)->object, \
+ (m)->offset, \
+ (m)->offset + PAGE_SIZE_64, \
+ FALSE); \
+ } \
+ MACRO_END
+
+
+#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT 64
+#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX 1024
+
+#define FCS_IDLE 0
+#define FCS_DELAYED 1
+#define FCS_DEADLOCK_DETECTED 2
+
+struct flow_control {
+ int state;
+ mach_timespec_t ts;
+};
+
+uint32_t vm_pageout_considered_page = 0;
+uint32_t vm_page_filecache_min = 0;
+
+#define ANONS_GRABBED_LIMIT 2
+
+/*
+ * vm_pageout_scan does the dirty work for the pageout daemon.
+ * It returns with both vm_page_queue_free_lock and vm_page_queue_lock
+ * held and vm_page_free_wanted == 0.
+ */
void
vm_pageout_scan(void)
{
vm_page_t local_freeq = NULL;
int local_freed = 0;
int delayed_unlock;
+ int delayed_unlock_limit = 0;
int refmod_state = 0;
int vm_pageout_deadlock_target = 0;
struct vm_pageout_queue *iq;
struct flow_control flow_control = { 0, { 0, 0 } };
boolean_t inactive_throttled = FALSE;
boolean_t try_failed;
- mach_timespec_t ts;
- unsigned int msecs = 0;
+ mach_timespec_t ts;
+ unsigned int msecs = 0;
vm_object_t object;
vm_object_t last_object_tried;
-#if defined(__ppc__) /* On ppc, vm statistics are still 32-bit */
- unsigned int zf_ratio;
- unsigned int zf_run_count;
-#else
- uint64_t zf_ratio;
- uint64_t zf_run_count;
-#endif
uint32_t catch_up_count = 0;
uint32_t inactive_reclaim_run;
boolean_t forced_reclaim;
+ boolean_t exceeded_burst_throttle;
+ boolean_t grab_anonymous = FALSE;
+ boolean_t force_anonymous = FALSE;
+ int anons_grabbed = 0;
int page_prev_state = 0;
+ int cache_evict_throttle = 0;
+ uint32_t vm_pageout_inactive_external_forced_reactivate_limit = 0;
+ int force_purge = 0;
+
+#if VM_PRESSURE_EVENTS
+ vm_pressure_level_t pressure_level;
+#endif /* VM_PRESSURE_EVENTS */
+
+ VM_DEBUG_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_START,
+ vm_pageout_speculative_clean, vm_pageout_inactive_clean,
+ vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external);
flow_control.state = FCS_IDLE;
iq = &vm_pageout_queue_internal;
vm_page_inactive_count);
inactive_reclaim_run = 0;
+ vm_pageout_inactive_external_forced_reactivate_limit = vm_page_active_count + vm_page_inactive_count;
-/*???*/ /*
+ /*
* We want to gradually dribble pages from the active queue
* to the inactive queue. If we let the inactive queue get
* very small, and then suddenly dump many pages into it,
* those pages won't get a sufficient chance to be referenced
* before we start taking them from the inactive queue.
*
- * We must limit the rate at which we send pages to the pagers.
- * data_write messages consume memory, for message buffers and
- * for map-copy objects. If we get too far ahead of the pagers,
- * we can potentially run out of memory.
- *
- * We can use the laundry count to limit directly the number
- * of pages outstanding to the default pager. A similar
- * strategy for external pagers doesn't work, because
- * external pagers don't have to deallocate the pages sent them,
- * and because we might have to send pages to external pagers
- * even if they aren't processing writes. So we also
- * use a burst count to limit writes to external pagers.
- *
- * When memory is very tight, we can't rely on external pagers to
- * clean pages. They probably aren't running, because they
- * aren't vm-privileged. If we kept sending dirty pages to them,
- * we could exhaust the free list.
+ * We must limit the rate at which we send pages to the pagers
+ * so that we don't tie up too many pages in the I/O queues.
+ * We implement a throttling mechanism using the laundry count
+ * to limit the number of pages outstanding to the default
+ * and external pagers. We can bypass the throttles and look
+ * for clean pages if the pageout queues don't drain in a timely
+ * fashion since this may indicate that the pageout paths are
+ * stalled waiting for memory, which only we can provide.
*/
Restart:
assert(delayed_unlock!=0);
- /*
- * A page is "zero-filled" if it was not paged in from somewhere,
- * and it belongs to an object at least VM_ZF_OBJECT_SIZE_THRESHOLD big.
- * Recalculate the zero-filled page ratio. We use this to apportion
- * victimized pages between the normal and zero-filled inactive
- * queues according to their relative abundance in memory. Thus if a task
- * is flooding memory with zf pages, we begin to hunt them down.
- * It would be better to throttle greedy tasks at a higher level,
- * but at the moment mach vm cannot do this.
- */
- {
-#if defined(__ppc__) /* On ppc, vm statistics are still 32-bit */
- uint32_t total = vm_page_active_count + vm_page_inactive_count;
- uint32_t normal = total - vm_zf_count;
-#else
- uint64_t total = vm_page_active_count + vm_page_inactive_count;
- uint64_t normal = total - vm_zf_count;
-#endif
-
- /* zf_ratio is the number of zf pages we victimize per normal page */
-
- if (vm_zf_count < vm_accellerate_zf_pageout_trigger)
- zf_ratio = 0;
- else if ((vm_zf_count <= normal) || (normal == 0))
- zf_ratio = 1;
- else
- zf_ratio = vm_zf_count / normal;
-
- zf_run_count = 0;
- }
-
/*
* Recalculate vm_page_inactivate_target.
*/
vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
vm_page_inactive_count +
vm_page_speculative_count);
+
+ vm_page_anonymous_min = vm_page_inactive_target / 20;
+
+
/*
* don't want to wake the pageout_scan thread up everytime we fall below
* the targets... set a low water mark at 0.25% below the target
*/
vm_page_inactive_min = vm_page_inactive_target - (vm_page_inactive_target / 400);
+ if (vm_page_speculative_percentage > 50)
+ vm_page_speculative_percentage = 50;
+ else if (vm_page_speculative_percentage <= 0)
+ vm_page_speculative_percentage = 1;
+
vm_page_speculative_target = VM_PAGE_SPECULATIVE_TARGET(vm_page_active_count +
vm_page_inactive_count);
+
object = NULL;
last_object_tried = NULL;
try_failed = FALSE;
vm_page_lock_queues();
delayed_unlock = 1;
}
+ if (vm_upl_wait_for_pages < 0)
+ vm_upl_wait_for_pages = 0;
+
+ delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT + vm_upl_wait_for_pages;
+
+ if (delayed_unlock_limit > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX)
+ delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX;
/*
- * Don't sweep through active queue more than the throttle
- * which should be kept relatively low
+ * Move pages from active to inactive if we're below the target
*/
- active_burst_count = MIN(vm_pageout_burst_active_throttle,
- vm_page_active_count);
+ /* if we are trying to make clean, we need to make sure we actually have inactive - mj */
+ if ((vm_page_inactive_count + vm_page_speculative_count) >= vm_page_inactive_target)
+ goto done_moving_active_pages;
+ if (object != NULL) {
+ vm_object_unlock(object);
+ object = NULL;
+ vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+ }
/*
- * Move pages from active to inactive.
+ * Don't sweep through active queue more than the throttle
+ * which should be kept relatively low
*/
- if ((vm_page_inactive_count + vm_page_speculative_count) >= vm_page_inactive_target)
- goto done_moving_active_pages;
+ active_burst_count = MIN(vm_pageout_burst_active_throttle, vm_page_active_count);
- while (!queue_empty(&vm_page_queue_active) && active_burst_count) {
+ VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_START,
+ vm_pageout_inactive, vm_pageout_inactive_used, vm_page_free_count, local_freed);
- if (active_burst_count)
- active_burst_count--;
+ VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_NONE,
+ vm_pageout_speculative_clean, vm_pageout_inactive_clean,
+ vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external);
+ memoryshot(VM_PAGEOUT_BALANCE, DBG_FUNC_START);
+
+
+ while (!queue_empty(&vm_page_queue_active) && active_burst_count--) {
vm_pageout_active++;
DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
/*
- * Try to lock object; since we've already got the
- * page queues lock, we can only 'try' for this one.
- * if the 'try' fails, we need to do a mutex_pause
- * to allow the owner of the object lock a chance to
- * run... otherwise, we're likely to trip over this
- * object in the same state as we work our way through
- * the queue... clumps of pages associated with the same
- * object are fairly typical on the inactive and active queues
- */
- if (m->object != object) {
- if (object != NULL) {
- vm_object_unlock(object);
- object = NULL;
- vm_pageout_scan_wants_object = VM_OBJECT_NULL;
- }
- if (!vm_object_lock_try_scan(m->object)) {
- /*
- * move page to end of active queue and continue
- */
- queue_remove(&vm_page_queue_active, m,
- vm_page_t, pageq);
- queue_enter(&vm_page_queue_active, m,
- vm_page_t, pageq);
-
- try_failed = TRUE;
-
- m = (vm_page_t) queue_first(&vm_page_queue_active);
- /*
- * this is the next object we're going to be interested in
- * try to make sure it's available after the mutex_yield
- * returns control
- */
- vm_pageout_scan_wants_object = m->object;
-
- goto done_with_activepage;
- }
- object = m->object;
-
- try_failed = FALSE;
- }
-
- /*
- * if the page is BUSY, then we pull it
- * off the active queue and leave it alone.
- * when BUSY is cleared, it will get stuck
- * back on the appropriate queue
+ * by not passing in a pmap_flush_context we will forgo any TLB flushing, local or otherwise...
+ *
+ * a TLB flush isn't really needed here since at worst we'll miss the reference bit being
+ * updated in the PTE if a remote processor still has this mapping cached in its TLB when the
+ * new reference happens. If no futher references happen on the page after that remote TLB flushes
+ * we'll see a clean, non-referenced page when it eventually gets pulled out of the inactive queue
+ * by pageout_scan, which is just fine since the last reference would have happened quite far
+ * in the past (TLB caches don't hang around for very long), and of course could just as easily
+ * have happened before we moved the page
*/
- if (m->busy) {
- queue_remove(&vm_page_queue_active, m,
- vm_page_t, pageq);
- m->pageq.next = NULL;
- m->pageq.prev = NULL;
-
- if (!m->fictitious)
- vm_page_active_count--;
- m->active = FALSE;
-
- goto done_with_activepage;
- }
-
- /* deal with a rogue "reusable" page */
- VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m);
+ pmap_clear_refmod_options(m->phys_page, VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL);
/*
- * Deactivate the page while holding the object
- * locked, so we know the page is still not busy.
- * This should prevent races between pmap_enter
- * and pmap_clear_reference. The page might be
- * absent or fictitious, but vm_page_deactivate
- * can handle that.
+ * The page might be absent or busy,
+ * but vm_page_deactivate can handle that.
+ * FALSE indicates that we don't want a H/W clear reference
*/
- vm_page_deactivate(m);
+ vm_page_deactivate_internal(m, FALSE);
-done_with_activepage:
- if (delayed_unlock++ > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT || try_failed == TRUE) {
+ if (delayed_unlock++ > delayed_unlock_limit) {
- if (object != NULL) {
- vm_pageout_scan_wants_object = VM_OBJECT_NULL;
- vm_object_unlock(object);
- object = NULL;
- }
- if (local_freeq) {
+ if (local_freeq) {
vm_page_unlock_queues();
- vm_page_free_list(local_freeq, TRUE);
+ VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START,
+ vm_page_free_count, local_freed, delayed_unlock_limit, 1);
+
+ vm_page_free_list(local_freeq, TRUE);
+
+ VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END,
+ vm_page_free_count, 0, 0, 1);
+
local_freeq = NULL;
local_freed = 0;
vm_page_lock_queues();
- } else
+ } else {
lck_mtx_yield(&vm_page_queue_lock);
-
+ }
+
delayed_unlock = 1;
/*
}
}
-
+ VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_END,
+ vm_page_active_count, vm_page_inactive_count, vm_page_speculative_count, vm_page_inactive_target);
+ memoryshot(VM_PAGEOUT_BALANCE, DBG_FUNC_END);
/**********************************************************************
* above this point we're playing with the active queue
done_moving_active_pages:
- /*
- * We are done if we have met our target *and*
- * nobody is still waiting for a page.
- */
if (vm_page_free_count + local_freed >= vm_page_free_target) {
if (object != NULL) {
vm_object_unlock(object);
if (local_freeq) {
vm_page_unlock_queues();
- vm_page_free_list(local_freeq, TRUE);
+ VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START,
+ vm_page_free_count, local_freed, delayed_unlock_limit, 2);
+
+ vm_page_free_list(local_freeq, TRUE);
+
+ VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END,
+ vm_page_free_count, local_freed, 0, 2);
+
local_freeq = NULL;
local_freed = 0;
vm_page_lock_queues();
}
/*
- * inactive target still not met... keep going
- * until we get the queues balanced
+ * make sure the pageout I/O threads are running
+ * throttled in case there are still requests
+ * in the laundry... since we have met our targets
+ * we don't need the laundry to be cleaned in a timely
+ * fashion... so let's avoid interfering with foreground
+ * activity
*/
+ vm_pageout_adjust_io_throttles(iq, eq, TRUE);
/*
- * Recalculate vm_page_inactivate_target.
+ * recalculate vm_page_inactivate_target
*/
vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
vm_page_inactive_count +
vm_page_speculative_count);
-
-#ifndef CONFIG_EMBEDDED
- /*
- * XXX: if no active pages can be reclaimed, pageout scan can be stuck trying
- * to balance the queues
- */
if (((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) &&
- !queue_empty(&vm_page_queue_active))
+ !queue_empty(&vm_page_queue_active)) {
+ /*
+ * inactive target still not met... keep going
+ * until we get the queues balanced...
+ */
continue;
-#endif
-
+ }
lck_mtx_lock(&vm_page_queue_free_lock);
if ((vm_page_free_count >= vm_page_free_target) &&
(vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) {
-
- vm_page_unlock_queues();
-
- thread_wakeup((event_t) &vm_pageout_garbage_collect);
-
+ /*
+ * done - we have met our target *and*
+ * there is no one waiting for a page.
+ */
+return_from_scan:
assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
+ VM_DEBUG_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_NONE,
+ vm_pageout_inactive, vm_pageout_inactive_used, 0, 0);
+ VM_DEBUG_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_END,
+ vm_pageout_speculative_clean, vm_pageout_inactive_clean,
+ vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external);
+
return;
}
lck_mtx_unlock(&vm_page_queue_free_lock);
* If the purge succeeds, go back to the top and reevalute
* the new memory situation.
*/
+
assert (available_for_purge>=0);
- if (available_for_purge)
- {
+ force_purge = 0; /* no force-purging */
+
+#if VM_PRESSURE_EVENTS
+ pressure_level = memorystatus_vm_pressure_level;
+
+ if (pressure_level > kVMPressureNormal) {
+
+ if (pressure_level >= kVMPressureCritical) {
+ force_purge = memorystatus_purge_on_critical;
+ } else if (pressure_level >= kVMPressureUrgent) {
+ force_purge = memorystatus_purge_on_urgent;
+ } else if (pressure_level >= kVMPressureWarning) {
+ force_purge = memorystatus_purge_on_warning;
+ }
+ }
+#endif /* VM_PRESSURE_EVENTS */
+
+ if (available_for_purge || force_purge) {
+
if (object != NULL) {
vm_object_unlock(object);
object = NULL;
}
- if(TRUE == vm_purgeable_object_purge_one()) {
+
+ memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_START);
+
+ VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_START, vm_page_free_count, 0, 0, 0);
+ if (vm_purgeable_object_purge_one(force_purge, C_DONT_BLOCK)) {
+
+ VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_END, vm_page_free_count, 0, 0, 0);
+ memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_END);
continue;
}
+ VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_END, 0, 0, 0, -1);
+ memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_END);
}
-
+
if (queue_empty(&sq->age_q) && vm_page_speculative_count) {
/*
- * try to pull pages from the aging bins
+ * try to pull pages from the aging bins...
* see vm_page.h for an explanation of how
* this mechanism works
*/
aq = &vm_page_queue_speculative[speculative_steal_index];
}
- if (num_scanned_queues ==
- VM_PAGE_MAX_SPECULATIVE_AGE_Q + 1) {
+ if (num_scanned_queues == VM_PAGE_MAX_SPECULATIVE_AGE_Q + 1) {
/*
* XXX We've scanned all the speculative
* queues but still haven't found one
* that is not empty, even though
* vm_page_speculative_count is not 0.
+ *
+ * report the anomaly...
*/
- /* report the anomaly... */
printf("vm_pageout_scan: "
"all speculative queues empty "
"but count=%d. Re-adjusting.\n",
vm_page_speculative_count);
- if (vm_page_speculative_count >
- vm_page_speculative_count_drift_max)
+ if (vm_page_speculative_count > vm_page_speculative_count_drift_max)
vm_page_speculative_count_drift_max = vm_page_speculative_count;
vm_page_speculative_count_drifts++;
#if 6553678
if (vm_page_speculative_count > vm_page_speculative_target)
can_steal = TRUE;
else {
- ts_fully_aged.tv_sec = (VM_PAGE_MAX_SPECULATIVE_AGE_Q * VM_PAGE_SPECULATIVE_Q_AGE_MS) / 1000;
- ts_fully_aged.tv_nsec = ((VM_PAGE_MAX_SPECULATIVE_AGE_Q * VM_PAGE_SPECULATIVE_Q_AGE_MS) % 1000)
+ ts_fully_aged.tv_sec = (VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_page_speculative_q_age_ms) / 1000;
+ ts_fully_aged.tv_nsec = ((VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_page_speculative_q_age_ms) % 1000)
* 1000 * NSEC_PER_USEC;
ADD_MACH_TIMESPEC(&ts_fully_aged, &aq->age_ts);
if (can_steal == TRUE)
vm_page_speculate_ageit(aq);
}
+ if (queue_empty(&sq->age_q) && cache_evict_throttle == 0) {
+ int pages_evicted;
+ if (object != NULL) {
+ vm_object_unlock(object);
+ object = NULL;
+ }
+ pages_evicted = vm_object_cache_evict(100, 10);
+
+ if (pages_evicted) {
+
+ vm_pageout_cache_evicted += pages_evicted;
+
+ VM_DEBUG_EVENT(vm_pageout_cache_evict, VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE,
+ vm_page_free_count, pages_evicted, vm_pageout_cache_evicted, 0);
+ memoryshot(VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE);
+
+ /*
+ * we just freed up to 100 pages,
+ * so go back to the top of the main loop
+ * and re-evaulate the memory situation
+ */
+ continue;
+ } else
+ cache_evict_throttle = 100;
+ }
+ if (cache_evict_throttle)
+ cache_evict_throttle--;
+
+ /*
+ * don't let the filecache_min fall below 33% of available memory...
+ *
+ * on systems w/o the compressor/swapper, the filecache is always
+ * a very large percentage of the AVAILABLE_NON_COMPRESSED_MEMORY
+ * since most (if not all) of the anonymous pages are in the
+ * throttled queue (which isn't counted as available) which
+ * effectively disables this filter
+ */
+ vm_page_filecache_min = (AVAILABLE_NON_COMPRESSED_MEMORY / 3);
+
+ exceeded_burst_throttle = FALSE;
/*
* Sometimes we have to pause:
* 1) No inactive pages - nothing to do.
- * 2) Flow control - default pageout queue is full
- * 3) Loop control - no acceptable pages found on the inactive queue
+ * 2) Loop control - no acceptable pages found on the inactive queue
* within the last vm_pageout_burst_inactive_throttle iterations
+ * 3) Flow control - default pageout queue is full
*/
- if (queue_empty(&vm_page_queue_inactive) && queue_empty(&vm_page_queue_zf) && queue_empty(&sq->age_q) &&
- (VM_PAGE_Q_THROTTLED(iq) || queue_empty(&vm_page_queue_throttled))) {
+ if (queue_empty(&vm_page_queue_inactive) && queue_empty(&vm_page_queue_anonymous) && queue_empty(&sq->age_q)) {
vm_pageout_scan_empty_throttle++;
msecs = vm_pageout_empty_wait;
goto vm_pageout_scan_delay;
vm_page_speculative_count))) {
vm_pageout_scan_burst_throttle++;
msecs = vm_pageout_burst_wait;
+
+ exceeded_burst_throttle = TRUE;
+ goto vm_pageout_scan_delay;
+
+ } else if (vm_page_free_count > (vm_page_free_reserved / 4) &&
+ VM_PAGEOUT_SCAN_NEEDS_TO_THROTTLE()) {
+ vm_pageout_scan_swap_throttle++;
+ msecs = vm_pageout_swap_wait;
goto vm_pageout_scan_delay;
- } else if (VM_PAGE_Q_THROTTLED(iq) && IP_VALID(memory_manager_default)) {
+ } else if (VM_PAGE_Q_THROTTLED(iq) &&
+ VM_DYNAMIC_PAGING_ENABLED(memory_manager_default)) {
clock_sec_t sec;
clock_nsec_t nsec;
switch (flow_control.state) {
case FCS_IDLE:
+ if ((vm_page_free_count + local_freed) < vm_page_free_target) {
+
+ if (vm_page_pageable_external_count > vm_page_filecache_min && !queue_empty(&vm_page_queue_inactive)) {
+ anons_grabbed = ANONS_GRABBED_LIMIT;
+ goto consider_inactive;
+ }
+ if (((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) && vm_page_active_count)
+ continue;
+ }
reset_deadlock_timer:
ts.tv_sec = vm_pageout_deadlock_wait / 1000;
ts.tv_nsec = (vm_pageout_deadlock_wait % 1000) * 1000 * NSEC_PER_USEC;
vm_pageout_deadlock_target = vm_pageout_deadlock_relief + vm_page_free_wanted + vm_page_free_wanted_privileged;
vm_pageout_scan_deadlock_detected++;
flow_control.state = FCS_DEADLOCK_DETECTED;
-
thread_wakeup((event_t) &vm_pageout_garbage_collect);
goto consider_inactive;
}
goto reset_deadlock_timer;
}
- vm_pageout_scan_throttle++;
- iq->pgo_throttled = TRUE;
vm_pageout_scan_delay:
if (object != NULL) {
vm_object_unlock(object);
}
vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+ vm_page_unlock_queues();
+
if (local_freeq) {
- vm_page_unlock_queues();
- vm_page_free_list(local_freeq, TRUE);
+
+ VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START,
+ vm_page_free_count, local_freed, delayed_unlock_limit, 3);
+
+ vm_page_free_list(local_freeq, TRUE);
+ VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END,
+ vm_page_free_count, local_freed, 0, 3);
+
local_freeq = NULL;
local_freed = 0;
- vm_page_lock_queues();
+ }
+ if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE)
+ vm_consider_waking_compactor_swapper();
- if (flow_control.state == FCS_DELAYED &&
- !VM_PAGE_Q_THROTTLED(iq)) {
- flow_control.state = FCS_IDLE;
- vm_pageout_scan_throttle_aborted++;
- goto consider_inactive;
- }
+ vm_page_lock_queues();
+
+ if (flow_control.state == FCS_DELAYED &&
+ !VM_PAGE_Q_THROTTLED(iq)) {
+ flow_control.state = FCS_IDLE;
+ goto consider_inactive;
+ }
+
+ if (vm_page_free_count >= vm_page_free_target) {
+ /*
+ * we're here because
+ * 1) someone else freed up some pages while we had
+ * the queues unlocked above
+ * and we've hit one of the 3 conditions that
+ * cause us to pause the pageout scan thread
+ *
+ * since we already have enough free pages,
+ * let's avoid stalling and return normally
+ *
+ * before we return, make sure the pageout I/O threads
+ * are running throttled in case there are still requests
+ * in the laundry... since we have enough free pages
+ * we don't need the laundry to be cleaned in a timely
+ * fashion... so let's avoid interfering with foreground
+ * activity
+ *
+ * we don't want to hold vm_page_queue_free_lock when
+ * calling vm_pageout_adjust_io_throttles (since it
+ * may cause other locks to be taken), we do the intitial
+ * check outside of the lock. Once we take the lock,
+ * we recheck the condition since it may have changed.
+ * if it has, no problem, we will make the threads
+ * non-throttled before actually blocking
+ */
+ vm_pageout_adjust_io_throttles(iq, eq, TRUE);
}
-#if CONFIG_EMBEDDED
- {
- int percent_avail;
+ lck_mtx_lock(&vm_page_queue_free_lock);
- /*
- * Decide if we need to send a memory status notification.
- */
- percent_avail =
- (vm_page_active_count + vm_page_inactive_count +
- vm_page_speculative_count + vm_page_free_count +
- (IP_VALID(memory_manager_default)?0:vm_page_purgeable_count) ) * 100 /
- atop_64(max_mem);
- if (percent_avail >= (kern_memorystatus_level + 5) ||
- percent_avail <= (kern_memorystatus_level - 5)) {
- kern_memorystatus_level = percent_avail;
- thread_wakeup((event_t)&kern_memorystatus_wakeup);
+ if (vm_page_free_count >= vm_page_free_target &&
+ (vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) {
+ goto return_from_scan;
}
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+
+ if ((vm_page_free_count + vm_page_cleaned_count) < vm_page_free_target) {
+ /*
+ * we're most likely about to block due to one of
+ * the 3 conditions that cause vm_pageout_scan to
+ * not be able to make forward progress w/r
+ * to providing new pages to the free queue,
+ * so unthrottle the I/O threads in case we
+ * have laundry to be cleaned... it needs
+ * to be completed ASAP.
+ *
+ * even if we don't block, we want the io threads
+ * running unthrottled since the sum of free +
+ * clean pages is still under our free target
+ */
+ vm_pageout_adjust_io_throttles(iq, eq, FALSE);
}
-#endif
+ if (vm_page_cleaned_count > 0 && exceeded_burst_throttle == FALSE) {
+ /*
+ * if we get here we're below our free target and
+ * we're stalling due to a full laundry queue or
+ * we don't have any inactive pages other then
+ * those in the clean queue...
+ * however, we have pages on the clean queue that
+ * can be moved to the free queue, so let's not
+ * stall the pageout scan
+ */
+ flow_control.state = FCS_IDLE;
+ goto consider_inactive;
+ }
+ VM_CHECK_MEMORYSTATUS;
+
+ if (flow_control.state != FCS_IDLE)
+ vm_pageout_scan_throttle++;
+ iq->pgo_throttled = TRUE;
+
+ if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE)
+ vm_consider_waking_compactor_swapper();
+
assert_wait_timeout((event_t) &iq->pgo_laundry, THREAD_INTERRUPTIBLE, msecs, 1000*NSEC_PER_USEC);
counter(c_vm_pageout_scan_block++);
assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
+ VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START,
+ iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0);
+ memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START);
+
thread_block(THREAD_CONTINUE_NULL);
+ VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END,
+ iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0);
+ memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END);
+
vm_page_lock_queues();
delayed_unlock = 1;
flow_control.state = FCS_IDLE;
consider_inactive:
+ vm_pageout_inactive_external_forced_reactivate_limit = MIN((vm_page_active_count + vm_page_inactive_count),
+ vm_pageout_inactive_external_forced_reactivate_limit);
loop_count++;
inactive_burst_count++;
vm_pageout_inactive++;
- /* Choose a victim. */
-
- while (1) {
+
+ /*
+ * Choose a victim.
+ */
+ while (1) {
m = NULL;
- if (IP_VALID(memory_manager_default)) {
+ if (VM_DYNAMIC_PAGING_ENABLED(memory_manager_default)) {
assert(vm_page_throttled_count == 0);
assert(queue_empty(&vm_page_queue_throttled));
}
-
/*
* The most eligible pages are ones we paged in speculatively,
* but which have not yet been touched.
*/
- if ( !queue_empty(&sq->age_q) ) {
- m = (vm_page_t) queue_first(&sq->age_q);
+ if (!queue_empty(&sq->age_q) && force_anonymous == FALSE) {
+ m = (vm_page_t) queue_first(&sq->age_q);
+
+ page_prev_state = PAGE_STATE_SPECULATIVE;
+
break;
}
/*
- * Time for a zero-filled inactive page?
+ * Try a clean-queue inactive page.
*/
- if ( ((zf_run_count < zf_ratio) && vm_zf_queue_count >= zf_queue_min_count) ||
- queue_empty(&vm_page_queue_inactive)) {
- if ( !queue_empty(&vm_page_queue_zf) ) {
- m = (vm_page_t) queue_first(&vm_page_queue_zf);
- zf_run_count++;
+ if (!queue_empty(&vm_page_queue_cleaned)) {
+ m = (vm_page_t) queue_first(&vm_page_queue_cleaned);
+
+ page_prev_state = PAGE_STATE_CLEAN;
+
+ break;
+ }
+
+ grab_anonymous = (vm_page_anonymous_count > vm_page_anonymous_min);
+
+ if (vm_page_pageable_external_count < vm_page_filecache_min || force_anonymous == TRUE) {
+ grab_anonymous = TRUE;
+ anons_grabbed = 0;
+ }
+
+ if (grab_anonymous == FALSE || anons_grabbed >= ANONS_GRABBED_LIMIT || queue_empty(&vm_page_queue_anonymous)) {
+
+ if ( !queue_empty(&vm_page_queue_inactive) ) {
+ m = (vm_page_t) queue_first(&vm_page_queue_inactive);
+
+ page_prev_state = PAGE_STATE_INACTIVE;
+ anons_grabbed = 0;
+
+ if (vm_page_pageable_external_count < vm_page_filecache_min) {
+ if ((++reactivated_this_call % 100))
+ goto must_activate_page;
+ /*
+ * steal 1% of the file backed pages even if
+ * we are under the limit that has been set
+ * for a healthy filecache
+ */
+ }
break;
}
}
+ if ( !queue_empty(&vm_page_queue_anonymous) ) {
+ m = (vm_page_t) queue_first(&vm_page_queue_anonymous);
+
+ page_prev_state = PAGE_STATE_ANONYMOUS;
+ anons_grabbed++;
+
+ break;
+ }
+
/*
- * It's either a normal inactive page or nothing.
+ * if we've gotten here, we have no victim page.
+ * if making clean, free the local freed list and return.
+ * if making free, check to see if we've finished balancing the queues
+ * yet, if we haven't just continue, else panic
*/
- if ( !queue_empty(&vm_page_queue_inactive) ) {
- m = (vm_page_t) queue_first(&vm_page_queue_inactive);
- zf_run_count = 0;
- break;
- }
+ vm_page_unlock_queues();
+
+ if (object != NULL) {
+ vm_object_unlock(object);
+ object = NULL;
+ }
+ vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+
+ if (local_freeq) {
+ VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START,
+ vm_page_free_count, local_freed, delayed_unlock_limit, 5);
+
+ vm_page_free_list(local_freeq, TRUE);
+
+ VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END,
+ vm_page_free_count, local_freed, 0, 5);
+
+ local_freeq = NULL;
+ local_freed = 0;
+ }
+ vm_page_lock_queues();
+ delayed_unlock = 1;
+
+ force_anonymous = FALSE;
+
+ if ((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target)
+ goto Restart;
- panic("vm_pageout: no victim");
+ if (!queue_empty(&sq->age_q))
+ goto Restart;
+
+ panic("vm_pageout: no victim");
+
+ /* NOTREACHED */
}
+ force_anonymous = FALSE;
+
+ /*
+ * we just found this page on one of our queues...
+ * it can't also be on the pageout queue, so safe
+ * to call VM_PAGE_QUEUES_REMOVE
+ */
+ assert(!m->pageout_queue);
+
+ VM_PAGE_QUEUES_REMOVE(m);
- assert(!m->active && (m->inactive || m->speculative || m->throttled));
assert(!m->laundry);
+ assert(!m->private);
+ assert(!m->fictitious);
assert(m->object != kernel_object);
assert(m->phys_page != vm_page_guard_addr);
- if (!m->speculative) {
+
+ if (page_prev_state != PAGE_STATE_SPECULATIVE)
vm_pageout_stats[vm_pageout_stat_now].considered++;
- }
DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
* object are fairly typical on the inactive and active queues
*/
if (!vm_object_lock_try_scan(m->object)) {
+ vm_page_t m_want = NULL;
+
vm_pageout_inactive_nolock++;
- requeue_page:
- /*
- * Move page to end and continue.
- * Don't re-issue ticket
- */
- if (m->zero_fill) {
- if (m->speculative) {
- panic("vm_pageout_scan(): page %p speculative and zero-fill !?\n", m);
- }
- assert(!m->speculative);
- queue_remove(&vm_page_queue_zf, m,
- vm_page_t, pageq);
- queue_enter(&vm_page_queue_zf, m,
- vm_page_t, pageq);
- } else if (m->speculative) {
- remque(&m->pageq);
- m->speculative = FALSE;
- vm_page_speculative_count--;
-
- /*
- * move to the head of the inactive queue
- * to get it out of the way... the speculative
- * queue is generally too small to depend
- * on there being enough pages from other
- * objects to make cycling it back on the
- * same queue a winning proposition
- */
- queue_enter_first(&vm_page_queue_inactive, m,
- vm_page_t, pageq);
- m->inactive = TRUE;
- vm_page_inactive_count++;
- token_new_pagecount++;
- } else if (m->throttled) {
- queue_remove(&vm_page_queue_throttled, m,
- vm_page_t, pageq);
- m->throttled = FALSE;
- vm_page_throttled_count--;
-
- /*
- * not throttled any more, so can stick
- * it on the inactive queue.
- */
- queue_enter(&vm_page_queue_inactive, m,
- vm_page_t, pageq);
- m->inactive = TRUE;
- vm_page_inactive_count++;
- token_new_pagecount++;
- } else {
- queue_remove(&vm_page_queue_inactive, m,
- vm_page_t, pageq);
-#if MACH_ASSERT
- vm_page_inactive_count--; /* balance for purgeable queue asserts */
-#endif
- vm_purgeable_q_advance_all();
+ if (page_prev_state == PAGE_STATE_CLEAN)
+ vm_pageout_cleaned_nolock++;
+
+ if (page_prev_state == PAGE_STATE_SPECULATIVE)
+ page_prev_state = PAGE_STATE_INACTIVE_FIRST;
- queue_enter(&vm_page_queue_inactive, m,
- vm_page_t, pageq);
-#if MACH_ASSERT
- vm_page_inactive_count++; /* balance for purgeable queue asserts */
-#endif
- token_new_pagecount++;
- }
pmap_clear_reference(m->phys_page);
m->reference = FALSE;
+ /*
+ * m->object must be stable since we hold the page queues lock...
+ * we can update the scan_collisions field sans the object lock
+ * since it is a separate field and this is the only spot that does
+ * a read-modify-write operation and it is never executed concurrently...
+ * we can asynchronously set this field to 0 when creating a UPL, so it
+ * is possible for the value to be a bit non-determistic, but that's ok
+ * since it's only used as a hint
+ */
+ m->object->scan_collisions++;
+
if ( !queue_empty(&sq->age_q) )
- m = (vm_page_t) queue_first(&sq->age_q);
- else if ( ((zf_run_count < zf_ratio) && vm_zf_queue_count >= zf_queue_min_count) ||
- queue_empty(&vm_page_queue_inactive)) {
- if ( !queue_empty(&vm_page_queue_zf) )
- m = (vm_page_t) queue_first(&vm_page_queue_zf);
- } else if ( !queue_empty(&vm_page_queue_inactive) ) {
- m = (vm_page_t) queue_first(&vm_page_queue_inactive);
- }
+ m_want = (vm_page_t) queue_first(&sq->age_q);
+ else if ( !queue_empty(&vm_page_queue_cleaned))
+ m_want = (vm_page_t) queue_first(&vm_page_queue_cleaned);
+ else if (anons_grabbed >= ANONS_GRABBED_LIMIT || queue_empty(&vm_page_queue_anonymous))
+ m_want = (vm_page_t) queue_first(&vm_page_queue_inactive);
+ else if ( !queue_empty(&vm_page_queue_anonymous))
+ m_want = (vm_page_t) queue_first(&vm_page_queue_anonymous);
+
/*
* this is the next object we're going to be interested in
* try to make sure its available after the mutex_yield
* returns control
*/
- vm_pageout_scan_wants_object = m->object;
+ if (m_want)
+ vm_pageout_scan_wants_object = m_want->object;
/*
* force us to dump any collected free pages
*/
try_failed = TRUE;
- goto done_with_inactivepage;
+ goto requeue_page;
}
object = m->object;
vm_pageout_scan_wants_object = VM_OBJECT_NULL;
try_failed = FALSE;
}
+ if (catch_up_count)
+ catch_up_count--;
- /*
- * Paging out pages of external objects which
- * are currently being created must be avoided.
- * The pager may claim for memory, thus leading to a
- * possible dead lock between it and the pageout thread,
- * if such pages are finally chosen. The remaining assumption
- * is that there will finally be enough available pages in the
- * inactive pool to page out in order to satisfy all memory
- * claimed by the thread which concurrently creates the pager.
- */
- if (!object->pager_initialized && object->pager_created) {
- /*
- * Move page to end and continue, hoping that
- * there will be enough other inactive pages to
- * page out so that the thread which currently
- * initializes the pager will succeed.
- * Don't re-grant the ticket, the page should
- * pulled from the queue and paged out whenever
- * one of its logically adjacent fellows is
- * targeted.
- */
- vm_pageout_inactive_avoid++;
- goto requeue_page;
- }
- /*
- * Remove the page from its list.
- */
- if (m->speculative) {
- remque(&m->pageq);
- page_prev_state = PAGE_STATE_SPECULATIVE;
- m->speculative = FALSE;
- vm_page_speculative_count--;
- } else if (m->throttled) {
- queue_remove(&vm_page_queue_throttled, m, vm_page_t, pageq);
- page_prev_state = PAGE_STATE_THROTTLED;
- m->throttled = FALSE;
- vm_page_throttled_count--;
- } else {
- if (m->zero_fill) {
- queue_remove(&vm_page_queue_zf, m, vm_page_t, pageq);
- page_prev_state = PAGE_STATE_ZEROFILL;
- vm_zf_queue_count--;
- } else {
- page_prev_state = PAGE_STATE_INACTIVE;
- queue_remove(&vm_page_queue_inactive, m, vm_page_t, pageq);
+ if (m->busy) {
+ if (m->encrypted_cleaning) {
+ /*
+ * ENCRYPTED SWAP:
+ * if this page has already been picked up as
+ * part of a page-out cluster, it will be busy
+ * because it is being encrypted (see
+ * vm_object_upl_request()). But we still
+ * want to demote it from "clean-in-place"
+ * (aka "adjacent") to "clean-and-free" (aka
+ * "target"), so let's ignore its "busy" bit
+ * here and proceed to check for "cleaning" a
+ * little bit below...
+ *
+ * CAUTION CAUTION:
+ * A "busy" page should still be left alone for
+ * most purposes, so we have to be very careful
+ * not to process that page too much.
+ */
+ assert(m->cleaning);
+ goto consider_inactive_page;
}
- m->inactive = FALSE;
- if (!m->fictitious)
- vm_page_inactive_count--;
- vm_purgeable_q_advance_all();
- }
-
- m->pageq.next = NULL;
- m->pageq.prev = NULL;
-
- if ( !m->fictitious && catch_up_count)
- catch_up_count--;
- /*
- * ENCRYPTED SWAP:
- * if this page has already been picked up as part of a
- * page-out cluster, it will be busy because it is being
- * encrypted (see vm_object_upl_request()). But we still
- * want to demote it from "clean-in-place" (aka "adjacent")
- * to "clean-and-free" (aka "target"), so let's ignore its
- * "busy" bit here and proceed to check for "cleaning" a
- * little bit below...
- */
- if ( !m->encrypted_cleaning && (m->busy || !object->alive)) {
/*
* Somebody is already playing with this page.
- * Leave it off the pageout queues.
+ * Put it back on the appropriate queue
*
*/
vm_pageout_inactive_busy++;
+ if (page_prev_state == PAGE_STATE_CLEAN)
+ vm_pageout_cleaned_busy++;
+
+requeue_page:
+ switch (page_prev_state) {
+
+ case PAGE_STATE_SPECULATIVE:
+ vm_page_speculate(m, FALSE);
+ break;
+
+ case PAGE_STATE_ANONYMOUS:
+ case PAGE_STATE_CLEAN:
+ case PAGE_STATE_INACTIVE:
+ VM_PAGE_ENQUEUE_INACTIVE(m, FALSE);
+ break;
+
+ case PAGE_STATE_INACTIVE_FIRST:
+ VM_PAGE_ENQUEUE_INACTIVE(m, TRUE);
+ break;
+ }
goto done_with_inactivepage;
}
+
/*
- * If it's absent or in error, we can reclaim the page.
+ * If it's absent, in error or the object is no longer alive,
+ * we can reclaim the page... in the no longer alive case,
+ * there are 2 states the page can be in that preclude us
+ * from reclaiming it - busy or cleaning - that we've already
+ * dealt with
*/
+ if (m->absent || m->error || !object->alive) {
- if (m->absent || m->error) {
- vm_pageout_inactive_absent++;
-reclaim_page:
+ if (m->absent)
+ vm_pageout_inactive_absent++;
+ else if (!object->alive)
+ vm_pageout_inactive_notalive++;
+ else
+ vm_pageout_inactive_error++;
+reclaim_page:
if (vm_pageout_deadlock_target) {
vm_pageout_scan_inactive_throttle_success++;
vm_pageout_deadlock_target--;
} else {
DTRACE_VM2(fsfree, int, 1, (uint64_t *), NULL);
}
- vm_page_free_prepare_queues(m);
+ assert(!m->cleaning);
+ assert(!m->laundry);
+
+ m->busy = TRUE;
/*
* remove page from object here since we're already
m->pageq.next = (queue_entry_t)local_freeq;
local_freeq = m;
local_freed++;
+
+ if (page_prev_state == PAGE_STATE_SPECULATIVE)
+ vm_pageout_freed_from_speculative++;
+ else if (page_prev_state == PAGE_STATE_CLEAN)
+ vm_pageout_freed_from_cleaned++;
+ else
+ vm_pageout_freed_from_inactive_clean++;
- inactive_burst_count = 0;
-
- if(page_prev_state != PAGE_STATE_SPECULATIVE) {
+ if (page_prev_state != PAGE_STATE_SPECULATIVE)
vm_pageout_stats[vm_pageout_stat_now].reclaimed++;
- page_prev_state = 0;
- }
-
- goto done_with_inactivepage;
- }
-
- assert(!m->private);
- assert(!m->fictitious);
-
- /*
- * If already cleaning this page in place, convert from
- * "adjacent" to "target". We can leave the page mapped,
- * and vm_pageout_object_terminate will determine whether
- * to free or reactivate.
- */
-
- if (m->cleaning) {
- m->busy = TRUE;
- m->pageout = TRUE;
- m->dump_cleaning = TRUE;
- vm_page_wire(m);
-
- CLUSTER_STAT(vm_pageout_cluster_conversions++);
inactive_burst_count = 0;
-
goto done_with_inactivepage;
}
-
/*
* If the object is empty, the page must be reclaimed even
* if dirty or used.
*/
if (object->copy == VM_OBJECT_NULL) {
if (object->purgable == VM_PURGABLE_EMPTY) {
- m->busy = TRUE;
if (m->pmapped == TRUE) {
/* unmap the page */
refmod_state = pmap_disconnect(m->phys_page);
if (refmod_state & VM_MEM_MODIFIED) {
- m->dirty = TRUE;
+ SET_PAGE_DIRTY(m, FALSE);
}
}
if (m->dirty || m->precious) {
}
goto reclaim_page;
}
- if (object->purgable == VM_PURGABLE_VOLATILE) {
+
+ if (COMPRESSED_PAGER_IS_ACTIVE) {
+ /*
+ * With the VM compressor, the cost of
+ * reclaiming a page is much lower (no I/O),
+ * so if we find a "volatile" page, it's better
+ * to let it get compressed rather than letting
+ * it occupy a full page until it gets purged.
+ * So no need to check for "volatile" here.
+ */
+ } else if (object->purgable == VM_PURGABLE_VOLATILE) {
+ /*
+ * Avoid cleaning a "volatile" page which might
+ * be purged soon.
+ */
+
/* if it's wired, we can't put it on our queue */
assert(!VM_PAGE_WIRED(m));
+
/* just stick it back on! */
+ reactivated_this_call++;
+
+ if (page_prev_state == PAGE_STATE_CLEAN)
+ vm_pageout_cleaned_volatile_reactivated++;
+
goto reactivate_page;
}
}
+consider_inactive_page:
+ if (m->busy) {
+ /*
+ * CAUTION CAUTION:
+ * A "busy" page should always be left alone, except...
+ */
+ if (m->cleaning && m->encrypted_cleaning) {
+ /*
+ * ENCRYPTED_SWAP:
+ * We could get here with a "busy" page
+ * if it's being encrypted during a
+ * "clean-in-place" operation. We'll deal
+ * with it right away by testing if it has been
+ * referenced and either reactivating it or
+ * promoting it from "clean-in-place" to
+ * "clean-and-free".
+ */
+ } else {
+ panic("\"busy\" page considered for pageout\n");
+ }
+ }
+
/*
* If it's being used, reactivate.
* (Fictitious pages are either busy or absent.)
if (refmod_state & VM_MEM_REFERENCED)
m->reference = TRUE;
- if (refmod_state & VM_MEM_MODIFIED)
- m->dirty = TRUE;
+ if (refmod_state & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(m, FALSE);
+ }
}
-
- if (m->reference || m->dirty) {
- /* deal with a rogue "reusable" page */
- VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m);
+
+ /*
+ * if (m->cleaning && !m->pageout)
+ * If already cleaning this page in place and it hasn't
+ * been recently referenced, just pull off the queue.
+ * We can leave the page mapped, and upl_commit_range
+ * will put it on the clean queue.
+ *
+ * note: if m->encrypted_cleaning == TRUE, then
+ * m->cleaning == TRUE
+ * and we'll handle it here
+ *
+ * if (m->pageout && !m->cleaning)
+ * an msync INVALIDATE is in progress...
+ * this page has been marked for destruction
+ * after it has been cleaned,
+ * but not yet gathered into a UPL
+ * where 'cleaning' will be set...
+ * just leave it off the paging queues
+ *
+ * if (m->pageout && m->clenaing)
+ * an msync INVALIDATE is in progress
+ * and the UPL has already gathered this page...
+ * just leave it off the paging queues
+ */
+
+ /*
+ * page with m->pageout and still on the queues means that an
+ * MS_INVALIDATE is in progress on this page... leave it alone
+ */
+ if (m->pageout) {
+ goto done_with_inactivepage;
}
+
+ /* if cleaning, reactivate if referenced. otherwise, just pull off queue */
+ if (m->cleaning) {
+ if (m->reference == TRUE) {
+ reactivated_this_call++;
+ goto reactivate_page;
+ } else {
+ goto done_with_inactivepage;
+ }
+ }
+
+ if (m->reference || m->dirty) {
+ /* deal with a rogue "reusable" page */
+ VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m);
+ }
- if (m->reference && !m->no_cache) {
+ if (!m->no_cache &&
+ (m->reference ||
+ (m->xpmapped && !object->internal && (vm_page_xpmapped_external_count < (vm_page_external_count / 4))))) {
/*
* The page we pulled off the inactive list has
* been referenced. It is possible for other
vm_pageout_inactive_force_reclaim++;
} else {
uint32_t isinuse;
+
+ if (page_prev_state == PAGE_STATE_CLEAN)
+ vm_pageout_cleaned_reference_reactivated++;
+
reactivate_page:
if ( !object->internal && object->pager != MEMORY_OBJECT_NULL &&
vnode_pager_get_isinuse(object->pager, &isinuse) == KERN_SUCCESS && !isinuse) {
vm_page_deactivate(m);
vm_pageout_inactive_deactivated++;
} else {
+must_activate_page:
/*
* The page was/is being used, so put back on active list.
*/
vm_page_activate(m);
VM_STAT_INCR(reactivations);
+ inactive_burst_count = 0;
}
+
+ if (page_prev_state == PAGE_STATE_CLEAN)
+ vm_pageout_cleaned_reactivated++;
+
vm_pageout_inactive_used++;
- inactive_burst_count = 0;
goto done_with_inactivepage;
}
*/
if ((refmod_state == -1) && !m->dirty && m->pmapped) {
refmod_state = pmap_get_refmod(m->phys_page);
- if (refmod_state & VM_MEM_MODIFIED)
- m->dirty = TRUE;
+ if (refmod_state & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(m, FALSE);
+ }
}
forced_reclaim = TRUE;
} else {
inactive_throttled = TRUE;
}
}
- if (inactive_throttled == TRUE) {
throttle_inactive:
- if (!IP_VALID(memory_manager_default) &&
- object->internal && m->dirty &&
- (object->purgable == VM_PURGABLE_DENY ||
- object->purgable == VM_PURGABLE_NONVOLATILE ||
- object->purgable == VM_PURGABLE_VOLATILE)) {
- queue_enter(&vm_page_queue_throttled, m,
- vm_page_t, pageq);
- m->throttled = TRUE;
- vm_page_throttled_count++;
- } else {
- if (m->zero_fill) {
- queue_enter(&vm_page_queue_zf, m,
- vm_page_t, pageq);
- vm_zf_queue_count++;
- } else
- queue_enter(&vm_page_queue_inactive, m,
- vm_page_t, pageq);
- m->inactive = TRUE;
- if (!m->fictitious) {
- vm_page_inactive_count++;
- token_new_pagecount++;
- }
- }
- vm_pageout_scan_inactive_throttled++;
+ if (!VM_DYNAMIC_PAGING_ENABLED(memory_manager_default) &&
+ object->internal && m->dirty &&
+ (object->purgable == VM_PURGABLE_DENY ||
+ object->purgable == VM_PURGABLE_NONVOLATILE ||
+ object->purgable == VM_PURGABLE_VOLATILE)) {
+ queue_enter(&vm_page_queue_throttled, m,
+ vm_page_t, pageq);
+ m->throttled = TRUE;
+ vm_page_throttled_count++;
+
+ vm_pageout_scan_reclaimed_throttled++;
+
+ inactive_burst_count = 0;
goto done_with_inactivepage;
}
+ if (inactive_throttled == TRUE) {
- /*
- * we've got a page that we can steal...
- * eliminate all mappings and make sure
- * we have the up-to-date modified state
- * first take the page BUSY, so that no new
- * mappings can be made
- */
- m->busy = TRUE;
-
- /*
- * if we need to do a pmap_disconnect then we
- * need to re-evaluate m->dirty since the pmap_disconnect
- * provides the true state atomically... the
- * page was still mapped up to the pmap_disconnect
- * and may have been dirtied at the last microsecond
+ if (object->internal == FALSE) {
+ /*
+ * we need to break up the following potential deadlock case...
+ * a) The external pageout thread is stuck on the truncate lock for a file that is being extended i.e. written.
+ * b) The thread doing the writing is waiting for pages while holding the truncate lock
+ * c) Most of the pages in the inactive queue belong to this file.
+ *
+ * we are potentially in this deadlock because...
+ * a) the external pageout queue is throttled
+ * b) we're done with the active queue and moved on to the inactive queue
+ * c) we've got a dirty external page
+ *
+ * since we don't know the reason for the external pageout queue being throttled we
+ * must suspect that we are deadlocked, so move the current page onto the active queue
+ * in an effort to cause a page from the active queue to 'age' to the inactive queue
+ *
+ * if we don't have jetsam configured (i.e. we have a dynamic pager), set
+ * 'force_anonymous' to TRUE to cause us to grab a page from the cleaned/anonymous
+ * pool the next time we select a victim page... if we can make enough new free pages,
+ * the deadlock will break, the external pageout queue will empty and it will no longer
+ * be throttled
+ *
+ * if we have jestam configured, keep a count of the pages reactivated this way so
+ * that we can try to find clean pages in the active/inactive queues before
+ * deciding to jetsam a process
+ */
+ vm_pageout_scan_inactive_throttled_external++;
+
+ queue_enter(&vm_page_queue_active, m, vm_page_t, pageq);
+ m->active = TRUE;
+ vm_page_active_count++;
+ vm_page_pageable_external_count++;
+
+ vm_pageout_adjust_io_throttles(iq, eq, FALSE);
+
+#if CONFIG_MEMORYSTATUS && CONFIG_JETSAM
+ vm_pageout_inactive_external_forced_reactivate_limit--;
+
+ if (vm_pageout_inactive_external_forced_reactivate_limit <= 0) {
+ vm_pageout_inactive_external_forced_reactivate_limit = vm_page_active_count + vm_page_inactive_count;
+ /*
+ * Possible deadlock scenario so request jetsam action
+ */
+ assert(object);
+ vm_object_unlock(object);
+ object = VM_OBJECT_NULL;
+ vm_page_unlock_queues();
+
+ VM_DEBUG_EVENT(vm_pageout_jetsam, VM_PAGEOUT_JETSAM, DBG_FUNC_START,
+ vm_page_active_count, vm_page_inactive_count, vm_page_free_count, vm_page_free_count);
+
+ /* Kill first suitable process */
+ if (memorystatus_kill_on_VM_page_shortage(FALSE) == FALSE) {
+ panic("vm_pageout_scan: Jetsam request failed\n");
+ }
+
+ VM_DEBUG_EVENT(vm_pageout_jetsam, VM_PAGEOUT_JETSAM, DBG_FUNC_END, 0, 0, 0, 0);
+
+ vm_pageout_inactive_external_forced_jetsam_count++;
+ vm_page_lock_queues();
+ delayed_unlock = 1;
+ }
+#else /* CONFIG_MEMORYSTATUS && CONFIG_JETSAM */
+ force_anonymous = TRUE;
+#endif
+ inactive_burst_count = 0;
+ goto done_with_inactivepage;
+ } else {
+ if (page_prev_state == PAGE_STATE_SPECULATIVE)
+ page_prev_state = PAGE_STATE_INACTIVE;
+
+ vm_pageout_scan_inactive_throttled_internal++;
+
+ goto requeue_page;
+ }
+ }
+
+ /*
+ * we've got a page that we can steal...
+ * eliminate all mappings and make sure
+ * we have the up-to-date modified state
*
- * we also check for the page being referenced 'late'
- * if it was, we first need to do a WAKEUP_DONE on it
- * since we already set m->busy = TRUE, before
- * going off to reactivate it
+ * if we need to do a pmap_disconnect then we
+ * need to re-evaluate m->dirty since the pmap_disconnect
+ * provides the true state atomically... the
+ * page was still mapped up to the pmap_disconnect
+ * and may have been dirtied at the last microsecond
*
* Note that if 'pmapped' is FALSE then the page is not
* and has not been in any map, so there is no point calling
- * pmap_disconnect(). m->dirty and/or m->reference could
- * have been set in anticipation of likely usage of the page.
+ * pmap_disconnect(). m->dirty could have been set in anticipation
+ * of likely usage of the page.
*/
if (m->pmapped == TRUE) {
- refmod_state = pmap_disconnect(m->phys_page);
- if (refmod_state & VM_MEM_MODIFIED)
- m->dirty = TRUE;
- if (refmod_state & VM_MEM_REFERENCED) {
-
- /* If m->reference is already set, this page must have
- * already failed the reactivate_limit test, so don't
- * bump the counts twice.
+ if (DEFAULT_PAGER_IS_ACTIVE || DEFAULT_FREEZER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE || object->internal == FALSE) {
+ /*
+ * Don't count this page as going into the compressor if any of these are true:
+ * 1) We have the dynamic pager i.e. no compressed pager
+ * 2) Freezer enabled device with a freezer file to hold the app data i.e. no compressed pager
+ * 3) Freezer enabled device with compressed pager backend (exclusive use) i.e. most of the VM system
+ (including vm_pageout_scan) has no knowledge of the compressor
+ * 4) This page belongs to a file and hence will not be sent into the compressor
*/
- if ( ! m->reference ) {
- m->reference = TRUE;
- if (forced_reclaim ||
- ++reactivated_this_call >= reactivate_limit)
- vm_pageout_reactivation_limit_exceeded++;
- else {
- PAGE_WAKEUP_DONE(m);
- goto reactivate_page;
- }
- }
+
+ refmod_state = pmap_disconnect_options(m->phys_page, 0, NULL);
+ } else {
+ refmod_state = pmap_disconnect_options(m->phys_page, PMAP_OPTIONS_COMPRESSOR, NULL);
+ }
+
+ if (refmod_state & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(m, FALSE);
}
}
/*
* If it's clean and not precious, we can free the page.
*/
if (!m->dirty && !m->precious) {
- if (m->zero_fill)
- vm_pageout_inactive_zf++;
- vm_pageout_inactive_clean++;
+ if (page_prev_state == PAGE_STATE_SPECULATIVE)
+ vm_pageout_speculative_clean++;
+ else {
+ if (page_prev_state == PAGE_STATE_ANONYMOUS)
+ vm_pageout_inactive_anonymous++;
+ else if (page_prev_state == PAGE_STATE_CLEAN)
+ vm_pageout_cleaned_reclaimed++;
+
+ vm_pageout_inactive_clean++;
+ }
+
+ /*
+ * OK, at this point we have found a page we are going to free.
+ */
+#if CONFIG_PHANTOM_CACHE
+ if (!object->internal)
+ vm_phantom_cache_add_ghost(m);
+#endif
goto reclaim_page;
}
* if the page was clean then). With the dirty page
* disconnected here, we can make one final check.
*/
- {
- boolean_t disconnect_throttled = FALSE;
- if (object->internal) {
- if (VM_PAGE_Q_THROTTLED(iq))
- disconnect_throttled = TRUE;
- } else if (VM_PAGE_Q_THROTTLED(eq)) {
- disconnect_throttled = TRUE;
- }
-
- if (disconnect_throttled == TRUE) {
- PAGE_WAKEUP_DONE(m);
- goto throttle_inactive;
- }
+ if (object->internal) {
+ if (VM_PAGE_Q_THROTTLED(iq))
+ inactive_throttled = TRUE;
+ } else if (VM_PAGE_Q_THROTTLED(eq)) {
+ inactive_throttled = TRUE;
}
- vm_pageout_stats[vm_pageout_stat_now].reclaimed++;
+ if (inactive_throttled == TRUE)
+ goto throttle_inactive;
+
+#if VM_PRESSURE_EVENTS
+#if CONFIG_JETSAM
+
+ /*
+ * If Jetsam is enabled, then the sending
+ * of memory pressure notifications is handled
+ * from the same thread that takes care of high-water
+ * and other jetsams i.e. the memorystatus_thread.
+ */
+
+#else /* CONFIG_JETSAM */
+
+ vm_pressure_response();
- vm_pageout_cluster(m);
+#endif /* CONFIG_JETSAM */
+#endif /* VM_PRESSURE_EVENTS */
+
+ /*
+ * do NOT set the pageout bit!
+ * sure, we might need free pages, but this page is going to take time to become free
+ * anyway, so we may as well put it on the clean queue first and take it from there later
+ * if necessary. that way, we'll ensure we don't free up too much. -mj
+ */
+ vm_pageout_cluster(m, FALSE);
- if (m->zero_fill)
- vm_pageout_inactive_zf++;
- vm_pageout_inactive_dirty++;
+ if (page_prev_state == PAGE_STATE_ANONYMOUS)
+ vm_pageout_inactive_anonymous++;
+ if (object->internal)
+ vm_pageout_inactive_dirty_internal++;
+ else
+ vm_pageout_inactive_dirty_external++;
- inactive_burst_count = 0;
done_with_inactivepage:
- if (delayed_unlock++ > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT || try_failed == TRUE) {
+
+ if (delayed_unlock++ > delayed_unlock_limit || try_failed == TRUE) {
+ boolean_t need_delay = TRUE;
if (object != NULL) {
vm_pageout_scan_wants_object = VM_OBJECT_NULL;
vm_object_unlock(object);
object = NULL;
}
+ vm_page_unlock_queues();
+
if (local_freeq) {
- vm_page_unlock_queues();
- vm_page_free_list(local_freeq, TRUE);
+
+ VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START,
+ vm_page_free_count, local_freed, delayed_unlock_limit, 4);
+
+ vm_page_free_list(local_freeq, TRUE);
+ VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END,
+ vm_page_free_count, local_freed, 0, 4);
+
local_freeq = NULL;
local_freed = 0;
- vm_page_lock_queues();
- } else
+ need_delay = FALSE;
+ }
+ if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) {
+ vm_consider_waking_compactor_swapper();
+ need_delay = FALSE;
+ }
+ vm_page_lock_queues();
+
+ if (need_delay == TRUE)
lck_mtx_yield(&vm_page_queue_lock);
delayed_unlock = 1;
}
+ vm_pageout_considered_page++;
+
/*
* back to top of pageout scan loop
*/
{
int free_after_reserve;
- vm_page_free_reserved += pages;
+ if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) {
+
+ if ((vm_page_free_reserved + pages + COMPRESSOR_FREE_RESERVED_LIMIT) >= (VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT))
+ vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT;
+ else
+ vm_page_free_reserved += (pages + COMPRESSOR_FREE_RESERVED_LIMIT);
+ } else {
+ if ((vm_page_free_reserved + pages) >= VM_PAGE_FREE_RESERVED_LIMIT)
+ vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT;
+ else
+ vm_page_free_reserved += pages;
+ }
free_after_reserve = vm_page_free_count_init - vm_page_free_reserved;
vm_page_free_min = vm_page_free_reserved +
vm_page_free_target = vm_page_free_min + 5;
vm_page_throttle_limit = vm_page_free_target - (vm_page_free_target / 3);
- vm_page_creation_throttle = vm_page_free_target / 2;
}
/*
{
DTRACE_VM2(pgrrun, int, 1, (uint64_t *), NULL);
vm_pageout_scan_event_counter++;
+
vm_pageout_scan();
- /* we hold vm_page_queue_free_lock now */
+ /*
+ * we hold both the vm_page_queue_free_lock
+ * and the vm_page_queues_lock at this point
+ */
assert(vm_page_free_wanted == 0);
assert(vm_page_free_wanted_privileged == 0);
assert_wait((event_t) &vm_page_free_wanted, THREAD_UNINT);
+
lck_mtx_unlock(&vm_page_queue_free_lock);
+ vm_page_unlock_queues();
counter(c_vm_pageout_block++);
thread_block((thread_continue_t)vm_pageout_continue);
{
vm_page_t m = NULL;
vm_object_t object;
+ vm_object_offset_t offset;
memory_object_t pager;
thread_t self = current_thread();
q->pgo_busy = TRUE;
queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq);
+ if (m->object->object_slid) {
+ panic("slid page %p not allowed on this path\n", m);
+ }
VM_PAGE_CHECK(m);
m->pageout_queue = FALSE;
m->pageq.next = NULL;
m->pageq.prev = NULL;
+
+ /*
+ * grab a snapshot of the object and offset this
+ * page is tabled in so that we can relookup this
+ * page after we've taken the object lock - these
+ * fields are stable while we hold the page queues lock
+ * but as soon as we drop it, there is nothing to keep
+ * this page in this object... we hold an activity_in_progress
+ * on this object which will keep it from terminating
+ */
+ object = m->object;
+ offset = m->offset;
+
vm_page_unlock_queues();
#ifdef FAKE_DEADLOCK
}
}
#endif
- object = m->object;
-
vm_object_lock(object);
+ m = vm_page_lookup(object, offset);
+
+ if (m == NULL ||
+ m->busy || m->cleaning || m->pageout_queue || !m->laundry) {
+ /*
+ * it's either the same page that someone else has
+ * started cleaning (or it's finished cleaning or
+ * been put back on the pageout queue), or
+ * the page has been freed or we have found a
+ * new page at this offset... in all of these cases
+ * we merely need to release the activity_in_progress
+ * we took when we put the page on the pageout queue
+ */
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+
+ vm_page_lockspin_queues();
+ continue;
+ }
if (!object->pager_initialized) {
/*
* Should only happen if there is no
* default pager.
*/
+ m->pageout = FALSE;
+
vm_page_lockspin_queues();
- vm_pageout_queue_steal(m, TRUE);
- vm_pageout_dirty_no_pager++;
+ vm_pageout_throttle_up(m);
vm_page_activate(m);
+ vm_pageout_dirty_no_pager++;
vm_page_unlock_queues();
/*
* And we are done with it.
*/
- PAGE_WAKEUP_DONE(m);
-
- vm_object_paging_end(object);
+ vm_object_activity_end(object);
vm_object_unlock(object);
vm_page_lockspin_queues();
}
}
pager = object->pager;
+
if (pager == MEMORY_OBJECT_NULL) {
/*
* This pager has been destroyed by either
} else {
vm_page_lockspin_queues();
- vm_pageout_queue_steal(m, TRUE);
+ vm_pageout_throttle_up(m);
vm_page_activate(m);
vm_page_unlock_queues();
/*
* And we are done with it.
*/
- PAGE_WAKEUP_DONE(m);
}
- vm_object_paging_end(object);
+ vm_object_activity_end(object);
vm_object_unlock(object);
vm_page_lockspin_queues();
continue;
}
+#if 0
+ /*
+ * we don't hold the page queue lock
+ * so this check isn't safe to make
+ */
VM_PAGE_CHECK(m);
- vm_object_unlock(object);
+#endif
/*
- * we expect the paging_in_progress reference to have
- * already been taken on the object before it was added
- * to the appropriate pageout I/O queue... this will
- * keep the object from being terminated and/or the
- * paging_offset from changing until the I/O has
- * completed... therefore no need to lock the object to
- * pull the paging_offset from it.
- *
+ * give back the activity_in_progress reference we
+ * took when we queued up this page and replace it
+ * it with a paging_in_progress reference that will
+ * also hold the paging offset from changing and
+ * prevent the object from terminating
+ */
+ vm_object_activity_end(object);
+ vm_object_paging_begin(object);
+ vm_object_unlock(object);
+
+ /*
* Send the data to the pager.
* any pageout clustering happens there
*/
vm_object_paging_end(object);
vm_object_unlock(object);
+ vm_pageout_io_throttle();
+
vm_page_lockspin_queues();
}
- assert_wait((event_t) q, THREAD_UNINT);
+ q->pgo_busy = FALSE;
+ q->pgo_idle = TRUE;
+
+ assert_wait((event_t) &q->pgo_pending, THREAD_UNINT);
+ vm_page_unlock_queues();
+
+ thread_block_parameter((thread_continue_t)vm_pageout_iothread_continue, (void *) q);
+ /*NOTREACHED*/
+}
- if (q->pgo_throttled == TRUE && !VM_PAGE_Q_THROTTLED(q)) {
- q->pgo_throttled = FALSE;
- thread_wakeup((event_t) &q->pgo_laundry);
+
+static void
+vm_pageout_iothread_external_continue(struct vm_pageout_queue *q)
+{
+ vm_page_t m = NULL;
+ vm_object_t object;
+ vm_object_offset_t offset;
+ memory_object_t pager;
+
+
+ if (vm_pageout_internal_iothread != THREAD_NULL)
+ current_thread()->options &= ~TH_OPT_VMPRIV;
+
+ vm_page_lockspin_queues();
+
+ while ( !queue_empty(&q->pgo_pending) ) {
+
+ q->pgo_busy = TRUE;
+ queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq);
+ if (m->object->object_slid) {
+ panic("slid page %p not allowed on this path\n", m);
+ }
+ VM_PAGE_CHECK(m);
+ m->pageout_queue = FALSE;
+ m->pageq.next = NULL;
+ m->pageq.prev = NULL;
+
+ /*
+ * grab a snapshot of the object and offset this
+ * page is tabled in so that we can relookup this
+ * page after we've taken the object lock - these
+ * fields are stable while we hold the page queues lock
+ * but as soon as we drop it, there is nothing to keep
+ * this page in this object... we hold an activity_in_progress
+ * on this object which will keep it from terminating
+ */
+ object = m->object;
+ offset = m->offset;
+
+ vm_page_unlock_queues();
+
+ vm_object_lock(object);
+
+ m = vm_page_lookup(object, offset);
+
+ if (m == NULL ||
+ m->busy || m->cleaning || m->pageout_queue || !m->laundry) {
+ /*
+ * it's either the same page that someone else has
+ * started cleaning (or it's finished cleaning or
+ * been put back on the pageout queue), or
+ * the page has been freed or we have found a
+ * new page at this offset... in all of these cases
+ * we merely need to release the activity_in_progress
+ * we took when we put the page on the pageout queue
+ */
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+
+ vm_page_lockspin_queues();
+ continue;
+ }
+ pager = object->pager;
+
+ if (pager == MEMORY_OBJECT_NULL) {
+ /*
+ * This pager has been destroyed by either
+ * memory_object_destroy or vm_object_destroy, and
+ * so there is nowhere for the page to go.
+ */
+ if (m->pageout) {
+ /*
+ * Just free the page... VM_PAGE_FREE takes
+ * care of cleaning up all the state...
+ * including doing the vm_pageout_throttle_up
+ */
+ VM_PAGE_FREE(m);
+ } else {
+ vm_page_lockspin_queues();
+
+ vm_pageout_throttle_up(m);
+ vm_page_activate(m);
+
+ vm_page_unlock_queues();
+
+ /*
+ * And we are done with it.
+ */
+ }
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+
+ vm_page_lockspin_queues();
+ continue;
+ }
+#if 0
+ /*
+ * we don't hold the page queue lock
+ * so this check isn't safe to make
+ */
+ VM_PAGE_CHECK(m);
+#endif
+ /*
+ * give back the activity_in_progress reference we
+ * took when we queued up this page and replace it
+ * it with a paging_in_progress reference that will
+ * also hold the paging offset from changing and
+ * prevent the object from terminating
+ */
+ vm_object_activity_end(object);
+ vm_object_paging_begin(object);
+ vm_object_unlock(object);
+
+ /*
+ * Send the data to the pager.
+ * any pageout clustering happens there
+ */
+ memory_object_data_return(pager,
+ m->offset + object->paging_offset,
+ PAGE_SIZE,
+ NULL,
+ NULL,
+ FALSE,
+ FALSE,
+ 0);
+
+ vm_object_lock(object);
+ vm_object_paging_end(object);
+ vm_object_unlock(object);
+
+ vm_pageout_io_throttle();
+
+ vm_page_lockspin_queues();
}
- if (q->pgo_draining == TRUE && q->pgo_laundry == 0) {
- q->pgo_draining = FALSE;
- thread_wakeup((event_t) (&q->pgo_laundry+1));
+ q->pgo_busy = FALSE;
+ q->pgo_idle = TRUE;
+
+ assert_wait((event_t) &q->pgo_pending, THREAD_UNINT);
+ vm_page_unlock_queues();
+
+ thread_block_parameter((thread_continue_t)vm_pageout_iothread_external_continue, (void *) q);
+ /*NOTREACHED*/
+}
+
+
+uint32_t vm_compressor_failed;
+
+static void
+vm_pageout_iothread_internal_continue(struct cq *cq)
+{
+ struct vm_pageout_queue *q;
+ vm_page_t m = NULL;
+ vm_object_t object;
+ memory_object_t pager;
+ boolean_t pgo_draining;
+ vm_page_t local_q;
+ int local_cnt;
+ vm_page_t local_freeq = NULL;
+ int local_freed = 0;
+ int local_batch_size;
+ kern_return_t retval;
+ int compressed_count_delta;
+
+
+ KERNEL_DEBUG(0xe040000c | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+ q = cq->q;
+ local_batch_size = q->pgo_maxlaundry / (vm_compressor_thread_count * 4);
+
+ while (TRUE) {
+
+ local_cnt = 0;
+ local_q = NULL;
+
+ KERNEL_DEBUG(0xe0400014 | DBG_FUNC_START, 0, 0, 0, 0, 0);
+
+ vm_page_lock_queues();
+
+ KERNEL_DEBUG(0xe0400014 | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+ KERNEL_DEBUG(0xe0400018 | DBG_FUNC_START, 0, 0, 0, 0, 0);
+
+ while ( !queue_empty(&q->pgo_pending) && local_cnt < local_batch_size) {
+
+ queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq);
+
+ VM_PAGE_CHECK(m);
+
+ m->pageout_queue = FALSE;
+ m->pageq.prev = NULL;
+
+ m->pageq.next = (queue_entry_t)local_q;
+ local_q = m;
+ local_cnt++;
+ }
+ if (local_q == NULL)
+ break;
+
+ q->pgo_busy = TRUE;
+
+ if ((pgo_draining = q->pgo_draining) == FALSE)
+ vm_pageout_throttle_up_batch(q, local_cnt);
+
+ vm_page_unlock_queues();
+
+ KERNEL_DEBUG(0xe0400018 | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+ while (local_q) {
+
+ m = local_q;
+ local_q = (vm_page_t)m->pageq.next;
+ m->pageq.next = NULL;
+
+ if (m->object->object_slid) {
+ panic("slid page %p not allowed on this path\n", m);
+ }
+
+ object = m->object;
+ pager = object->pager;
+
+ if (!object->pager_initialized || pager == MEMORY_OBJECT_NULL) {
+
+ KERNEL_DEBUG(0xe0400010 | DBG_FUNC_START, object, pager, 0, 0, 0);
+
+ vm_object_lock(object);
+
+ /*
+ * If there is no memory object for the page, create
+ * one and hand it to the compression pager.
+ */
+
+ if (!object->pager_initialized)
+ vm_object_collapse(object, (vm_object_offset_t) 0, TRUE);
+ if (!object->pager_initialized)
+ vm_object_compressor_pager_create(object);
+
+ if (!object->pager_initialized) {
+ /*
+ * Still no pager for the object.
+ * Reactivate the page.
+ *
+ * Should only happen if there is no
+ * compression pager
+ */
+ m->pageout = FALSE;
+ m->laundry = FALSE;
+ PAGE_WAKEUP_DONE(m);
+
+ vm_page_lockspin_queues();
+ vm_page_activate(m);
+ vm_pageout_dirty_no_pager++;
+ vm_page_unlock_queues();
+
+ /*
+ * And we are done with it.
+ */
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+
+ continue;
+ }
+ pager = object->pager;
+
+ if (pager == MEMORY_OBJECT_NULL) {
+ /*
+ * This pager has been destroyed by either
+ * memory_object_destroy or vm_object_destroy, and
+ * so there is nowhere for the page to go.
+ */
+ if (m->pageout) {
+ /*
+ * Just free the page... VM_PAGE_FREE takes
+ * care of cleaning up all the state...
+ * including doing the vm_pageout_throttle_up
+ */
+ VM_PAGE_FREE(m);
+ } else {
+ m->laundry = FALSE;
+ PAGE_WAKEUP_DONE(m);
+
+ vm_page_lockspin_queues();
+ vm_page_activate(m);
+ vm_page_unlock_queues();
+
+ /*
+ * And we are done with it.
+ */
+ }
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+
+ continue;
+ }
+ vm_object_unlock(object);
+
+ KERNEL_DEBUG(0xe0400010 | DBG_FUNC_END, object, pager, 0, 0, 0);
+ }
+ while (vm_page_free_count < (vm_page_free_reserved - COMPRESSOR_FREE_RESERVED_LIMIT)) {
+ kern_return_t wait_result;
+ int need_wakeup = 0;
+
+ if (local_freeq) {
+ vm_page_free_list(local_freeq, TRUE);
+
+ local_freeq = NULL;
+ local_freed = 0;
+
+ continue;
+ }
+ lck_mtx_lock_spin(&vm_page_queue_free_lock);
+
+ if (vm_page_free_count < (vm_page_free_reserved - COMPRESSOR_FREE_RESERVED_LIMIT)) {
+
+ if (vm_page_free_wanted_privileged++ == 0)
+ need_wakeup = 1;
+ wait_result = assert_wait((event_t)&vm_page_free_wanted_privileged, THREAD_UNINT);
+
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+
+ if (need_wakeup)
+ thread_wakeup((event_t)&vm_page_free_wanted);
+
+ if (wait_result == THREAD_WAITING)
+ thread_block(THREAD_CONTINUE_NULL);
+ } else
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+ }
+
+ assert(object->activity_in_progress > 0);
+
+ retval = vm_compressor_pager_put(
+ pager,
+ m->offset + object->paging_offset,
+ m->phys_page,
+ &cq->current_chead,
+ cq->scratch_buf,
+ &compressed_count_delta);
+
+ vm_object_lock(object);
+ assert(object->activity_in_progress > 0);
+
+ assert(m->object == object);
+
+ vm_compressor_pager_count(pager,
+ compressed_count_delta,
+ FALSE, /* shared_lock */
+ object);
+
+ m->laundry = FALSE;
+ m->pageout = FALSE;
+
+ if (retval == KERN_SUCCESS) {
+ /*
+ * If the object is purgeable, its owner's
+ * purgeable ledgers will be updated in
+ * vm_page_remove() but the page still
+ * contributes to the owner's memory footprint,
+ * so account for it as such.
+ */
+ if (object->purgable != VM_PURGABLE_DENY &&
+ object->vo_purgeable_owner != NULL) {
+ /* one more compressed purgeable page */
+ vm_purgeable_compressed_update(object,
+ +1);
+ }
+
+ vm_page_compressions_failing = FALSE;
+
+ VM_STAT_INCR(compressions);
+
+ if (m->tabled)
+ vm_page_remove(m, TRUE);
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+
+ m->pageq.next = (queue_entry_t)local_freeq;
+ local_freeq = m;
+ local_freed++;
+
+ } else {
+ PAGE_WAKEUP_DONE(m);
+
+ vm_page_lockspin_queues();
+
+ vm_page_activate(m);
+ vm_compressor_failed++;
+
+ vm_page_compressions_failing = TRUE;
+
+ vm_page_unlock_queues();
+
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+ }
+ }
+ if (local_freeq) {
+ vm_page_free_list(local_freeq, TRUE);
+
+ local_freeq = NULL;
+ local_freed = 0;
+ }
+ if (pgo_draining == TRUE) {
+ vm_page_lockspin_queues();
+ vm_pageout_throttle_up_batch(q, local_cnt);
+ vm_page_unlock_queues();
+ }
}
+ KERNEL_DEBUG(0xe040000c | DBG_FUNC_START, 0, 0, 0, 0, 0);
+
+ /*
+ * queue lock is held and our q is empty
+ */
q->pgo_busy = FALSE;
q->pgo_idle = TRUE;
+
+ assert_wait((event_t) &q->pgo_pending, THREAD_UNINT);
vm_page_unlock_queues();
- thread_block_parameter((thread_continue_t)vm_pageout_iothread_continue, (void *) &q->pgo_pending);
- /*NOTREACHED*/
+ KERNEL_DEBUG(0xe0400018 | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+ thread_block_parameter((thread_continue_t)vm_pageout_iothread_internal_continue, (void *) cq);
+ /*NOTREACHED*/
+}
+
+
+
+static void
+vm_pageout_adjust_io_throttles(struct vm_pageout_queue *iq, struct vm_pageout_queue *eq, boolean_t req_lowpriority)
+{
+ uint32_t policy;
+ boolean_t set_iq = FALSE;
+ boolean_t set_eq = FALSE;
+
+ if (hibernate_cleaning_in_progress == TRUE)
+ req_lowpriority = FALSE;
+
+ if ((DEFAULT_PAGER_IS_ACTIVE || DEFAULT_FREEZER_IS_ACTIVE) && iq->pgo_inited == TRUE && iq->pgo_lowpriority != req_lowpriority)
+ set_iq = TRUE;
+
+ if (eq->pgo_inited == TRUE && eq->pgo_lowpriority != req_lowpriority)
+ set_eq = TRUE;
+
+ if (set_iq == TRUE || set_eq == TRUE) {
+
+ vm_page_unlock_queues();
+
+ if (req_lowpriority == TRUE) {
+ policy = THROTTLE_LEVEL_PAGEOUT_THROTTLED;
+ DTRACE_VM(laundrythrottle);
+ } else {
+ policy = THROTTLE_LEVEL_PAGEOUT_UNTHROTTLED;
+ DTRACE_VM(laundryunthrottle);
+ }
+ if (set_iq == TRUE) {
+ proc_set_task_policy_thread(kernel_task, iq->pgo_tid, TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy);
+
+ iq->pgo_lowpriority = req_lowpriority;
+ }
+ if (set_eq == TRUE) {
+ proc_set_task_policy_thread(kernel_task, eq->pgo_tid, TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy);
+
+ eq->pgo_lowpriority = req_lowpriority;
+ }
+ vm_page_lock_queues();
+ }
}
self->options |= TH_OPT_VMPRIV;
- vm_pageout_iothread_continue(&vm_pageout_queue_external);
+ DTRACE_VM2(laundrythrottle, int, 1, (uint64_t *), NULL);
+
+ proc_set_task_policy_thread(kernel_task, self->thread_id, TASK_POLICY_EXTERNAL,
+ TASK_POLICY_IO, THROTTLE_LEVEL_PAGEOUT_THROTTLED);
+
+ vm_page_lock_queues();
+
+ vm_pageout_queue_external.pgo_tid = self->thread_id;
+ vm_pageout_queue_external.pgo_lowpriority = TRUE;
+ vm_pageout_queue_external.pgo_inited = TRUE;
+
+ vm_page_unlock_queues();
+
+ if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE)
+ vm_pageout_iothread_external_continue(&vm_pageout_queue_external);
+ else
+ vm_pageout_iothread_continue(&vm_pageout_queue_external);
+
/*NOTREACHED*/
}
static void
-vm_pageout_iothread_internal(void)
+vm_pageout_iothread_internal(struct cq *cq)
{
thread_t self = current_thread();
self->options |= TH_OPT_VMPRIV;
- vm_pageout_iothread_continue(&vm_pageout_queue_internal);
+ if (DEFAULT_PAGER_IS_ACTIVE || DEFAULT_FREEZER_IS_ACTIVE) {
+ DTRACE_VM2(laundrythrottle, int, 1, (uint64_t *), NULL);
+
+ proc_set_task_policy_thread(kernel_task, self->thread_id, TASK_POLICY_EXTERNAL,
+ TASK_POLICY_IO, THROTTLE_LEVEL_PAGEOUT_THROTTLED);
+ }
+ vm_page_lock_queues();
+
+ vm_pageout_queue_internal.pgo_tid = self->thread_id;
+ vm_pageout_queue_internal.pgo_lowpriority = TRUE;
+ vm_pageout_queue_internal.pgo_inited = TRUE;
+
+ vm_page_unlock_queues();
+
+ if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) {
+ cq->q = &vm_pageout_queue_internal;
+ cq->current_chead = NULL;
+ cq->scratch_buf = kalloc(COMPRESSOR_SCRATCH_BUF_SIZE);
+
+ vm_pageout_iothread_internal_continue(cq);
+ } else
+ vm_pageout_iothread_continue(&vm_pageout_queue_internal);
+
/*NOTREACHED*/
}
}
}
+extern boolean_t memorystatus_manual_testing_on;
+extern unsigned int memorystatus_level;
+
+
+#if VM_PRESSURE_EVENTS
+
+boolean_t vm_pressure_events_enabled = FALSE;
+
+void
+vm_pressure_response(void)
+{
+
+ vm_pressure_level_t old_level = kVMPressureNormal;
+ int new_level = -1;
+
+ uint64_t available_memory = 0;
+
+ if (vm_pressure_events_enabled == FALSE)
+ return;
+
+
+ available_memory = (((uint64_t) AVAILABLE_NON_COMPRESSED_MEMORY) * 100);
+
+
+ memorystatus_level = (unsigned int) (available_memory / atop_64(max_mem));
+
+ if (memorystatus_manual_testing_on) {
+ return;
+ }
+
+ old_level = memorystatus_vm_pressure_level;
+
+ switch (memorystatus_vm_pressure_level) {
+
+ case kVMPressureNormal:
+ {
+ if (VM_PRESSURE_WARNING_TO_CRITICAL()) {
+ new_level = kVMPressureCritical;
+ } else if (VM_PRESSURE_NORMAL_TO_WARNING()) {
+ new_level = kVMPressureWarning;
+ }
+ break;
+ }
+
+ case kVMPressureWarning:
+ case kVMPressureUrgent:
+ {
+ if (VM_PRESSURE_WARNING_TO_NORMAL()) {
+ new_level = kVMPressureNormal;
+ } else if (VM_PRESSURE_WARNING_TO_CRITICAL()) {
+ new_level = kVMPressureCritical;
+ }
+ break;
+ }
+
+ case kVMPressureCritical:
+ {
+ if (VM_PRESSURE_WARNING_TO_NORMAL()) {
+ new_level = kVMPressureNormal;
+ } else if (VM_PRESSURE_CRITICAL_TO_WARNING()) {
+ new_level = kVMPressureWarning;
+ }
+ break;
+ }
+
+ default:
+ return;
+ }
+
+ if (new_level != -1) {
+ memorystatus_vm_pressure_level = (vm_pressure_level_t) new_level;
+
+ if ((memorystatus_vm_pressure_level != kVMPressureNormal) || (old_level != new_level)) {
+ if (vm_pressure_thread_running == FALSE) {
+ thread_wakeup(&vm_pressure_thread);
+ }
+
+ if (old_level != new_level) {
+ thread_wakeup(&vm_pressure_changed);
+ }
+ }
+ }
+
+}
+#endif /* VM_PRESSURE_EVENTS */
+
+kern_return_t
+mach_vm_pressure_level_monitor(__unused boolean_t wait_for_pressure, __unused unsigned int *pressure_level) {
+
+#if !VM_PRESSURE_EVENTS
+
+ return KERN_FAILURE;
+
+#else /* VM_PRESSURE_EVENTS */
+
+ kern_return_t kr = KERN_SUCCESS;
+
+ if (pressure_level != NULL) {
+
+ vm_pressure_level_t old_level = memorystatus_vm_pressure_level;
+
+ if (wait_for_pressure == TRUE) {
+ wait_result_t wr = 0;
+
+ while (old_level == *pressure_level) {
+ wr = assert_wait((event_t) &vm_pressure_changed,
+ THREAD_INTERRUPTIBLE);
+ if (wr == THREAD_WAITING) {
+ wr = thread_block(THREAD_CONTINUE_NULL);
+ }
+ if (wr == THREAD_INTERRUPTED) {
+ return KERN_ABORTED;
+ }
+ if (wr == THREAD_AWAKENED) {
+
+ old_level = memorystatus_vm_pressure_level;
+
+ if (old_level != *pressure_level) {
+ break;
+ }
+ }
+ }
+ }
+
+ *pressure_level = old_level;
+ kr = KERN_SUCCESS;
+ } else {
+ kr = KERN_INVALID_ARGUMENT;
+ }
+
+ return kr;
+#endif /* VM_PRESSURE_EVENTS */
+}
+
+#if VM_PRESSURE_EVENTS
+void
+vm_pressure_thread(void) {
+ static boolean_t thread_initialized = FALSE;
+
+ if (thread_initialized == TRUE) {
+ vm_pressure_thread_running = TRUE;
+ consider_vm_pressure_events();
+ vm_pressure_thread_running = FALSE;
+ }
+
+ thread_initialized = TRUE;
+ assert_wait((event_t) &vm_pressure_thread, THREAD_UNINT);
+ thread_block((thread_continue_t)vm_pressure_thread);
+}
+#endif /* VM_PRESSURE_EVENTS */
+
+
+uint32_t vm_pageout_considered_page_last = 0;
+
+/*
+ * called once per-second via "compute_averages"
+ */
+void
+compute_pageout_gc_throttle()
+{
+ if (vm_pageout_considered_page != vm_pageout_considered_page_last) {
+
+ vm_pageout_considered_page_last = vm_pageout_considered_page;
+
+ thread_wakeup((event_t) &vm_pageout_garbage_collect);
+ }
+}
+
+
static void
vm_pageout_garbage_collect(int collect)
{
+
if (collect) {
boolean_t buf_large_zfree = FALSE;
+ boolean_t first_try = TRUE;
+
stack_collect();
- /*
- * consider_zone_gc should be last, because the other operations
- * might return memory to zones.
- */
consider_machine_collect();
- if (consider_buffer_cache_collect != NULL) {
- buf_large_zfree = (*consider_buffer_cache_collect)(0);
- }
- consider_zone_gc(buf_large_zfree);
+ m_drain();
+
+ do {
+ if (consider_buffer_cache_collect != NULL) {
+ buf_large_zfree = (*consider_buffer_cache_collect)(0);
+ }
+ if (first_try == TRUE || buf_large_zfree == TRUE) {
+ /*
+ * consider_zone_gc should be last, because the other operations
+ * might return memory to zones.
+ */
+ consider_zone_gc(buf_large_zfree);
+ }
+ first_try = FALSE;
+
+ } while (buf_large_zfree == TRUE && vm_page_free_count < vm_page_free_target);
consider_machine_adjust();
}
-
assert_wait((event_t) &vm_pageout_garbage_collect, THREAD_UNINT);
thread_block_parameter((thread_continue_t) vm_pageout_garbage_collect, (void *)1);
}
+void vm_pageout_reinit_tuneables(void);
+
+void
+vm_pageout_reinit_tuneables(void)
+{
+ vm_compressor_minorcompact_threshold_divisor = 18;
+ vm_compressor_majorcompact_threshold_divisor = 22;
+ vm_compressor_unthrottle_threshold_divisor = 32;
+}
+
+
+#if VM_PAGE_BUCKETS_CHECK
+#if VM_PAGE_FAKE_BUCKETS
+extern vm_map_offset_t vm_page_fake_buckets_start, vm_page_fake_buckets_end;
+#endif /* VM_PAGE_FAKE_BUCKETS */
+#endif /* VM_PAGE_BUCKETS_CHECK */
+
+#define FBDP_TEST_COLLAPSE_COMPRESSOR 0
+#if FBDP_TEST_COLLAPSE_COMPRESSOR
+extern boolean_t vm_object_collapse_compressor_allowed;
+#include <IOKit/IOLib.h>
+#endif /* FBDP_TEST_COLLAPSE_COMPRESSOR */
+
+#define FBDP_TEST_WIRE_AND_EXTRACT 0
+#if FBDP_TEST_WIRE_AND_EXTRACT
+extern ledger_template_t task_ledger_template;
+#include <mach/mach_vm.h>
+extern ppnum_t vm_map_get_phys_page(vm_map_t map,
+ vm_offset_t offset);
+#endif /* FBDP_TEST_WIRE_AND_EXTRACT */
void
vm_pageout(void)
* Initialize some paging parameters.
*/
+ if (vm_pageout_swap_wait == 0)
+ vm_pageout_swap_wait = VM_PAGEOUT_SWAP_WAIT;
+
if (vm_pageout_idle_wait == 0)
vm_pageout_idle_wait = VM_PAGEOUT_IDLE_WAIT;
vm_pageout_queue_external.pgo_busy = FALSE;
vm_pageout_queue_external.pgo_throttled = FALSE;
vm_pageout_queue_external.pgo_draining = FALSE;
+ vm_pageout_queue_external.pgo_lowpriority = FALSE;
+ vm_pageout_queue_external.pgo_tid = -1;
+ vm_pageout_queue_external.pgo_inited = FALSE;
+
queue_init(&vm_pageout_queue_internal.pgo_pending);
vm_pageout_queue_internal.pgo_maxlaundry = 0;
vm_pageout_queue_internal.pgo_busy = FALSE;
vm_pageout_queue_internal.pgo_throttled = FALSE;
vm_pageout_queue_internal.pgo_draining = FALSE;
-
+ vm_pageout_queue_internal.pgo_lowpriority = FALSE;
+ vm_pageout_queue_internal.pgo_tid = -1;
+ vm_pageout_queue_internal.pgo_inited = FALSE;
/* internal pageout thread started when default pager registered first time */
/* external pageout and garbage collection threads started here */
thread_deallocate(vm_pageout_external_iothread);
result = kernel_thread_start_priority((thread_continue_t)vm_pageout_garbage_collect, NULL,
- MINPRI_KERNEL,
+ BASEPRI_DEFAULT,
&thread);
if (result != KERN_SUCCESS)
panic("vm_pageout_garbage_collect: create failed");
thread_deallocate(thread);
+#if VM_PRESSURE_EVENTS
+ result = kernel_thread_start_priority((thread_continue_t)vm_pressure_thread, NULL,
+ BASEPRI_DEFAULT,
+ &thread);
+
+ if (result != KERN_SUCCESS)
+ panic("vm_pressure_thread: create failed");
+
+ thread_deallocate(thread);
+#endif
+
vm_object_reaper_init();
+
+ if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE)
+ vm_compressor_pager_init();
+
+#if VM_PRESSURE_EVENTS
+ vm_pressure_events_enabled = TRUE;
+#endif /* VM_PRESSURE_EVENTS */
+
+#if CONFIG_PHANTOM_CACHE
+ vm_phantom_cache_init();
+#endif
+#if VM_PAGE_BUCKETS_CHECK
+#if VM_PAGE_FAKE_BUCKETS
+ printf("**** DEBUG: protecting fake buckets [0x%llx:0x%llx]\n",
+ (uint64_t) vm_page_fake_buckets_start,
+ (uint64_t) vm_page_fake_buckets_end);
+ pmap_protect(kernel_pmap,
+ vm_page_fake_buckets_start,
+ vm_page_fake_buckets_end,
+ VM_PROT_READ);
+// *(char *) vm_page_fake_buckets_start = 'x'; /* panic! */
+#endif /* VM_PAGE_FAKE_BUCKETS */
+#endif /* VM_PAGE_BUCKETS_CHECK */
+
+#if VM_OBJECT_TRACKING
+ vm_object_tracking_init();
+#endif /* VM_OBJECT_TRACKING */
+
+
+#if FBDP_TEST_COLLAPSE_COMPRESSOR
+ vm_object_size_t backing_size, top_size;
+ vm_object_t backing_object, top_object;
+ vm_map_offset_t backing_offset, top_offset;
+ unsigned char *backing_address, *top_address;
+ kern_return_t kr;
+
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR:\n");
+
+ /* create backing object */
+ backing_size = 15 * PAGE_SIZE;
+ backing_object = vm_object_allocate(backing_size);
+ assert(backing_object != VM_OBJECT_NULL);
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: created backing object %p\n",
+ backing_object);
+ /* map backing object */
+ backing_offset = 0;
+ kr = vm_map_enter(kernel_map, &backing_offset, backing_size, 0,
+ VM_FLAGS_ANYWHERE, backing_object, 0, FALSE,
+ VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
+ assert(kr == KERN_SUCCESS);
+ backing_address = (unsigned char *) backing_offset;
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: "
+ "mapped backing object %p at 0x%llx\n",
+ backing_object, (uint64_t) backing_offset);
+ /* populate with pages to be compressed in backing object */
+ backing_address[0x1*PAGE_SIZE] = 0xB1;
+ backing_address[0x4*PAGE_SIZE] = 0xB4;
+ backing_address[0x7*PAGE_SIZE] = 0xB7;
+ backing_address[0xa*PAGE_SIZE] = 0xBA;
+ backing_address[0xd*PAGE_SIZE] = 0xBD;
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: "
+ "populated pages to be compressed in "
+ "backing_object %p\n", backing_object);
+ /* compress backing object */
+ vm_object_pageout(backing_object);
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: compressing backing_object %p\n",
+ backing_object);
+ /* wait for all the pages to be gone */
+ while (*(volatile int *)&backing_object->resident_page_count != 0)
+ IODelay(10);
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: backing_object %p compressed\n",
+ backing_object);
+ /* populate with pages to be resident in backing object */
+ backing_address[0x0*PAGE_SIZE] = 0xB0;
+ backing_address[0x3*PAGE_SIZE] = 0xB3;
+ backing_address[0x6*PAGE_SIZE] = 0xB6;
+ backing_address[0x9*PAGE_SIZE] = 0xB9;
+ backing_address[0xc*PAGE_SIZE] = 0xBC;
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: "
+ "populated pages to be resident in "
+ "backing_object %p\n", backing_object);
+ /* leave the other pages absent */
+ /* mess with the paging_offset of the backing_object */
+ assert(backing_object->paging_offset == 0);
+ backing_object->paging_offset = 0x3000;
+
+ /* create top object */
+ top_size = 9 * PAGE_SIZE;
+ top_object = vm_object_allocate(top_size);
+ assert(top_object != VM_OBJECT_NULL);
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: created top object %p\n",
+ top_object);
+ /* map top object */
+ top_offset = 0;
+ kr = vm_map_enter(kernel_map, &top_offset, top_size, 0,
+ VM_FLAGS_ANYWHERE, top_object, 0, FALSE,
+ VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
+ assert(kr == KERN_SUCCESS);
+ top_address = (unsigned char *) top_offset;
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: "
+ "mapped top object %p at 0x%llx\n",
+ top_object, (uint64_t) top_offset);
+ /* populate with pages to be compressed in top object */
+ top_address[0x3*PAGE_SIZE] = 0xA3;
+ top_address[0x4*PAGE_SIZE] = 0xA4;
+ top_address[0x5*PAGE_SIZE] = 0xA5;
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: "
+ "populated pages to be compressed in "
+ "top_object %p\n", top_object);
+ /* compress top object */
+ vm_object_pageout(top_object);
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: compressing top_object %p\n",
+ top_object);
+ /* wait for all the pages to be gone */
+ while (top_object->resident_page_count != 0);
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: top_object %p compressed\n",
+ top_object);
+ /* populate with pages to be resident in top object */
+ top_address[0x0*PAGE_SIZE] = 0xA0;
+ top_address[0x1*PAGE_SIZE] = 0xA1;
+ top_address[0x2*PAGE_SIZE] = 0xA2;
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: "
+ "populated pages to be resident in "
+ "top_object %p\n", top_object);
+ /* leave the other pages absent */
+
+ /* link the 2 objects */
+ vm_object_reference(backing_object);
+ top_object->shadow = backing_object;
+ top_object->vo_shadow_offset = 0x3000;
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: linked %p and %p\n",
+ top_object, backing_object);
+
+ /* unmap backing object */
+ vm_map_remove(kernel_map,
+ backing_offset,
+ backing_offset + backing_size,
+ 0);
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: "
+ "unmapped backing_object %p [0x%llx:0x%llx]\n",
+ backing_object,
+ (uint64_t) backing_offset,
+ (uint64_t) (backing_offset + backing_size));
+
+ /* collapse */
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: collapsing %p\n", top_object);
+ vm_object_lock(top_object);
+ vm_object_collapse(top_object, 0, FALSE);
+ vm_object_unlock(top_object);
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: collapsed %p\n", top_object);
+
+ /* did it work? */
+ if (top_object->shadow != VM_OBJECT_NULL) {
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: not collapsed\n");
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: FAIL\n");
+ if (vm_object_collapse_compressor_allowed) {
+ panic("FBDP_TEST_COLLAPSE_COMPRESSOR: FAIL\n");
+ }
+ } else {
+ /* check the contents of the mapping */
+ unsigned char expect[9] =
+ { 0xA0, 0xA1, 0xA2, /* resident in top */
+ 0xA3, 0xA4, 0xA5, /* compressed in top */
+ 0xB9, /* resident in backing + shadow_offset */
+ 0xBD, /* compressed in backing + shadow_offset + paging_offset */
+ 0x00 }; /* absent in both */
+ unsigned char actual[9];
+ unsigned int i, errors;
+
+ errors = 0;
+ for (i = 0; i < sizeof (actual); i++) {
+ actual[i] = (unsigned char) top_address[i*PAGE_SIZE];
+ if (actual[i] != expect[i]) {
+ errors++;
+ }
+ }
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: "
+ "actual [%x %x %x %x %x %x %x %x %x] "
+ "expect [%x %x %x %x %x %x %x %x %x] "
+ "%d errors\n",
+ actual[0], actual[1], actual[2], actual[3],
+ actual[4], actual[5], actual[6], actual[7],
+ actual[8],
+ expect[0], expect[1], expect[2], expect[3],
+ expect[4], expect[5], expect[6], expect[7],
+ expect[8],
+ errors);
+ if (errors) {
+ panic("FBDP_TEST_COLLAPSE_COMPRESSOR: FAIL\n");
+ } else {
+ printf("FBDP_TEST_COLLAPSE_COMPRESSOR: PASS\n");
+ }
+ }
+#endif /* FBDP_TEST_COLLAPSE_COMPRESSOR */
+
+#if FBDP_TEST_WIRE_AND_EXTRACT
+ ledger_t ledger;
+ vm_map_t user_map, wire_map;
+ mach_vm_address_t user_addr, wire_addr;
+ mach_vm_size_t user_size, wire_size;
+ mach_vm_offset_t cur_offset;
+ vm_prot_t cur_prot, max_prot;
+ ppnum_t user_ppnum, wire_ppnum;
+ kern_return_t kr;
+
+ ledger = ledger_instantiate(task_ledger_template,
+ LEDGER_CREATE_ACTIVE_ENTRIES);
+ user_map = vm_map_create(pmap_create(ledger, 0, TRUE),
+ 0x100000000ULL,
+ 0x200000000ULL,
+ TRUE);
+ wire_map = vm_map_create(NULL,
+ 0x100000000ULL,
+ 0x200000000ULL,
+ TRUE);
+ user_addr = 0;
+ user_size = 0x10000;
+ kr = mach_vm_allocate(user_map,
+ &user_addr,
+ user_size,
+ VM_FLAGS_ANYWHERE);
+ assert(kr == KERN_SUCCESS);
+ wire_addr = 0;
+ wire_size = user_size;
+ kr = mach_vm_remap(wire_map,
+ &wire_addr,
+ wire_size,
+ 0,
+ VM_FLAGS_ANYWHERE,
+ user_map,
+ user_addr,
+ FALSE,
+ &cur_prot,
+ &max_prot,
+ VM_INHERIT_NONE);
+ assert(kr == KERN_SUCCESS);
+ for (cur_offset = 0;
+ cur_offset < wire_size;
+ cur_offset += PAGE_SIZE) {
+ kr = vm_map_wire_and_extract(wire_map,
+ wire_addr + cur_offset,
+ VM_PROT_DEFAULT,
+ TRUE,
+ &wire_ppnum);
+ assert(kr == KERN_SUCCESS);
+ user_ppnum = vm_map_get_phys_page(user_map,
+ user_addr + cur_offset);
+ printf("FBDP_TEST_WIRE_AND_EXTRACT: kr=0x%x "
+ "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n",
+ kr,
+ user_map, user_addr + cur_offset, user_ppnum,
+ wire_map, wire_addr + cur_offset, wire_ppnum);
+ if (kr != KERN_SUCCESS ||
+ wire_ppnum == 0 ||
+ wire_ppnum != user_ppnum) {
+ panic("FBDP_TEST_WIRE_AND_EXTRACT: FAIL\n");
+ }
+ }
+ cur_offset -= PAGE_SIZE;
+ kr = vm_map_wire_and_extract(wire_map,
+ wire_addr + cur_offset,
+ VM_PROT_DEFAULT,
+ TRUE,
+ &wire_ppnum);
+ assert(kr == KERN_SUCCESS);
+ printf("FBDP_TEST_WIRE_AND_EXTRACT: re-wire kr=0x%x "
+ "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n",
+ kr,
+ user_map, user_addr + cur_offset, user_ppnum,
+ wire_map, wire_addr + cur_offset, wire_ppnum);
+ if (kr != KERN_SUCCESS ||
+ wire_ppnum == 0 ||
+ wire_ppnum != user_ppnum) {
+ panic("FBDP_TEST_WIRE_AND_EXTRACT: FAIL\n");
+ }
+
+ printf("FBDP_TEST_WIRE_AND_EXTRACT: PASS\n");
+#endif /* FBDP_TEST_WIRE_AND_EXTRACT */
vm_pageout_continue();
/*NOTREACHED*/
}
+
+
+#define MAX_COMRPESSOR_THREAD_COUNT 8
+
+struct cq ciq[MAX_COMRPESSOR_THREAD_COUNT];
+
+int vm_compressor_thread_count = 2;
+
kern_return_t
vm_pageout_internal_start(void)
{
- kern_return_t result;
+ kern_return_t result;
+ int i;
+ host_basic_info_data_t hinfo;
+
+ if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) {
+ mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
+#define BSD_HOST 1
+ host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
+
+ assert(hinfo.max_cpus > 0);
+
+ if (vm_compressor_thread_count >= hinfo.max_cpus)
+ vm_compressor_thread_count = hinfo.max_cpus - 1;
+ if (vm_compressor_thread_count <= 0)
+ vm_compressor_thread_count = 1;
+ else if (vm_compressor_thread_count > MAX_COMRPESSOR_THREAD_COUNT)
+ vm_compressor_thread_count = MAX_COMRPESSOR_THREAD_COUNT;
- vm_pageout_queue_internal.pgo_maxlaundry = VM_PAGE_LAUNDRY_MAX;
- result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal, NULL, BASEPRI_PREEMPT - 1, &vm_pageout_internal_iothread);
- if (result == KERN_SUCCESS)
- thread_deallocate(vm_pageout_internal_iothread);
+ vm_pageout_queue_internal.pgo_maxlaundry = (vm_compressor_thread_count * 4) * VM_PAGE_LAUNDRY_MAX;
+ } else {
+ vm_compressor_thread_count = 1;
+ vm_pageout_queue_internal.pgo_maxlaundry = VM_PAGE_LAUNDRY_MAX;
+ }
+
+ for (i = 0; i < vm_compressor_thread_count; i++) {
+
+ result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal, (void *)&ciq[i], BASEPRI_PREEMPT - 1, &vm_pageout_internal_iothread);
+ if (result == KERN_SUCCESS)
+ thread_deallocate(vm_pageout_internal_iothread);
+ else
+ break;
+ }
return result;
}
-
+#if CONFIG_IOSCHED
/*
- * when marshalling pages into a UPL and subsequently committing
- * or aborting them, it is necessary to hold
- * the vm_page_queue_lock (a hot global lock) for certain operations
- * on the page... however, the majority of the work can be done
- * while merely holding the object lock... in fact there are certain
- * collections of pages that don't require any work brokered by the
- * vm_page_queue_lock... to mitigate the time spent behind the global
- * lock, go to a 2 pass algorithm... collect pages up to DELAYED_WORK_LIMIT
- * while doing all of the work that doesn't require the vm_page_queue_lock...
- * then call dw_do_work to acquire the vm_page_queue_lock and do the
- * necessary work for each page... we will grab the busy bit on the page
- * if it's not already held so that dw_do_work can drop the object lock
- * if it can't immediately take the vm_page_queue_lock in order to compete
- * for the locks in the same order that vm_pageout_scan takes them.
- * the operation names are modeled after the names of the routines that
- * need to be called in order to make the changes very obvious in the
- * original loop
+ * To support I/O Expedite for compressed files we mark the upls with special flags.
+ * The way decmpfs works is that we create a big upl which marks all the pages needed to
+ * represent the compressed file as busy. We tag this upl with the flag UPL_DECMP_REQ. Decmpfs
+ * then issues smaller I/Os for compressed I/Os, deflates them and puts the data into the pages
+ * being held in the big original UPL. We mark each of these smaller UPLs with the flag
+ * UPL_DECMP_REAL_IO. Any outstanding real I/O UPL is tracked by the big req upl using the
+ * decmp_io_upl field (in the upl structure). This link is protected in the forward direction
+ * by the req upl lock (the reverse link doesnt need synch. since we never inspect this link
+ * unless the real I/O upl is being destroyed).
*/
-#define DELAYED_WORK_LIMIT 32
-
-#define DW_vm_page_unwire 0x01
-#define DW_vm_page_wire 0x02
-#define DW_vm_page_free 0x04
-#define DW_vm_page_activate 0x08
-#define DW_vm_page_deactivate_internal 0x10
-#define DW_vm_page_speculate 0x20
-#define DW_vm_page_lru 0x40
-#define DW_vm_pageout_throttle_up 0x80
-#define DW_PAGE_WAKEUP 0x100
-#define DW_clear_busy 0x200
-#define DW_clear_reference 0x400
-#define DW_set_reference 0x800
-
-struct dw {
- vm_page_t dw_m;
- int dw_mask;
-};
-
-
-static void dw_do_work(vm_object_t object, struct dw *dwp, int dw_count);
+static void
+upl_set_decmp_info(upl_t upl, upl_t src_upl)
+{
+ assert((src_upl->flags & UPL_DECMP_REQ) != 0);
+
+ upl_lock(src_upl);
+ if (src_upl->decmp_io_upl) {
+ /*
+ * If there is already an alive real I/O UPL, ignore this new UPL.
+ * This case should rarely happen and even if it does, it just means
+ * that we might issue a spurious expedite which the driver is expected
+ * to handle.
+ */
+ upl_unlock(src_upl);
+ return;
+ }
+ src_upl->decmp_io_upl = (void *)upl;
+ src_upl->ref_count++;
+
+ upl->flags |= UPL_DECMP_REAL_IO;
+ upl->decmp_io_upl = (void *)src_upl;
+ upl_unlock(src_upl);
+}
+#endif /* CONFIG_IOSCHED */
+#if UPL_DEBUG
+int upl_debug_enabled = 1;
+#else
+int upl_debug_enabled = 0;
+#endif
static upl_t
upl_create(int type, int flags, upl_size_t size)
{
upl_t upl;
- int page_field_size = 0;
+ vm_size_t page_field_size = 0;
int upl_flags = 0;
- int upl_size = sizeof(struct upl);
+ vm_size_t upl_size = sizeof(struct upl);
size = round_page_32(size);
upl_flags |= UPL_LITE;
}
if (type & UPL_CREATE_INTERNAL) {
- upl_size += (int) sizeof(struct upl_page_info) * atop(size);
+ upl_size += sizeof(struct upl_page_info) * atop(size);
upl_flags |= UPL_INTERNAL;
}
upl->size = 0;
upl->map_object = NULL;
upl->ref_count = 1;
+ upl->ext_ref_count = 0;
upl->highest_page = 0;
upl_lock_init(upl);
upl->vector_upl = NULL;
+#if CONFIG_IOSCHED
+ if (type & UPL_CREATE_IO_TRACKING) {
+ upl->upl_priority = proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO);
+ }
+
+ upl->upl_reprio_info = 0;
+ upl->decmp_io_upl = 0;
+ if ((type & UPL_CREATE_INTERNAL) && (type & UPL_CREATE_EXPEDITE_SUP)) {
+ /* Only support expedite on internal UPLs */
+ thread_t curthread = current_thread();
+ upl->upl_reprio_info = (uint64_t *)kalloc(sizeof(uint64_t) * atop(size));
+ bzero(upl->upl_reprio_info, (sizeof(uint64_t) * atop(size)));
+ upl->flags |= UPL_EXPEDITE_SUPPORTED;
+ if (curthread->decmp_upl != NULL)
+ upl_set_decmp_info(upl, curthread->decmp_upl);
+ }
+#endif
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if ((type & UPL_CREATE_IO_TRACKING) || upl_debug_enabled) {
+ upl->upl_creator = current_thread();
+ upl->uplq.next = 0;
+ upl->uplq.prev = 0;
+ upl->flags |= UPL_TRACKED_BY_OBJECT;
+ }
+#endif
+
#if UPL_DEBUG
upl->ubc_alias1 = 0;
upl->ubc_alias2 = 0;
- upl->upl_creator = current_thread();
upl->upl_state = 0;
upl->upl_commit_index = 0;
bzero(&upl->upl_commit_records[0], sizeof(upl->upl_commit_records));
int page_field_size; /* bit field in word size buf */
int size;
-#if UPL_DEBUG
- {
+ if (upl->ext_ref_count) {
+ panic("upl(%p) ext_ref_count", upl);
+ }
+
+#if CONFIG_IOSCHED
+ if ((upl->flags & UPL_DECMP_REAL_IO) && upl->decmp_io_upl) {
+ upl_t src_upl;
+ src_upl = upl->decmp_io_upl;
+ assert((src_upl->flags & UPL_DECMP_REQ) != 0);
+ upl_lock(src_upl);
+ src_upl->decmp_io_upl = NULL;
+ upl_unlock(src_upl);
+ upl_deallocate(src_upl);
+ }
+#endif /* CONFIG_IOSCHED */
+
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if ((upl->flags & UPL_TRACKED_BY_OBJECT) && !(upl->flags & UPL_VECTOR)) {
vm_object_t object;
if (upl->flags & UPL_SHADOWED) {
} else {
object = upl->map_object;
}
+
vm_object_lock(object);
queue_remove(&object->uplq, upl, upl_t, uplq);
+ vm_object_activity_end(object);
+ vm_object_collapse(object, 0, TRUE);
vm_object_unlock(object);
}
-#endif /* UPL_DEBUG */
+#endif
/*
* drop a reference on the map_object whether or
* not a pageout object is inserted
}
upl_lock_destroy(upl);
upl->vector_upl = (vector_upl_t) 0xfeedbeef;
+
+#if CONFIG_IOSCHED
+ if (upl->flags & UPL_EXPEDITE_SUPPORTED)
+ kfree(upl->upl_reprio_info, sizeof(uint64_t) * (size/PAGE_SIZE));
+#endif
+
if (upl->flags & UPL_INTERNAL) {
kfree(upl,
sizeof(struct upl) +
}
}
-void uc_upl_dealloc(upl_t upl);
-__private_extern__ void
-uc_upl_dealloc(upl_t upl)
-{
- if (--upl->ref_count == 0)
- upl_destroy(upl);
-}
-
void
upl_deallocate(upl_t upl)
{
+ upl_lock(upl);
if (--upl->ref_count == 0) {
if(vector_upl_is_valid(upl))
vector_upl_deallocate(upl);
+ upl_unlock(upl);
upl_destroy(upl);
}
+ else
+ upl_unlock(upl);
+}
+
+#if CONFIG_IOSCHED
+void
+upl_mark_decmp(upl_t upl)
+{
+ if (upl->flags & UPL_TRACKED_BY_OBJECT) {
+ upl->flags |= UPL_DECMP_REQ;
+ upl->upl_creator->decmp_upl = (void *)upl;
+ }
+}
+
+void
+upl_unmark_decmp(upl_t upl)
+{
+ if(upl && (upl->flags & UPL_DECMP_REQ)) {
+ upl->upl_creator->decmp_upl = NULL;
+ }
+}
+
+#endif /* CONFIG_IOSCHED */
+
+#define VM_PAGE_Q_BACKING_UP(q) \
+ ((q)->pgo_laundry >= (((q)->pgo_maxlaundry * 8) / 10))
+
+boolean_t must_throttle_writes(void);
+
+boolean_t
+must_throttle_writes()
+{
+ if (VM_PAGE_Q_BACKING_UP(&vm_pageout_queue_external) &&
+ vm_page_pageable_external_count > (AVAILABLE_NON_COMPRESSED_MEMORY * 6) / 10)
+ return (TRUE);
+
+ return (FALSE);
}
+
#if DEVELOPMENT || DEBUG
/*/*
* Statistics about UPL enforcement of copy-on-write obligations.
vm_page_t dst_page = VM_PAGE_NULL;
vm_object_offset_t dst_offset;
upl_size_t xfer_size;
+ unsigned int size_in_pages;
boolean_t dirty;
boolean_t hw_dirty;
upl_t upl = NULL;
int refmod_state = 0;
wpl_array_t lite_list = NULL;
vm_object_t last_copy_object;
- struct dw dw_array[DELAYED_WORK_LIMIT];
- struct dw *dwp;
+ struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT];
+ struct vm_page_delayed_work *dwp;
int dw_count;
+ int dw_limit;
+ int io_tracking_flag = 0;
if (cntrl_flags & ~UPL_VALID_FLAGS) {
/*
panic("vm_object_upl_request: contiguous object specified\n");
- if ((size / PAGE_SIZE) > MAX_UPL_SIZE)
- size = MAX_UPL_SIZE * PAGE_SIZE;
+ if (size > MAX_UPL_SIZE_BYTES)
+ size = MAX_UPL_SIZE_BYTES;
if ( (cntrl_flags & UPL_SET_INTERNAL) && page_list_count != NULL)
- *page_list_count = MAX_UPL_SIZE;
+ *page_list_count = MAX_UPL_SIZE_BYTES >> PAGE_SHIFT;
+
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if (object->io_tracking || upl_debug_enabled)
+ io_tracking_flag |= UPL_CREATE_IO_TRACKING;
+#endif
+#if CONFIG_IOSCHED
+ if (object->io_tracking)
+ io_tracking_flag |= UPL_CREATE_EXPEDITE_SUP;
+#endif
if (cntrl_flags & UPL_SET_INTERNAL) {
if (cntrl_flags & UPL_SET_LITE) {
- upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE, 0, size);
+ upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE | io_tracking_flag, 0, size);
user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
lite_list = (wpl_array_t)
lite_list = NULL;
}
} else {
- upl = upl_create(UPL_CREATE_INTERNAL, 0, size);
+ upl = upl_create(UPL_CREATE_INTERNAL | io_tracking_flag, 0, size);
user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
if (size == 0) {
} else {
if (cntrl_flags & UPL_SET_LITE) {
- upl = upl_create(UPL_CREATE_EXTERNAL | UPL_CREATE_LITE, 0, size);
+ upl = upl_create(UPL_CREATE_EXTERNAL | UPL_CREATE_LITE | io_tracking_flag, 0, size);
lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));
if (size == 0) {
lite_list = NULL;
}
} else {
- upl = upl_create(UPL_CREATE_EXTERNAL, 0, size);
+ upl = upl_create(UPL_CREATE_EXTERNAL | io_tracking_flag, 0, size);
}
}
*upl_ptr = upl;
upl->map_object->pageout = TRUE;
upl->map_object->can_persist = FALSE;
upl->map_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
- upl->map_object->shadow_offset = offset;
+ upl->map_object->vo_shadow_offset = offset;
upl->map_object->wimg_bits = object->wimg_bits;
VM_PAGE_GRAB_FICTITIOUS(alias_page);
upl->size = size;
upl->offset = offset + object->paging_offset;
-#if UPL_DEBUG
- queue_enter(&object->uplq, upl, upl_t, uplq);
-#endif /* UPL_DEBUG */
-
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if (object->io_tracking || upl_debug_enabled) {
+ vm_object_activity_begin(object);
+ queue_enter(&object->uplq, upl, upl_t, uplq);
+ }
+#endif
if ((cntrl_flags & UPL_WILL_MODIFY) && object->copy != VM_OBJECT_NULL) {
/*
* Honor copy-on-write obligations
xfer_size = size;
dst_offset = offset;
+ size_in_pages = size / PAGE_SIZE;
dwp = &dw_array[0];
dw_count = 0;
+ dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
+
+ if (vm_page_free_count > (vm_page_free_target + size_in_pages) ||
+ object->resident_page_count < ((MAX_UPL_SIZE_BYTES * 2) >> PAGE_SHIFT))
+ object->scan_collisions = 0;
+
+ if ((cntrl_flags & UPL_WILL_MODIFY) && must_throttle_writes() == TRUE) {
+ boolean_t isSSD = FALSE;
+
+ vnode_pager_get_isSSD(object->pager, &isSSD);
+ vm_object_unlock(object);
+
+ OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages);
+
+ if (isSSD == TRUE)
+ delay(1000 * size_in_pages);
+ else
+ delay(5000 * size_in_pages);
+ OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
+
+ vm_object_lock(object);
+ }
while (xfer_size) {
dst_page->fictitious ||
dst_page->absent ||
dst_page->error ||
- (VM_PAGE_WIRED(dst_page) && !dst_page->pageout && !dst_page->list_req_pending)) {
-
+ dst_page->cleaning ||
+ (VM_PAGE_WIRED(dst_page))) {
+
if (user_page_list)
user_page_list[entry].phys_addr = 0;
/*
* we're only asking for DIRTY pages to be returned
*/
- if (dst_page->list_req_pending || !(cntrl_flags & UPL_FOR_PAGEOUT)) {
+ if (dst_page->laundry || !(cntrl_flags & UPL_FOR_PAGEOUT)) {
/*
* if we were the page stolen by vm_pageout_scan to be
* cleaned (as opposed to a buddy being clustered in
* this is a request for a PAGEOUT cluster and this page
* is merely along for the ride as a 'buddy'... not only
* does it have to be dirty to be returned, but it also
- * can't have been referenced recently... note that we've
- * already filtered above based on whether this page is
- * currently on the inactive queue or it meets the page
- * ticket (generation count) check
+ * can't have been referenced recently...
*/
- if ( (cntrl_flags & UPL_CLEAN_IN_PLACE || !(refmod_state & VM_MEM_REFERENCED)) &&
- ((refmod_state & VM_MEM_MODIFIED) || dst_page->dirty || dst_page->precious) ) {
+ if ( (hibernate_cleaning_in_progress == TRUE ||
+ (!((refmod_state & VM_MEM_REFERENCED) || dst_page->reference) || dst_page->throttled)) &&
+ ((refmod_state & VM_MEM_MODIFIED) || dst_page->dirty || dst_page->precious) ) {
goto check_busy;
}
dont_return:
* if we reach here, we're not to return
* the page... go on to the next one
*/
+ if (dst_page->laundry == TRUE) {
+ /*
+ * if we get here, the page is not 'cleaning' (filtered out above).
+ * since it has been referenced, remove it from the laundry
+ * so we don't pay the cost of an I/O to clean a page
+ * we're just going to take back
+ */
+ vm_page_lockspin_queues();
+
+ vm_pageout_steal_laundry(dst_page, TRUE);
+ vm_page_activate(dst_page);
+
+ vm_page_unlock_queues();
+ }
if (user_page_list)
user_page_list[entry].phys_addr = 0;
goto try_next_page;
}
check_busy:
- if (dst_page->busy && (!(dst_page->list_req_pending && (dst_page->pageout || dst_page->cleaning)))) {
- if (cntrl_flags & UPL_NOBLOCK) {
- if (user_page_list)
+ if (dst_page->busy) {
+ if (cntrl_flags & UPL_NOBLOCK) {
+ if (user_page_list)
user_page_list[entry].phys_addr = 0;
goto try_next_page;
continue;
}
- /*
- * Someone else already cleaning the page?
- */
- if ((dst_page->cleaning || dst_page->absent || VM_PAGE_WIRED(dst_page)) && !dst_page->list_req_pending) {
- if (user_page_list)
- user_page_list[entry].phys_addr = 0;
-
- goto try_next_page;
- }
/*
* ENCRYPTED SWAP:
* The caller is gathering this page and might
vm_page_lockspin_queues();
-#if CONFIG_EMBEDDED
- if (dst_page->laundry)
-#else
- if (dst_page->pageout_queue == TRUE)
-#endif
- {
+ if (dst_page->pageout_queue == TRUE) {
/*
* we've buddied up a page for a clustered pageout
* that has already been moved to the pageout
* were not counted in the initial
* vm_pageout_scan work
*/
- if (dst_page->list_req_pending)
+ if (dst_page->pageout)
encountered_lrp = TRUE;
- if ((dst_page->dirty || (dst_page->object->internal && dst_page->precious)) && !dst_page->list_req_pending) {
+ if ((dst_page->dirty || (dst_page->object->internal && dst_page->precious))) {
if (encountered_lrp)
CLUSTER_STAT(pages_at_higher_offsets++;)
else
CLUSTER_STAT(pages_at_lower_offsets++;)
}
#endif
- /*
- * Turn off busy indication on pending
- * pageout. Note: we can only get here
- * in the request pending case.
- */
- dst_page->list_req_pending = FALSE;
- dst_page->busy = FALSE;
-
hw_dirty = refmod_state & VM_MEM_MODIFIED;
dirty = hw_dirty ? TRUE : dst_page->dirty;
*/
vm_external_state_set(object->existence_map, dst_page->offset);
#endif /*MACH_PAGEMAP*/
- dst_page->dirty = dirty;
+ if (dirty) {
+ SET_PAGE_DIRTY(dst_page, FALSE);
+ } else {
+ dst_page->dirty = FALSE;
+ }
if (!dirty)
dst_page->precious = TRUE;
- if (dst_page->pageout)
- dst_page->busy = TRUE;
-
if ( (cntrl_flags & UPL_ENCRYPT) ) {
/*
* ENCRYPTED SWAP:
dst_page->encrypted_cleaning = TRUE;
}
if ( !(cntrl_flags & UPL_CLEAN_IN_PLACE) ) {
- /*
- * deny access to the target page
- * while it is being worked on
- */
- if ((!dst_page->pageout) && ( !VM_PAGE_WIRED(dst_page))) {
- dst_page->busy = TRUE;
+ if ( !VM_PAGE_WIRED(dst_page))
dst_page->pageout = TRUE;
-
- dwp->dw_mask |= DW_vm_page_wire;
- }
}
} else {
if ((cntrl_flags & UPL_WILL_MODIFY) && object->copy != last_copy_object) {
if (dst_page != VM_PAGE_NULL) {
if ((cntrl_flags & UPL_RET_ONLY_ABSENT)) {
+ /*
+ * skip over pages already present in the cache
+ */
+ if (user_page_list)
+ user_page_list[entry].phys_addr = 0;
- if ( !(dst_page->absent && dst_page->list_req_pending) ) {
- /*
- * skip over pages already present in the cache
- */
- if (user_page_list)
- user_page_list[entry].phys_addr = 0;
-
- goto try_next_page;
- }
+ goto try_next_page;
+ }
+ if (dst_page->fictitious) {
+ panic("need corner case for fictitious page");
}
- if ( !(dst_page->list_req_pending) ) {
-
- if (dst_page->cleaning) {
- /*
- * someone else is writing to the page... wait...
- */
- PAGE_SLEEP(object, dst_page, THREAD_UNINT);
-
- continue;
- }
- } else {
- if (dst_page->fictitious &&
- dst_page->phys_page == vm_page_fictitious_addr) {
- assert( !dst_page->speculative);
- /*
- * dump the fictitious page
- */
- dst_page->list_req_pending = FALSE;
-
- VM_PAGE_FREE(dst_page);
-
- dst_page = NULL;
- } else if (dst_page->absent) {
- /*
- * the default_pager case
- */
- dst_page->list_req_pending = FALSE;
- dst_page->busy = FALSE;
+ if (dst_page->busy || dst_page->cleaning) {
+ /*
+ * someone else is playing with the
+ * page. We will have to wait.
+ */
+ PAGE_SLEEP(object, dst_page, THREAD_UNINT);
- } else if (dst_page->pageout || dst_page->cleaning) {
- /*
- * page was earmarked by vm_pageout_scan
- * to be cleaned and stolen... we're going
- * to take it back since we are not attempting
- * to read that page and we don't want to stall
- * waiting for it to be cleaned for 2 reasons...
- * 1 - no use paging it out and back in
- * 2 - if we stall, we may casue a deadlock in
- * the FS trying to acquire the its locks
- * on the VNOP_PAGEOUT path presuming that
- * those locks are already held on the read
- * path before trying to create this UPL
- *
- * so undo all of the state that vm_pageout_scan
- * hung on this page
- */
- dst_page->busy = FALSE;
+ continue;
+ }
+ if (dst_page->laundry) {
+ dst_page->pageout = FALSE;
- vm_pageout_queue_steal(dst_page, FALSE);
- }
+ vm_pageout_steal_laundry(dst_page, FALSE);
}
- }
- if (dst_page == VM_PAGE_NULL) {
+ } else {
if (object->private) {
/*
* This is a nasty wrinkle for users
goto try_next_page;
}
- /*
- * need to allocate a page
- */
- dst_page = vm_page_grab();
+ if (object->scan_collisions) {
+ /*
+ * the pageout_scan thread is trying to steal
+ * pages from this object, but has run into our
+ * lock... grab 2 pages from the head of the object...
+ * the first is freed on behalf of pageout_scan, the
+ * 2nd is for our own use... we use vm_object_page_grab
+ * in both cases to avoid taking pages from the free
+ * list since we are under memory pressure and our
+ * lock on this object is getting in the way of
+ * relieving it
+ */
+ dst_page = vm_object_page_grab(object);
+
+ if (dst_page != VM_PAGE_NULL)
+ vm_page_release(dst_page);
+ dst_page = vm_object_page_grab(object);
+ }
+ if (dst_page == VM_PAGE_NULL) {
+ /*
+ * need to allocate a page
+ */
+ dst_page = vm_page_grab();
+ }
if (dst_page == VM_PAGE_NULL) {
if ( (cntrl_flags & (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) == (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) {
/*
* offset...
*/
vm_object_unlock(object);
+
+ OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages);
+
+ VM_DEBUG_EVENT(vm_upl_page_wait, VM_UPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);
+
VM_PAGE_WAIT();
+ OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
+
+ VM_DEBUG_EVENT(vm_upl_page_wait, VM_UPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
+
vm_object_lock(object);
continue;
* speculative list
*/
dst_page->clustered = TRUE;
- }
- }
- if (dst_page->fictitious) {
- panic("need corner case for fictitious page");
- }
- if (dst_page->busy) {
- /*
- * someone else is playing with the
- * page. We will have to wait.
- */
- PAGE_SLEEP(object, dst_page, THREAD_UNINT);
- continue;
+ if ( !(cntrl_flags & UPL_FILE_IO))
+ VM_STAT_INCR(pageins);
+ }
}
/*
* ENCRYPTED SWAP:
alias_page = NULL;
}
- if (cntrl_flags & UPL_CLEAN_IN_PLACE) {
+ if (cntrl_flags & UPL_REQUEST_SET_DIRTY) {
+ upl->flags &= ~UPL_CLEAR_DIRTY;
+ upl->flags |= UPL_SET_DIRTY;
+ dirty = TRUE;
+ upl->flags |= UPL_SET_DIRTY;
+ } else if (cntrl_flags & UPL_CLEAN_IN_PLACE) {
/*
* clean in place for read implies
* that a write will be done on all
*/
dwp->dw_mask |= DW_set_reference;
}
- dst_page->precious = (cntrl_flags & UPL_PRECIOUS) ? TRUE : FALSE;
+ if (cntrl_flags & UPL_PRECIOUS) {
+ if (dst_page->object->internal) {
+ SET_PAGE_DIRTY(dst_page, FALSE);
+ dst_page->precious = FALSE;
+ } else {
+ dst_page->precious = TRUE;
+ }
+ } else {
+ dst_page->precious = FALSE;
+ }
}
if (dst_page->busy)
upl->flags |= UPL_HAS_BUSY;
user_page_list[entry].dirty = dst_page->dirty;
user_page_list[entry].precious = dst_page->precious;
user_page_list[entry].device = FALSE;
+ user_page_list[entry].needed = FALSE;
if (dst_page->clustered == TRUE)
user_page_list[entry].speculative = dst_page->speculative;
else
* update clustered and speculative state
*
*/
- VM_PAGE_CONSUME_CLUSTERED(dst_page);
+ if (dst_page->clustered)
+ VM_PAGE_CONSUME_CLUSTERED(dst_page);
}
try_next_page:
if (dwp->dw_mask) {
if (dwp->dw_mask & DW_vm_page_activate)
VM_STAT_INCR(reactivations);
- if (dst_page->busy == FALSE) {
- /*
- * dw_do_work may need to drop the object lock
- * if it does, we need the pages it's looking at to
- * be held stable via the busy bit.
- */
- dst_page->busy = TRUE;
- dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
- }
- dwp->dw_m = dst_page;
- dwp++;
- dw_count++;
+ VM_PAGE_ADD_DELAYED_WORK(dwp, dst_page, dw_count);
- if (dw_count >= DELAYED_WORK_LIMIT) {
- dw_do_work(object, &dw_array[0], dw_count);
+ if (dw_count >= dw_limit) {
+ vm_page_do_delayed_work(object, &dw_array[0], dw_count);
dwp = &dw_array[0];
dw_count = 0;
xfer_size -= PAGE_SIZE;
}
if (dw_count)
- dw_do_work(object, &dw_array[0], dw_count);
+ vm_page_do_delayed_work(object, &dw_array[0], dw_count);
if (alias_page != NULL) {
VM_PAGE_FREE(alias_page);
base_offset = (offset & ~((vm_object_offset_t) super_cluster - 1));
super_size = (offset + size) > (base_offset + super_cluster) ? super_cluster<<1 : super_cluster;
- super_size_64 = ((base_offset + super_size) > object->size) ? (object->size - base_offset) : super_size;
+ super_size_64 = ((base_offset + super_size) > object->vo_size) ? (object->vo_size - base_offset) : super_size;
super_size = (upl_size_t) super_size_64;
assert(super_size == super_size_64);
return KERN_SUCCESS;
}
+
+ if (entry->is_sub_map) {
+ vm_map_t submap;
+
+ submap = entry->object.sub_map;
+ local_start = entry->vme_start;
+ local_offset = entry->offset;
+
+ vm_map_reference(submap);
+ vm_map_unlock_read(map);
+
+ ret = vm_map_create_upl(submap,
+ local_offset + (offset - local_start),
+ upl_size, upl, page_list, count, flags);
+ vm_map_deallocate(submap);
+
+ return ret;
+ }
+
if (entry->object.vm_object == VM_OBJECT_NULL || !entry->object.vm_object->phys_contiguous) {
- if ((*upl_size/PAGE_SIZE) > MAX_UPL_SIZE)
- *upl_size = MAX_UPL_SIZE * PAGE_SIZE;
+ if (*upl_size > MAX_UPL_SIZE_BYTES)
+ *upl_size = MAX_UPL_SIZE_BYTES;
}
/*
* Create an object if necessary.
vm_map_unlock_read(map);
return KERN_PROTECTION_FAILURE;
}
- if (entry->needs_copy) {
- /*
- * Honor copy-on-write for COPY_SYMMETRIC
- * strategy.
- */
- vm_map_t local_map;
- vm_object_t object;
- vm_object_offset_t new_offset;
- vm_prot_t prot;
- boolean_t wired;
- vm_map_version_t version;
- vm_map_t real_map;
-
- local_map = map;
-
- if (vm_map_lookup_locked(&local_map,
- offset, VM_PROT_WRITE,
- OBJECT_LOCK_EXCLUSIVE,
- &version, &object,
- &new_offset, &prot, &wired,
- NULL,
- &real_map) != KERN_SUCCESS) {
- vm_map_unlock_read(local_map);
- return KERN_FAILURE;
- }
- if (real_map != map)
- vm_map_unlock(real_map);
- vm_map_unlock_read(local_map);
+ }
- vm_object_unlock(object);
+ local_object = entry->object.vm_object;
+ if (vm_map_entry_should_cow_for_true_share(entry) &&
+ local_object->vo_size > *upl_size &&
+ *upl_size != 0) {
+ vm_prot_t prot;
+
+ /*
+ * Set up the targeted range for copy-on-write to avoid
+ * applying true_share/copy_delay to the entire object.
+ */
+ if (vm_map_lock_read_to_write(map)) {
goto REDISCOVER_ENTRY;
}
- }
- if (entry->is_sub_map) {
- vm_map_t submap;
- submap = entry->object.sub_map;
- local_start = entry->vme_start;
- local_offset = entry->offset;
+ vm_map_clip_start(map,
+ entry,
+ vm_map_trunc_page(offset,
+ VM_MAP_PAGE_MASK(map)));
+ vm_map_clip_end(map,
+ entry,
+ vm_map_round_page(offset + *upl_size,
+ VM_MAP_PAGE_MASK(map)));
+ if ((entry->vme_end - offset) < *upl_size) {
+ *upl_size = (upl_size_t) (entry->vme_end - offset);
+ assert(*upl_size == entry->vme_end - offset);
+ }
- vm_map_reference(submap);
- vm_map_unlock_read(map);
+ prot = entry->protection & ~VM_PROT_WRITE;
+ if (override_nx(map, entry->alias) && prot)
+ prot |= VM_PROT_EXECUTE;
+ vm_object_pmap_protect(local_object,
+ entry->offset,
+ entry->vme_end - entry->vme_start,
+ ((entry->is_shared || map->mapped_in_other_pmaps)
+ ? PMAP_NULL
+ : map->pmap),
+ entry->vme_start,
+ prot);
+ entry->needs_copy = TRUE;
- ret = vm_map_create_upl(submap,
- local_offset + (offset - local_start),
- upl_size, upl, page_list, count, flags);
- vm_map_deallocate(submap);
+ vm_map_lock_write_to_read(map);
+ }
- return ret;
+ if (entry->needs_copy) {
+ /*
+ * Honor copy-on-write for COPY_SYMMETRIC
+ * strategy.
+ */
+ vm_map_t local_map;
+ vm_object_t object;
+ vm_object_offset_t new_offset;
+ vm_prot_t prot;
+ boolean_t wired;
+ vm_map_version_t version;
+ vm_map_t real_map;
+ vm_prot_t fault_type;
+
+ local_map = map;
+
+ if (caller_flags & UPL_COPYOUT_FROM) {
+ fault_type = VM_PROT_READ | VM_PROT_COPY;
+ vm_counters.create_upl_extra_cow++;
+ vm_counters.create_upl_extra_cow_pages += (entry->vme_end - entry->vme_start) / PAGE_SIZE;
+ } else {
+ fault_type = VM_PROT_WRITE;
+ }
+ if (vm_map_lookup_locked(&local_map,
+ offset, fault_type,
+ OBJECT_LOCK_EXCLUSIVE,
+ &version, &object,
+ &new_offset, &prot, &wired,
+ NULL,
+ &real_map) != KERN_SUCCESS) {
+ if (fault_type == VM_PROT_WRITE) {
+ vm_counters.create_upl_lookup_failure_write++;
+ } else {
+ vm_counters.create_upl_lookup_failure_copy++;
+ }
+ vm_map_unlock_read(local_map);
+ return KERN_FAILURE;
+ }
+ if (real_map != map)
+ vm_map_unlock(real_map);
+ vm_map_unlock_read(local_map);
+
+ vm_object_unlock(object);
+
+ goto REDISCOVER_ENTRY;
}
+
if (sync_cow_data) {
if (entry->object.vm_object->shadow || entry->object.vm_object->copy) {
local_object = entry->object.vm_object;
(vm_object_offset_t)
((offset - local_start) +
local_offset) +
- local_object->shadow_offset,
+ local_object->vo_shadow_offset,
*upl_size, FALSE,
MEMORY_OBJECT_DATA_SYNC,
VM_PROT_NO_CHANGE);
upl = vector_upl_subupl_byindex(vector_upl, curr_upl++ );
if(upl == NULL)
goto process_upl_to_enter;
+
vector_upl_get_iostate(vector_upl, upl, &subupl_offset, &subupl_size);
*dst_addr = (vm_map_offset_t)(vector_upl_dst_addr + (vm_map_offset_t)subupl_offset);
} else {
upl->map_object->pageout = TRUE;
upl->map_object->can_persist = FALSE;
upl->map_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
- upl->map_object->shadow_offset = upl->offset - object->paging_offset;
+ upl->map_object->vo_shadow_offset = upl->offset - object->paging_offset;
upl->map_object->wimg_bits = object->wimg_bits;
- offset = upl->map_object->shadow_offset;
+ offset = upl->map_object->vo_shadow_offset;
new_offset = 0;
size = upl->size;
offset = 0;
else
offset = upl->offset - upl->map_object->paging_offset;
+
size = upl->size;
vm_object_reference(upl->map_object);
m = vm_page_lookup(upl->map_object, offset);
if (m) {
- unsigned int cache_attr;
- cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK;
-
m->pmapped = TRUE;
/* CODE SIGNING ENFORCEMENT: page has been wpmapped,
* but only in kernel space. If this was on a user map,
* we'd have to set the wpmapped bit. */
/* m->wpmapped = TRUE; */
- assert(map==kernel_map);
+ assert(map->pmap == kernel_pmap);
- PMAP_ENTER(map->pmap, addr, m, VM_PROT_ALL, cache_attr, TRUE);
+ PMAP_ENTER(map->pmap, addr, m, VM_PROT_DEFAULT, VM_PROT_NONE, 0, TRUE);
}
offset += PAGE_SIZE_64;
}
if(!isVectorUPL) {
upl_unlock(upl);
- vm_map_remove(map,
- vm_map_trunc_page(addr),
- vm_map_round_page(addr + size),
+ vm_map_remove(
+ map,
+ vm_map_trunc_page(addr,
+ VM_MAP_PAGE_MASK(map)),
+ vm_map_round_page(addr + size,
+ VM_MAP_PAGE_MASK(map)),
VM_MAP_NO_FLAGS);
return KERN_SUCCESS;
return KERN_FAILURE;
}
-static void
-dw_do_work(
- vm_object_t object,
- struct dw *dwp,
- int dw_count)
-{
- int j;
- boolean_t held_as_spin = TRUE;
-
- /*
- * pageout_scan takes the vm_page_lock_queues first
- * then tries for the object lock... to avoid what
- * is effectively a lock inversion, we'll go to the
- * trouble of taking them in that same order... otherwise
- * if this object contains the majority of the pages resident
- * in the UBC (or a small set of large objects actively being
- * worked on contain the majority of the pages), we could
- * cause the pageout_scan thread to 'starve' in its attempt
- * to find pages to move to the free queue, since it has to
- * successfully acquire the object lock of any candidate page
- * before it can steal/clean it.
- */
- if (!vm_page_trylockspin_queues()) {
- vm_object_unlock(object);
-
- vm_page_lockspin_queues();
-
- for (j = 0; ; j++) {
- if (!vm_object_lock_avoid(object) &&
- _vm_object_lock_try(object))
- break;
- vm_page_unlock_queues();
- mutex_pause(j);
- vm_page_lockspin_queues();
- }
- }
- for (j = 0; j < dw_count; j++, dwp++) {
-
- if (dwp->dw_mask & DW_vm_pageout_throttle_up)
- vm_pageout_throttle_up(dwp->dw_m);
-
- if (dwp->dw_mask & DW_vm_page_wire)
- vm_page_wire(dwp->dw_m);
- else if (dwp->dw_mask & DW_vm_page_unwire) {
- boolean_t queueit;
-
- queueit = (dwp->dw_mask & DW_vm_page_free) ? FALSE : TRUE;
-
- vm_page_unwire(dwp->dw_m, queueit);
- }
- if (dwp->dw_mask & DW_vm_page_free) {
- if (held_as_spin == TRUE) {
- vm_page_lockconvert_queues();
- held_as_spin = FALSE;
- }
- vm_page_free(dwp->dw_m);
- } else {
- if (dwp->dw_mask & DW_vm_page_deactivate_internal)
- vm_page_deactivate_internal(dwp->dw_m, FALSE);
- else if (dwp->dw_mask & DW_vm_page_activate)
- vm_page_activate(dwp->dw_m);
- else if (dwp->dw_mask & DW_vm_page_speculate)
- vm_page_speculate(dwp->dw_m, TRUE);
- else if (dwp->dw_mask & DW_vm_page_lru)
- vm_page_lru(dwp->dw_m);
-
- if (dwp->dw_mask & DW_set_reference)
- dwp->dw_m->reference = TRUE;
- else if (dwp->dw_mask & DW_clear_reference)
- dwp->dw_m->reference = FALSE;
-
- if (dwp->dw_mask & DW_clear_busy)
- dwp->dw_m->busy = FALSE;
-
- if (dwp->dw_mask & DW_PAGE_WAKEUP)
- PAGE_WAKEUP(dwp->dw_m);
- }
- }
- vm_page_unlock_queues();
-}
-
-
-
kern_return_t
upl_commit_range(
upl_t upl,
int occupied;
int clear_refmod = 0;
int pgpgout_count = 0;
- struct dw dw_array[DELAYED_WORK_LIMIT];
- struct dw *dwp;
- int dw_count, isVectorUPL = 0;
+ struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT];
+ struct vm_page_delayed_work *dwp;
+ int dw_count;
+ int dw_limit;
+ int isVectorUPL = 0;
upl_t vector_upl = NULL;
+ boolean_t should_be_throttled = FALSE;
+
+ vm_page_t nxt_page = VM_PAGE_NULL;
+ int fast_path_possible = 0;
+ int fast_path_full_commit = 0;
+ int throttle_page = 0;
+ int unwired_count = 0;
+ int local_queue_count = 0;
+ queue_head_t local_queue;
*empty = FALSE;
}
return KERN_FAILURE;
}
+ if (upl->flags & UPL_SET_DIRTY)
+ flags |= UPL_COMMIT_SET_DIRTY;
if (upl->flags & UPL_CLEAR_DIRTY)
flags |= UPL_COMMIT_CLEAR_DIRTY;
*/
flags &= ~UPL_COMMIT_CS_VALIDATED;
}
+ if (!VM_DYNAMIC_PAGING_ENABLED(memory_manager_default) && shadow_object->internal)
+ should_be_throttled = TRUE;
dwp = &dw_array[0];
dw_count = 0;
+ dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
+
+ if ((upl->flags & UPL_IO_WIRE) &&
+ !(flags & UPL_COMMIT_FREE_ABSENT) &&
+ !isVectorUPL &&
+ shadow_object->purgable != VM_PURGABLE_VOLATILE &&
+ shadow_object->purgable != VM_PURGABLE_EMPTY) {
+
+ if (!queue_empty(&shadow_object->memq)) {
+ queue_init(&local_queue);
+ if (size == shadow_object->vo_size) {
+ nxt_page = (vm_page_t)queue_first(&shadow_object->memq);
+ fast_path_full_commit = 1;
+ }
+ fast_path_possible = 1;
+
+ if (!VM_DYNAMIC_PAGING_ENABLED(memory_manager_default) && shadow_object->internal &&
+ (shadow_object->purgable == VM_PURGABLE_DENY ||
+ shadow_object->purgable == VM_PURGABLE_NONVOLATILE ||
+ shadow_object->purgable == VM_PURGABLE_VOLATILE)) {
+ throttle_page = 1;
+ }
+ }
+ }
while (xfer_size) {
vm_page_t t, m;
if (upl->flags & UPL_LITE) {
unsigned int pg_num;
+ if (nxt_page != VM_PAGE_NULL) {
+ m = nxt_page;
+ nxt_page = (vm_page_t)queue_next(&nxt_page->listq);
+ target_offset = m->offset;
+ }
pg_num = (unsigned int) (target_offset/PAGE_SIZE);
assert(pg_num == target_offset/PAGE_SIZE);
if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
lite_list[pg_num>>5] &= ~(1 << (pg_num & 31));
- if (!(upl->flags & UPL_KERNEL_OBJECT))
+ if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL)
m = vm_page_lookup(shadow_object, target_offset + (upl->offset - shadow_object->paging_offset));
- }
+ } else
+ m = NULL;
}
if (upl->flags & UPL_SHADOWED) {
if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) {
VM_PAGE_FREE(t);
- if (m == VM_PAGE_NULL)
- m = vm_page_lookup(shadow_object, target_offset + object->shadow_offset);
+ if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL)
+ m = vm_page_lookup(shadow_object, target_offset + object->vo_shadow_offset);
}
}
- if ((upl->flags & UPL_KERNEL_OBJECT) || m == VM_PAGE_NULL)
+ if (m == VM_PAGE_NULL)
+ goto commit_next_page;
+
+ if (m->compressor) {
+ assert(m->busy);
+
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
goto commit_next_page;
+ }
if (flags & UPL_COMMIT_CS_VALIDATED) {
/*
m->cs_validated = page_list[entry].cs_validated;
m->cs_tainted = page_list[entry].cs_tainted;
}
+ if (flags & UPL_COMMIT_WRITTEN_BY_KERNEL)
+ m->written_by_kernel = TRUE;
+
if (upl->flags & UPL_IO_WIRE) {
if (page_list)
page_list[entry].phys_addr = 0;
- if (flags & UPL_COMMIT_SET_DIRTY)
- m->dirty = TRUE;
- else if (flags & UPL_COMMIT_CLEAR_DIRTY) {
+ if (flags & UPL_COMMIT_SET_DIRTY) {
+ SET_PAGE_DIRTY(m, FALSE);
+ } else if (flags & UPL_COMMIT_CLEAR_DIRTY) {
m->dirty = FALSE;
if (! (flags & UPL_COMMIT_CS_VALIDATED) &&
* so it will need to be
* re-validated.
*/
+ if (m->slid) {
+ panic("upl_commit_range(%p): page %p was slid\n",
+ upl, m);
+ }
+ assert(!m->slid);
m->cs_validated = FALSE;
#if DEVELOPMENT || DEBUG
vm_cs_validated_resets++;
}
clear_refmod |= VM_MEM_MODIFIED;
}
- if (flags & UPL_COMMIT_INACTIVATE) {
- dwp->dw_mask |= DW_vm_page_deactivate_internal;
- clear_refmod |= VM_MEM_REFERENCED;
- }
if (upl->flags & UPL_ACCESS_BLOCKED) {
/*
* We blocked access to the pages in this UPL.
*/
dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
}
- if (m->absent) {
- if (flags & UPL_COMMIT_FREE_ABSENT)
- dwp->dw_mask |= DW_vm_page_free;
- else {
+ if (fast_path_possible) {
+ assert(m->object->purgable != VM_PURGABLE_EMPTY);
+ assert(m->object->purgable != VM_PURGABLE_VOLATILE);
+ if (m->absent) {
+ assert(m->wire_count == 0);
+ assert(m->busy);
+
m->absent = FALSE;
dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
+ } else {
+ if (m->wire_count == 0)
+ panic("wire_count == 0, m = %p, obj = %p\n", m, shadow_object);
+
+ /*
+ * XXX FBDP need to update some other
+ * counters here (purgeable_wired_count)
+ * (ledgers), ...
+ */
+ assert(m->wire_count);
+ m->wire_count--;
+
+ if (m->wire_count == 0)
+ unwired_count++;
}
- } else
- dwp->dw_mask |= DW_vm_page_unwire;
+ if (m->wire_count == 0) {
+ queue_enter(&local_queue, m, vm_page_t, pageq);
+ local_queue_count++;
+
+ if (throttle_page) {
+ m->throttled = TRUE;
+ } else {
+ if (flags & UPL_COMMIT_INACTIVATE)
+ m->inactive = TRUE;
+ else
+ m->active = TRUE;
+ }
+ }
+ } else {
+ if (flags & UPL_COMMIT_INACTIVATE) {
+ dwp->dw_mask |= DW_vm_page_deactivate_internal;
+ clear_refmod |= VM_MEM_REFERENCED;
+ }
+ if (m->absent) {
+ if (flags & UPL_COMMIT_FREE_ABSENT)
+ dwp->dw_mask |= DW_vm_page_free;
+ else {
+ m->absent = FALSE;
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
+ if ( !(dwp->dw_mask & DW_vm_page_deactivate_internal))
+ dwp->dw_mask |= DW_vm_page_activate;
+ }
+ } else
+ dwp->dw_mask |= DW_vm_page_unwire;
+ }
goto commit_next_page;
}
+ assert(!m->compressor);
+
+ if (page_list)
+ page_list[entry].phys_addr = 0;
+
/*
* make sure to clear the hardware
* modify or reference bits before
if (flags & UPL_COMMIT_CLEAR_DIRTY) {
m->dirty = FALSE;
- if (! (flags & UPL_COMMIT_CS_VALIDATED) &&
- m->cs_validated && !m->cs_tainted) {
+ clear_refmod |= VM_MEM_MODIFIED;
+ }
+ if (m->laundry)
+ dwp->dw_mask |= DW_vm_pageout_throttle_up;
+
+ if (VM_PAGE_WIRED(m))
+ m->pageout = FALSE;
+
+ if (! (flags & UPL_COMMIT_CS_VALIDATED) &&
+ m->cs_validated && !m->cs_tainted) {
+ /*
+ * CODE SIGNING:
+ * This page is no longer dirty
+ * but could have been modified,
+ * so it will need to be
+ * re-validated.
+ */
+ if (m->slid) {
+ panic("upl_commit_range(%p): page %p was slid\n",
+ upl, m);
+ }
+ assert(!m->slid);
+ m->cs_validated = FALSE;
+#if DEVELOPMENT || DEBUG
+ vm_cs_validated_resets++;
+#endif
+ pmap_disconnect(m->phys_page);
+ }
+ if (m->overwriting) {
+ /*
+ * the (COPY_OUT_FROM == FALSE) request_page_list case
+ */
+ if (m->busy) {
+#if CONFIG_PHANTOM_CACHE
+ if (m->absent && !m->object->internal)
+ dwp->dw_mask |= DW_vm_phantom_cache_update;
+#endif
+ m->absent = FALSE;
+
+ dwp->dw_mask |= DW_clear_busy;
+ } else {
/*
- * CODE SIGNING:
- * This page is no longer dirty
- * but could have been modified,
- * so it will need to be
- * re-validated.
+ * alternate (COPY_OUT_FROM == FALSE) page_list case
+ * Occurs when the original page was wired
+ * at the time of the list request
*/
- m->cs_validated = FALSE;
-#if DEVELOPMENT || DEBUG
- vm_cs_validated_resets++;
-#endif
- pmap_disconnect(m->phys_page);
- }
- clear_refmod |= VM_MEM_MODIFIED;
- }
- if (page_list) {
- upl_page_info_t *p;
-
- p = &(page_list[entry]);
-
- if (p->phys_addr && p->pageout && !m->pageout) {
- m->busy = TRUE;
- m->pageout = TRUE;
-
- dwp->dw_mask |= DW_vm_page_wire;
-
- } else if (p->phys_addr &&
- !p->pageout && m->pageout &&
- !m->dump_cleaning) {
- m->pageout = FALSE;
- m->absent = FALSE;
- m->overwriting = FALSE;
+ assert(VM_PAGE_WIRED(m));
- dwp->dw_mask |= (DW_vm_page_unwire | DW_clear_busy | DW_PAGE_WAKEUP);
+ dwp->dw_mask |= DW_vm_page_unwire; /* reactivates */
}
- page_list[entry].phys_addr = 0;
+ m->overwriting = FALSE;
}
- m->dump_cleaning = FALSE;
+ if (m->encrypted_cleaning == TRUE) {
+ m->encrypted_cleaning = FALSE;
- if (m->laundry)
- dwp->dw_mask |= DW_vm_pageout_throttle_up;
+ dwp->dw_mask |= DW_clear_busy | DW_PAGE_WAKEUP;
+ }
+ m->cleaning = FALSE;
if (m->pageout) {
- m->cleaning = FALSE;
- m->encrypted_cleaning = FALSE;
+ /*
+ * With the clean queue enabled, UPL_PAGEOUT should
+ * no longer set the pageout bit. It's pages now go
+ * to the clean queue.
+ */
+ assert(!(flags & UPL_PAGEOUT));
+
m->pageout = FALSE;
#if MACH_CLUSTER_STATS
if (m->wanted) vm_pageout_target_collisions++;
#endif
- m->dirty = FALSE;
-
- if (! (flags & UPL_COMMIT_CS_VALIDATED) &&
- m->cs_validated && !m->cs_tainted) {
- /*
- * CODE SIGNING:
- * This page is no longer dirty
- * but could have been modified,
- * so it will need to be
- * re-validated.
- */
- m->cs_validated = FALSE;
-#if DEVELOPMENT || DEBUG
- vm_cs_validated_resets++;
-#endif
- pmap_disconnect(m->phys_page);
- }
-
if ((flags & UPL_COMMIT_SET_DIRTY) ||
- (m->pmapped && (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED)))
- m->dirty = TRUE;
-
- if (m->dirty) {
+ (m->pmapped && (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED))) {
/*
* page was re-dirtied after we started
* the pageout... reactivate it since
* we don't know whether the on-disk
* copy matches what is now in memory
*/
- dwp->dw_mask |= (DW_vm_page_unwire | DW_clear_busy | DW_PAGE_WAKEUP);
+ SET_PAGE_DIRTY(m, FALSE);
+
+ dwp->dw_mask |= DW_vm_page_activate | DW_PAGE_WAKEUP;
if (upl->flags & UPL_PAGEOUT) {
CLUSTER_STAT(vm_pageout_target_page_dirtied++;)
* page has been successfully cleaned
* go ahead and free it for other use
*/
-
if (m->object->internal) {
DTRACE_VM2(anonpgout, int, 1, (uint64_t *), NULL);
} else {
DTRACE_VM2(fspgout, int, 1, (uint64_t *), NULL);
}
- dwp->dw_mask |= DW_vm_page_free;
-
- if (upl->flags & UPL_PAGEOUT) {
- CLUSTER_STAT(vm_pageout_target_page_freed++;)
+ m->dirty = FALSE;
+ m->busy = TRUE;
- if (page_list[entry].dirty) {
- VM_STAT_INCR(pageouts);
- DTRACE_VM2(pgout, int, 1, (uint64_t *), NULL);
- pgpgout_count++;
- }
- }
+ dwp->dw_mask |= DW_vm_page_free;
}
goto commit_next_page;
}
else vm_pageout_cluster_cleaned++;
if (m->wanted) vm_pageout_cluster_collisions++;
#endif
- m->dirty = FALSE;
-
- if (! (flags & UPL_COMMIT_CS_VALIDATED) &&
- m->cs_validated && !m->cs_tainted) {
- /*
- * CODE SIGNING:
- * This page is no longer dirty
- * but could have been modified,
- * so it will need to be
- * re-validated.
- */
- m->cs_validated = FALSE;
-#if DEVELOPMENT || DEBUG
- vm_cs_validated_resets++;
-#endif
- pmap_disconnect(m->phys_page);
- }
-
- if ((m->busy) && (m->cleaning)) {
- /*
- * the request_page_list case
- */
- m->absent = FALSE;
- m->overwriting = FALSE;
-
- dwp->dw_mask |= DW_clear_busy;
-
- } else if (m->overwriting) {
- /*
- * alternate request page list, write to
- * page_list case. Occurs when the original
- * page was wired at the time of the list
- * request
- */
- assert(VM_PAGE_WIRED(m));
- m->overwriting = FALSE;
-
- dwp->dw_mask |= DW_vm_page_unwire; /* reactivates */
- }
- m->cleaning = FALSE;
- m->encrypted_cleaning = FALSE;
-
/*
* It is a part of the semantic of COPYOUT_FROM
* UPLs that a commit implies cache sync
if ((upl->flags & UPL_PAGE_SYNC_DONE) || (flags & UPL_COMMIT_CLEAR_PRECIOUS))
m->precious = FALSE;
- if (flags & UPL_COMMIT_SET_DIRTY)
- m->dirty = TRUE;
+ if (flags & UPL_COMMIT_SET_DIRTY) {
+ SET_PAGE_DIRTY(m, FALSE);
+ } else {
+ m->dirty = FALSE;
+ }
- if ((flags & UPL_COMMIT_INACTIVATE) && !m->clustered && !m->speculative) {
- dwp->dw_mask |= DW_vm_page_deactivate_internal;
- clear_refmod |= VM_MEM_REFERENCED;
+ /* with the clean queue on, move *all* cleaned pages to the clean queue */
+ if (hibernate_cleaning_in_progress == FALSE && !m->dirty && (upl->flags & UPL_PAGEOUT)) {
+ pgpgout_count++;
- } else if (!m->active && !m->inactive && !m->speculative) {
+ VM_STAT_INCR(pageouts);
+ DTRACE_VM2(pgout, int, 1, (uint64_t *), NULL);
- if (m->clustered || (flags & UPL_COMMIT_SPECULATE))
- dwp->dw_mask |= DW_vm_page_speculate;
- else if (m->reference)
- dwp->dw_mask |= DW_vm_page_activate;
- else {
+ dwp->dw_mask |= DW_enqueue_cleaned;
+ vm_pageout_enqueued_cleaned_from_inactive_dirty++;
+ } else if (should_be_throttled == TRUE && !m->active && !m->inactive && !m->speculative && !m->throttled) {
+ /*
+ * page coming back in from being 'frozen'...
+ * it was dirty before it was frozen, so keep it so
+ * the vm_page_activate will notice that it really belongs
+ * on the throttle queue and put it there
+ */
+ SET_PAGE_DIRTY(m, FALSE);
+ dwp->dw_mask |= DW_vm_page_activate;
+
+ } else {
+ if ((flags & UPL_COMMIT_INACTIVATE) && !m->clustered && !m->speculative) {
dwp->dw_mask |= DW_vm_page_deactivate_internal;
clear_refmod |= VM_MEM_REFERENCED;
+ } else if (!m->active && !m->inactive && !m->speculative) {
+
+ if (m->clustered || (flags & UPL_COMMIT_SPECULATE))
+ dwp->dw_mask |= DW_vm_page_speculate;
+ else if (m->reference)
+ dwp->dw_mask |= DW_vm_page_activate;
+ else {
+ dwp->dw_mask |= DW_vm_page_deactivate_internal;
+ clear_refmod |= VM_MEM_REFERENCED;
+ }
}
}
if (upl->flags & UPL_ACCESS_BLOCKED) {
if (dwp->dw_mask) {
if (dwp->dw_mask & ~(DW_clear_busy | DW_PAGE_WAKEUP)) {
- if (m->busy == FALSE) {
- /*
- * dw_do_work may need to drop the object lock
- * if it does, we need the pages it's looking at to
- * be held stable via the busy bit.
- */
- m->busy = TRUE;
- dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
- }
- dwp->dw_m = m;
- dwp++;
- dw_count++;
+ VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count);
- if (dw_count >= DELAYED_WORK_LIMIT) {
- dw_do_work(shadow_object, &dw_array[0], dw_count);
+ if (dw_count >= dw_limit) {
+ vm_page_do_delayed_work(shadow_object, &dw_array[0], dw_count);
dwp = &dw_array[0];
dw_count = 0;
}
}
if (dw_count)
- dw_do_work(shadow_object, &dw_array[0], dw_count);
+ vm_page_do_delayed_work(shadow_object, &dw_array[0], dw_count);
+
+ if (fast_path_possible) {
+ assert(shadow_object->purgable != VM_PURGABLE_VOLATILE);
+ assert(shadow_object->purgable != VM_PURGABLE_EMPTY);
+
+ if (local_queue_count || unwired_count) {
+
+ if (local_queue_count) {
+ vm_page_t first_local, last_local;
+ vm_page_t first_target;
+ queue_head_t *target_queue;
+
+ if (throttle_page)
+ target_queue = &vm_page_queue_throttled;
+ else {
+ if (flags & UPL_COMMIT_INACTIVATE) {
+ if (shadow_object->internal)
+ target_queue = &vm_page_queue_anonymous;
+ else
+ target_queue = &vm_page_queue_inactive;
+ } else
+ target_queue = &vm_page_queue_active;
+ }
+ /*
+ * Transfer the entire local queue to a regular LRU page queues.
+ */
+ first_local = (vm_page_t) queue_first(&local_queue);
+ last_local = (vm_page_t) queue_last(&local_queue);
+
+ vm_page_lockspin_queues();
+
+ first_target = (vm_page_t) queue_first(target_queue);
+
+ if (queue_empty(target_queue))
+ queue_last(target_queue) = (queue_entry_t) last_local;
+ else
+ queue_prev(&first_target->pageq) = (queue_entry_t) last_local;
+
+ queue_first(target_queue) = (queue_entry_t) first_local;
+ queue_prev(&first_local->pageq) = (queue_entry_t) target_queue;
+ queue_next(&last_local->pageq) = (queue_entry_t) first_target;
+
+ /*
+ * Adjust the global page counts.
+ */
+ if (throttle_page) {
+ vm_page_throttled_count += local_queue_count;
+ } else {
+ if (flags & UPL_COMMIT_INACTIVATE) {
+ if (shadow_object->internal)
+ vm_page_anonymous_count += local_queue_count;
+ vm_page_inactive_count += local_queue_count;
+
+ token_new_pagecount += local_queue_count;
+ } else
+ vm_page_active_count += local_queue_count;
+
+ if (shadow_object->internal)
+ vm_page_pageable_internal_count += local_queue_count;
+ else
+ vm_page_pageable_external_count += local_queue_count;
+ }
+ } else {
+ vm_page_lockspin_queues();
+ }
+ if (unwired_count) {
+ vm_page_wire_count -= unwired_count;
+ VM_CHECK_MEMORYSTATUS;
+ }
+ vm_page_unlock_queues();
+
+ shadow_object->wired_page_count -= unwired_count;
+ }
+ }
occupied = 1;
if (upl->flags & UPL_DEVICE_MEMORY) {
int pg_num;
int i;
- pg_num = upl->size/PAGE_SIZE;
- pg_num = (pg_num + 31) >> 5;
occupied = 0;
- for (i = 0; i < pg_num; i++) {
- if (lite_list[i] != 0) {
- occupied = 1;
- break;
+ if (!fast_path_full_commit) {
+ pg_num = upl->size/PAGE_SIZE;
+ pg_num = (pg_num + 31) >> 5;
+
+ for (i = 0; i < pg_num; i++) {
+ if (lite_list[i] != 0) {
+ occupied = 1;
+ break;
+ }
}
}
} else {
* against this object
*/
vm_object_activity_end(shadow_object);
+ vm_object_collapse(shadow_object, 0, TRUE);
} else {
/*
* we dontated the paging reference to
int error,
boolean_t *empty)
{
+ upl_page_info_t *user_page_list = NULL;
upl_size_t xfer_size, subupl_size = size;
vm_object_t shadow_object;
vm_object_t object;
int entry;
wpl_array_t lite_list;
int occupied;
- struct dw dw_array[DELAYED_WORK_LIMIT];
- struct dw *dwp;
- int dw_count, isVectorUPL = 0;
+ struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT];
+ struct vm_page_delayed_work *dwp;
+ int dw_count;
+ int dw_limit;
+ int isVectorUPL = 0;
upl_t vector_upl = NULL;
*empty = FALSE;
lite_list = (wpl_array_t)
((((uintptr_t)upl) + sizeof(struct upl))
+ ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t)));
+
+ user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
} else {
lite_list = (wpl_array_t)
(((uintptr_t)upl) + sizeof(struct upl));
dwp = &dw_array[0];
dw_count = 0;
+ dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
if ((error & UPL_ABORT_DUMP_PAGES) && (upl->flags & UPL_KERNEL_OBJECT))
panic("upl_abort_range: kernel_object being DUMPED");
while (xfer_size) {
vm_page_t t, m;
+ unsigned int pg_num;
+ boolean_t needed;
- dwp->dw_mask = 0;
+ pg_num = (unsigned int) (target_offset/PAGE_SIZE);
+ assert(pg_num == target_offset/PAGE_SIZE);
+
+ needed = FALSE;
+ if (user_page_list)
+ needed = user_page_list[pg_num].needed;
+
+ dwp->dw_mask = 0;
m = VM_PAGE_NULL;
if (upl->flags & UPL_LITE) {
- unsigned int pg_num;
-
- pg_num = (unsigned int) (target_offset/PAGE_SIZE);
- assert(pg_num == target_offset/PAGE_SIZE);
-
if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
lite_list[pg_num>>5] &= ~(1 << (pg_num & 31));
VM_PAGE_FREE(t);
if (m == VM_PAGE_NULL)
- m = vm_page_lookup(shadow_object, target_offset + object->shadow_offset);
+ m = vm_page_lookup(shadow_object, target_offset + object->vo_shadow_offset);
}
}
if ((upl->flags & UPL_KERNEL_OBJECT))
if (m != VM_PAGE_NULL) {
+ assert(!m->compressor);
+
if (m->absent) {
boolean_t must_free = TRUE;
- m->clustered = FALSE;
/*
* COPYOUT = FALSE case
* check for error conditions which must
m->unusual = TRUE;
must_free = FALSE;
}
+ if (m->clustered && needed == FALSE) {
+ /*
+ * This page was a part of a speculative
+ * read-ahead initiated by the kernel
+ * itself. No one is expecting this
+ * page and no one will clean up its
+ * error state if it ever becomes valid
+ * in the future.
+ * We have to free it here.
+ */
+ must_free = TRUE;
+ }
/*
* ENCRYPTED SWAP:
m->cleaning = FALSE;
m->encrypted_cleaning = FALSE;
+
+ if (m->overwriting && !m->busy) {
+ /*
+ * this shouldn't happen since
+ * this is an 'absent' page, but
+ * it doesn't hurt to check for
+ * the 'alternate' method of
+ * stabilizing the page...
+ * we will mark 'busy' to be cleared
+ * in the following code which will
+ * take care of the primary stabilzation
+ * method (i.e. setting 'busy' to TRUE)
+ */
+ dwp->dw_mask |= DW_vm_page_unwire;
+ }
m->overwriting = FALSE;
dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
if (m->laundry)
dwp->dw_mask |= DW_vm_pageout_throttle_up;
- if (m->pageout) {
- assert(m->busy);
- assert(m->wire_count == 1);
- m->pageout = FALSE;
+ if (upl->flags & UPL_ACCESS_BLOCKED) {
+ /*
+ * We blocked access to the pages in this UPL.
+ * Clear the "busy" bit and wake up any waiter
+ * for this page.
+ */
+ dwp->dw_mask |= DW_clear_busy;
+ }
+ if (m->overwriting) {
+ if (m->busy)
+ dwp->dw_mask |= DW_clear_busy;
+ else {
+ /*
+ * deal with the 'alternate' method
+ * of stabilizing the page...
+ * we will either free the page
+ * or mark 'busy' to be cleared
+ * in the following code which will
+ * take care of the primary stabilzation
+ * method (i.e. setting 'busy' to TRUE)
+ */
+ dwp->dw_mask |= DW_vm_page_unwire;
+ }
+ m->overwriting = FALSE;
+ }
+ if (m->encrypted_cleaning == TRUE) {
+ m->encrypted_cleaning = FALSE;
- dwp->dw_mask |= DW_vm_page_unwire;
+ dwp->dw_mask |= DW_clear_busy;
}
- m->dump_cleaning = FALSE;
+ m->pageout = FALSE;
m->cleaning = FALSE;
- m->encrypted_cleaning = FALSE;
- m->overwriting = FALSE;
#if MACH_PAGEMAP
vm_external_state_clr(m->object->existence_map, m->offset);
#endif /* MACH_PAGEMAP */
dwp->dw_mask |= DW_vm_page_free;
} else {
- if (error & UPL_ABORT_REFERENCE) {
- /*
- * we've been told to explictly
- * reference this page... for
- * file I/O, this is done by
- * implementing an LRU on the inactive q
- */
- dwp->dw_mask |= DW_vm_page_lru;
+ if (!(dwp->dw_mask & DW_vm_page_unwire)) {
+ if (error & UPL_ABORT_REFERENCE) {
+ /*
+ * we've been told to explictly
+ * reference this page... for
+ * file I/O, this is done by
+ * implementing an LRU on the inactive q
+ */
+ dwp->dw_mask |= DW_vm_page_lru;
+
+ } else if (!m->active && !m->inactive && !m->speculative)
+ dwp->dw_mask |= DW_vm_page_deactivate_internal;
}
- dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
+ dwp->dw_mask |= DW_PAGE_WAKEUP;
}
}
}
if (dwp->dw_mask) {
if (dwp->dw_mask & ~(DW_clear_busy | DW_PAGE_WAKEUP)) {
- if (m->busy == FALSE) {
- /*
- * dw_do_work may need to drop the object lock
- * if it does, we need the pages it's looking at to
- * be held stable via the busy bit.
- */
- m->busy = TRUE;
- dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
- }
- dwp->dw_m = m;
- dwp++;
- dw_count++;
+ VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count);
- if (dw_count >= DELAYED_WORK_LIMIT) {
- dw_do_work(shadow_object, &dw_array[0], dw_count);
+ if (dw_count >= dw_limit) {
+ vm_page_do_delayed_work(shadow_object, &dw_array[0], dw_count);
dwp = &dw_array[0];
dw_count = 0;
}
}
if (dw_count)
- dw_do_work(shadow_object, &dw_array[0], dw_count);
+ vm_page_do_delayed_work(shadow_object, &dw_array[0], dw_count);
occupied = 1;
* against this object
*/
vm_object_activity_end(shadow_object);
+ vm_object_collapse(shadow_object, 0, TRUE);
} else {
/*
* we dontated the paging reference to
}
+void
+iopl_valid_data(
+ upl_t upl)
+{
+ vm_object_t object;
+ vm_offset_t offset;
+ vm_page_t m, nxt_page = VM_PAGE_NULL;
+ upl_size_t size;
+ int wired_count = 0;
+
+ if (upl == NULL)
+ panic("iopl_valid_data: NULL upl");
+ if (vector_upl_is_valid(upl))
+ panic("iopl_valid_data: vector upl");
+ if ((upl->flags & (UPL_DEVICE_MEMORY|UPL_SHADOWED|UPL_ACCESS_BLOCKED|UPL_IO_WIRE|UPL_INTERNAL)) != UPL_IO_WIRE)
+ panic("iopl_valid_data: unsupported upl, flags = %x", upl->flags);
+
+ object = upl->map_object;
+
+ if (object == kernel_object || object == compressor_object)
+ panic("iopl_valid_data: object == kernel or compressor");
+
+ if (object->purgable == VM_PURGABLE_VOLATILE)
+ panic("iopl_valid_data: object == VM_PURGABLE_VOLATILE");
+
+ size = upl->size;
+
+ vm_object_lock(object);
+
+ if (object->vo_size == size && object->resident_page_count == (size / PAGE_SIZE))
+ nxt_page = (vm_page_t)queue_first(&object->memq);
+ else
+ offset = 0 + upl->offset - object->paging_offset;
+
+ while (size) {
+
+ if (nxt_page != VM_PAGE_NULL) {
+ m = nxt_page;
+ nxt_page = (vm_page_t)queue_next(&nxt_page->listq);
+ } else {
+ m = vm_page_lookup(object, offset);
+ offset += PAGE_SIZE;
+
+ if (m == VM_PAGE_NULL)
+ panic("iopl_valid_data: missing expected page at offset %lx", (long)offset);
+ }
+ if (m->busy) {
+ if (!m->absent)
+ panic("iopl_valid_data: busy page w/o absent");
+
+ if (m->pageq.next || m->pageq.prev)
+ panic("iopl_valid_data: busy+absent page on page queue");
+
+ m->absent = FALSE;
+ m->dirty = TRUE;
+ m->wire_count++;
+ wired_count++;
+
+ PAGE_WAKEUP_DONE(m);
+ }
+ size -= PAGE_SIZE;
+ }
+ if (wired_count) {
+ object->wired_page_count += wired_count;
+
+ vm_page_lockspin_queues();
+ vm_page_wire_count += wired_count;
+ vm_page_unlock_queues();
+ }
+ vm_object_unlock(object);
+}
+
+
+
+
+void
+vm_object_set_pmap_cache_attr(
+ vm_object_t object,
+ upl_page_info_array_t user_page_list,
+ unsigned int num_pages,
+ boolean_t batch_pmap_op)
+{
+ unsigned int cache_attr = 0;
+
+ cache_attr = object->wimg_bits & VM_WIMG_MASK;
+ assert(user_page_list);
+ if (cache_attr != VM_WIMG_USE_DEFAULT) {
+ PMAP_BATCH_SET_CACHE_ATTR(object, user_page_list, cache_attr, num_pages, batch_pmap_op);
+ }
+}
+
unsigned int vm_object_iopl_request_sleep_for_cleaning = 0;
kern_return_t
unsigned int entry;
wpl_array_t lite_list = NULL;
int no_zero_fill = FALSE;
+ unsigned int size_in_pages;
u_int32_t psize;
kern_return_t ret;
vm_prot_t prot;
struct vm_object_fault_info fault_info;
- struct dw dw_array[DELAYED_WORK_LIMIT];
- struct dw *dwp;
+ struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT];
+ struct vm_page_delayed_work *dwp;
int dw_count;
+ int dw_limit;
int dw_index;
+ boolean_t caller_lookup;
+ int io_tracking_flag = 0;
+ int interruptible;
+
+ boolean_t set_cache_attr_needed = FALSE;
+ boolean_t free_wired_pages = FALSE;
+ int fast_path_possible = 0;
+
if (cntrl_flags & ~UPL_VALID_FLAGS) {
/*
return KERN_INVALID_VALUE;
if (object->phys_contiguous) {
- if ((offset + object->shadow_offset) >= (vm_object_offset_t)max_valid_dma_address)
+ if ((offset + object->vo_shadow_offset) >= (vm_object_offset_t)max_valid_dma_address)
return KERN_INVALID_ADDRESS;
- if (((offset + object->shadow_offset) + size) >= (vm_object_offset_t)max_valid_dma_address)
+ if (((offset + object->vo_shadow_offset) + size) >= (vm_object_offset_t)max_valid_dma_address)
return KERN_INVALID_ADDRESS;
}
}
*/
assert(! (cntrl_flags & UPL_ENCRYPT));
}
- if (cntrl_flags & UPL_NOZEROFILL)
+ if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO))
no_zero_fill = TRUE;
if (cntrl_flags & UPL_COPYOUT_FROM)
else
prot = VM_PROT_READ | VM_PROT_WRITE;
- if (((size/PAGE_SIZE) > MAX_UPL_SIZE) && !object->phys_contiguous)
- size = MAX_UPL_SIZE * PAGE_SIZE;
-
- if (cntrl_flags & UPL_SET_INTERNAL) {
- if (page_list_count != NULL)
- *page_list_count = MAX_UPL_SIZE;
- }
- if (((cntrl_flags & UPL_SET_INTERNAL) && !(object->phys_contiguous)) &&
- ((page_list_count != NULL) && (*page_list_count != 0) && *page_list_count < (size/page_size)))
- return KERN_INVALID_ARGUMENT;
-
if ((!object->internal) && (object->paging_offset != 0))
panic("vm_object_iopl_request: external object with non-zero paging offset\n");
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if ((object->io_tracking && object != kernel_object) || upl_debug_enabled)
+ io_tracking_flag |= UPL_CREATE_IO_TRACKING;
+#endif
+
+#if CONFIG_IOSCHED
+ if (object->io_tracking) {
+ /* Check if we're dealing with the kernel object. We do not support expedite on kernel object UPLs */
+ if (object != kernel_object)
+ io_tracking_flag |= UPL_CREATE_EXPEDITE_SUP;
+ }
+#endif
if (object->phys_contiguous)
psize = PAGE_SIZE;
psize = size;
if (cntrl_flags & UPL_SET_INTERNAL) {
- upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE, UPL_IO_WIRE, psize);
+ upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE | io_tracking_flag, UPL_IO_WIRE, psize);
user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
lite_list = (wpl_array_t) (((uintptr_t)user_page_list) +
lite_list = NULL;
}
} else {
- upl = upl_create(UPL_CREATE_LITE, UPL_IO_WIRE, psize);
+ upl = upl_create(UPL_CREATE_LITE | io_tracking_flag, UPL_IO_WIRE, psize);
lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));
if (size == 0) {
upl->map_object = object;
upl->size = size;
+ size_in_pages = size / PAGE_SIZE;
+
if (object == kernel_object &&
!(cntrl_flags & (UPL_NEED_32BIT_ADDR | UPL_BLOCK_ACCESS))) {
upl->flags |= UPL_KERNEL_OBJECT;
if (cntrl_flags & UPL_BLOCK_ACCESS) {
/*
- * The user requested that access to the pages in this URL
+ * The user requested that access to the pages in this UPL
* be blocked until the UPL is commited or aborted.
*/
upl->flags |= UPL_ACCESS_BLOCKED;
}
- if (object->phys_contiguous) {
-#if UPL_DEBUG
+ if (!(cntrl_flags & (UPL_NEED_32BIT_ADDR | UPL_BLOCK_ACCESS)) &&
+ object->purgable != VM_PURGABLE_VOLATILE &&
+ object->purgable != VM_PURGABLE_EMPTY &&
+ object->copy == NULL &&
+ size == object->vo_size &&
+ offset == 0 &&
+ object->resident_page_count == 0 &&
+ object->shadow == NULL &&
+ object->pager == NULL)
+ {
+ fast_path_possible = 1;
+ set_cache_attr_needed = TRUE;
+ }
+
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if (upl->flags & UPL_TRACKED_BY_OBJECT) {
+ vm_object_activity_begin(object);
queue_enter(&object->uplq, upl, upl_t, uplq);
-#endif /* UPL_DEBUG */
+ }
+#endif
+
+ if (object->phys_contiguous) {
if (upl->flags & UPL_ACCESS_BLOCKED) {
assert(!object->blocked_access);
*/
upl->flags |= UPL_DEVICE_MEMORY;
- upl->highest_page = (ppnum_t) ((offset + object->shadow_offset + size - 1)>>PAGE_SHIFT);
+ upl->highest_page = (ppnum_t) ((offset + object->vo_shadow_offset + size - 1)>>PAGE_SHIFT);
if (user_page_list) {
- user_page_list[0].phys_addr = (ppnum_t) ((offset + object->shadow_offset)>>PAGE_SHIFT);
+ user_page_list[0].phys_addr = (ppnum_t) ((offset + object->vo_shadow_offset)>>PAGE_SHIFT);
user_page_list[0].device = TRUE;
}
if (page_list_count != NULL) {
}
return KERN_SUCCESS;
}
- if (object != kernel_object) {
+ if (object != kernel_object && object != compressor_object) {
/*
* Protect user space from future COW operations
*/
+#if VM_OBJECT_TRACKING_OP_TRUESHARE
+ if (!object->true_share &&
+ vm_object_tracking_inited) {
+ void *bt[VM_OBJECT_TRACKING_BTDEPTH];
+ int num = 0;
+
+ num = OSBacktrace(bt,
+ VM_OBJECT_TRACKING_BTDEPTH);
+ btlog_add_entry(vm_object_tracking_btlog,
+ object,
+ VM_OBJECT_TRACKING_OP_TRUESHARE,
+ bt,
+ num);
+ }
+#endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
+
object->true_share = TRUE;
if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
}
-#if UPL_DEBUG
- queue_enter(&object->uplq, upl, upl_t, uplq);
-#endif /* UPL_DEBUG */
-
if (!(cntrl_flags & UPL_COPYOUT_FROM) &&
object->copy != VM_OBJECT_NULL) {
/*
iopl_cow_pages += size >> PAGE_SHIFT;
#endif
}
+ if (cntrl_flags & UPL_SET_INTERRUPTIBLE)
+ interruptible = THREAD_ABORTSAFE;
+ else
+ interruptible = THREAD_UNINT;
+
+ entry = 0;
+
+ xfer_size = size;
+ dst_offset = offset;
+ dw_count = 0;
+
+ if (fast_path_possible) {
+ int wired_count = 0;
+
+ while (xfer_size) {
+
+ while ( (dst_page = vm_page_grab()) == VM_PAGE_NULL) {
+ OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages);
+
+ VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);
+
+ if (vm_page_wait(interruptible) == FALSE) {
+ /*
+ * interrupted case
+ */
+ OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
+ VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1);
- entry = 0;
+ if (wired_count) {
+ vm_page_lockspin_queues();
+ vm_page_wire_count += wired_count;
+ vm_page_unlock_queues();
- xfer_size = size;
- dst_offset = offset;
+ free_wired_pages = TRUE;
+ }
+ ret = MACH_SEND_INTERRUPTED;
+
+ goto return_err;
+ }
+ OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
+
+ VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
+ }
+ if (no_zero_fill == FALSE)
+ vm_page_zero_fill(dst_page);
+ else
+ dst_page->absent = TRUE;
+
+ dst_page->reference = TRUE;
+
+ if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
+ SET_PAGE_DIRTY(dst_page, FALSE);
+ }
+ if (dst_page->absent == FALSE) {
+ assert(object->purgable != VM_PURGABLE_VOLATILE);
+ assert(object->purgable != VM_PURGABLE_EMPTY);
+ dst_page->wire_count++;
+ wired_count++;
+
+ PAGE_WAKEUP_DONE(dst_page);
+ }
+ vm_page_insert_internal(dst_page, object, dst_offset, FALSE, TRUE, TRUE);
+
+ lite_list[entry>>5] |= 1 << (entry & 31);
+
+ if (dst_page->phys_page > upl->highest_page)
+ upl->highest_page = dst_page->phys_page;
+
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = dst_page->phys_page;
+ user_page_list[entry].absent = dst_page->absent;
+ user_page_list[entry].dirty = dst_page->dirty;
+ user_page_list[entry].precious = FALSE;
+ user_page_list[entry].pageout = FALSE;
+ user_page_list[entry].device = FALSE;
+ user_page_list[entry].needed = FALSE;
+ user_page_list[entry].speculative = FALSE;
+ user_page_list[entry].cs_validated = FALSE;
+ user_page_list[entry].cs_tainted = FALSE;
+ }
+ entry++;
+ dst_offset += PAGE_SIZE_64;
+ xfer_size -= PAGE_SIZE;
+ size_in_pages--;
+ }
+ if (wired_count) {
+ vm_page_lockspin_queues();
+ vm_page_wire_count += wired_count;
+ vm_page_unlock_queues();
+ }
+ goto finish;
+ }
fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL;
fault_info.user_tag = 0;
fault_info.hi_offset = offset + xfer_size;
fault_info.no_cache = FALSE;
fault_info.stealth = FALSE;
+ fault_info.io_sync = FALSE;
+ fault_info.cs_bypass = FALSE;
fault_info.mark_zf_absent = TRUE;
+ fault_info.interruptible = interruptible;
+ fault_info.batch_pmap_op = TRUE;
dwp = &dw_array[0];
- dw_count = 0;
+ dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
while (xfer_size) {
vm_fault_return_t result;
if (object == kernel_object)
panic("vm_object_iopl_request: missing/bad page in kernel object\n");
+ if (object == compressor_object)
+ panic("vm_object_iopl_request: missing/bad page in compressor object\n");
+
+ if (cntrl_flags & UPL_REQUEST_NO_FAULT) {
+ ret = KERN_MEMORY_ERROR;
+ goto return_err;
+ }
+ set_cache_attr_needed = TRUE;
+
+ /*
+ * We just looked up the page and the result remains valid
+ * until the object lock is release, so send it to
+ * vm_fault_page() (as "dst_page"), to avoid having to
+ * look it up again there.
+ */
+ caller_lookup = TRUE;
do {
vm_page_t top_page;
kern_return_t error_code;
- int interruptible;
-
- if (cntrl_flags & UPL_SET_INTERRUPTIBLE)
- interruptible = THREAD_ABORTSAFE;
- else
- interruptible = THREAD_UNINT;
- fault_info.interruptible = interruptible;
fault_info.cluster_size = xfer_size;
vm_object_paging_begin(object);
result = vm_fault_page(object, dst_offset,
- prot | VM_PROT_WRITE, FALSE,
+ prot | VM_PROT_WRITE, FALSE,
+ caller_lookup,
&prot, &dst_page, &top_page,
(int *)0,
&error_code, no_zero_fill,
FALSE, &fault_info);
+ /* our lookup is no longer valid at this point */
+ caller_lookup = FALSE;
+
switch (result) {
case VM_FAULT_SUCCESS:
vm_object_lock(object);
break;
- case VM_FAULT_FICTITIOUS_SHORTAGE:
- vm_page_more_fictitious();
+ case VM_FAULT_MEMORY_SHORTAGE:
+ OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages);
- vm_object_lock(object);
- break;
+ VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);
- case VM_FAULT_MEMORY_SHORTAGE:
if (vm_page_wait(interruptible)) {
+ OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
+
+ VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
vm_object_lock(object);
+
break;
}
+ OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
+
+ VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1);
+
/* fall thru */
case VM_FAULT_INTERRUPTED:
} while (result != VM_FAULT_SUCCESS);
}
-
if (upl->flags & UPL_KERNEL_OBJECT)
goto record_phys_addr;
+ if (dst_page->compressor) {
+ dst_page->busy = TRUE;
+ goto record_phys_addr;
+ }
+
if (dst_page->cleaning) {
/*
- * Someone else is cleaning this page in place.as
+ * Someone else is cleaning this page in place.
* In theory, we should be able to proceed and use this
* page but they'll probably end up clearing the "busy"
* bit on it in upl_commit_range() but they didn't set
PAGE_SLEEP(object, dst_page, THREAD_UNINT);
continue;
}
+ if (dst_page->laundry) {
+ dst_page->pageout = FALSE;
+
+ vm_pageout_steal_laundry(dst_page, FALSE);
+ }
if ( (cntrl_flags & UPL_NEED_32BIT_ADDR) &&
dst_page->phys_page >= (max_valid_dma_address >> PAGE_SHIFT) ) {
vm_page_t low_page;
else
refmod = 0;
- if ( !dst_page->absent)
+ if (!dst_page->absent)
vm_page_copy(dst_page, low_page);
low_page->reference = dst_page->reference;
if (refmod & VM_MEM_REFERENCED)
low_page->reference = TRUE;
- if (refmod & VM_MEM_MODIFIED)
- low_page->dirty = TRUE;
+ if (refmod & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(low_page, FALSE);
+ }
vm_page_replace(low_page, object, dst_offset);
if (cntrl_flags & UPL_BLOCK_ACCESS) {
/*
* Mark the page "busy" to block any future page fault
- * on this page. We'll also remove the mapping
+ * on this page in addition to wiring it.
+ * We'll also remove the mapping
* of all these pages before leaving this routine.
*/
assert(!dst_page->fictitious);
*/
dwp->dw_mask |= DW_set_reference;
- if (!(cntrl_flags & UPL_COPYOUT_FROM))
- dst_page->dirty = TRUE;
+ if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
+ SET_PAGE_DIRTY(dst_page, TRUE);
+ }
+ if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->written_by_kernel == TRUE) {
+ pmap_sync_page_attributes_phys(dst_page->phys_page);
+ dst_page->written_by_kernel = FALSE;
+ }
+
record_phys_addr:
if (dst_page->busy)
upl->flags |= UPL_HAS_BUSY;
user_page_list[entry].dirty = dst_page->dirty;
user_page_list[entry].precious = dst_page->precious;
user_page_list[entry].device = FALSE;
+ user_page_list[entry].needed = FALSE;
if (dst_page->clustered == TRUE)
user_page_list[entry].speculative = dst_page->speculative;
else
user_page_list[entry].cs_validated = dst_page->cs_validated;
user_page_list[entry].cs_tainted = dst_page->cs_tainted;
}
- if (object != kernel_object) {
+ if (object != kernel_object && object != compressor_object) {
/*
* someone is explicitly grabbing this page...
* update clustered and speculative state
*
*/
- VM_PAGE_CONSUME_CLUSTERED(dst_page);
+ if (dst_page->clustered)
+ VM_PAGE_CONSUME_CLUSTERED(dst_page);
}
entry++;
dst_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
+ size_in_pages--;
if (dwp->dw_mask) {
- if (dst_page->busy == FALSE) {
- /*
- * dw_do_work may need to drop the object lock
- * if it does, we need the pages it's looking at to
- * be held stable via the busy bit.
- */
- dst_page->busy = TRUE;
- dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
- }
- dwp->dw_m = dst_page;
- dwp++;
- dw_count++;
+ VM_PAGE_ADD_DELAYED_WORK(dwp, dst_page, dw_count);
- if (dw_count >= DELAYED_WORK_LIMIT) {
- dw_do_work(object, &dw_array[0], dw_count);
+ if (dw_count >= dw_limit) {
+ vm_page_do_delayed_work(object, &dw_array[0], dw_count);
dwp = &dw_array[0];
dw_count = 0;
}
}
if (dw_count)
- dw_do_work(object, &dw_array[0], dw_count);
+ vm_page_do_delayed_work(object, &dw_array[0], dw_count);
+
+finish:
+ if (user_page_list && set_cache_attr_needed == TRUE)
+ vm_object_set_pmap_cache_attr(object, user_page_list, entry, TRUE);
if (page_list_count != NULL) {
if (upl->flags & UPL_INTERNAL)
}
vm_page_lock_queues();
- if (dst_page->absent) {
+ if (dst_page->absent || free_wired_pages == TRUE) {
vm_page_free(dst_page);
need_unwire = FALSE;
vm_page_unwire(dst_page, TRUE);
PAGE_WAKEUP_DONE(dst_page);
- }
+ }
vm_page_unlock_queues();
if (need_unwire == TRUE)
#endif
if (! (upl->flags & UPL_KERNEL_OBJECT)) {
vm_object_activity_end(object);
+ vm_object_collapse(object, 0, TRUE);
}
vm_object_unlock(object);
upl_destroy(upl);
* Make each UPL point to the correct VM object, i.e. the
* object holding the pages that the UPL refers to...
*/
-#if UPL_DEBUG
- queue_remove(&object1->uplq, upl1, upl_t, uplq);
- queue_remove(&object2->uplq, upl2, upl_t, uplq);
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if ((upl1->flags & UPL_TRACKED_BY_OBJECT) || (upl2->flags & UPL_TRACKED_BY_OBJECT)) {
+ vm_object_lock(object1);
+ vm_object_lock(object2);
+ }
+ if (upl1->flags & UPL_TRACKED_BY_OBJECT)
+ queue_remove(&object1->uplq, upl1, upl_t, uplq);
+ if (upl2->flags & UPL_TRACKED_BY_OBJECT)
+ queue_remove(&object2->uplq, upl2, upl_t, uplq);
#endif
upl1->map_object = object2;
upl2->map_object = object1;
-#if UPL_DEBUG
- queue_enter(&object1->uplq, upl2, upl_t, uplq);
- queue_enter(&object2->uplq, upl1, upl_t, uplq);
+
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if (upl1->flags & UPL_TRACKED_BY_OBJECT)
+ queue_enter(&object2->uplq, upl1, upl_t, uplq);
+ if (upl2->flags & UPL_TRACKED_BY_OBJECT)
+ queue_enter(&object1->uplq, upl2, upl_t, uplq);
+ if ((upl1->flags & UPL_TRACKED_BY_OBJECT) || (upl2->flags & UPL_TRACKED_BY_OBJECT)) {
+ vm_object_unlock(object2);
+ vm_object_unlock(object1);
+ }
#endif
}
return retval;
}
+void
+upl_range_needed(
+ upl_t upl,
+ int index,
+ int count)
+{
+ upl_page_info_t *user_page_list;
+ int size_in_pages;
+
+ if ( !(upl->flags & UPL_INTERNAL) || count <= 0)
+ return;
+
+ size_in_pages = upl->size / PAGE_SIZE;
+
+ user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
+
+ while (count-- && index < size_in_pages)
+ user_page_list[index++].needed = TRUE;
+}
+
+
/*
* ENCRYPTED SWAP:
*
}
map_entry->object.vm_object = kernel_object;
map_entry->offset = page_map_offset;
+ map_entry->protection = VM_PROT_NONE;
+ map_entry->max_protection = VM_PROT_NONE;
+ map_entry->permanent = TRUE;
vm_object_reference(kernel_object);
vm_map_unlock(kernel_map);
*/
kern_return_t
vm_paging_map_object(
- vm_map_offset_t *address,
vm_page_t page,
vm_object_t object,
vm_object_offset_t offset,
- vm_map_size_t *size,
vm_prot_t protection,
- boolean_t can_unlock_object)
+ boolean_t can_unlock_object,
+ vm_map_size_t *size, /* IN/OUT */
+ vm_map_offset_t *address, /* OUT */
+ boolean_t *need_unmap) /* OUT */
{
kern_return_t kr;
vm_map_offset_t page_map_offset;
vm_object_offset_t object_offset;
int i;
-
if (page != VM_PAGE_NULL && *size == PAGE_SIZE) {
+ /* use permanent 1-to-1 kernel mapping of physical memory ? */
+#if __x86_64__
+ *address = (vm_map_offset_t)
+ PHYSMAP_PTOV((pmap_paddr_t)page->phys_page <<
+ PAGE_SHIFT);
+ *need_unmap = FALSE;
+ return KERN_SUCCESS;
+#else
+#warn "vm_paging_map_object: no 1-to-1 kernel mapping of physical memory..."
+#endif
+
assert(page->busy);
/*
* Use one of the pre-allocated kernel virtual addresses
*/
vm_paging_page_waiter_total++;
vm_paging_page_waiter++;
- thread_sleep_fast_usimple_lock(&vm_paging_page_waiter,
- &vm_paging_lock,
- THREAD_UNINT);
+ kr = assert_wait((event_t)&vm_paging_page_waiter, THREAD_UNINT);
+ if (kr == THREAD_WAITING) {
+ simple_unlock(&vm_paging_lock);
+ kr = thread_block(THREAD_CONTINUE_NULL);
+ simple_lock(&vm_paging_lock);
+ }
vm_paging_page_waiter--;
/* ... and try again */
}
vm_paging_page_inuse[i] = TRUE;
simple_unlock(&vm_paging_lock);
- if (page->pmapped == FALSE) {
- pmap_sync_page_data_phys(page->phys_page);
- }
page->pmapped = TRUE;
/*
page_map_offset,
page,
protection,
- ((int) page->object->wimg_bits &
- VM_WIMG_MASK),
+ VM_PROT_NONE,
+ 0,
TRUE);
vm_paging_objects_mapped++;
vm_paging_pages_mapped++;
*address = page_map_offset;
+ *need_unmap = TRUE;
/* all done and mapped, ready to use ! */
return KERN_SUCCESS;
}
if (! can_unlock_object) {
+ *address = 0;
+ *size = 0;
+ *need_unmap = FALSE;
return KERN_NOT_SUPPORTED;
}
object_offset = vm_object_trunc_page(offset);
- map_size = vm_map_round_page(*size);
+ map_size = vm_map_round_page(*size,
+ VM_MAP_PAGE_MASK(kernel_map));
/*
* Try and map the required range of the object
if (kr != KERN_SUCCESS) {
*address = 0;
*size = 0;
+ *need_unmap = FALSE;
vm_object_deallocate(object); /* for the map entry */
vm_object_lock(object);
return kr;
for (page_map_offset = 0;
map_size != 0;
map_size -= PAGE_SIZE_64, page_map_offset += PAGE_SIZE_64) {
- unsigned int cache_attr;
page = vm_page_lookup(object, offset + page_map_offset);
if (page == VM_PAGE_NULL) {
assert(kr == KERN_SUCCESS);
*address = 0;
*size = 0;
+ *need_unmap = FALSE;
vm_object_lock(object);
return KERN_MEMORY_ERROR;
}
- if (page->pmapped == FALSE) {
- pmap_sync_page_data_phys(page->phys_page);
- }
page->pmapped = TRUE;
- cache_attr = ((unsigned int) object->wimg_bits) & VM_WIMG_MASK;
//assert(pmap_verify_free(page->phys_page));
PMAP_ENTER(kernel_pmap,
*address + page_map_offset,
page,
protection,
- cache_attr,
+ VM_PROT_NONE,
+ 0,
TRUE);
}
vm_paging_objects_mapped_slow++;
vm_paging_pages_mapped_slow += (unsigned long) (map_size / PAGE_SIZE_64);
+ *need_unmap = TRUE;
+
return KERN_SUCCESS;
}
}
}
-#if CRYPTO
+#if ENCRYPTED_SWAP
/*
* Encryption data.
* "iv" is the "initial vector". Ideally, we want to
*/
#define SWAP_CRYPT_AES_KEY_SIZE 128 /* XXX 192 and 256 don't work ! */
boolean_t swap_crypt_ctx_initialized = FALSE;
-aes_32t swap_crypt_key[8]; /* big enough for a 256 key */
+uint32_t swap_crypt_key[8]; /* big enough for a 256 key */
aes_ctx swap_crypt_ctx;
const unsigned char swap_crypt_null_iv[AES_BLOCK_SIZE] = {0xa, };
{
kern_return_t kr;
vm_map_size_t kernel_mapping_size;
+ boolean_t kernel_mapping_needs_unmap;
vm_offset_t kernel_vaddr;
union {
unsigned char aes_iv[AES_BLOCK_SIZE];
}
assert(page->busy);
- assert(page->dirty || page->precious);
if (page->encrypted) {
/*
vm_page_encrypt_already_encrypted_counter++;
return;
}
+ assert(page->dirty || page->precious);
+
ASSERT_PAGE_DECRYPTED(page);
/*
* its contents and encrypt them.
*/
kernel_mapping_size = PAGE_SIZE;
- kr = vm_paging_map_object(&kernel_mapping_offset,
- page,
+ kernel_mapping_needs_unmap = FALSE;
+ kr = vm_paging_map_object(page,
page->object,
page->offset,
- &kernel_mapping_size,
VM_PROT_READ | VM_PROT_WRITE,
- FALSE);
+ FALSE,
+ &kernel_mapping_size,
+ &kernel_mapping_offset,
+ &kernel_mapping_needs_unmap);
if (kr != KERN_SUCCESS) {
panic("vm_page_encrypt: "
"could not map page in kernel: 0x%x\n",
}
} else {
kernel_mapping_size = 0;
+ kernel_mapping_needs_unmap = FALSE;
}
kernel_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping_offset);
* if we had to map it ourselves. Otherwise, let
* the caller undo the mapping if needed.
*/
- if (kernel_mapping_size != 0) {
+ if (kernel_mapping_needs_unmap) {
vm_paging_unmap_object(page->object,
kernel_mapping_offset,
kernel_mapping_offset + kernel_mapping_size);
kern_return_t kr;
vm_map_size_t kernel_mapping_size;
vm_offset_t kernel_vaddr;
+ boolean_t kernel_mapping_needs_unmap;
union {
unsigned char aes_iv[AES_BLOCK_SIZE];
struct {
vm_object_offset_t paging_offset;
} vm;
} decrypt_iv;
+ boolean_t was_dirty;
assert(page->busy);
assert(page->encrypted);
+ was_dirty = page->dirty;
+
/*
* Take a paging-in-progress reference to keep the object
* alive even if we have to unlock it (in vm_paging_map_object()
* its contents and decrypt them.
*/
kernel_mapping_size = PAGE_SIZE;
- kr = vm_paging_map_object(&kernel_mapping_offset,
- page,
+ kernel_mapping_needs_unmap = FALSE;
+ kr = vm_paging_map_object(page,
page->object,
page->offset,
- &kernel_mapping_size,
VM_PROT_READ | VM_PROT_WRITE,
- FALSE);
+ FALSE,
+ &kernel_mapping_size,
+ &kernel_mapping_offset,
+ &kernel_mapping_needs_unmap);
if (kr != KERN_SUCCESS) {
panic("vm_page_decrypt: "
"could not map page in kernel: 0x%x\n",
}
} else {
kernel_mapping_size = 0;
+ kernel_mapping_needs_unmap = FALSE;
}
kernel_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping_offset);
* if we had to map it ourselves. Otherwise, let
* the caller undo the mapping if needed.
*/
- if (kernel_mapping_size != 0) {
+ if (kernel_mapping_needs_unmap) {
vm_paging_unmap_object(page->object,
kernel_vaddr,
kernel_vaddr + PAGE_SIZE);
}
- /*
- * After decryption, the page is actually clean.
- * It was encrypted as part of paging, which "cleans"
- * the "dirty" pages.
- * Noone could access it after it was encrypted
- * and the decryption doesn't count.
- */
- page->dirty = FALSE;
- assert (page->cs_validated == FALSE);
- pmap_clear_refmod(page->phys_page, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
+ if (was_dirty) {
+ /*
+ * The pager did not specify that the page would be
+ * clean when it got paged in, so let's not clean it here
+ * either.
+ */
+ } else {
+ /*
+ * After decryption, the page is actually still clean.
+ * It was encrypted as part of paging, which "cleans"
+ * the "dirty" pages.
+ * Noone could access it after it was encrypted
+ * and the decryption doesn't count.
+ */
+ page->dirty = FALSE;
+ assert (page->cs_validated == FALSE);
+ pmap_clear_refmod(page->phys_page, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
+ }
page->encrypted = FALSE;
/*
base_offset + offset_in_upl);
if (page == VM_PAGE_NULL) {
panic("upl_encrypt: "
- "no page for (obj=%p,off=%lld+%d)!\n",
+ "no page for (obj=%p,off=0x%llx+0x%x)!\n",
shadow_object,
base_offset,
offset_in_upl);
goto process_upl_to_encrypt;
}
-#else /* CRYPTO */
+#else /* ENCRYPTED_SWAP */
void
upl_encrypt(
__unused upl_t upl,
{
}
-#endif /* CRYPTO */
+#endif /* ENCRYPTED_SWAP */
+/*
+ * page->object must be locked
+ */
void
-vm_pageout_queue_steal(vm_page_t page, boolean_t queues_locked)
+vm_pageout_steal_laundry(vm_page_t page, boolean_t queues_locked)
{
- boolean_t pageout;
-
- pageout = page->pageout;
-
- page->list_req_pending = FALSE;
- page->cleaning = FALSE;
- page->pageout = FALSE;
-
if (!queues_locked) {
vm_page_lockspin_queues();
}
*/
vm_pageout_throttle_up(page);
- if (pageout == TRUE) {
- /*
- * toss the wire count we picked up
- * when we intially set this page up
- * to be cleaned...
- */
- vm_page_unwire(page, TRUE);
- }
vm_page_steal_pageout_page++;
if (!queues_locked) {
vector_upl->size = 0;
vector_upl->offset = 0;
kfree(vector_upl, sizeof(struct _vector_upl));
- vector_upl = (vector_upl_t)0xdeadbeef;
+ vector_upl = (vector_upl_t)0xfeedfeed;
}
else
panic("vector_upl_deallocate was passed a non-vectored upl\n");
{
if(upl && ((upl->flags & UPL_VECTOR)==UPL_VECTOR)) {
vector_upl_t vector_upl = upl->vector_upl;
- if(vector_upl == NULL || vector_upl == (vector_upl_t)0xdeadbeef || vector_upl == (vector_upl_t)0xfeedbeef)
+ if(vector_upl == NULL || vector_upl == (vector_upl_t)0xfeedfeed || vector_upl == (vector_upl_t)0xfeedbeef)
return FALSE;
else
return TRUE;
}
}
+void
+upl_set_referenced(
+ upl_t upl,
+ boolean_t value)
+{
+ upl_lock(upl);
+ if (value) {
+ upl->ext_ref_count++;
+ } else {
+ if (!upl->ext_ref_count) {
+ panic("upl_set_referenced not %p\n", upl);
+ }
+ upl->ext_ref_count--;
+ }
+ upl_unlock(upl);
+}
+
+#if CONFIG_IOSCHED
+void
+upl_set_blkno(
+ upl_t upl,
+ vm_offset_t upl_offset,
+ int io_size,
+ int64_t blkno)
+{
+ int i,j;
+ if ((upl->flags & UPL_EXPEDITE_SUPPORTED) == 0)
+ return;
+
+ assert(upl->upl_reprio_info != 0);
+ for(i = (int)(upl_offset / PAGE_SIZE), j = 0; j < io_size; i++, j += PAGE_SIZE) {
+ UPL_SET_REPRIO_INFO(upl, i, blkno, io_size);
+ }
+}
+#endif
+
+boolean_t
+vm_page_is_slideable(vm_page_t m)
+{
+ boolean_t result = FALSE;
+ vm_shared_region_slide_info_t si;
+
+ vm_object_lock_assert_held(m->object);
+
+ /* make sure our page belongs to the one object allowed to do this */
+ if (!m->object->object_slid) {
+ goto done;
+ }
+
+ si = m->object->vo_slide_info;
+ if (si == NULL) {
+ goto done;
+ }
+
+ if(!m->slid && (si->start <= m->offset && si->end > m->offset)) {
+ result = TRUE;
+ }
+
+done:
+ return result;
+}
+
+int vm_page_slide_counter = 0;
+int vm_page_slide_errors = 0;
+kern_return_t
+vm_page_slide(
+ vm_page_t page,
+ vm_map_offset_t kernel_mapping_offset)
+{
+ kern_return_t kr;
+ vm_map_size_t kernel_mapping_size;
+ boolean_t kernel_mapping_needs_unmap;
+ vm_offset_t kernel_vaddr;
+ uint32_t pageIndex = 0;
+
+ assert(!page->slid);
+ assert(page->object->object_slid);
+ vm_object_lock_assert_exclusive(page->object);
+
+ if (page->error)
+ return KERN_FAILURE;
+
+ /*
+ * Take a paging-in-progress reference to keep the object
+ * alive even if we have to unlock it (in vm_paging_map_object()
+ * for example)...
+ */
+ vm_object_paging_begin(page->object);
+
+ if (kernel_mapping_offset == 0) {
+ /*
+ * The page hasn't already been mapped in kernel space
+ * by the caller. Map it now, so that we can access
+ * its contents and decrypt them.
+ */
+ kernel_mapping_size = PAGE_SIZE;
+ kernel_mapping_needs_unmap = FALSE;
+ kr = vm_paging_map_object(page,
+ page->object,
+ page->offset,
+ VM_PROT_READ | VM_PROT_WRITE,
+ FALSE,
+ &kernel_mapping_size,
+ &kernel_mapping_offset,
+ &kernel_mapping_needs_unmap);
+ if (kr != KERN_SUCCESS) {
+ panic("vm_page_slide: "
+ "could not map page in kernel: 0x%x\n",
+ kr);
+ }
+ } else {
+ kernel_mapping_size = 0;
+ kernel_mapping_needs_unmap = FALSE;
+ }
+ kernel_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping_offset);
+
+ /*
+ * Slide the pointers on the page.
+ */
+
+ /*assert that slide_file_info.start/end are page-aligned?*/
+
+ assert(!page->slid);
+ assert(page->object->object_slid);
+
+ /* on some platforms this is an extern int, on others it's a cpp macro */
+ __unreachable_ok_push
+ /* TODO: Consider this */
+ if (!TEST_PAGE_SIZE_4K) {
+ for (int i = 0; i < 4; i++) {
+ pageIndex = (uint32_t)((page->offset - page->object->vo_slide_info->start)/0x1000);
+ kr = vm_shared_region_slide_page(page->object->vo_slide_info, kernel_vaddr + (0x1000*i), pageIndex + i);
+ }
+ } else {
+ pageIndex = (uint32_t)((page->offset - page->object->vo_slide_info->start)/PAGE_SIZE);
+ kr = vm_shared_region_slide_page(page->object->vo_slide_info, kernel_vaddr, pageIndex);
+ }
+ __unreachable_ok_pop
+
+ vm_page_slide_counter++;
+
+ /*
+ * Unmap the page from the kernel's address space,
+ */
+ if (kernel_mapping_needs_unmap) {
+ vm_paging_unmap_object(page->object,
+ kernel_vaddr,
+ kernel_vaddr + PAGE_SIZE);
+ }
+
+ page->dirty = FALSE;
+ pmap_clear_refmod(page->phys_page, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
+
+ if (kr != KERN_SUCCESS || cs_debug > 1) {
+ printf("vm_page_slide(%p): "
+ "obj %p off 0x%llx mobj %p moff 0x%llx\n",
+ page,
+ page->object, page->offset,
+ page->object->pager,
+ page->offset + page->object->paging_offset);
+ }
+
+ if (kr == KERN_SUCCESS) {
+ page->slid = TRUE;
+ } else {
+ page->error = TRUE;
+ vm_page_slide_errors++;
+ }
+
+ vm_object_paging_end(page->object);
+
+ return kr;
+}
+
+void inline memoryshot(unsigned int event, unsigned int control)
+{
+ if (vm_debug_events) {
+ KERNEL_DEBUG_CONSTANT1((MACHDBG_CODE(DBG_MACH_VM_PRESSURE, event)) | control,
+ vm_page_active_count, vm_page_inactive_count,
+ vm_page_free_count, vm_page_speculative_count,
+ vm_page_throttled_count);
+ } else {
+ (void) event;
+ (void) control;
+ }
+
+}
#ifdef MACH_BSD
return(UPL_PHYS_PAGE(upl, index));
}
-
void
vm_countdirtypages(void)
{
vm_page_unlock_queues();
vm_page_lock_queues();
- m = (vm_page_t) queue_first(&vm_page_queue_zf);
+ m = (vm_page_t) queue_first(&vm_page_queue_anonymous);
do {
if (m ==(vm_page_t )0) break;
m = (vm_page_t) queue_next(&m->pageq);
if (m ==(vm_page_t )0) break;
- } while (!queue_end(&vm_page_queue_zf,(queue_entry_t) m));
+ } while (!queue_end(&vm_page_queue_anonymous,(queue_entry_t) m));
vm_page_unlock_queues();
printf("IN Q: %d : %d : %d\n", dpages, pgopages, precpages);
}
#endif /* UPL_DEBUG */
+#if VM_PRESSURE_EVENTS
+/*
+ * Upward trajectory.
+ */
+extern boolean_t vm_compressor_low_on_space(void);
+
+boolean_t
+VM_PRESSURE_NORMAL_TO_WARNING(void) {
+
+ if (DEFAULT_PAGER_IS_ACTIVE || DEFAULT_FREEZER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPLESS) {
+
+ /* Available pages below our threshold */
+ if (memorystatus_available_pages < memorystatus_available_pages_pressure) {
+ /* No frozen processes to kill */
+ if (memorystatus_frozen_count == 0) {
+ /* Not enough suspended processes available. */
+ if (memorystatus_suspended_count < MEMORYSTATUS_SUSPENDED_THRESHOLD) {
+ return TRUE;
+ }
+ }
+ }
+ return FALSE;
+ } else {
+ return ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0);
+ }
+}
-#if MACH_KDB
-#include <ddb/db_output.h>
-#include <ddb/db_print.h>
-#include <vm/vm_print.h>
+boolean_t
+VM_PRESSURE_WARNING_TO_CRITICAL(void) {
-#define printf kdbprintf
-void db_pageout(void);
+ if (DEFAULT_PAGER_IS_ACTIVE || DEFAULT_FREEZER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPLESS) {
+ /* Available pages below our threshold */
+ if (memorystatus_available_pages < memorystatus_available_pages_critical) {
+ return TRUE;
+ }
+ return FALSE;
+ } else {
+ return (vm_compressor_low_on_space() || (AVAILABLE_NON_COMPRESSED_MEMORY < ((12 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0);
+ }
+}
-void
-db_vm(void)
-{
+/*
+ * Downward trajectory.
+ */
+boolean_t
+VM_PRESSURE_WARNING_TO_NORMAL(void) {
- iprintf("VM Statistics:\n");
- db_indent += 2;
- iprintf("pages:\n");
- db_indent += 2;
- iprintf("activ %5d inact %5d free %5d",
- vm_page_active_count, vm_page_inactive_count,
- vm_page_free_count);
- printf(" wire %5d gobbl %5d\n",
- vm_page_wire_count, vm_page_gobble_count);
- db_indent -= 2;
- iprintf("target:\n");
- db_indent += 2;
- iprintf("min %5d inact %5d free %5d",
- vm_page_free_min, vm_page_inactive_target,
- vm_page_free_target);
- printf(" resrv %5d\n", vm_page_free_reserved);
- db_indent -= 2;
- iprintf("pause:\n");
- db_pageout();
- db_indent -= 2;
+ if (DEFAULT_PAGER_IS_ACTIVE || DEFAULT_FREEZER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPLESS) {
+ /* Available pages above our threshold */
+ unsigned int target_threshold = memorystatus_available_pages_pressure + ((15 * memorystatus_available_pages_pressure) / 100);
+ if (memorystatus_available_pages > target_threshold) {
+ return TRUE;
+ }
+ return FALSE;
+ } else {
+ return ((AVAILABLE_NON_COMPRESSED_MEMORY > ((12 * VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) / 10)) ? 1 : 0);
+ }
}
-#if MACH_COUNTERS
-extern int c_laundry_pages_freed;
-#endif /* MACH_COUNTERS */
+boolean_t
+VM_PRESSURE_CRITICAL_TO_WARNING(void) {
-void
-db_pageout(void)
-{
- iprintf("Pageout Statistics:\n");
- db_indent += 2;
- iprintf("active %5d inactv %5d\n",
- vm_pageout_active, vm_pageout_inactive);
- iprintf("nolock %5d avoid %5d busy %5d absent %5d\n",
- vm_pageout_inactive_nolock, vm_pageout_inactive_avoid,
- vm_pageout_inactive_busy, vm_pageout_inactive_absent);
- iprintf("used %5d clean %5d dirty %5d\n",
- vm_pageout_inactive_used, vm_pageout_inactive_clean,
- vm_pageout_inactive_dirty);
-#if MACH_COUNTERS
- iprintf("laundry_pages_freed %d\n", c_laundry_pages_freed);
-#endif /* MACH_COUNTERS */
-#if MACH_CLUSTER_STATS
- iprintf("Cluster Statistics:\n");
- db_indent += 2;
- iprintf("dirtied %5d cleaned %5d collisions %5d\n",
- vm_pageout_cluster_dirtied, vm_pageout_cluster_cleaned,
- vm_pageout_cluster_collisions);
- iprintf("clusters %5d conversions %5d\n",
- vm_pageout_cluster_clusters, vm_pageout_cluster_conversions);
- db_indent -= 2;
- iprintf("Target Statistics:\n");
- db_indent += 2;
- iprintf("collisions %5d page_dirtied %5d page_freed %5d\n",
- vm_pageout_target_collisions, vm_pageout_target_page_dirtied,
- vm_pageout_target_page_freed);
- db_indent -= 2;
-#endif /* MACH_CLUSTER_STATS */
- db_indent -= 2;
+ if (DEFAULT_PAGER_IS_ACTIVE || DEFAULT_FREEZER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPLESS) {
+ /* Available pages above our threshold */
+ unsigned int target_threshold = memorystatus_available_pages_critical + ((15 * memorystatus_available_pages_critical) / 100);
+ if (memorystatus_available_pages > target_threshold) {
+ return TRUE;
+ }
+ return FALSE;
+ } else {
+ return ((AVAILABLE_NON_COMPRESSED_MEMORY > ((14 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0);
+ }
}
+#endif /* VM_PRESSURE_EVENTS */
-#endif /* MACH_KDB */
projects / apple / xnu.git / blobdiff