#include <kern/thread.h>
#include <kern/xpr.h>
#include <kern/kalloc.h>
+#include <kern/policy_internal.h>
+#include <kern/thread_group.h>
#include <machine/vm_tuning.h>
#include <machine/commpage.h>
#include <vm/vm_shared_region.h>
#include <vm/vm_compressor.h>
+#include <san/kasan.h>
+
#if CONFIG_PHANTOM_CACHE
#include <vm/vm_phantom_cache.h>
#endif
-/*
- * ENCRYPTED SWAP:
- */
-#include <libkern/crypto/aes.h>
-extern u_int32_t random(void); /* from <libkern/libkern.h> */
extern int cs_debug;
extern void m_drain(void);
#if VM_PRESSURE_EVENTS
+#if CONFIG_JETSAM
extern unsigned int memorystatus_available_pages;
extern unsigned int memorystatus_available_pages_pressure;
extern unsigned int memorystatus_available_pages_critical;
+#else /* CONFIG_JETSAM */
+extern uint64_t memorystatus_available_pages;
+extern uint64_t memorystatus_available_pages_pressure;
+extern uint64_t memorystatus_available_pages_critical;
+#endif /* CONFIG_JETSAM */
+
extern unsigned int memorystatus_frozen_count;
extern unsigned int memorystatus_suspended_count;
#endif
#ifndef VM_PAGEOUT_BURST_INACTIVE_THROTTLE /* maximum iterations of the inactive queue w/o stealing/cleaning a page */
+#ifdef CONFIG_EMBEDDED
+#define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 1024
+#else
#define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 4096
#endif
+#endif
#ifndef VM_PAGEOUT_DEADLOCK_RELIEF
#define VM_PAGEOUT_DEADLOCK_RELIEF 100 /* number of pages to move to break deadlock */
#endif /* VM_PAGE_SPECULATIVE_TARGET */
-#ifndef VM_PAGE_INACTIVE_HEALTHY_LIMIT
-#define VM_PAGE_INACTIVE_HEALTHY_LIMIT(total) ((total) * 1 / 200)
-#endif /* VM_PAGE_INACTIVE_HEALTHY_LIMIT */
-
-
/*
* To obtain a reasonable LRU approximation, the inactive queue
* needs to be large enough to give pages on it a chance to be
*/
#ifndef VM_PAGE_INACTIVE_TARGET
+#ifdef CONFIG_EMBEDDED
+#define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 1 / 3)
+#else
#define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 1 / 2)
+#endif
#endif /* VM_PAGE_INACTIVE_TARGET */
/*
*/
#ifndef VM_PAGE_FREE_TARGET
+#ifdef CONFIG_EMBEDDED
+#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 100)
+#else
#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80)
+#endif
#endif /* VM_PAGE_FREE_TARGET */
*/
#ifndef VM_PAGE_FREE_MIN
+#ifdef CONFIG_EMBEDDED
+#define VM_PAGE_FREE_MIN(free) (10 + (free) / 200)
+#else
#define VM_PAGE_FREE_MIN(free) (10 + (free) / 100)
+#endif
#endif /* VM_PAGE_FREE_MIN */
+#ifdef CONFIG_EMBEDDED
+#define VM_PAGE_FREE_RESERVED_LIMIT 100
+#define VM_PAGE_FREE_MIN_LIMIT 1500
+#define VM_PAGE_FREE_TARGET_LIMIT 2000
+#else
#define VM_PAGE_FREE_RESERVED_LIMIT 1700
#define VM_PAGE_FREE_MIN_LIMIT 3500
#define VM_PAGE_FREE_TARGET_LIMIT 4000
+#endif
/*
* When vm_page_free_count falls below vm_page_free_reserved,
*/
#define VM_PAGE_REACTIVATE_LIMIT_MAX 20000
#ifndef VM_PAGE_REACTIVATE_LIMIT
+#ifdef CONFIG_EMBEDDED
+#define VM_PAGE_REACTIVATE_LIMIT(avail) (VM_PAGE_INACTIVE_TARGET(avail) / 2)
+#else
#define VM_PAGE_REACTIVATE_LIMIT(avail) (MAX((avail) * 1 / 20,VM_PAGE_REACTIVATE_LIMIT_MAX))
+#endif
#endif /* VM_PAGE_REACTIVATE_LIMIT */
-#define VM_PAGEOUT_INACTIVE_FORCE_RECLAIM 100
+#define VM_PAGEOUT_INACTIVE_FORCE_RECLAIM 1000
extern boolean_t hibernate_cleaning_in_progress;
struct vm_pageout_queue *q;
void *current_chead;
char *scratch_buf;
+ int id;
};
+struct cq ciq[MAX_COMPRESSOR_THREAD_COUNT];
+
#if VM_PRESSURE_EVENTS
void vm_pressure_thread(void);
boolean_t VM_PRESSURE_WARNING_TO_NORMAL(void);
boolean_t VM_PRESSURE_CRITICAL_TO_WARNING(void);
#endif
-static void vm_pageout_garbage_collect(int);
-static void vm_pageout_iothread_continue(struct vm_pageout_queue *);
+void vm_pageout_garbage_collect(int);
static void vm_pageout_iothread_external(void);
static void vm_pageout_iothread_internal(struct cq *cq);
-static void vm_pageout_adjust_io_throttles(struct vm_pageout_queue *, struct vm_pageout_queue *, boolean_t);
+static void vm_pageout_adjust_eq_iothrottle(struct vm_pageout_queue *, boolean_t);
extern void vm_pageout_continue(void);
extern void vm_pageout_scan(void);
+void vm_tests(void); /* forward */
+
+boolean_t vm_restricted_to_single_processor = FALSE;
+#if !CONFIG_EMBEDDED
+static boolean_t vm_pageout_waiter = FALSE;
+static boolean_t vm_pageout_running = FALSE;
+#endif /* !CONFIG_EMBEDDED */
+
static thread_t vm_pageout_external_iothread = THREAD_NULL;
static thread_t vm_pageout_internal_iothread = THREAD_NULL;
*/
unsigned int vm_pageout_active = 0; /* debugging */
-unsigned int vm_pageout_active_busy = 0; /* debugging */
unsigned int vm_pageout_inactive = 0; /* debugging */
unsigned int vm_pageout_inactive_throttled = 0; /* debugging */
unsigned int vm_pageout_inactive_forced = 0; /* debugging */
unsigned int vm_pageout_cache_evicted = 0; /* debugging */
unsigned int vm_pageout_inactive_clean = 0; /* debugging */
unsigned int vm_pageout_speculative_clean = 0; /* debugging */
+unsigned int vm_pageout_speculative_dirty = 0; /* debugging */
unsigned int vm_pageout_freed_from_cleaned = 0;
unsigned int vm_pageout_freed_from_speculative = 0;
unsigned int vm_pageout_freed_from_inactive_clean = 0;
+unsigned int vm_pageout_freed_after_compression = 0;
+
+extern uint32_t vm_compressor_pages_grabbed;
+extern uint32_t c_segment_pages_compressed;
-unsigned int vm_pageout_enqueued_cleaned_from_inactive_clean = 0;
unsigned int vm_pageout_enqueued_cleaned_from_inactive_dirty = 0;
unsigned int vm_pageout_cleaned_reclaimed = 0; /* debugging; how many cleaned pages are reclaimed by the pageout scan */
unsigned int vm_pageout_inactive_deactivated = 0; /* debugging */
unsigned int vm_pageout_inactive_anonymous = 0; /* debugging */
unsigned int vm_pageout_dirty_no_pager = 0; /* debugging */
-unsigned int vm_pageout_purged_objects = 0; /* debugging */
+unsigned int vm_pageout_purged_objects = 0; /* used for sysctl vm stats */
unsigned int vm_stat_discard = 0; /* debugging */
unsigned int vm_stat_discard_sent = 0; /* debugging */
unsigned int vm_stat_discard_failure = 0; /* debugging */
unsigned int vm_stat_discard_throttle = 0; /* debugging */
unsigned int vm_pageout_reactivation_limit_exceeded = 0; /* debugging */
-unsigned int vm_pageout_catch_ups = 0; /* debugging */
unsigned int vm_pageout_inactive_force_reclaim = 0; /* debugging */
+unsigned int vm_pageout_skipped_external = 0; /* debugging */
unsigned int vm_pageout_scan_reclaimed_throttled = 0;
unsigned int vm_pageout_scan_active_throttled = 0;
unsigned int vm_pageout_scan_active_throttle_success = 0; /* debugging */
unsigned int vm_pageout_scan_inactive_throttle_success = 0; /* debugging */
unsigned int vm_pageout_inactive_external_forced_jetsam_count = 0; /* debugging */
+unsigned int vm_pageout_scan_throttle_deferred = 0; /* debugging */
+unsigned int vm_pageout_scan_yield_unthrottled = 0; /* debugging */
unsigned int vm_page_speculative_count_drifts = 0;
unsigned int vm_page_speculative_count_drift_max = 0;
+uint32_t vm_compressor_failed;
/*
* Backing store throttle when BS is exhausted
unsigned int vm_page_steal_pageout_page = 0;
-/*
- * ENCRYPTED SWAP:
- * counters and statistics...
- */
-unsigned long vm_page_decrypt_counter = 0;
-unsigned long vm_page_decrypt_for_upl_counter = 0;
-unsigned long vm_page_encrypt_counter = 0;
-unsigned long vm_page_encrypt_abort_counter = 0;
-unsigned long vm_page_encrypt_already_encrypted_counter = 0;
-boolean_t vm_pages_encrypted = FALSE; /* are there encrypted pages ? */
+struct vm_config vm_config;
-struct vm_pageout_queue vm_pageout_queue_internal;
-struct vm_pageout_queue vm_pageout_queue_external;
+struct vm_pageout_queue vm_pageout_queue_internal __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
+struct vm_pageout_queue vm_pageout_queue_external __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
unsigned int vm_page_speculative_target = 0;
#endif
extern boolean_t memorystatus_kill_on_VM_page_shortage(boolean_t async);
extern void memorystatus_on_pageout_scan_end(void);
-#endif
-boolean_t vm_page_compressions_failing = FALSE;
-
-/*
- * Routine: vm_backing_store_disable
- * Purpose:
- * Suspend non-privileged threads wishing to extend
- * backing store when we are low on backing store
- * (Synchronized by caller)
- */
-void
-vm_backing_store_disable(
- boolean_t disable)
-{
- if(disable) {
- vm_backing_store_low = 1;
- } else {
- if(vm_backing_store_low) {
- vm_backing_store_low = 0;
- thread_wakeup((event_t) &vm_backing_store_low);
- }
- }
-}
+uint32_t vm_pageout_memorystatus_fb_factor_nr = 5;
+uint32_t vm_pageout_memorystatus_fb_factor_dr = 2;
+#if DEVELOPMENT || DEBUG
+uint32_t vm_grab_anon_overrides = 0;
+uint32_t vm_grab_anon_nops = 0;
+#endif
+#endif
#if MACH_CLUSTER_STATS
unsigned long vm_pageout_cluster_dirtied = 0;
#define CLUSTER_STAT(clause)
#endif /* MACH_CLUSTER_STATS */
+
+#if DEVELOPMENT || DEBUG
+vmct_stats_t vmct_stats;
+#endif
+
/*
* Routine: vm_pageout_object_terminate
* Purpose:
shadow_object = object->shadow;
vm_object_lock(shadow_object);
- while (!queue_empty(&object->memq)) {
+ while (!vm_page_queue_empty(&object->memq)) {
vm_page_t p, m;
vm_object_offset_t offset;
- p = (vm_page_t) queue_first(&object->memq);
+ p = (vm_page_t) vm_page_queue_first(&object->memq);
assert(p->private);
- assert(p->pageout);
- p->pageout = FALSE;
+ assert(p->free_when_done);
+ p->free_when_done = FALSE;
assert(!p->cleaning);
assert(!p->laundry);
* Also decrement the burst throttle (if external).
*/
vm_page_lock_queues();
- if (m->pageout_queue)
+ if (m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q)
vm_pageout_throttle_up(m);
/*
* pages may have been modified between the selection as an
* adjacent page and conversion to a target.
*/
- if (m->pageout) {
+ if (m->free_when_done) {
assert(m->busy);
+ assert(m->vm_page_q_state == VM_PAGE_IS_WIRED);
assert(m->wire_count == 1);
m->cleaning = FALSE;
- m->encrypted_cleaning = FALSE;
- m->pageout = FALSE;
+ m->free_when_done = FALSE;
#if MACH_CLUSTER_STATS
if (m->wanted) vm_pageout_target_collisions++;
#endif
* can detect whether the page was redirtied during
* pageout by checking the modify state.
*/
- if (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED) {
+ if (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED) {
SET_PAGE_DIRTY(m, FALSE);
} else {
m->dirty = FALSE;
* If prep_pin_count is nonzero, then someone is using the
* page, so make it active.
*/
- if (!m->active && !m->inactive && !m->throttled && !m->private) {
+ if ((m->vm_page_q_state == VM_PAGE_NOT_ON_Q) && !m->private) {
if (m->reference)
vm_page_activate(m);
else
* will take care of resetting dirty. We clear the
* modify however for the Programmed I/O case.
*/
- pmap_clear_modify(m->phys_page);
+ pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m));
m->busy = FALSE;
m->absent = FALSE;
* consulted if m->dirty is false.
*/
#if MACH_CLUSTER_STATS
- m->dirty = pmap_is_modified(m->phys_page);
+ m->dirty = pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m));
if (m->dirty) vm_pageout_cluster_dirtied++;
else vm_pageout_cluster_cleaned++;
m->dirty = FALSE;
#endif
}
- if (m->encrypted_cleaning == TRUE) {
- m->encrypted_cleaning = FALSE;
- m->busy = FALSE;
- }
m->cleaning = FALSE;
/*
* must be locked.
*
*/
-void
+static void
vm_pageclean_setup(
vm_page_t m,
vm_page_t new_m,
#endif
XPR(XPR_VM_PAGEOUT,
- "vm_pageclean_setup, obj 0x%X off 0x%X page 0x%X new 0x%X new_off 0x%X\n",
- m->object, m->offset, m,
+ "vm_pageclean_setup, obj 0x%X off 0x%X page 0x%X new 0x%X new_off 0x%X\n",
+ VM_PAGE_OBJECT(m), m->offset, m,
new_m, new_offset);
- pmap_clear_modify(m->phys_page);
+ pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m));
/*
* Mark original page as cleaning in place.
* the real page.
*/
assert(new_m->fictitious);
- assert(new_m->phys_page == vm_page_fictitious_addr);
+ assert(VM_PAGE_GET_PHYS_PAGE(new_m) == vm_page_fictitious_addr);
new_m->fictitious = FALSE;
new_m->private = TRUE;
- new_m->pageout = TRUE;
- new_m->phys_page = m->phys_page;
+ new_m->free_when_done = TRUE;
+ VM_PAGE_SET_PHYS_PAGE(new_m, VM_PAGE_GET_PHYS_PAGE(m));
vm_page_lockspin_queues();
- vm_page_wire(new_m);
+ vm_page_wire(new_m, VM_KERN_MEMORY_NONE, TRUE);
vm_page_unlock_queues();
- vm_page_insert(new_m, new_object, new_offset);
+ vm_page_insert_wired(new_m, new_object, new_offset, VM_KERN_MEMORY_NONE);
assert(!new_m->wanted);
new_m->busy = FALSE;
}
XPR(XPR_VM_PAGEOUT,
"vm_pageout_initialize_page, page 0x%X\n",
m, 0, 0, 0, 0);
+
+ assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
+
+ object = VM_PAGE_OBJECT(m);
+
assert(m->busy);
+ assert(object->internal);
/*
* Verify that we really want to clean this page
/*
* Create a paging reference to let us play with the object.
*/
- object = m->object;
paging_offset = m->offset + object->paging_offset;
if (m->absent || m->error || m->restart || (!m->dirty && !m->precious)) {
- VM_PAGE_FREE(m);
panic("reservation without pageout?"); /* alan */
+
+ VM_PAGE_FREE(m);
vm_object_unlock(object);
return;
pager = object->pager;
if (pager == MEMORY_OBJECT_NULL) {
- VM_PAGE_FREE(m);
panic("missing pager for copy object");
+
+ VM_PAGE_FREE(m);
return;
}
/*
* set the page for future call to vm_fault_list_request
*/
- pmap_clear_modify(m->phys_page);
+ pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m));
SET_PAGE_DIRTY(m, FALSE);
- m->pageout = TRUE;
/*
* keep the object from collapsing or terminating
*
* The page must not be on any pageout queue.
*/
+int32_t vmct_active = 0;
+typedef enum vmct_state_t {
+ VMCT_IDLE,
+ VMCT_AWAKENED,
+ VMCT_ACTIVE,
+} vmct_state_t;
+vmct_state_t vmct_state[MAX_COMPRESSOR_THREAD_COUNT];
void
-vm_pageout_cluster(vm_page_t m, boolean_t pageout)
+vm_pageout_cluster(vm_page_t m)
{
- vm_object_t object = m->object;
+ vm_object_t object = VM_PAGE_OBJECT(m);
struct vm_pageout_queue *q;
object, m->offset, m, 0, 0);
VM_PAGE_CHECK(m);
-#if DEBUG
- lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
-#endif
+ LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
vm_object_lock_assert_exclusive(object);
/*
* Only a certain kind of page is appreciated here.
*/
assert((m->dirty || m->precious) && (!VM_PAGE_WIRED(m)));
- assert(!m->cleaning && !m->pageout && !m->laundry);
-#ifndef CONFIG_FREEZE
- assert(!m->inactive && !m->active);
- assert(!m->throttled);
-#endif
+ assert(!m->cleaning && !m->laundry);
+ assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q);
/*
* protect the object from collapse or termination
*/
vm_object_activity_begin(object);
- m->pageout = pageout;
-
if (object->internal == TRUE) {
- if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE)
- m->busy = TRUE;
+ assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
+
+ m->busy = TRUE;
q = &vm_pageout_queue_internal;
} else
m->laundry = TRUE;
q->pgo_laundry++;
- m->pageout_queue = TRUE;
- queue_enter(&q->pgo_pending, m, vm_page_t, pageq);
-
+ m->vm_page_q_state = VM_PAGE_ON_PAGEOUT_Q;
+ vm_page_queue_enter(&q->pgo_pending, m, vm_page_t, pageq);
+
if (q->pgo_idle == TRUE) {
q->pgo_idle = FALSE;
thread_wakeup((event_t) &q->pgo_pending);
vm_page_t m)
{
struct vm_pageout_queue *q;
+ vm_object_t m_object;
- assert(m->object != VM_OBJECT_NULL);
- assert(m->object != kernel_object);
+ m_object = VM_PAGE_OBJECT(m);
-#if DEBUG
- lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
- vm_object_lock_assert_exclusive(m->object);
-#endif
+ assert(m_object != VM_OBJECT_NULL);
+ assert(m_object != kernel_object);
+
+ LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+ vm_object_lock_assert_exclusive(m_object);
vm_pageout_throttle_up_count++;
- if (m->object->internal == TRUE)
+ if (m_object->internal == TRUE)
q = &vm_pageout_queue_internal;
else
q = &vm_pageout_queue_external;
- if (m->pageout_queue == TRUE) {
+ if (m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) {
- queue_remove(&q->pgo_pending, m, vm_page_t, pageq);
- m->pageout_queue = FALSE;
+ vm_page_queue_remove(&q->pgo_pending, m, vm_page_t, pageq);
+ m->vm_page_q_state = VM_PAGE_NOT_ON_Q;
- m->pageq.next = NULL;
- m->pageq.prev = NULL;
+ VM_PAGE_ZERO_PAGEQ_ENTRY(m);
- vm_object_activity_end(m->object);
+ vm_object_activity_end(m_object);
}
if (m->laundry == TRUE) {
struct vm_pageout_queue *q,
int batch_cnt)
{
-#if DEBUG
- lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
-#endif
+ LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
vm_pageout_throttle_up_count += batch_cnt;
#define VM_PAGEOUT_STAT_SIZE 31
struct vm_pageout_stat {
unsigned int considered;
- unsigned int reclaimed;
-} vm_pageout_stats[VM_PAGEOUT_STAT_SIZE] = {{0,0}, };
+ unsigned int reclaimed_clean;
+ unsigned int pages_compressed;
+ unsigned int pages_grabbed_by_compressor;
+ unsigned int cleaned_dirty_external;
+ unsigned int throttled_internal_q;
+ unsigned int throttled_external_q;
+ unsigned int failed_compressions;
+} vm_pageout_stats[VM_PAGEOUT_STAT_SIZE] = {{0,0,0,0,0,0,0,0}, };
+
unsigned int vm_pageout_stat_now = 0;
unsigned int vm_memory_pressure = 0;
#endif /* VM_PAGE_BUCKETS_CHECK */
vm_memory_pressure =
- vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].reclaimed;
+ vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].reclaimed_clean;
commpage_set_memory_pressure( vm_memory_pressure );
/* move "now" forward */
vm_pageout_next = VM_PAGEOUT_STAT_AFTER(vm_pageout_stat_now);
vm_pageout_stats[vm_pageout_next].considered = 0;
- vm_pageout_stats[vm_pageout_next].reclaimed = 0;
+ vm_pageout_stats[vm_pageout_next].reclaimed_clean = 0;
+ vm_pageout_stats[vm_pageout_next].throttled_internal_q = 0;
+ vm_pageout_stats[vm_pageout_next].throttled_external_q = 0;
+ vm_pageout_stats[vm_pageout_next].cleaned_dirty_external = 0;
+ vm_pageout_stats[vm_pageout_next].pages_compressed = 0;
+ vm_pageout_stats[vm_pageout_next].pages_grabbed_by_compressor = 0;
+ vm_pageout_stats[vm_pageout_next].failed_compressions = 0;
+
vm_pageout_stat_now = vm_pageout_next;
}
}
/* provide number of pages reclaimed in the last "nsecs_monitored" */
- do {
- vm_pageout_now = vm_pageout_stat_now;
- pages_reclaimed = 0;
- for (vm_pageout_then =
- VM_PAGEOUT_STAT_BEFORE(vm_pageout_now);
- vm_pageout_then != vm_pageout_now &&
- nsecs_monitored-- != 0;
- vm_pageout_then =
- VM_PAGEOUT_STAT_BEFORE(vm_pageout_then)) {
- pages_reclaimed += vm_pageout_stats[vm_pageout_then].reclaimed;
- }
- } while (vm_pageout_now != vm_pageout_stat_now);
+ vm_pageout_now = vm_pageout_stat_now;
+ pages_reclaimed = 0;
+ for (vm_pageout_then =
+ VM_PAGEOUT_STAT_BEFORE(vm_pageout_now);
+ vm_pageout_then != vm_pageout_now &&
+ nsecs_monitored-- != 0;
+ vm_pageout_then =
+ VM_PAGEOUT_STAT_BEFORE(vm_pageout_then)) {
+ pages_reclaimed += vm_pageout_stats[vm_pageout_then].reclaimed_clean;
+ }
*pages_reclaimed_p = pages_reclaimed;
return KERN_SUCCESS;
-/*
- * function in BSD to apply I/O throttle to the pageout thread
- */
-extern void vm_pageout_io_throttle(void);
+#if DEVELOPMENT || DEBUG
+
+static void
+vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *, int);
/*
- * Page States: Used below to maintain the page state
- * before it's removed from it's Q. This saved state
- * helps us do the right accounting in certain cases
+ * condition variable used to make sure there is
+ * only a single sweep going on at a time
*/
-#define PAGE_STATE_SPECULATIVE 1
-#define PAGE_STATE_ANONYMOUS 2
-#define PAGE_STATE_INACTIVE 3
-#define PAGE_STATE_INACTIVE_FIRST 4
-#define PAGE_STATE_CLEAN 5
-
-
-#define VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m) \
- MACRO_BEGIN \
- /* \
- * If a "reusable" page somehow made it back into \
- * the active queue, it's been re-used and is not \
- * quite re-usable. \
- * If the VM object was "all_reusable", consider it \
- * as "all re-used" instead of converting it to \
- * "partially re-used", which could be expensive. \
- */ \
- if ((m)->reusable || \
- (m)->object->all_reusable) { \
- vm_object_reuse_pages((m)->object, \
- (m)->offset, \
- (m)->offset + PAGE_SIZE_64, \
- FALSE); \
- } \
- MACRO_END
+boolean_t vm_pageout_disconnect_all_pages_active = FALSE;
-#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT 64
-#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX 1024
+void
+vm_pageout_disconnect_all_pages()
+{
+ vm_page_lock_queues();
-#define FCS_IDLE 0
-#define FCS_DELAYED 1
-#define FCS_DEADLOCK_DETECTED 2
+ if (vm_pageout_disconnect_all_pages_active == TRUE) {
+ vm_page_unlock_queues();
+ return;
+ }
+ vm_pageout_disconnect_all_pages_active = TRUE;
+ vm_page_unlock_queues();
-struct flow_control {
- int state;
- mach_timespec_t ts;
-};
+ vm_pageout_disconnect_all_pages_in_queue(&vm_page_queue_throttled, vm_page_throttled_count);
+ vm_pageout_disconnect_all_pages_in_queue(&vm_page_queue_anonymous, vm_page_anonymous_count);
+ vm_pageout_disconnect_all_pages_in_queue(&vm_page_queue_active, vm_page_active_count);
-uint32_t vm_pageout_considered_page = 0;
-uint32_t vm_page_filecache_min = 0;
+ vm_pageout_disconnect_all_pages_active = FALSE;
+}
-#define ANONS_GRABBED_LIMIT 2
-/*
- * vm_pageout_scan does the dirty work for the pageout daemon.
- * It returns with both vm_page_queue_free_lock and vm_page_queue_lock
- * held and vm_page_free_wanted == 0.
- */
void
-vm_pageout_scan(void)
+vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *q, int qcount)
{
- unsigned int loop_count = 0;
- unsigned int inactive_burst_count = 0;
- unsigned int active_burst_count = 0;
- unsigned int reactivated_this_call;
- unsigned int reactivate_limit;
- vm_page_t local_freeq = NULL;
- int local_freed = 0;
- int delayed_unlock;
- int delayed_unlock_limit = 0;
- int refmod_state = 0;
- int vm_pageout_deadlock_target = 0;
- struct vm_pageout_queue *iq;
- struct vm_pageout_queue *eq;
- struct vm_speculative_age_q *sq;
- struct flow_control flow_control = { 0, { 0, 0 } };
- boolean_t inactive_throttled = FALSE;
- boolean_t try_failed;
- mach_timespec_t ts;
- unsigned int msecs = 0;
- vm_object_t object;
- vm_object_t last_object_tried;
- uint32_t catch_up_count = 0;
- uint32_t inactive_reclaim_run;
- boolean_t forced_reclaim;
- boolean_t exceeded_burst_throttle;
- boolean_t grab_anonymous = FALSE;
- boolean_t force_anonymous = FALSE;
- int anons_grabbed = 0;
- int page_prev_state = 0;
- int cache_evict_throttle = 0;
- uint32_t vm_pageout_inactive_external_forced_reactivate_limit = 0;
- int force_purge = 0;
+ vm_page_t m;
+ vm_object_t t_object = NULL;
+ vm_object_t l_object = NULL;
+ vm_object_t m_object = NULL;
+ int delayed_unlock = 0;
+ int try_failed_count = 0;
+ int disconnected_count = 0;
+ int paused_count = 0;
+ int object_locked_count = 0;
+
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_DISCONNECT_ALL_PAGE_MAPPINGS)) | DBG_FUNC_START,
+ q, qcount, 0, 0, 0);
-#if VM_PRESSURE_EVENTS
- vm_pressure_level_t pressure_level;
-#endif /* VM_PRESSURE_EVENTS */
+ vm_page_lock_queues();
- VM_DEBUG_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_START,
- vm_pageout_speculative_clean, vm_pageout_inactive_clean,
- vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external);
+ while (qcount && !vm_page_queue_empty(q)) {
- flow_control.state = FCS_IDLE;
- iq = &vm_pageout_queue_internal;
- eq = &vm_pageout_queue_external;
- sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
+ LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+ m = (vm_page_t) vm_page_queue_first(q);
+ m_object = VM_PAGE_OBJECT(m);
- XPR(XPR_VM_PAGEOUT, "vm_pageout_scan\n", 0, 0, 0, 0, 0);
+ /*
+ * check to see if we currently are working
+ * with the same object... if so, we've
+ * already got the lock
+ */
+ if (m_object != l_object) {
+ /*
+ * the object associated with candidate page is
+ * different from the one we were just working
+ * with... dump the lock if we still own it
+ */
+ if (l_object != NULL) {
+ vm_object_unlock(l_object);
+ l_object = NULL;
+ }
+ if (m_object != t_object)
+ try_failed_count = 0;
-
- vm_page_lock_queues();
- delayed_unlock = 1; /* must be nonzero if Qs are locked, 0 if unlocked */
+ /*
+ * Try to lock object; since we've alread got the
+ * page queues lock, we can only 'try' for this one.
+ * if the 'try' fails, we need to do a mutex_pause
+ * to allow the owner of the object lock a chance to
+ * run...
+ */
+ if ( !vm_object_lock_try_scan(m_object)) {
- /*
- * Calculate the max number of referenced pages on the inactive
- * queue that we will reactivate.
- */
- reactivated_this_call = 0;
- reactivate_limit = VM_PAGE_REACTIVATE_LIMIT(vm_page_active_count +
- vm_page_inactive_count);
- inactive_reclaim_run = 0;
+ if (try_failed_count > 20) {
+ goto reenter_pg_on_q;
+ }
+ vm_page_unlock_queues();
+ mutex_pause(try_failed_count++);
+ vm_page_lock_queues();
+ delayed_unlock = 0;
- vm_pageout_inactive_external_forced_reactivate_limit = vm_page_active_count + vm_page_inactive_count;
+ paused_count++;
- /*
- * We want to gradually dribble pages from the active queue
- * to the inactive queue. If we let the inactive queue get
- * very small, and then suddenly dump many pages into it,
- * those pages won't get a sufficient chance to be referenced
- * before we start taking them from the inactive queue.
- *
- * We must limit the rate at which we send pages to the pagers
- * so that we don't tie up too many pages in the I/O queues.
- * We implement a throttling mechanism using the laundry count
- * to limit the number of pages outstanding to the default
- * and external pagers. We can bypass the throttles and look
- * for clean pages if the pageout queues don't drain in a timely
- * fashion since this may indicate that the pageout paths are
- * stalled waiting for memory, which only we can provide.
- */
+ t_object = m_object;
+ continue;
+ }
+ object_locked_count++;
+ l_object = m_object;
+ }
+ if ( !m_object->alive || m->cleaning || m->laundry || m->busy || m->absent || m->error || m->free_when_done) {
+ /*
+ * put it back on the head of its queue
+ */
+ goto reenter_pg_on_q;
+ }
+ if (m->pmapped == TRUE) {
-Restart:
- assert(delayed_unlock!=0);
-
- /*
- * Recalculate vm_page_inactivate_target.
- */
- vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
- vm_page_inactive_count +
- vm_page_speculative_count);
+ pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
- vm_page_anonymous_min = vm_page_inactive_target / 20;
+ disconnected_count++;
+ }
+reenter_pg_on_q:
+ vm_page_queue_remove(q, m, vm_page_t, pageq);
+ vm_page_queue_enter(q, m, vm_page_t, pageq);
+ qcount--;
+ try_failed_count = 0;
- /*
- * don't want to wake the pageout_scan thread up everytime we fall below
- * the targets... set a low water mark at 0.25% below the target
- */
- vm_page_inactive_min = vm_page_inactive_target - (vm_page_inactive_target / 400);
+ if (delayed_unlock++ > 128) {
- if (vm_page_speculative_percentage > 50)
- vm_page_speculative_percentage = 50;
- else if (vm_page_speculative_percentage <= 0)
- vm_page_speculative_percentage = 1;
+ if (l_object != NULL) {
+ vm_object_unlock(l_object);
+ l_object = NULL;
+ }
+ lck_mtx_yield(&vm_page_queue_lock);
+ delayed_unlock = 0;
+ }
+ }
+ if (l_object != NULL) {
+ vm_object_unlock(l_object);
+ l_object = NULL;
+ }
+ vm_page_unlock_queues();
- vm_page_speculative_target = VM_PAGE_SPECULATIVE_TARGET(vm_page_active_count +
- vm_page_inactive_count);
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_DISCONNECT_ALL_PAGE_MAPPINGS)) | DBG_FUNC_END,
+ q, disconnected_count, object_locked_count, paused_count, 0);
+}
- object = NULL;
- last_object_tried = NULL;
- try_failed = FALSE;
-
- if ((vm_page_inactive_count + vm_page_speculative_count) < VM_PAGE_INACTIVE_HEALTHY_LIMIT(vm_page_active_count))
- catch_up_count = vm_page_inactive_count + vm_page_speculative_count;
- else
- catch_up_count = 0;
-
- for (;;) {
- vm_page_t m;
+#endif
- DTRACE_VM2(rev, int, 1, (uint64_t *), NULL);
- if (delayed_unlock == 0) {
- vm_page_lock_queues();
- delayed_unlock = 1;
- }
- if (vm_upl_wait_for_pages < 0)
- vm_upl_wait_for_pages = 0;
+static void
+vm_pageout_page_queue(vm_page_queue_head_t *, int);
- delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT + vm_upl_wait_for_pages;
+/*
+ * condition variable used to make sure there is
+ * only a single sweep going on at a time
+ */
+boolean_t vm_pageout_anonymous_pages_active = FALSE;
- if (delayed_unlock_limit > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX)
- delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX;
- /*
- * Move pages from active to inactive if we're below the target
- */
- /* if we are trying to make clean, we need to make sure we actually have inactive - mj */
- if ((vm_page_inactive_count + vm_page_speculative_count) >= vm_page_inactive_target)
- goto done_moving_active_pages;
-
- if (object != NULL) {
- vm_object_unlock(object);
- object = NULL;
- vm_pageout_scan_wants_object = VM_OBJECT_NULL;
- }
- /*
- * Don't sweep through active queue more than the throttle
- * which should be kept relatively low
- */
- active_burst_count = MIN(vm_pageout_burst_active_throttle, vm_page_active_count);
-
- VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_START,
- vm_pageout_inactive, vm_pageout_inactive_used, vm_page_free_count, local_freed);
-
- VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_NONE,
- vm_pageout_speculative_clean, vm_pageout_inactive_clean,
- vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external);
- memoryshot(VM_PAGEOUT_BALANCE, DBG_FUNC_START);
+void
+vm_pageout_anonymous_pages()
+{
+ if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {
+ vm_page_lock_queues();
- while (!queue_empty(&vm_page_queue_active) && active_burst_count--) {
+ if (vm_pageout_anonymous_pages_active == TRUE) {
+ vm_page_unlock_queues();
+ return;
+ }
+ vm_pageout_anonymous_pages_active = TRUE;
+ vm_page_unlock_queues();
- vm_pageout_active++;
+ vm_pageout_page_queue(&vm_page_queue_throttled, vm_page_throttled_count);
+ vm_pageout_page_queue(&vm_page_queue_anonymous, vm_page_anonymous_count);
+ vm_pageout_page_queue(&vm_page_queue_active, vm_page_active_count);
- m = (vm_page_t) queue_first(&vm_page_queue_active);
+ if (VM_CONFIG_SWAP_IS_PRESENT)
+ vm_consider_swapping();
- assert(m->active && !m->inactive);
- assert(!m->laundry);
- assert(m->object != kernel_object);
- assert(m->phys_page != vm_page_guard_addr);
+ vm_page_lock_queues();
+ vm_pageout_anonymous_pages_active = FALSE;
+ vm_page_unlock_queues();
+ }
+}
- DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
- /*
- * by not passing in a pmap_flush_context we will forgo any TLB flushing, local or otherwise...
- *
- * a TLB flush isn't really needed here since at worst we'll miss the reference bit being
- * updated in the PTE if a remote processor still has this mapping cached in its TLB when the
- * new reference happens. If no futher references happen on the page after that remote TLB flushes
- * we'll see a clean, non-referenced page when it eventually gets pulled out of the inactive queue
- * by pageout_scan, which is just fine since the last reference would have happened quite far
- * in the past (TLB caches don't hang around for very long), and of course could just as easily
- * have happened before we moved the page
- */
- pmap_clear_refmod_options(m->phys_page, VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL);
+void
+vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount)
+{
+ vm_page_t m;
+ vm_object_t t_object = NULL;
+ vm_object_t l_object = NULL;
+ vm_object_t m_object = NULL;
+ int delayed_unlock = 0;
+ int try_failed_count = 0;
+ int refmod_state;
+ int pmap_options;
+ struct vm_pageout_queue *iq;
+ ppnum_t phys_page;
- /*
- * The page might be absent or busy,
- * but vm_page_deactivate can handle that.
- * FALSE indicates that we don't want a H/W clear reference
- */
- vm_page_deactivate_internal(m, FALSE);
- if (delayed_unlock++ > delayed_unlock_limit) {
+ iq = &vm_pageout_queue_internal;
+
+ vm_page_lock_queues();
- if (local_freeq) {
- vm_page_unlock_queues();
-
- VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START,
- vm_page_free_count, local_freed, delayed_unlock_limit, 1);
+ while (qcount && !vm_page_queue_empty(q)) {
- vm_page_free_list(local_freeq, TRUE);
-
- VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END,
- vm_page_free_count, 0, 0, 1);
+ LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
- local_freeq = NULL;
- local_freed = 0;
- vm_page_lock_queues();
- } else {
- lck_mtx_yield(&vm_page_queue_lock);
- }
-
- delayed_unlock = 1;
+ if (VM_PAGE_Q_THROTTLED(iq)) {
- /*
- * continue the while loop processing
- * the active queue... need to hold
- * the page queues lock
- */
+ if (l_object != NULL) {
+ vm_object_unlock(l_object);
+ l_object = NULL;
}
+ iq->pgo_draining = TRUE;
+
+ assert_wait((event_t) (&iq->pgo_laundry + 1), THREAD_INTERRUPTIBLE);
+ vm_page_unlock_queues();
+
+ thread_block(THREAD_CONTINUE_NULL);
+
+ vm_page_lock_queues();
+ delayed_unlock = 0;
+ continue;
}
+ m = (vm_page_t) vm_page_queue_first(q);
+ m_object = VM_PAGE_OBJECT(m);
- VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_END,
- vm_page_active_count, vm_page_inactive_count, vm_page_speculative_count, vm_page_inactive_target);
- memoryshot(VM_PAGEOUT_BALANCE, DBG_FUNC_END);
-
- /**********************************************************************
- * above this point we're playing with the active queue
- * below this point we're playing with the throttling mechanisms
- * and the inactive queue
- **********************************************************************/
-
-done_moving_active_pages:
+ /*
+ * check to see if we currently are working
+ * with the same object... if so, we've
+ * already got the lock
+ */
+ if (m_object != l_object) {
+ if ( !m_object->internal)
+ goto reenter_pg_on_q;
- if (vm_page_free_count + local_freed >= vm_page_free_target) {
- if (object != NULL) {
- vm_object_unlock(object);
- object = NULL;
+ /*
+ * the object associated with candidate page is
+ * different from the one we were just working
+ * with... dump the lock if we still own it
+ */
+ if (l_object != NULL) {
+ vm_object_unlock(l_object);
+ l_object = NULL;
}
- vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+ if (m_object != t_object)
+ try_failed_count = 0;
- if (local_freeq) {
- vm_page_unlock_queues();
-
- VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START,
- vm_page_free_count, local_freed, delayed_unlock_limit, 2);
-
- vm_page_free_list(local_freeq, TRUE);
-
- VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END,
- vm_page_free_count, local_freed, 0, 2);
+ /*
+ * Try to lock object; since we've alread got the
+ * page queues lock, we can only 'try' for this one.
+ * if the 'try' fails, we need to do a mutex_pause
+ * to allow the owner of the object lock a chance to
+ * run...
+ */
+ if ( !vm_object_lock_try_scan(m_object)) {
- local_freeq = NULL;
- local_freed = 0;
+ if (try_failed_count > 20) {
+ goto reenter_pg_on_q;
+ }
+ vm_page_unlock_queues();
+ mutex_pause(try_failed_count++);
vm_page_lock_queues();
+ delayed_unlock = 0;
+
+ t_object = m_object;
+ continue;
}
+ l_object = m_object;
+ }
+ if ( !m_object->alive || m->cleaning || m->laundry || m->busy || m->absent || m->error || m->free_when_done) {
/*
- * make sure the pageout I/O threads are running
- * throttled in case there are still requests
- * in the laundry... since we have met our targets
- * we don't need the laundry to be cleaned in a timely
- * fashion... so let's avoid interfering with foreground
- * activity
+ * page is not to be cleaned
+ * put it back on the head of its queue
*/
- vm_pageout_adjust_io_throttles(iq, eq, TRUE);
+ goto reenter_pg_on_q;
+ }
+ phys_page = VM_PAGE_GET_PHYS_PAGE(m);
- /*
- * recalculate vm_page_inactivate_target
- */
- vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
- vm_page_inactive_count +
- vm_page_speculative_count);
- if (((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) &&
- !queue_empty(&vm_page_queue_active)) {
- /*
- * inactive target still not met... keep going
- * until we get the queues balanced...
- */
- continue;
+ if (m->reference == FALSE && m->pmapped == TRUE) {
+ refmod_state = pmap_get_refmod(phys_page);
+
+ if (refmod_state & VM_MEM_REFERENCED)
+ m->reference = TRUE;
+ if (refmod_state & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(m, FALSE);
}
- lck_mtx_lock(&vm_page_queue_free_lock);
+ }
+ if (m->reference == TRUE) {
+ m->reference = FALSE;
+ pmap_clear_refmod_options(phys_page, VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL);
+ goto reenter_pg_on_q;
+ }
+ if (m->pmapped == TRUE) {
+ if (m->dirty || m->precious) {
+ pmap_options = PMAP_OPTIONS_COMPRESSOR;
+ } else {
+ pmap_options = PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED;
+ }
+ refmod_state = pmap_disconnect_options(phys_page, pmap_options, NULL);
+ if (refmod_state & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(m, FALSE);
+ }
+ }
+ if ( !m->dirty && !m->precious) {
+ vm_page_unlock_queues();
+ VM_PAGE_FREE(m);
+ vm_page_lock_queues();
+ delayed_unlock = 0;
- if ((vm_page_free_count >= vm_page_free_target) &&
- (vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) {
- /*
- * done - we have met our target *and*
- * there is no one waiting for a page.
- */
-return_from_scan:
- assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
+ goto next_pg;
+ }
+ if (!m_object->pager_initialized || m_object->pager == MEMORY_OBJECT_NULL) {
+
+ if (!m_object->pager_initialized) {
- VM_DEBUG_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_NONE,
- vm_pageout_inactive, vm_pageout_inactive_used, 0, 0);
- VM_DEBUG_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_END,
- vm_pageout_speculative_clean, vm_pageout_inactive_clean,
- vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external);
+ vm_page_unlock_queues();
- return;
+ vm_object_collapse(m_object, (vm_object_offset_t) 0, TRUE);
+
+ if (!m_object->pager_initialized)
+ vm_object_compressor_pager_create(m_object);
+
+ vm_page_lock_queues();
+ delayed_unlock = 0;
}
- lck_mtx_unlock(&vm_page_queue_free_lock);
+ if (!m_object->pager_initialized || m_object->pager == MEMORY_OBJECT_NULL)
+ goto reenter_pg_on_q;
+ /*
+ * vm_object_compressor_pager_create will drop the object lock
+ * which means 'm' may no longer be valid to use
+ */
+ continue;
}
-
/*
- * Before anything, we check if we have any ripe volatile
- * objects around. If so, try to purge the first object.
- * If the purge fails, fall through to reclaim a page instead.
- * If the purge succeeds, go back to the top and reevalute
- * the new memory situation.
+ * we've already factored out pages in the laundry which
+ * means this page can't be on the pageout queue so it's
+ * safe to do the vm_page_queues_remove
*/
-
- assert (available_for_purge>=0);
- force_purge = 0; /* no force-purging */
-
-#if VM_PRESSURE_EVENTS
- pressure_level = memorystatus_vm_pressure_level;
+ vm_page_queues_remove(m, TRUE);
- if (pressure_level > kVMPressureNormal) {
-
- if (pressure_level >= kVMPressureCritical) {
- force_purge = memorystatus_purge_on_critical;
- } else if (pressure_level >= kVMPressureUrgent) {
- force_purge = memorystatus_purge_on_urgent;
- } else if (pressure_level >= kVMPressureWarning) {
- force_purge = memorystatus_purge_on_warning;
- }
- }
-#endif /* VM_PRESSURE_EVENTS */
+ LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
- if (available_for_purge || force_purge) {
+ vm_pageout_cluster(m);
- if (object != NULL) {
- vm_object_unlock(object);
- object = NULL;
- }
+ goto next_pg;
- memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_START);
+reenter_pg_on_q:
+ vm_page_queue_remove(q, m, vm_page_t, pageq);
+ vm_page_queue_enter(q, m, vm_page_t, pageq);
+next_pg:
+ qcount--;
+ try_failed_count = 0;
- VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_START, vm_page_free_count, 0, 0, 0);
- if (vm_purgeable_object_purge_one(force_purge, C_DONT_BLOCK)) {
+ if (delayed_unlock++ > 128) {
- VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_END, vm_page_free_count, 0, 0, 0);
- memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_END);
- continue;
+ if (l_object != NULL) {
+ vm_object_unlock(l_object);
+ l_object = NULL;
}
- VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_END, 0, 0, 0, -1);
- memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_END);
+ lck_mtx_yield(&vm_page_queue_lock);
+ delayed_unlock = 0;
}
+ }
+ if (l_object != NULL) {
+ vm_object_unlock(l_object);
+ l_object = NULL;
+ }
+ vm_page_unlock_queues();
+}
- if (queue_empty(&sq->age_q) && vm_page_speculative_count) {
- /*
- * try to pull pages from the aging bins...
- * see vm_page.h for an explanation of how
- * this mechanism works
- */
- struct vm_speculative_age_q *aq;
- mach_timespec_t ts_fully_aged;
- boolean_t can_steal = FALSE;
- int num_scanned_queues;
-
- aq = &vm_page_queue_speculative[speculative_steal_index];
- num_scanned_queues = 0;
- while (queue_empty(&aq->age_q) &&
- num_scanned_queues++ != VM_PAGE_MAX_SPECULATIVE_AGE_Q) {
- speculative_steal_index++;
+/*
+ * function in BSD to apply I/O throttle to the pageout thread
+ */
+extern void vm_pageout_io_throttle(void);
- if (speculative_steal_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q)
- speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
-
- aq = &vm_page_queue_speculative[speculative_steal_index];
- }
+#define VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, obj) \
+ MACRO_BEGIN \
+ /* \
+ * If a "reusable" page somehow made it back into \
+ * the active queue, it's been re-used and is not \
+ * quite re-usable. \
+ * If the VM object was "all_reusable", consider it \
+ * as "all re-used" instead of converting it to \
+ * "partially re-used", which could be expensive. \
+ */ \
+ assert(VM_PAGE_OBJECT((m)) == (obj)); \
+ if ((m)->reusable || \
+ (obj)->all_reusable) { \
+ vm_object_reuse_pages((obj), \
+ (m)->offset, \
+ (m)->offset + PAGE_SIZE_64, \
+ FALSE); \
+ } \
+ MACRO_END
- if (num_scanned_queues == VM_PAGE_MAX_SPECULATIVE_AGE_Q + 1) {
- /*
- * XXX We've scanned all the speculative
- * queues but still haven't found one
- * that is not empty, even though
- * vm_page_speculative_count is not 0.
- *
- * report the anomaly...
- */
- printf("vm_pageout_scan: "
- "all speculative queues empty "
- "but count=%d. Re-adjusting.\n",
- vm_page_speculative_count);
- if (vm_page_speculative_count > vm_page_speculative_count_drift_max)
- vm_page_speculative_count_drift_max = vm_page_speculative_count;
- vm_page_speculative_count_drifts++;
-#if 6553678
- Debugger("vm_pageout_scan: no speculative pages");
-#endif
- /* readjust... */
- vm_page_speculative_count = 0;
- /* ... and continue */
- continue;
- }
-
- if (vm_page_speculative_count > vm_page_speculative_target)
- can_steal = TRUE;
- else {
- ts_fully_aged.tv_sec = (VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_page_speculative_q_age_ms) / 1000;
- ts_fully_aged.tv_nsec = ((VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_page_speculative_q_age_ms) % 1000)
- * 1000 * NSEC_PER_USEC;
-
- ADD_MACH_TIMESPEC(&ts_fully_aged, &aq->age_ts);
-
- clock_sec_t sec;
- clock_nsec_t nsec;
- clock_get_system_nanotime(&sec, &nsec);
- ts.tv_sec = (unsigned int) sec;
- ts.tv_nsec = nsec;
- if (CMP_MACH_TIMESPEC(&ts, &ts_fully_aged) >= 0)
- can_steal = TRUE;
- }
- if (can_steal == TRUE)
- vm_page_speculate_ageit(aq);
- }
- if (queue_empty(&sq->age_q) && cache_evict_throttle == 0) {
- int pages_evicted;
+#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT 64
+#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX 1024
- if (object != NULL) {
- vm_object_unlock(object);
- object = NULL;
- }
- pages_evicted = vm_object_cache_evict(100, 10);
+#define FCS_IDLE 0
+#define FCS_DELAYED 1
+#define FCS_DEADLOCK_DETECTED 2
- if (pages_evicted) {
+struct flow_control {
+ int state;
+ mach_timespec_t ts;
+};
- vm_pageout_cache_evicted += pages_evicted;
+#if CONFIG_BACKGROUND_QUEUE
+uint64_t vm_pageout_skipped_bq_internal = 0;
+uint64_t vm_pageout_considered_bq_internal = 0;
+uint64_t vm_pageout_considered_bq_external = 0;
+uint64_t vm_pageout_rejected_bq_internal = 0;
+uint64_t vm_pageout_rejected_bq_external = 0;
+#endif
- VM_DEBUG_EVENT(vm_pageout_cache_evict, VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE,
- vm_page_free_count, pages_evicted, vm_pageout_cache_evicted, 0);
- memoryshot(VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE);
+uint32_t vm_pageout_no_victim = 0;
+uint32_t vm_pageout_considered_page = 0;
+uint32_t vm_page_filecache_min = 0;
- /*
- * we just freed up to 100 pages,
- * so go back to the top of the main loop
- * and re-evaulate the memory situation
- */
- continue;
- } else
- cache_evict_throttle = 100;
- }
- if (cache_evict_throttle)
- cache_evict_throttle--;
+#define ANONS_GRABBED_LIMIT 2
- /*
- * don't let the filecache_min fall below 33% of available memory...
- *
- * on systems w/o the compressor/swapper, the filecache is always
- * a very large percentage of the AVAILABLE_NON_COMPRESSED_MEMORY
- * since most (if not all) of the anonymous pages are in the
- * throttled queue (which isn't counted as available) which
- * effectively disables this filter
- */
- vm_page_filecache_min = (AVAILABLE_NON_COMPRESSED_MEMORY / 3);
+#if CONFIG_SECLUDED_MEMORY
+extern vm_page_t vm_page_grab_secluded(void);
+uint64_t vm_pageout_secluded_burst_count = 0;
+#endif /* CONFIG_SECLUDED_MEMORY */
- exceeded_burst_throttle = FALSE;
- /*
- * Sometimes we have to pause:
- * 1) No inactive pages - nothing to do.
- * 2) Loop control - no acceptable pages found on the inactive queue
- * within the last vm_pageout_burst_inactive_throttle iterations
- * 3) Flow control - default pageout queue is full
- */
- if (queue_empty(&vm_page_queue_inactive) && queue_empty(&vm_page_queue_anonymous) && queue_empty(&sq->age_q)) {
- vm_pageout_scan_empty_throttle++;
- msecs = vm_pageout_empty_wait;
- goto vm_pageout_scan_delay;
- } else if (inactive_burst_count >=
- MIN(vm_pageout_burst_inactive_throttle,
- (vm_page_inactive_count +
- vm_page_speculative_count))) {
- vm_pageout_scan_burst_throttle++;
- msecs = vm_pageout_burst_wait;
+static void vm_pageout_delayed_unlock(int *, int *, vm_page_t *);
+static void vm_pageout_prepare_to_block(vm_object_t *, int *, vm_page_t *, int *, int);
- exceeded_burst_throttle = TRUE;
- goto vm_pageout_scan_delay;
+#define VM_PAGEOUT_PB_NO_ACTION 0
+#define VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER 1
+#define VM_PAGEOUT_PB_THREAD_YIELD 2
- } else if (vm_page_free_count > (vm_page_free_reserved / 4) &&
- VM_PAGEOUT_SCAN_NEEDS_TO_THROTTLE()) {
- vm_pageout_scan_swap_throttle++;
- msecs = vm_pageout_swap_wait;
- goto vm_pageout_scan_delay;
- } else if (VM_PAGE_Q_THROTTLED(iq) &&
- VM_DYNAMIC_PAGING_ENABLED(memory_manager_default)) {
- clock_sec_t sec;
- clock_nsec_t nsec;
+static void
+vm_pageout_delayed_unlock(int *delayed_unlock, int *local_freed, vm_page_t *local_freeq)
+{
+ if (*local_freeq) {
+ vm_page_unlock_queues();
- switch (flow_control.state) {
+ VM_DEBUG_EVENT(
+ vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START,
+ vm_page_free_count, *local_freed, 0, 1);
- case FCS_IDLE:
- if ((vm_page_free_count + local_freed) < vm_page_free_target) {
+ vm_page_free_list(*local_freeq, TRUE);
- if (vm_page_pageable_external_count > vm_page_filecache_min && !queue_empty(&vm_page_queue_inactive)) {
- anons_grabbed = ANONS_GRABBED_LIMIT;
- goto consider_inactive;
- }
- if (((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) && vm_page_active_count)
- continue;
- }
-reset_deadlock_timer:
- ts.tv_sec = vm_pageout_deadlock_wait / 1000;
- ts.tv_nsec = (vm_pageout_deadlock_wait % 1000) * 1000 * NSEC_PER_USEC;
- clock_get_system_nanotime(&sec, &nsec);
- flow_control.ts.tv_sec = (unsigned int) sec;
- flow_control.ts.tv_nsec = nsec;
- ADD_MACH_TIMESPEC(&flow_control.ts, &ts);
-
- flow_control.state = FCS_DELAYED;
- msecs = vm_pageout_deadlock_wait;
+ VM_DEBUG_EVENT(vm_pageout_freelist,VM_PAGEOUT_FREELIST, DBG_FUNC_END,
+ vm_page_free_count, 0, 0, 1);
- break;
-
- case FCS_DELAYED:
- clock_get_system_nanotime(&sec, &nsec);
- ts.tv_sec = (unsigned int) sec;
- ts.tv_nsec = nsec;
+ *local_freeq = NULL;
+ *local_freed = 0;
- if (CMP_MACH_TIMESPEC(&ts, &flow_control.ts) >= 0) {
- /*
- * the pageout thread for the default pager is potentially
- * deadlocked since the
- * default pager queue has been throttled for more than the
- * allowable time... we need to move some clean pages or dirty
- * pages belonging to the external pagers if they aren't throttled
- * vm_page_free_wanted represents the number of threads currently
- * blocked waiting for pages... we'll move one page for each of
- * these plus a fixed amount to break the logjam... once we're done
- * moving this number of pages, we'll re-enter the FSC_DELAYED state
- * with a new timeout target since we have no way of knowing
- * whether we've broken the deadlock except through observation
- * of the queue associated with the default pager... we need to
- * stop moving pages and allow the system to run to see what
- * state it settles into.
- */
- vm_pageout_deadlock_target = vm_pageout_deadlock_relief + vm_page_free_wanted + vm_page_free_wanted_privileged;
- vm_pageout_scan_deadlock_detected++;
- flow_control.state = FCS_DEADLOCK_DETECTED;
- thread_wakeup((event_t) &vm_pageout_garbage_collect);
- goto consider_inactive;
- }
- /*
- * just resniff instead of trying
- * to compute a new delay time... we're going to be
- * awakened immediately upon a laundry completion,
- * so we won't wait any longer than necessary
- */
- msecs = vm_pageout_idle_wait;
- break;
+ vm_page_lock_queues();
+ } else {
+ lck_mtx_yield(&vm_page_queue_lock);
+ }
+ *delayed_unlock = 1;
+}
- case FCS_DEADLOCK_DETECTED:
- if (vm_pageout_deadlock_target)
- goto consider_inactive;
- goto reset_deadlock_timer;
- }
-vm_pageout_scan_delay:
- if (object != NULL) {
- vm_object_unlock(object);
- object = NULL;
- }
- vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+static void
+vm_pageout_prepare_to_block(vm_object_t *object, int *delayed_unlock,
+ vm_page_t *local_freeq, int *local_freed, int action)
+{
+ vm_page_unlock_queues();
- vm_page_unlock_queues();
+ if (*object != NULL) {
+ vm_object_unlock(*object);
+ *object = NULL;
+ }
+ vm_pageout_scan_wants_object = VM_OBJECT_NULL;
- if (local_freeq) {
+ if (*local_freeq) {
- VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START,
- vm_page_free_count, local_freed, delayed_unlock_limit, 3);
+ VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START,
+ vm_page_free_count, *local_freed, 0, 2);
- vm_page_free_list(local_freeq, TRUE);
+ vm_page_free_list(*local_freeq, TRUE);
- VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END,
- vm_page_free_count, local_freed, 0, 3);
+ VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END,
+ vm_page_free_count, 0, 0, 2);
- local_freeq = NULL;
- local_freed = 0;
- }
- if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE)
- vm_consider_waking_compactor_swapper();
+ *local_freeq = NULL;
+ *local_freed = 0;
+ }
+ *delayed_unlock = 1;
- vm_page_lock_queues();
+ switch (action) {
- if (flow_control.state == FCS_DELAYED &&
- !VM_PAGE_Q_THROTTLED(iq)) {
- flow_control.state = FCS_IDLE;
- goto consider_inactive;
- }
-
- if (vm_page_free_count >= vm_page_free_target) {
- /*
- * we're here because
- * 1) someone else freed up some pages while we had
- * the queues unlocked above
- * and we've hit one of the 3 conditions that
- * cause us to pause the pageout scan thread
- *
- * since we already have enough free pages,
- * let's avoid stalling and return normally
- *
- * before we return, make sure the pageout I/O threads
- * are running throttled in case there are still requests
- * in the laundry... since we have enough free pages
- * we don't need the laundry to be cleaned in a timely
- * fashion... so let's avoid interfering with foreground
- * activity
- *
- * we don't want to hold vm_page_queue_free_lock when
- * calling vm_pageout_adjust_io_throttles (since it
- * may cause other locks to be taken), we do the intitial
- * check outside of the lock. Once we take the lock,
- * we recheck the condition since it may have changed.
- * if it has, no problem, we will make the threads
- * non-throttled before actually blocking
- */
- vm_pageout_adjust_io_throttles(iq, eq, TRUE);
- }
- lck_mtx_lock(&vm_page_queue_free_lock);
+ case VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER:
+ vm_consider_waking_compactor_swapper();
+ break;
+ case VM_PAGEOUT_PB_THREAD_YIELD:
+ thread_yield_internal(1);
+ break;
+ case VM_PAGEOUT_PB_NO_ACTION:
+ default:
+ break;
+ }
+ vm_page_lock_queues();
+}
- if (vm_page_free_count >= vm_page_free_target &&
- (vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) {
- goto return_from_scan;
- }
- lck_mtx_unlock(&vm_page_queue_free_lock);
-
- if ((vm_page_free_count + vm_page_cleaned_count) < vm_page_free_target) {
- /*
- * we're most likely about to block due to one of
- * the 3 conditions that cause vm_pageout_scan to
- * not be able to make forward progress w/r
- * to providing new pages to the free queue,
- * so unthrottle the I/O threads in case we
- * have laundry to be cleaned... it needs
- * to be completed ASAP.
- *
- * even if we don't block, we want the io threads
- * running unthrottled since the sum of free +
- * clean pages is still under our free target
- */
- vm_pageout_adjust_io_throttles(iq, eq, FALSE);
- }
- if (vm_page_cleaned_count > 0 && exceeded_burst_throttle == FALSE) {
- /*
- * if we get here we're below our free target and
- * we're stalling due to a full laundry queue or
- * we don't have any inactive pages other then
- * those in the clean queue...
- * however, we have pages on the clean queue that
- * can be moved to the free queue, so let's not
- * stall the pageout scan
- */
- flow_control.state = FCS_IDLE;
- goto consider_inactive;
- }
- VM_CHECK_MEMORYSTATUS;
- if (flow_control.state != FCS_IDLE)
- vm_pageout_scan_throttle++;
- iq->pgo_throttled = TRUE;
+int last_vm_pageout_freed_from_inactive_clean = 0;
+int last_vm_pageout_freed_from_cleaned = 0;
+int last_vm_pageout_freed_from_speculative = 0;
+int last_vm_pageout_freed_after_compression = 0;
+int last_vm_pageout_enqueued_cleaned_from_inactive_dirty = 0;
+int last_vm_pageout_inactive_force_reclaim = 0;
+int last_vm_pageout_scan_inactive_throttled_external = 0;
+int last_vm_pageout_scan_inactive_throttled_internal = 0;
+int last_vm_pageout_reactivation_limit_exceeded = 0;
+int last_vm_pageout_considered_page = 0;
+int last_vm_compressor_pages_grabbed = 0;
+int last_vm_compressor_failed = 0;
+int last_vm_pageout_skipped_external = 0;
- if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE)
- vm_consider_waking_compactor_swapper();
- assert_wait_timeout((event_t) &iq->pgo_laundry, THREAD_INTERRUPTIBLE, msecs, 1000*NSEC_PER_USEC);
- counter(c_vm_pageout_scan_block++);
+void update_vm_info(void)
+{
+ int tmp1, tmp2, tmp3, tmp4;
- vm_page_unlock_queues();
+ if (!kdebug_enable)
+ return;
+
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO1)) | DBG_FUNC_NONE,
+ vm_page_active_count,
+ vm_page_speculative_count,
+ vm_page_inactive_count,
+ vm_page_anonymous_count,
+ 0);
+
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO2)) | DBG_FUNC_NONE,
+ vm_page_free_count,
+ vm_page_wire_count,
+ VM_PAGE_COMPRESSOR_COUNT,
+ 0, 0);
+
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO3)) | DBG_FUNC_NONE,
+ c_segment_pages_compressed,
+ vm_page_internal_count,
+ vm_page_external_count,
+ vm_page_xpmapped_external_count,
+ 0);
+
+
+ if ((vm_pageout_considered_page - last_vm_pageout_considered_page) == 0 &&
+ (vm_pageout_enqueued_cleaned_from_inactive_dirty - last_vm_pageout_enqueued_cleaned_from_inactive_dirty == 0) &&
+ (vm_pageout_freed_after_compression - last_vm_pageout_freed_after_compression == 0))
+ return;
+
+
+ tmp1 = vm_pageout_considered_page;
+ tmp2 = vm_pageout_freed_from_speculative;
+ tmp3 = vm_pageout_freed_from_inactive_clean;
+
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO4)) | DBG_FUNC_NONE,
+ tmp1 - last_vm_pageout_considered_page,
+ tmp2 - last_vm_pageout_freed_from_speculative,
+ tmp3 - last_vm_pageout_freed_from_inactive_clean,
+ 0, 0);
+
+ last_vm_pageout_considered_page = tmp1;
+ last_vm_pageout_freed_from_speculative = tmp2;
+ last_vm_pageout_freed_from_inactive_clean = tmp3;
+
+
+ tmp1 = vm_pageout_scan_inactive_throttled_external;
+ tmp2 = vm_pageout_enqueued_cleaned_from_inactive_dirty;
+ tmp3 = vm_pageout_freed_from_cleaned;
+
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO5)) | DBG_FUNC_NONE,
+ tmp1 - last_vm_pageout_scan_inactive_throttled_external,
+ tmp2 - last_vm_pageout_enqueued_cleaned_from_inactive_dirty,
+ tmp3 - last_vm_pageout_freed_from_cleaned,
+ 0, 0);
+
+ vm_pageout_stats[vm_pageout_stat_now].throttled_external_q += (tmp1 - last_vm_pageout_scan_inactive_throttled_external);
+ vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_external += (tmp2 - last_vm_pageout_enqueued_cleaned_from_inactive_dirty);
+
+ last_vm_pageout_scan_inactive_throttled_external = tmp1;
+ last_vm_pageout_enqueued_cleaned_from_inactive_dirty = tmp2;
+ last_vm_pageout_freed_from_cleaned = tmp3;
+
+
+ tmp1 = vm_pageout_scan_inactive_throttled_internal;
+ tmp2 = vm_pageout_freed_after_compression;
+ tmp3 = vm_compressor_pages_grabbed;
+ tmp4 = vm_pageout_skipped_external;
+
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO6)) | DBG_FUNC_NONE,
+ tmp1 - last_vm_pageout_scan_inactive_throttled_internal,
+ tmp2 - last_vm_pageout_freed_after_compression,
+ tmp3 - last_vm_compressor_pages_grabbed,
+ tmp4 - last_vm_pageout_skipped_external,
+ 0);
+
+ vm_pageout_stats[vm_pageout_stat_now].throttled_internal_q += (tmp1 - last_vm_pageout_scan_inactive_throttled_internal);
+ vm_pageout_stats[vm_pageout_stat_now].pages_compressed += (tmp2 - last_vm_pageout_freed_after_compression);
+ vm_pageout_stats[vm_pageout_stat_now].pages_grabbed_by_compressor += (tmp3 - last_vm_compressor_pages_grabbed);
+
+ last_vm_pageout_scan_inactive_throttled_internal = tmp1;
+ last_vm_pageout_freed_after_compression = tmp2;
+ last_vm_compressor_pages_grabbed = tmp3;
+ last_vm_pageout_skipped_external = tmp4;
+
+
+ if ((vm_pageout_reactivation_limit_exceeded - last_vm_pageout_reactivation_limit_exceeded) == 0 &&
+ (vm_pageout_inactive_force_reclaim - last_vm_pageout_inactive_force_reclaim) == 0 &&
+ (vm_compressor_failed - last_vm_compressor_failed) == 0)
+ return;
+
+ tmp1 = vm_pageout_reactivation_limit_exceeded;
+ tmp2 = vm_pageout_inactive_force_reclaim;
+ tmp3 = vm_compressor_failed;
+
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO7)) | DBG_FUNC_NONE,
+ tmp1 - last_vm_pageout_reactivation_limit_exceeded,
+ tmp2 - last_vm_pageout_inactive_force_reclaim,
+ tmp3 - last_vm_compressor_failed,
+ 0, 0);
+
+ vm_pageout_stats[vm_pageout_stat_now].failed_compressions += (tmp3 - last_vm_compressor_failed);
+
+ last_vm_pageout_reactivation_limit_exceeded = tmp1;
+ last_vm_pageout_inactive_force_reclaim = tmp2;
+ last_vm_compressor_failed = tmp3;
+}
- assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
- VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START,
- iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0);
- memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START);
+/*
+ * vm_pageout_scan does the dirty work for the pageout daemon.
+ * It returns with both vm_page_queue_free_lock and vm_page_queue_lock
+ * held and vm_page_free_wanted == 0.
+ */
+void
+vm_pageout_scan(void)
+{
+ unsigned int loop_count = 0;
+ unsigned int inactive_burst_count = 0;
+ unsigned int active_burst_count = 0;
+ unsigned int reactivated_this_call;
+ unsigned int reactivate_limit;
+ vm_page_t local_freeq = NULL;
+ int local_freed = 0;
+ int delayed_unlock;
+ int delayed_unlock_limit = 0;
+ int refmod_state = 0;
+ int vm_pageout_deadlock_target = 0;
+ struct vm_pageout_queue *iq;
+ struct vm_pageout_queue *eq;
+ struct vm_speculative_age_q *sq;
+ struct flow_control flow_control = { 0, { 0, 0 } };
+ boolean_t inactive_throttled = FALSE;
+ boolean_t try_failed;
+ mach_timespec_t ts;
+ unsigned int msecs = 0;
+ vm_object_t object = NULL;
+ uint32_t inactive_reclaim_run;
+ boolean_t exceeded_burst_throttle;
+ boolean_t grab_anonymous = FALSE;
+ boolean_t force_anonymous = FALSE;
+ boolean_t force_speculative_aging = FALSE;
+ int anons_grabbed = 0;
+ int page_prev_q_state = 0;
+#if CONFIG_BACKGROUND_QUEUE
+ boolean_t page_from_bg_q = FALSE;
+#endif
+ int cache_evict_throttle = 0;
+ uint32_t vm_pageout_inactive_external_forced_reactivate_limit = 0;
+ int force_purge = 0;
+#define DELAY_SPECULATIVE_AGE 1000
+ int delay_speculative_age = 0;
+ vm_object_t m_object = VM_OBJECT_NULL;
- thread_block(THREAD_CONTINUE_NULL);
+#if VM_PRESSURE_EVENTS
+ vm_pressure_level_t pressure_level;
+#endif /* VM_PRESSURE_EVENTS */
- VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END,
- iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0);
- memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END);
+ VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_START,
+ vm_pageout_speculative_clean, vm_pageout_inactive_clean,
+ vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external);
- vm_page_lock_queues();
- delayed_unlock = 1;
+ flow_control.state = FCS_IDLE;
+ iq = &vm_pageout_queue_internal;
+ eq = &vm_pageout_queue_external;
+ sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
- iq->pgo_throttled = FALSE;
- if (loop_count >= vm_page_inactive_count)
- loop_count = 0;
- inactive_burst_count = 0;
+ XPR(XPR_VM_PAGEOUT, "vm_pageout_scan\n", 0, 0, 0, 0, 0);
- goto Restart;
- /*NOTREACHED*/
- }
+ /* Ask the pmap layer to return any pages it no longer needs. */
+ pmap_release_pages_fast();
+ vm_page_lock_queues();
+ delayed_unlock = 1;
- flow_control.state = FCS_IDLE;
-consider_inactive:
- vm_pageout_inactive_external_forced_reactivate_limit = MIN((vm_page_active_count + vm_page_inactive_count),
- vm_pageout_inactive_external_forced_reactivate_limit);
- loop_count++;
- inactive_burst_count++;
- vm_pageout_inactive++;
+ /*
+ * Calculate the max number of referenced pages on the inactive
+ * queue that we will reactivate.
+ */
+ reactivated_this_call = 0;
+ reactivate_limit = VM_PAGE_REACTIVATE_LIMIT(vm_page_active_count +
+ vm_page_inactive_count);
+ inactive_reclaim_run = 0;
+ vm_pageout_inactive_external_forced_reactivate_limit = vm_page_active_count + vm_page_inactive_count;
- /*
- * Choose a victim.
- */
- while (1) {
- m = NULL;
-
- if (VM_DYNAMIC_PAGING_ENABLED(memory_manager_default)) {
- assert(vm_page_throttled_count == 0);
- assert(queue_empty(&vm_page_queue_throttled));
- }
- /*
- * The most eligible pages are ones we paged in speculatively,
- * but which have not yet been touched.
- */
- if (!queue_empty(&sq->age_q) && force_anonymous == FALSE) {
- m = (vm_page_t) queue_first(&sq->age_q);
+ /*
+ * We want to gradually dribble pages from the active queue
+ * to the inactive queue. If we let the inactive queue get
+ * very small, and then suddenly dump many pages into it,
+ * those pages won't get a sufficient chance to be referenced
+ * before we start taking them from the inactive queue.
+ *
+ * We must limit the rate at which we send pages to the pagers
+ * so that we don't tie up too many pages in the I/O queues.
+ * We implement a throttling mechanism using the laundry count
+ * to limit the number of pages outstanding to the default
+ * and external pagers. We can bypass the throttles and look
+ * for clean pages if the pageout queues don't drain in a timely
+ * fashion since this may indicate that the pageout paths are
+ * stalled waiting for memory, which only we can provide.
+ */
- page_prev_state = PAGE_STATE_SPECULATIVE;
- break;
- }
- /*
- * Try a clean-queue inactive page.
- */
- if (!queue_empty(&vm_page_queue_cleaned)) {
- m = (vm_page_t) queue_first(&vm_page_queue_cleaned);
-
- page_prev_state = PAGE_STATE_CLEAN;
-
- break;
- }
+Restart:
- grab_anonymous = (vm_page_anonymous_count > vm_page_anonymous_min);
+ assert(object == NULL);
+ assert(delayed_unlock != 0);
+
+ /*
+ * Recalculate vm_page_inactivate_target.
+ */
+ vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
+ vm_page_inactive_count +
+ vm_page_speculative_count);
- if (vm_page_pageable_external_count < vm_page_filecache_min || force_anonymous == TRUE) {
- grab_anonymous = TRUE;
- anons_grabbed = 0;
- }
+ vm_page_anonymous_min = vm_page_inactive_target / 20;
- if (grab_anonymous == FALSE || anons_grabbed >= ANONS_GRABBED_LIMIT || queue_empty(&vm_page_queue_anonymous)) {
- if ( !queue_empty(&vm_page_queue_inactive) ) {
- m = (vm_page_t) queue_first(&vm_page_queue_inactive);
-
- page_prev_state = PAGE_STATE_INACTIVE;
- anons_grabbed = 0;
+ /*
+ * don't want to wake the pageout_scan thread up everytime we fall below
+ * the targets... set a low water mark at 0.25% below the target
+ */
+ vm_page_inactive_min = vm_page_inactive_target - (vm_page_inactive_target / 400);
- if (vm_page_pageable_external_count < vm_page_filecache_min) {
- if ((++reactivated_this_call % 100))
- goto must_activate_page;
- /*
- * steal 1% of the file backed pages even if
- * we are under the limit that has been set
- * for a healthy filecache
- */
- }
- break;
- }
- }
- if ( !queue_empty(&vm_page_queue_anonymous) ) {
- m = (vm_page_t) queue_first(&vm_page_queue_anonymous);
+ if (vm_page_speculative_percentage > 50)
+ vm_page_speculative_percentage = 50;
+ else if (vm_page_speculative_percentage <= 0)
+ vm_page_speculative_percentage = 1;
- page_prev_state = PAGE_STATE_ANONYMOUS;
- anons_grabbed++;
+ vm_page_speculative_target = VM_PAGE_SPECULATIVE_TARGET(vm_page_active_count +
+ vm_page_inactive_count);
- break;
- }
+ try_failed = FALSE;
+
+ for (;;) {
+ vm_page_t m;
+
+ DTRACE_VM2(rev, int, 1, (uint64_t *), NULL);
+
+ if (vm_upl_wait_for_pages < 0)
+ vm_upl_wait_for_pages = 0;
+
+ delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT + vm_upl_wait_for_pages;
+
+ if (delayed_unlock_limit > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX)
+ delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX;
+
+#if CONFIG_SECLUDED_MEMORY
+ /*
+ * Deal with secluded_q overflow.
+ */
+ if (vm_page_secluded_count > vm_page_secluded_target) {
+ unsigned int secluded_overflow;
+ vm_page_t secluded_page;
- /*
- * if we've gotten here, we have no victim page.
- * if making clean, free the local freed list and return.
- * if making free, check to see if we've finished balancing the queues
- * yet, if we haven't just continue, else panic
- */
- vm_page_unlock_queues();
-
if (object != NULL) {
vm_object_unlock(object);
object = NULL;
+ vm_pageout_scan_wants_object = VM_OBJECT_NULL;
}
- vm_pageout_scan_wants_object = VM_OBJECT_NULL;
-
- if (local_freeq) {
- VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START,
- vm_page_free_count, local_freed, delayed_unlock_limit, 5);
-
- vm_page_free_list(local_freeq, TRUE);
-
- VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END,
- vm_page_free_count, local_freed, 0, 5);
-
- local_freeq = NULL;
- local_freed = 0;
- }
- vm_page_lock_queues();
- delayed_unlock = 1;
+ /*
+ * SECLUDED_AGING_BEFORE_ACTIVE:
+ * Excess secluded pages go to the active queue and
+ * will later go to the inactive queue.
+ */
+ active_burst_count = MIN(vm_pageout_burst_active_throttle,
+ vm_page_secluded_count_inuse);
+ secluded_overflow = (vm_page_secluded_count -
+ vm_page_secluded_target);
+ while (secluded_overflow-- > 0 &&
+ vm_page_secluded_count > vm_page_secluded_target) {
+ assert((vm_page_secluded_count_free +
+ vm_page_secluded_count_inuse) ==
+ vm_page_secluded_count);
+ secluded_page = (vm_page_t)vm_page_queue_first(&vm_page_queue_secluded);
+ assert(secluded_page->vm_page_q_state ==
+ VM_PAGE_ON_SECLUDED_Q);
+ vm_page_queues_remove(secluded_page, FALSE);
+ assert(!secluded_page->fictitious);
+ assert(!VM_PAGE_WIRED(secluded_page));
+ if (secluded_page->vm_page_object == 0) {
+ /* transfer to free queue */
+ assert(secluded_page->busy);
+ secluded_page->snext = local_freeq;
+ local_freeq = secluded_page;
+ local_freed++;
+ } else {
+ /* transfer to head of active queue */
+ vm_page_enqueue_active(secluded_page, FALSE);
+ if (active_burst_count-- == 0) {
+ vm_pageout_secluded_burst_count++;
+ break;
+ }
+ }
+ secluded_page = VM_PAGE_NULL;
- force_anonymous = FALSE;
+ if (delayed_unlock++ > delayed_unlock_limit) {
+ vm_pageout_delayed_unlock(&delayed_unlock, &local_freed, &local_freeq);
+ }
+ }
+ }
+#endif /* CONFIG_SECLUDED_MEMORY */
- if ((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target)
- goto Restart;
+ assert(delayed_unlock);
- if (!queue_empty(&sq->age_q))
- goto Restart;
+ /*
+ * Move pages from active to inactive if we're below the target
+ */
+ if ((vm_page_inactive_count + vm_page_speculative_count) >= vm_page_inactive_target)
+ goto done_moving_active_pages;
- panic("vm_pageout: no victim");
-
- /* NOTREACHED */
+ if (object != NULL) {
+ vm_object_unlock(object);
+ object = NULL;
+ vm_pageout_scan_wants_object = VM_OBJECT_NULL;
}
- force_anonymous = FALSE;
-
/*
- * we just found this page on one of our queues...
- * it can't also be on the pageout queue, so safe
- * to call VM_PAGE_QUEUES_REMOVE
+ * Don't sweep through active queue more than the throttle
+ * which should be kept relatively low
*/
- assert(!m->pageout_queue);
+ active_burst_count = MIN(vm_pageout_burst_active_throttle, vm_page_active_count);
- VM_PAGE_QUEUES_REMOVE(m);
+ VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_START,
+ vm_pageout_inactive, vm_pageout_inactive_used, vm_page_free_count, local_freed);
- assert(!m->laundry);
- assert(!m->private);
- assert(!m->fictitious);
- assert(m->object != kernel_object);
- assert(m->phys_page != vm_page_guard_addr);
+ VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_NONE,
+ vm_pageout_speculative_clean, vm_pageout_inactive_clean,
+ vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external);
+ memoryshot(VM_PAGEOUT_BALANCE, DBG_FUNC_START);
- if (page_prev_state != PAGE_STATE_SPECULATIVE)
- vm_pageout_stats[vm_pageout_stat_now].considered++;
+ while (!vm_page_queue_empty(&vm_page_queue_active) && active_burst_count--) {
- DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
+ vm_pageout_active++;
+
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
+
+ assert(m->vm_page_q_state == VM_PAGE_ON_ACTIVE_Q);
+ assert(!m->laundry);
+ assert(VM_PAGE_OBJECT(m) != kernel_object);
+ assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr);
+
+ DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
- /*
- * check to see if we currently are working
- * with the same object... if so, we've
- * already got the lock
- */
- if (m->object != object) {
- /*
- * the object associated with candidate page is
- * different from the one we were just working
- * with... dump the lock if we still own it
- */
- if (object != NULL) {
- vm_object_unlock(object);
- object = NULL;
- vm_pageout_scan_wants_object = VM_OBJECT_NULL;
- }
/*
- * Try to lock object; since we've alread got the
- * page queues lock, we can only 'try' for this one.
- * if the 'try' fails, we need to do a mutex_pause
- * to allow the owner of the object lock a chance to
- * run... otherwise, we're likely to trip over this
- * object in the same state as we work our way through
- * the queue... clumps of pages associated with the same
- * object are fairly typical on the inactive and active queues
+ * by not passing in a pmap_flush_context we will forgo any TLB flushing, local or otherwise...
+ *
+ * a TLB flush isn't really needed here since at worst we'll miss the reference bit being
+ * updated in the PTE if a remote processor still has this mapping cached in its TLB when the
+ * new reference happens. If no futher references happen on the page after that remote TLB flushes
+ * we'll see a clean, non-referenced page when it eventually gets pulled out of the inactive queue
+ * by pageout_scan, which is just fine since the last reference would have happened quite far
+ * in the past (TLB caches don't hang around for very long), and of course could just as easily
+ * have happened before we moved the page
*/
- if (!vm_object_lock_try_scan(m->object)) {
- vm_page_t m_want = NULL;
+ pmap_clear_refmod_options(VM_PAGE_GET_PHYS_PAGE(m), VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL);
- vm_pageout_inactive_nolock++;
+ /*
+ * The page might be absent or busy,
+ * but vm_page_deactivate can handle that.
+ * FALSE indicates that we don't want a H/W clear reference
+ */
+ vm_page_deactivate_internal(m, FALSE);
- if (page_prev_state == PAGE_STATE_CLEAN)
- vm_pageout_cleaned_nolock++;
+ if (delayed_unlock++ > delayed_unlock_limit) {
+ vm_pageout_delayed_unlock(&delayed_unlock, &local_freed, &local_freeq);
+ }
+ }
- if (page_prev_state == PAGE_STATE_SPECULATIVE)
- page_prev_state = PAGE_STATE_INACTIVE_FIRST;
+ VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_END,
+ vm_page_active_count, vm_page_inactive_count, vm_page_speculative_count, vm_page_inactive_target);
+ memoryshot(VM_PAGEOUT_BALANCE, DBG_FUNC_END);
- pmap_clear_reference(m->phys_page);
- m->reference = FALSE;
+ /**********************************************************************
+ * above this point we're playing with the active and secluded queues
+ * below this point we're playing with the throttling mechanisms
+ * and the inactive queue
+ **********************************************************************/
- /*
- * m->object must be stable since we hold the page queues lock...
- * we can update the scan_collisions field sans the object lock
- * since it is a separate field and this is the only spot that does
- * a read-modify-write operation and it is never executed concurrently...
- * we can asynchronously set this field to 0 when creating a UPL, so it
- * is possible for the value to be a bit non-determistic, but that's ok
- * since it's only used as a hint
- */
- m->object->scan_collisions++;
+done_moving_active_pages:
- if ( !queue_empty(&sq->age_q) )
- m_want = (vm_page_t) queue_first(&sq->age_q);
- else if ( !queue_empty(&vm_page_queue_cleaned))
- m_want = (vm_page_t) queue_first(&vm_page_queue_cleaned);
- else if (anons_grabbed >= ANONS_GRABBED_LIMIT || queue_empty(&vm_page_queue_anonymous))
- m_want = (vm_page_t) queue_first(&vm_page_queue_inactive);
- else if ( !queue_empty(&vm_page_queue_anonymous))
- m_want = (vm_page_t) queue_first(&vm_page_queue_anonymous);
+ if (vm_page_free_count + local_freed >= vm_page_free_target)
+ {
+ vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed,
+ VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER);
+ /*
+ * make sure the pageout I/O threads are running
+ * throttled in case there are still requests
+ * in the laundry... since we have met our targets
+ * we don't need the laundry to be cleaned in a timely
+ * fashion... so let's avoid interfering with foreground
+ * activity
+ */
+ vm_pageout_adjust_eq_iothrottle(eq, TRUE);
+ /*
+ * recalculate vm_page_inactivate_target
+ */
+ vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
+ vm_page_inactive_count +
+ vm_page_speculative_count);
+#ifndef CONFIG_EMBEDDED
+ if (((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) &&
+ !vm_page_queue_empty(&vm_page_queue_active)) {
/*
- * this is the next object we're going to be interested in
- * try to make sure its available after the mutex_yield
- * returns control
+ * inactive target still not met... keep going
+ * until we get the queues balanced...
*/
- if (m_want)
- vm_pageout_scan_wants_object = m_want->object;
+ continue;
+ }
+#endif
+ lck_mtx_lock(&vm_page_queue_free_lock);
+ if ((vm_page_free_count >= vm_page_free_target) &&
+ (vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) {
/*
- * force us to dump any collected free pages
- * and to pause before moving on
+ * done - we have met our target *and*
+ * there is no one waiting for a page.
*/
- try_failed = TRUE;
+return_from_scan:
+ assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
- goto requeue_page;
- }
- object = m->object;
- vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+ VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_NONE,
+ vm_pageout_inactive, vm_pageout_inactive_used, 0, 0);
+ VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_END,
+ vm_pageout_speculative_clean, vm_pageout_inactive_clean,
+ vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external);
- try_failed = FALSE;
+ return;
+ }
+ lck_mtx_unlock(&vm_page_queue_free_lock);
}
- if (catch_up_count)
- catch_up_count--;
+
+ /*
+ * Before anything, we check if we have any ripe volatile
+ * objects around. If so, try to purge the first object.
+ * If the purge fails, fall through to reclaim a page instead.
+ * If the purge succeeds, go back to the top and reevalute
+ * the new memory situation.
+ */
+
+ assert (available_for_purge>=0);
+ force_purge = 0; /* no force-purging */
- if (m->busy) {
- if (m->encrypted_cleaning) {
- /*
- * ENCRYPTED SWAP:
- * if this page has already been picked up as
- * part of a page-out cluster, it will be busy
- * because it is being encrypted (see
- * vm_object_upl_request()). But we still
- * want to demote it from "clean-in-place"
- * (aka "adjacent") to "clean-and-free" (aka
- * "target"), so let's ignore its "busy" bit
- * here and proceed to check for "cleaning" a
- * little bit below...
- *
- * CAUTION CAUTION:
- * A "busy" page should still be left alone for
- * most purposes, so we have to be very careful
- * not to process that page too much.
- */
- assert(m->cleaning);
- goto consider_inactive_page;
- }
+#if VM_PRESSURE_EVENTS
+ pressure_level = memorystatus_vm_pressure_level;
- /*
- * Somebody is already playing with this page.
- * Put it back on the appropriate queue
- *
- */
- vm_pageout_inactive_busy++;
+ if (pressure_level > kVMPressureNormal) {
- if (page_prev_state == PAGE_STATE_CLEAN)
- vm_pageout_cleaned_busy++;
-
-requeue_page:
- switch (page_prev_state) {
+ if (pressure_level >= kVMPressureCritical) {
+ force_purge = memorystatus_purge_on_critical;
+ } else if (pressure_level >= kVMPressureUrgent) {
+ force_purge = memorystatus_purge_on_urgent;
+ } else if (pressure_level >= kVMPressureWarning) {
+ force_purge = memorystatus_purge_on_warning;
+ }
+ }
+#endif /* VM_PRESSURE_EVENTS */
- case PAGE_STATE_SPECULATIVE:
- vm_page_speculate(m, FALSE);
- break;
+ if (available_for_purge || force_purge) {
- case PAGE_STATE_ANONYMOUS:
- case PAGE_STATE_CLEAN:
- case PAGE_STATE_INACTIVE:
- VM_PAGE_ENQUEUE_INACTIVE(m, FALSE);
- break;
+ if (object != NULL) {
+ vm_object_unlock(object);
+ object = NULL;
+ }
- case PAGE_STATE_INACTIVE_FIRST:
- VM_PAGE_ENQUEUE_INACTIVE(m, TRUE);
- break;
+ memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_START);
+
+ VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_START, vm_page_free_count, 0, 0, 0);
+ if (vm_purgeable_object_purge_one(force_purge, C_DONT_BLOCK)) {
+ vm_pageout_purged_objects++;
+ VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_END, vm_page_free_count, 0, 0, 0);
+ memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_END);
+ continue;
}
- goto done_with_inactivepage;
+ VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_END, 0, 0, 0, -1);
+ memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_END);
}
+ if (vm_page_queue_empty(&sq->age_q) && vm_page_speculative_count) {
+ /*
+ * try to pull pages from the aging bins...
+ * see vm_page.h for an explanation of how
+ * this mechanism works
+ */
+ struct vm_speculative_age_q *aq;
+ boolean_t can_steal = FALSE;
+ int num_scanned_queues;
+
+ aq = &vm_page_queue_speculative[speculative_steal_index];
- /*
- * If it's absent, in error or the object is no longer alive,
- * we can reclaim the page... in the no longer alive case,
- * there are 2 states the page can be in that preclude us
- * from reclaiming it - busy or cleaning - that we've already
- * dealt with
- */
- if (m->absent || m->error || !object->alive) {
+ num_scanned_queues = 0;
+ while (vm_page_queue_empty(&aq->age_q) &&
+ num_scanned_queues++ != VM_PAGE_MAX_SPECULATIVE_AGE_Q) {
- if (m->absent)
- vm_pageout_inactive_absent++;
- else if (!object->alive)
- vm_pageout_inactive_notalive++;
- else
- vm_pageout_inactive_error++;
-reclaim_page:
- if (vm_pageout_deadlock_target) {
- vm_pageout_scan_inactive_throttle_success++;
- vm_pageout_deadlock_target--;
- }
+ speculative_steal_index++;
- DTRACE_VM2(dfree, int, 1, (uint64_t *), NULL);
+ if (speculative_steal_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q)
+ speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
+
+ aq = &vm_page_queue_speculative[speculative_steal_index];
+ }
- if (object->internal) {
- DTRACE_VM2(anonfree, int, 1, (uint64_t *), NULL);
- } else {
- DTRACE_VM2(fsfree, int, 1, (uint64_t *), NULL);
+ if (num_scanned_queues == VM_PAGE_MAX_SPECULATIVE_AGE_Q + 1) {
+ /*
+ * XXX We've scanned all the speculative
+ * queues but still haven't found one
+ * that is not empty, even though
+ * vm_page_speculative_count is not 0.
+ *
+ * report the anomaly...
+ */
+ printf("vm_pageout_scan: "
+ "all speculative queues empty "
+ "but count=%d. Re-adjusting.\n",
+ vm_page_speculative_count);
+ if (vm_page_speculative_count > vm_page_speculative_count_drift_max)
+ vm_page_speculative_count_drift_max = vm_page_speculative_count;
+ vm_page_speculative_count_drifts++;
+#if DEVELOPMENT || DEBUG
+ panic("vm_pageout_scan: vm_page_speculative_count=%d but queues are empty", vm_page_speculative_count);
+#endif /* DEVELOPMENT || DEBUG */
+ /* readjust... */
+ vm_page_speculative_count = 0;
+ /* ... and continue */
+ continue;
}
- assert(!m->cleaning);
- assert(!m->laundry);
- m->busy = TRUE;
+ if (vm_page_speculative_count > vm_page_speculative_target || force_speculative_aging == TRUE)
+ can_steal = TRUE;
+ else {
+ if (!delay_speculative_age) {
+ mach_timespec_t ts_fully_aged;
- /*
- * remove page from object here since we're already
- * behind the object lock... defer the rest of the work
- * we'd normally do in vm_page_free_prepare_object
- * until 'vm_page_free_list' is called
- */
- if (m->tabled)
- vm_page_remove(m, TRUE);
+ ts_fully_aged.tv_sec = (VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_page_speculative_q_age_ms) / 1000;
+ ts_fully_aged.tv_nsec = ((VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_page_speculative_q_age_ms) % 1000)
+ * 1000 * NSEC_PER_USEC;
- assert(m->pageq.next == NULL &&
- m->pageq.prev == NULL);
- m->pageq.next = (queue_entry_t)local_freeq;
- local_freeq = m;
- local_freed++;
-
- if (page_prev_state == PAGE_STATE_SPECULATIVE)
- vm_pageout_freed_from_speculative++;
- else if (page_prev_state == PAGE_STATE_CLEAN)
- vm_pageout_freed_from_cleaned++;
- else
- vm_pageout_freed_from_inactive_clean++;
+ ADD_MACH_TIMESPEC(&ts_fully_aged, &aq->age_ts);
- if (page_prev_state != PAGE_STATE_SPECULATIVE)
- vm_pageout_stats[vm_pageout_stat_now].reclaimed++;
+ clock_sec_t sec;
+ clock_nsec_t nsec;
+ clock_get_system_nanotime(&sec, &nsec);
+ ts.tv_sec = (unsigned int) sec;
+ ts.tv_nsec = nsec;
- inactive_burst_count = 0;
- goto done_with_inactivepage;
- }
- /*
- * If the object is empty, the page must be reclaimed even
- * if dirty or used.
- * If the page belongs to a volatile object, we stick it back
- * on.
- */
- if (object->copy == VM_OBJECT_NULL) {
- if (object->purgable == VM_PURGABLE_EMPTY) {
- if (m->pmapped == TRUE) {
- /* unmap the page */
- refmod_state = pmap_disconnect(m->phys_page);
- if (refmod_state & VM_MEM_MODIFIED) {
- SET_PAGE_DIRTY(m, FALSE);
- }
- }
- if (m->dirty || m->precious) {
- /* we saved the cost of cleaning this page ! */
- vm_page_purged_count++;
+ if (CMP_MACH_TIMESPEC(&ts, &ts_fully_aged) >= 0)
+ can_steal = TRUE;
+ else
+ delay_speculative_age++;
+ } else {
+ delay_speculative_age++;
+ if (delay_speculative_age == DELAY_SPECULATIVE_AGE)
+ delay_speculative_age = 0;
}
- goto reclaim_page;
}
+ if (can_steal == TRUE)
+ vm_page_speculate_ageit(aq);
+ }
+ force_speculative_aging = FALSE;
- if (COMPRESSED_PAGER_IS_ACTIVE) {
- /*
- * With the VM compressor, the cost of
- * reclaiming a page is much lower (no I/O),
- * so if we find a "volatile" page, it's better
- * to let it get compressed rather than letting
+#if CONFIG_BACKGROUND_QUEUE
+ if (vm_page_queue_empty(&sq->age_q) && cache_evict_throttle == 0 &&
+ ((vm_page_background_mode == VM_PAGE_BG_DISABLED) || (vm_page_background_count <= vm_page_background_target)))
+#else
+ if (vm_page_queue_empty(&sq->age_q) && cache_evict_throttle == 0)
+#endif
+ {
+ int pages_evicted;
+
+ if (object != NULL) {
+ vm_object_unlock(object);
+ object = NULL;
+ }
+ pages_evicted = vm_object_cache_evict(100, 10);
+
+ if (pages_evicted) {
+
+ vm_pageout_cache_evicted += pages_evicted;
+
+ VM_DEBUG_EVENT(vm_pageout_cache_evict, VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE,
+ vm_page_free_count, pages_evicted, vm_pageout_cache_evicted, 0);
+ memoryshot(VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE);
+
+ /*
+ * we just freed up to 100 pages,
+ * so go back to the top of the main loop
+ * and re-evaulate the memory situation
+ */
+ continue;
+ } else
+ cache_evict_throttle = 1000;
+ }
+ if (cache_evict_throttle)
+ cache_evict_throttle--;
+
+#if CONFIG_JETSAM
+ /*
+ * don't let the filecache_min fall below 15% of available memory
+ * on systems with an active compressor that isn't nearing its
+ * limits w/r to accepting new data
+ *
+ * on systems w/o the compressor/swapper, the filecache is always
+ * a very large percentage of the AVAILABLE_NON_COMPRESSED_MEMORY
+ * since most (if not all) of the anonymous pages are in the
+ * throttled queue (which isn't counted as available) which
+ * effectively disables this filter
+ */
+ if (vm_compressor_low_on_space())
+ vm_page_filecache_min = 0;
+ else
+ vm_page_filecache_min = (AVAILABLE_NON_COMPRESSED_MEMORY / 7);
+#else
+ if (vm_compressor_out_of_space())
+ vm_page_filecache_min = 0;
+ else {
+ /*
+ * don't let the filecache_min fall below 33% of available memory...
+ */
+ vm_page_filecache_min = (AVAILABLE_NON_COMPRESSED_MEMORY / 3);
+ }
+#endif
+ if (vm_page_free_count < (vm_page_free_reserved / 4))
+ vm_page_filecache_min = 0;
+
+ exceeded_burst_throttle = FALSE;
+ /*
+ * Sometimes we have to pause:
+ * 1) No inactive pages - nothing to do.
+ * 2) Loop control - no acceptable pages found on the inactive queue
+ * within the last vm_pageout_burst_inactive_throttle iterations
+ * 3) Flow control - default pageout queue is full
+ */
+ if (vm_page_queue_empty(&vm_page_queue_inactive) &&
+ vm_page_queue_empty(&vm_page_queue_anonymous) &&
+ vm_page_queue_empty(&sq->age_q)) {
+ vm_pageout_scan_empty_throttle++;
+ msecs = vm_pageout_empty_wait;
+ goto vm_pageout_scan_delay;
+
+ } else if (inactive_burst_count >=
+ MIN(vm_pageout_burst_inactive_throttle,
+ (vm_page_inactive_count +
+ vm_page_speculative_count))) {
+ vm_pageout_scan_burst_throttle++;
+ msecs = vm_pageout_burst_wait;
+
+ exceeded_burst_throttle = TRUE;
+ goto vm_pageout_scan_delay;
+
+ } else if (vm_page_free_count > (vm_page_free_reserved / 4) &&
+ VM_PAGEOUT_SCAN_NEEDS_TO_THROTTLE()) {
+ vm_pageout_scan_swap_throttle++;
+ msecs = vm_pageout_swap_wait;
+ goto vm_pageout_scan_delay;
+
+ } else if (VM_PAGE_Q_THROTTLED(iq) &&
+ VM_DYNAMIC_PAGING_ENABLED()) {
+ clock_sec_t sec;
+ clock_nsec_t nsec;
+
+ switch (flow_control.state) {
+
+ case FCS_IDLE:
+ if ((vm_page_free_count + local_freed) < vm_page_free_target) {
+
+ vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed,
+ VM_PAGEOUT_PB_THREAD_YIELD);
+ if (!VM_PAGE_Q_THROTTLED(iq)) {
+ vm_pageout_scan_yield_unthrottled++;
+ continue;
+ }
+ if (vm_page_pageable_external_count > vm_page_filecache_min &&
+ !vm_page_queue_empty(&vm_page_queue_inactive)) {
+ anons_grabbed = ANONS_GRABBED_LIMIT;
+ vm_pageout_scan_throttle_deferred++;
+ goto consider_inactive;
+ }
+ if (((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) && vm_page_active_count)
+ continue;
+ }
+reset_deadlock_timer:
+ ts.tv_sec = vm_pageout_deadlock_wait / 1000;
+ ts.tv_nsec = (vm_pageout_deadlock_wait % 1000) * 1000 * NSEC_PER_USEC;
+ clock_get_system_nanotime(&sec, &nsec);
+ flow_control.ts.tv_sec = (unsigned int) sec;
+ flow_control.ts.tv_nsec = nsec;
+ ADD_MACH_TIMESPEC(&flow_control.ts, &ts);
+
+ flow_control.state = FCS_DELAYED;
+ msecs = vm_pageout_deadlock_wait;
+
+ break;
+
+ case FCS_DELAYED:
+ clock_get_system_nanotime(&sec, &nsec);
+ ts.tv_sec = (unsigned int) sec;
+ ts.tv_nsec = nsec;
+
+ if (CMP_MACH_TIMESPEC(&ts, &flow_control.ts) >= 0) {
+ /*
+ * the pageout thread for the default pager is potentially
+ * deadlocked since the
+ * default pager queue has been throttled for more than the
+ * allowable time... we need to move some clean pages or dirty
+ * pages belonging to the external pagers if they aren't throttled
+ * vm_page_free_wanted represents the number of threads currently
+ * blocked waiting for pages... we'll move one page for each of
+ * these plus a fixed amount to break the logjam... once we're done
+ * moving this number of pages, we'll re-enter the FSC_DELAYED state
+ * with a new timeout target since we have no way of knowing
+ * whether we've broken the deadlock except through observation
+ * of the queue associated with the default pager... we need to
+ * stop moving pages and allow the system to run to see what
+ * state it settles into.
+ */
+ vm_pageout_deadlock_target = vm_pageout_deadlock_relief + vm_page_free_wanted + vm_page_free_wanted_privileged;
+ vm_pageout_scan_deadlock_detected++;
+ flow_control.state = FCS_DEADLOCK_DETECTED;
+ thread_wakeup((event_t) &vm_pageout_garbage_collect);
+ goto consider_inactive;
+ }
+ /*
+ * just resniff instead of trying
+ * to compute a new delay time... we're going to be
+ * awakened immediately upon a laundry completion,
+ * so we won't wait any longer than necessary
+ */
+ msecs = vm_pageout_idle_wait;
+ break;
+
+ case FCS_DEADLOCK_DETECTED:
+ if (vm_pageout_deadlock_target)
+ goto consider_inactive;
+ goto reset_deadlock_timer;
+
+ }
+vm_pageout_scan_delay:
+ vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed,
+ VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER);
+
+ if (flow_control.state == FCS_DELAYED &&
+ !VM_PAGE_Q_THROTTLED(iq)) {
+ flow_control.state = FCS_IDLE;
+ goto consider_inactive;
+ }
+
+ if (vm_page_free_count >= vm_page_free_target) {
+ /*
+ * we're here because
+ * 1) someone else freed up some pages while we had
+ * the queues unlocked above
+ * and we've hit one of the 3 conditions that
+ * cause us to pause the pageout scan thread
+ *
+ * since we already have enough free pages,
+ * let's avoid stalling and return normally
+ *
+ * before we return, make sure the pageout I/O threads
+ * are running throttled in case there are still requests
+ * in the laundry... since we have enough free pages
+ * we don't need the laundry to be cleaned in a timely
+ * fashion... so let's avoid interfering with foreground
+ * activity
+ *
+ * we don't want to hold vm_page_queue_free_lock when
+ * calling vm_pageout_adjust_eq_iothrottle (since it
+ * may cause other locks to be taken), we do the intitial
+ * check outside of the lock. Once we take the lock,
+ * we recheck the condition since it may have changed.
+ * if it has, no problem, we will make the threads
+ * non-throttled before actually blocking
+ */
+ vm_pageout_adjust_eq_iothrottle(eq, TRUE);
+ }
+ lck_mtx_lock(&vm_page_queue_free_lock);
+
+ if (vm_page_free_count >= vm_page_free_target &&
+ (vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) {
+ goto return_from_scan;
+ }
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+
+ if ((vm_page_free_count + vm_page_cleaned_count) < vm_page_free_target) {
+ /*
+ * we're most likely about to block due to one of
+ * the 3 conditions that cause vm_pageout_scan to
+ * not be able to make forward progress w/r
+ * to providing new pages to the free queue,
+ * so unthrottle the I/O threads in case we
+ * have laundry to be cleaned... it needs
+ * to be completed ASAP.
+ *
+ * even if we don't block, we want the io threads
+ * running unthrottled since the sum of free +
+ * clean pages is still under our free target
+ */
+ vm_pageout_adjust_eq_iothrottle(eq, FALSE);
+ }
+ if (vm_page_cleaned_count > 0 && exceeded_burst_throttle == FALSE) {
+ /*
+ * if we get here we're below our free target and
+ * we're stalling due to a full laundry queue or
+ * we don't have any inactive pages other then
+ * those in the clean queue...
+ * however, we have pages on the clean queue that
+ * can be moved to the free queue, so let's not
+ * stall the pageout scan
+ */
+ flow_control.state = FCS_IDLE;
+ goto consider_inactive;
+ }
+ VM_CHECK_MEMORYSTATUS;
+
+ if (flow_control.state != FCS_IDLE)
+ vm_pageout_scan_throttle++;
+ iq->pgo_throttled = TRUE;
+
+ assert_wait_timeout((event_t) &iq->pgo_laundry, THREAD_INTERRUPTIBLE, msecs, 1000*NSEC_PER_USEC);
+ counter(c_vm_pageout_scan_block++);
+
+ vm_page_unlock_queues();
+
+ assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
+
+ VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START,
+ iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0);
+ memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START);
+
+ thread_block(THREAD_CONTINUE_NULL);
+
+ VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END,
+ iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0);
+ memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END);
+
+ vm_page_lock_queues();
+
+ iq->pgo_throttled = FALSE;
+
+ if (loop_count >= vm_page_inactive_count)
+ loop_count = 0;
+ inactive_burst_count = 0;
+
+ goto Restart;
+ /*NOTREACHED*/
+ }
+
+
+ flow_control.state = FCS_IDLE;
+consider_inactive:
+ vm_pageout_inactive_external_forced_reactivate_limit = MIN((vm_page_active_count + vm_page_inactive_count),
+ vm_pageout_inactive_external_forced_reactivate_limit);
+ loop_count++;
+ inactive_burst_count++;
+ vm_pageout_inactive++;
+
+
+ /*
+ * Choose a victim.
+ */
+ while (1) {
+ uint32_t inactive_external_count;
+
+#if CONFIG_BACKGROUND_QUEUE
+ page_from_bg_q = FALSE;
+#endif /* CONFIG_BACKGROUND_QUEUE */
+
+ m = NULL;
+ m_object = VM_OBJECT_NULL;
+
+ if (VM_DYNAMIC_PAGING_ENABLED()) {
+ assert(vm_page_throttled_count == 0);
+ assert(vm_page_queue_empty(&vm_page_queue_throttled));
+ }
+
+ /*
+ * Try for a clean-queue inactive page.
+ * These are pages that vm_pageout_scan tried to steal earlier, but
+ * were dirty and had to be cleaned. Pick them up now that they are clean.
+ */
+ if (!vm_page_queue_empty(&vm_page_queue_cleaned)) {
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);
+
+ assert(m->vm_page_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q);
+
+ break;
+ }
+
+ /*
+ * The next most eligible pages are ones we paged in speculatively,
+ * but which have not yet been touched and have been aged out.
+ */
+ if (!vm_page_queue_empty(&sq->age_q)) {
+ m = (vm_page_t) vm_page_queue_first(&sq->age_q);
+
+ assert(m->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q);
+
+ if (!m->dirty || force_anonymous == FALSE)
+ break;
+ else
+ m = NULL;
+ }
+
+#if CONFIG_BACKGROUND_QUEUE
+ if (vm_page_background_mode != VM_PAGE_BG_DISABLED && (vm_page_background_count > vm_page_background_target)) {
+ vm_object_t bg_m_object = NULL;
+
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_background);
+
+ bg_m_object = VM_PAGE_OBJECT(m);
+
+ if (!VM_PAGE_PAGEABLE(m)) {
+ /*
+ * This page is on the background queue
+ * but not on a pageable queue. This is
+ * likely a transient state and whoever
+ * took it out of its pageable queue
+ * will likely put it back on a pageable
+ * queue soon but we can't deal with it
+ * at this point, so let's ignore this
+ * page.
+ */
+ } else if (force_anonymous == FALSE || bg_m_object->internal) {
+
+ if (bg_m_object->internal &&
+ ((vm_compressor_out_of_space() == TRUE) ||
+ (vm_page_free_count < (vm_page_free_reserved / 4)))) {
+
+ vm_pageout_skipped_bq_internal++;
+ } else {
+ page_from_bg_q = TRUE;
+
+ if (bg_m_object->internal)
+ vm_pageout_considered_bq_internal++;
+ else
+ vm_pageout_considered_bq_external++;
+
+ break;
+ }
+ }
+ }
+#endif
+
+ grab_anonymous = (vm_page_anonymous_count > vm_page_anonymous_min);
+ inactive_external_count = vm_page_inactive_count - vm_page_anonymous_count;
+
+ if ((vm_page_pageable_external_count < vm_page_filecache_min || force_anonymous == TRUE) ||
+ ((inactive_external_count < vm_page_anonymous_count) && (inactive_external_count < (vm_page_pageable_external_count / 3)))) {
+ grab_anonymous = TRUE;
+ anons_grabbed = 0;
+
+ vm_pageout_skipped_external++;
+ goto want_anonymous;
+ }
+#if CONFIG_JETSAM
+ /* If the file-backed pool has accumulated
+ * significantly more pages than the jetsam
+ * threshold, prefer to reclaim those
+ * inline to minimise compute overhead of reclaiming
+ * anonymous pages.
+ * This calculation does not account for the CPU local
+ * external page queues, as those are expected to be
+ * much smaller relative to the global pools.
+ */
+ if (grab_anonymous == TRUE && !VM_PAGE_Q_THROTTLED(eq)) {
+ if (vm_page_pageable_external_count >
+ vm_page_filecache_min) {
+ if ((vm_page_pageable_external_count *
+ vm_pageout_memorystatus_fb_factor_dr) >
+ (memorystatus_available_pages_critical *
+ vm_pageout_memorystatus_fb_factor_nr)) {
+ grab_anonymous = FALSE;
+#if DEVELOPMENT || DEBUG
+ vm_grab_anon_overrides++;
+#endif
+ }
+ }
+#if DEVELOPMENT || DEBUG
+ if (grab_anonymous) {
+ vm_grab_anon_nops++;
+ }
+#endif
+ }
+#endif /* CONFIG_JETSAM */
+
+want_anonymous:
+ if (grab_anonymous == FALSE || anons_grabbed >= ANONS_GRABBED_LIMIT || vm_page_queue_empty(&vm_page_queue_anonymous)) {
+
+ if ( !vm_page_queue_empty(&vm_page_queue_inactive) ) {
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
+
+ assert(m->vm_page_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q);
+ anons_grabbed = 0;
+
+ if (vm_page_pageable_external_count < vm_page_filecache_min) {
+ if ((++reactivated_this_call % 100))
+ goto must_activate_page;
+ /*
+ * steal 1% of the file backed pages even if
+ * we are under the limit that has been set
+ * for a healthy filecache
+ */
+ }
+ break;
+ }
+ }
+ if ( !vm_page_queue_empty(&vm_page_queue_anonymous) ) {
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
+
+ assert(m->vm_page_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q);
+ anons_grabbed++;
+
+ break;
+ }
+
+ /*
+ * if we've gotten here, we have no victim page.
+ * check to see if we've not finished balancing the queues
+ * or we have a page on the aged speculative queue that we
+ * skipped due to force_anonymous == TRUE.. or we have
+ * speculative pages that we can prematurely age... if
+ * one of these cases we'll keep going, else panic
+ */
+ force_anonymous = FALSE;
+ vm_pageout_no_victim++;
+
+ if ((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target)
+ goto done_with_inactivepage;
+
+ if (!vm_page_queue_empty(&sq->age_q))
+ goto done_with_inactivepage;
+
+ if (vm_page_speculative_count) {
+ force_speculative_aging = TRUE;
+ goto done_with_inactivepage;
+ }
+ panic("vm_pageout: no victim");
+
+ /* NOTREACHED */
+ }
+ assert(VM_PAGE_PAGEABLE(m));
+ m_object = VM_PAGE_OBJECT(m);
+ force_anonymous = FALSE;
+
+ page_prev_q_state = m->vm_page_q_state;
+ /*
+ * we just found this page on one of our queues...
+ * it can't also be on the pageout queue, so safe
+ * to call vm_page_queues_remove
+ */
+ vm_page_queues_remove(m, TRUE);
+
+ assert(!m->laundry);
+ assert(!m->private);
+ assert(!m->fictitious);
+ assert(m_object != kernel_object);
+ assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr);
+
+ vm_pageout_stats[vm_pageout_stat_now].considered++;
+ vm_pageout_considered_page++;
+
+ DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
+
+ /*
+ * check to see if we currently are working
+ * with the same object... if so, we've
+ * already got the lock
+ */
+ if (m_object != object) {
+ /*
+ * the object associated with candidate page is
+ * different from the one we were just working
+ * with... dump the lock if we still own it
+ */
+ if (object != NULL) {
+ vm_object_unlock(object);
+ object = NULL;
+ vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+ }
+ /*
+ * Try to lock object; since we've alread got the
+ * page queues lock, we can only 'try' for this one.
+ * if the 'try' fails, we need to do a mutex_pause
+ * to allow the owner of the object lock a chance to
+ * run... otherwise, we're likely to trip over this
+ * object in the same state as we work our way through
+ * the queue... clumps of pages associated with the same
+ * object are fairly typical on the inactive and active queues
+ */
+ if (!vm_object_lock_try_scan(m_object)) {
+ vm_page_t m_want = NULL;
+
+ vm_pageout_inactive_nolock++;
+
+ if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q)
+ vm_pageout_cleaned_nolock++;
+
+ pmap_clear_reference(VM_PAGE_GET_PHYS_PAGE(m));
+ m->reference = FALSE;
+
+#if !CONFIG_EMBEDDED
+ /*
+ * m->object must be stable since we hold the page queues lock...
+ * we can update the scan_collisions field sans the object lock
+ * since it is a separate field and this is the only spot that does
+ * a read-modify-write operation and it is never executed concurrently...
+ * we can asynchronously set this field to 0 when creating a UPL, so it
+ * is possible for the value to be a bit non-determistic, but that's ok
+ * since it's only used as a hint
+ */
+
+ /*
+ * This is not used on EMBEDDED because having this variable set *could* lead
+ * us to self-cannibalize pages from m_object to fill a UPL for a pagein.
+ * And, there's a high probability that the object that vm_pageout_scan
+ * wants and collides on is a very popular object e.g. the shared cache on EMBEDDED.
+ * The older pages that we cannibalize from the shared cache could be really
+ * important text pages e.g. the system call stubs.
+ */
+ m_object->scan_collisions = 1;
+#endif /* !CONFIG_EMBEDDED */
+
+ if ( !vm_page_queue_empty(&sq->age_q) )
+ m_want = (vm_page_t) vm_page_queue_first(&sq->age_q);
+ else if ( !vm_page_queue_empty(&vm_page_queue_cleaned))
+ m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);
+ else if ( !vm_page_queue_empty(&vm_page_queue_inactive) &&
+ (anons_grabbed >= ANONS_GRABBED_LIMIT || vm_page_queue_empty(&vm_page_queue_anonymous)))
+ m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
+ else if ( !vm_page_queue_empty(&vm_page_queue_anonymous))
+ m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
+
+ /*
+ * this is the next object we're going to be interested in
+ * try to make sure its available after the mutex_yield
+ * returns control
+ */
+ if (m_want)
+ vm_pageout_scan_wants_object = VM_PAGE_OBJECT(m_want);
+
+ /*
+ * force us to dump any collected free pages
+ * and to pause before moving on
+ */
+ try_failed = TRUE;
+
+ goto requeue_page;
+ }
+ object = m_object;
+ vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+
+ try_failed = FALSE;
+ }
+ assert(m_object == object);
+ assert(VM_PAGE_OBJECT(m) == m_object);
+
+ if (m->busy) {
+ /*
+ * Somebody is already playing with this page.
+ * Put it back on the appropriate queue
+ *
+ */
+ vm_pageout_inactive_busy++;
+
+ if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q)
+ vm_pageout_cleaned_busy++;
+requeue_page:
+ if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q)
+ vm_page_enqueue_inactive(m, FALSE);
+ else
+ vm_page_activate(m);
+#if CONFIG_BACKGROUND_QUEUE
+ if (page_from_bg_q == TRUE) {
+ if (m_object->internal)
+ vm_pageout_rejected_bq_internal++;
+ else
+ vm_pageout_rejected_bq_external++;
+ }
+#endif
+ goto done_with_inactivepage;
+ }
+
+
+ /*
+ * If it's absent, in error or the object is no longer alive,
+ * we can reclaim the page... in the no longer alive case,
+ * there are 2 states the page can be in that preclude us
+ * from reclaiming it - busy or cleaning - that we've already
+ * dealt with
+ */
+ if (m->absent || m->error || !object->alive) {
+
+ if (m->absent)
+ vm_pageout_inactive_absent++;
+ else if (!object->alive)
+ vm_pageout_inactive_notalive++;
+ else
+ vm_pageout_inactive_error++;
+reclaim_page:
+ if (vm_pageout_deadlock_target) {
+ vm_pageout_scan_inactive_throttle_success++;
+ vm_pageout_deadlock_target--;
+ }
+
+ DTRACE_VM2(dfree, int, 1, (uint64_t *), NULL);
+
+ if (object->internal) {
+ DTRACE_VM2(anonfree, int, 1, (uint64_t *), NULL);
+ } else {
+ DTRACE_VM2(fsfree, int, 1, (uint64_t *), NULL);
+ }
+ assert(!m->cleaning);
+ assert(!m->laundry);
+
+ m->busy = TRUE;
+
+ /*
+ * remove page from object here since we're already
+ * behind the object lock... defer the rest of the work
+ * we'd normally do in vm_page_free_prepare_object
+ * until 'vm_page_free_list' is called
+ */
+ if (m->tabled)
+ vm_page_remove(m, TRUE);
+
+ assert(m->pageq.next == 0 && m->pageq.prev == 0);
+ m->snext = local_freeq;
+ local_freeq = m;
+ local_freed++;
+
+ if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q)
+ vm_pageout_freed_from_speculative++;
+ else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q)
+ vm_pageout_freed_from_cleaned++;
+ else
+ vm_pageout_freed_from_inactive_clean++;
+
+ vm_pageout_stats[vm_pageout_stat_now].reclaimed_clean++;
+
+ inactive_burst_count = 0;
+ goto done_with_inactivepage;
+ }
+ /*
+ * If the object is empty, the page must be reclaimed even
+ * if dirty or used.
+ * If the page belongs to a volatile object, we stick it back
+ * on.
+ */
+ if (object->copy == VM_OBJECT_NULL) {
+ if (object->purgable == VM_PURGABLE_EMPTY) {
+ if (m->pmapped == TRUE) {
+ /* unmap the page */
+ refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
+ if (refmod_state & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(m, FALSE);
+ }
+ }
+ if (m->dirty || m->precious) {
+ /* we saved the cost of cleaning this page ! */
+ vm_page_purged_count++;
+ }
+ goto reclaim_page;
+ }
+
+ if (VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
+ /*
+ * With the VM compressor, the cost of
+ * reclaiming a page is much lower (no I/O),
+ * so if we find a "volatile" page, it's better
+ * to let it get compressed rather than letting
* it occupy a full page until it gets purged.
* So no need to check for "volatile" here.
*/
/* just stick it back on! */
reactivated_this_call++;
- if (page_prev_state == PAGE_STATE_CLEAN)
+ if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q)
vm_pageout_cleaned_volatile_reactivated++;
goto reactivate_page;
}
}
-
-consider_inactive_page:
- if (m->busy) {
- /*
- * CAUTION CAUTION:
- * A "busy" page should always be left alone, except...
- */
- if (m->cleaning && m->encrypted_cleaning) {
- /*
- * ENCRYPTED_SWAP:
- * We could get here with a "busy" page
- * if it's being encrypted during a
- * "clean-in-place" operation. We'll deal
- * with it right away by testing if it has been
- * referenced and either reactivating it or
- * promoting it from "clean-in-place" to
- * "clean-and-free".
- */
- } else {
- panic("\"busy\" page considered for pageout\n");
- }
- }
-
/*
* If it's being used, reactivate.
* (Fictitious pages are either busy or absent.)
refmod_state = -1;
if (m->reference == FALSE && m->pmapped == TRUE) {
- refmod_state = pmap_get_refmod(m->phys_page);
+ refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
if (refmod_state & VM_MEM_REFERENCED)
m->reference = TRUE;
}
/*
- * if (m->cleaning && !m->pageout)
+ * if (m->cleaning && !m->free_when_done)
* If already cleaning this page in place and it hasn't
* been recently referenced, just pull off the queue.
* We can leave the page mapped, and upl_commit_range
* will put it on the clean queue.
*
- * note: if m->encrypted_cleaning == TRUE, then
- * m->cleaning == TRUE
- * and we'll handle it here
- *
- * if (m->pageout && !m->cleaning)
+ * if (m->free_when_done && !m->cleaning)
* an msync INVALIDATE is in progress...
* this page has been marked for destruction
* after it has been cleaned,
* where 'cleaning' will be set...
* just leave it off the paging queues
*
- * if (m->pageout && m->clenaing)
+ * if (m->free_when_done && m->clenaing)
* an msync INVALIDATE is in progress
* and the UPL has already gathered this page...
* just leave it off the paging queues
*/
/*
- * page with m->pageout and still on the queues means that an
+ * page with m->free_when_done and still on the queues means that an
* MS_INVALIDATE is in progress on this page... leave it alone
*/
- if (m->pageout) {
+ if (m->free_when_done) {
goto done_with_inactivepage;
}
if (m->reference || m->dirty) {
/* deal with a rogue "reusable" page */
- VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m);
+ VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, m_object);
}
if (!m->no_cache &&
+#if CONFIG_BACKGROUND_QUEUE
+ page_from_bg_q == FALSE &&
+#endif
(m->reference ||
(m->xpmapped && !object->internal && (vm_page_xpmapped_external_count < (vm_page_external_count / 4))))) {
/*
*/
if (++reactivated_this_call >= reactivate_limit) {
vm_pageout_reactivation_limit_exceeded++;
- } else if (catch_up_count) {
- vm_pageout_catch_ups++;
} else if (++inactive_reclaim_run >= VM_PAGEOUT_INACTIVE_FORCE_RECLAIM) {
vm_pageout_inactive_force_reclaim++;
} else {
uint32_t isinuse;
- if (page_prev_state == PAGE_STATE_CLEAN)
+ if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q)
vm_pageout_cleaned_reference_reactivated++;
-
reactivate_page:
if ( !object->internal && object->pager != MEMORY_OBJECT_NULL &&
vnode_pager_get_isinuse(object->pager, &isinuse) == KERN_SUCCESS && !isinuse) {
VM_STAT_INCR(reactivations);
inactive_burst_count = 0;
}
-
- if (page_prev_state == PAGE_STATE_CLEAN)
+#if CONFIG_BACKGROUND_QUEUE
+ if (page_from_bg_q == TRUE) {
+ if (m_object->internal)
+ vm_pageout_rejected_bq_internal++;
+ else
+ vm_pageout_rejected_bq_external++;
+ }
+#endif
+ if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q)
vm_pageout_cleaned_reactivated++;
-
vm_pageout_inactive_used++;
goto done_with_inactivepage;
* the dirty bit.
*/
if ((refmod_state == -1) && !m->dirty && m->pmapped) {
- refmod_state = pmap_get_refmod(m->phys_page);
+ refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
if (refmod_state & VM_MEM_MODIFIED) {
SET_PAGE_DIRTY(m, FALSE);
}
}
- forced_reclaim = TRUE;
- } else {
- forced_reclaim = FALSE;
}
XPR(XPR_VM_PAGEOUT,
}
}
throttle_inactive:
- if (!VM_DYNAMIC_PAGING_ENABLED(memory_manager_default) &&
+ if (!VM_DYNAMIC_PAGING_ENABLED() &&
object->internal && m->dirty &&
(object->purgable == VM_PURGABLE_DENY ||
object->purgable == VM_PURGABLE_NONVOLATILE ||
object->purgable == VM_PURGABLE_VOLATILE)) {
- queue_enter(&vm_page_queue_throttled, m,
- vm_page_t, pageq);
- m->throttled = TRUE;
+ vm_page_check_pageable_safe(m);
+ assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q);
+ vm_page_queue_enter(&vm_page_queue_throttled, m,
+ vm_page_t, pageq);
+ m->vm_page_q_state = VM_PAGE_ON_THROTTLED_Q;
vm_page_throttled_count++;
vm_pageout_scan_reclaimed_throttled++;
* the deadlock will break, the external pageout queue will empty and it will no longer
* be throttled
*
- * if we have jestam configured, keep a count of the pages reactivated this way so
+ * if we have jetsam configured, keep a count of the pages reactivated this way so
* that we can try to find clean pages in the active/inactive queues before
* deciding to jetsam a process
*/
- vm_pageout_scan_inactive_throttled_external++;
+ vm_pageout_scan_inactive_throttled_external++;
- queue_enter(&vm_page_queue_active, m, vm_page_t, pageq);
- m->active = TRUE;
+ vm_page_check_pageable_safe(m);
+ assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q);
+ vm_page_queue_enter(&vm_page_queue_active, m, vm_page_t, pageq);
+ m->vm_page_q_state = VM_PAGE_ON_ACTIVE_Q;
vm_page_active_count++;
vm_page_pageable_external_count++;
- vm_pageout_adjust_io_throttles(iq, eq, FALSE);
+ vm_pageout_adjust_eq_iothrottle(eq, FALSE);
#if CONFIG_MEMORYSTATUS && CONFIG_JETSAM
vm_pageout_inactive_external_forced_reactivate_limit--;
vm_object_unlock(object);
object = VM_OBJECT_NULL;
vm_page_unlock_queues();
-
- VM_DEBUG_EVENT(vm_pageout_jetsam, VM_PAGEOUT_JETSAM, DBG_FUNC_START,
+
+ VM_DEBUG_CONSTANT_EVENT(vm_pageout_jetsam, VM_PAGEOUT_JETSAM, DBG_FUNC_START,
vm_page_active_count, vm_page_inactive_count, vm_page_free_count, vm_page_free_count);
- /* Kill first suitable process */
- if (memorystatus_kill_on_VM_page_shortage(FALSE) == FALSE) {
- panic("vm_pageout_scan: Jetsam request failed\n");
+ /* Kill first suitable process. If this call returned FALSE, we might have simply purged a process instead. */
+ if (memorystatus_kill_on_VM_page_shortage(FALSE) == TRUE) {
+ vm_pageout_inactive_external_forced_jetsam_count++;
}
- VM_DEBUG_EVENT(vm_pageout_jetsam, VM_PAGEOUT_JETSAM, DBG_FUNC_END, 0, 0, 0, 0);
+ VM_DEBUG_CONSTANT_EVENT(vm_pageout_jetsam, VM_PAGEOUT_JETSAM, DBG_FUNC_END,
+ vm_page_active_count, vm_page_inactive_count, vm_page_free_count, vm_page_free_count);
- vm_pageout_inactive_external_forced_jetsam_count++;
vm_page_lock_queues();
delayed_unlock = 1;
}
inactive_burst_count = 0;
goto done_with_inactivepage;
} else {
- if (page_prev_state == PAGE_STATE_SPECULATIVE)
- page_prev_state = PAGE_STATE_INACTIVE;
-
vm_pageout_scan_inactive_throttled_internal++;
-
- goto requeue_page;
+ goto must_activate_page;
}
}
* of likely usage of the page.
*/
if (m->pmapped == TRUE) {
+ int pmap_options;
- if (DEFAULT_PAGER_IS_ACTIVE || DEFAULT_FREEZER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE || object->internal == FALSE) {
+ /*
+ * Don't count this page as going into the compressor
+ * if any of these are true:
+ * 1) compressed pager isn't enabled
+ * 2) Freezer enabled device with compressed pager
+ * backend (exclusive use) i.e. most of the VM system
+ * (including vm_pageout_scan) has no knowledge of
+ * the compressor
+ * 3) This page belongs to a file and hence will not be
+ * sent into the compressor
+ */
+ if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE ||
+ object->internal == FALSE) {
+ pmap_options = 0;
+ } else if (m->dirty || m->precious) {
/*
- * Don't count this page as going into the compressor if any of these are true:
- * 1) We have the dynamic pager i.e. no compressed pager
- * 2) Freezer enabled device with a freezer file to hold the app data i.e. no compressed pager
- * 3) Freezer enabled device with compressed pager backend (exclusive use) i.e. most of the VM system
- (including vm_pageout_scan) has no knowledge of the compressor
- * 4) This page belongs to a file and hence will not be sent into the compressor
+ * VM knows that this page is dirty (or
+ * precious) and needs to be compressed
+ * rather than freed.
+ * Tell the pmap layer to count this page
+ * as "compressed".
*/
-
- refmod_state = pmap_disconnect_options(m->phys_page, 0, NULL);
+ pmap_options = PMAP_OPTIONS_COMPRESSOR;
} else {
- refmod_state = pmap_disconnect_options(m->phys_page, PMAP_OPTIONS_COMPRESSOR, NULL);
+ /*
+ * VM does not know if the page needs to
+ * be preserved but the pmap layer might tell
+ * us if any mapping has "modified" it.
+ * Let's the pmap layer to count this page
+ * as compressed if and only if it has been
+ * modified.
+ */
+ pmap_options =
+ PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED;
}
-
+ refmod_state = pmap_disconnect_options(VM_PAGE_GET_PHYS_PAGE(m),
+ pmap_options,
+ NULL);
if (refmod_state & VM_MEM_MODIFIED) {
SET_PAGE_DIRTY(m, FALSE);
}
*/
if (!m->dirty && !m->precious) {
- if (page_prev_state == PAGE_STATE_SPECULATIVE)
+ if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q)
vm_pageout_speculative_clean++;
else {
- if (page_prev_state == PAGE_STATE_ANONYMOUS)
+ if (page_prev_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q)
vm_pageout_inactive_anonymous++;
- else if (page_prev_state == PAGE_STATE_CLEAN)
+ else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q)
vm_pageout_cleaned_reclaimed++;
vm_pageout_inactive_clean++;
}
-
/*
* OK, at this point we have found a page we are going to free.
*/
#endif /* CONFIG_JETSAM */
#endif /* VM_PRESSURE_EVENTS */
- /*
- * do NOT set the pageout bit!
- * sure, we might need free pages, but this page is going to take time to become free
- * anyway, so we may as well put it on the clean queue first and take it from there later
- * if necessary. that way, we'll ensure we don't free up too much. -mj
- */
- vm_pageout_cluster(m, FALSE);
-
- if (page_prev_state == PAGE_STATE_ANONYMOUS)
+ if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q)
+ vm_pageout_speculative_dirty++;
+ else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q)
vm_pageout_inactive_anonymous++;
+
if (object->internal)
vm_pageout_inactive_dirty_internal++;
else
vm_pageout_inactive_dirty_external++;
+ /*
+ * do NOT set the pageout bit!
+ * sure, we might need free pages, but this page is going to take time to become free
+ * anyway, so we may as well put it on the clean queue first and take it from there later
+ * if necessary. that way, we'll ensure we don't free up too much. -mj
+ */
+ vm_pageout_cluster(m);
done_with_inactivepage:
if (delayed_unlock++ > delayed_unlock_limit || try_failed == TRUE) {
- boolean_t need_delay = TRUE;
-
- if (object != NULL) {
- vm_pageout_scan_wants_object = VM_OBJECT_NULL;
- vm_object_unlock(object);
- object = NULL;
- }
- vm_page_unlock_queues();
-
- if (local_freeq) {
-
- VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START,
- vm_page_free_count, local_freed, delayed_unlock_limit, 4);
-
- vm_page_free_list(local_freeq, TRUE);
-
- VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END,
- vm_page_free_count, local_freed, 0, 4);
-
- local_freeq = NULL;
- local_freed = 0;
- need_delay = FALSE;
- }
- if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) {
- vm_consider_waking_compactor_swapper();
- need_delay = FALSE;
- }
- vm_page_lock_queues();
- if (need_delay == TRUE)
+ vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed,
+ VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER);
+ if (try_failed == TRUE)
lck_mtx_yield(&vm_page_queue_lock);
-
- delayed_unlock = 1;
}
- vm_pageout_considered_page++;
/*
* back to top of pageout scan loop
{
int free_after_reserve;
- if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) {
+ if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {
if ((vm_page_free_reserved + pages + COMPRESSOR_FREE_RESERVED_LIMIT) >= (VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT))
vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT;
VM_PAGE_FREE_TARGET(free_after_reserve);
if (vm_page_free_target > VM_PAGE_FREE_TARGET_LIMIT)
- vm_page_free_target = VM_PAGE_FREE_TARGET_LIMIT;
-
- if (vm_page_free_target < vm_page_free_min + 5)
- vm_page_free_target = vm_page_free_min + 5;
-
- vm_page_throttle_limit = vm_page_free_target - (vm_page_free_target / 3);
-}
-
-/*
- * vm_pageout is the high level pageout daemon.
- */
-
-void
-vm_pageout_continue(void)
-{
- DTRACE_VM2(pgrrun, int, 1, (uint64_t *), NULL);
- vm_pageout_scan_event_counter++;
-
- vm_pageout_scan();
- /*
- * we hold both the vm_page_queue_free_lock
- * and the vm_page_queues_lock at this point
- */
- assert(vm_page_free_wanted == 0);
- assert(vm_page_free_wanted_privileged == 0);
- assert_wait((event_t) &vm_page_free_wanted, THREAD_UNINT);
-
- lck_mtx_unlock(&vm_page_queue_free_lock);
- vm_page_unlock_queues();
-
- counter(c_vm_pageout_block++);
- thread_block((thread_continue_t)vm_pageout_continue);
- /*NOTREACHED*/
-}
-
-
-#ifdef FAKE_DEADLOCK
-
-#define FAKE_COUNT 5000
-
-int internal_count = 0;
-int fake_deadlock = 0;
-
-#endif
-
-static void
-vm_pageout_iothread_continue(struct vm_pageout_queue *q)
-{
- vm_page_t m = NULL;
- vm_object_t object;
- vm_object_offset_t offset;
- memory_object_t pager;
- thread_t self = current_thread();
-
- if ((vm_pageout_internal_iothread != THREAD_NULL)
- && (self == vm_pageout_external_iothread )
- && (self->options & TH_OPT_VMPRIV))
- self->options &= ~TH_OPT_VMPRIV;
-
- vm_page_lockspin_queues();
-
- while ( !queue_empty(&q->pgo_pending) ) {
-
- q->pgo_busy = TRUE;
- queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq);
- if (m->object->object_slid) {
- panic("slid page %p not allowed on this path\n", m);
- }
- VM_PAGE_CHECK(m);
- m->pageout_queue = FALSE;
- m->pageq.next = NULL;
- m->pageq.prev = NULL;
-
- /*
- * grab a snapshot of the object and offset this
- * page is tabled in so that we can relookup this
- * page after we've taken the object lock - these
- * fields are stable while we hold the page queues lock
- * but as soon as we drop it, there is nothing to keep
- * this page in this object... we hold an activity_in_progress
- * on this object which will keep it from terminating
- */
- object = m->object;
- offset = m->offset;
-
- vm_page_unlock_queues();
-
-#ifdef FAKE_DEADLOCK
- if (q == &vm_pageout_queue_internal) {
- vm_offset_t addr;
- int pg_count;
-
- internal_count++;
-
- if ((internal_count == FAKE_COUNT)) {
-
- pg_count = vm_page_free_count + vm_page_free_reserved;
-
- if (kmem_alloc(kernel_map, &addr, PAGE_SIZE * pg_count) == KERN_SUCCESS) {
- kmem_free(kernel_map, addr, PAGE_SIZE * pg_count);
- }
- internal_count = 0;
- fake_deadlock++;
- }
- }
-#endif
- vm_object_lock(object);
-
- m = vm_page_lookup(object, offset);
-
- if (m == NULL ||
- m->busy || m->cleaning || m->pageout_queue || !m->laundry) {
- /*
- * it's either the same page that someone else has
- * started cleaning (or it's finished cleaning or
- * been put back on the pageout queue), or
- * the page has been freed or we have found a
- * new page at this offset... in all of these cases
- * we merely need to release the activity_in_progress
- * we took when we put the page on the pageout queue
- */
- vm_object_activity_end(object);
- vm_object_unlock(object);
-
- vm_page_lockspin_queues();
- continue;
- }
- if (!object->pager_initialized) {
-
- /*
- * If there is no memory object for the page, create
- * one and hand it to the default pager.
- */
-
- if (!object->pager_initialized)
- vm_object_collapse(object,
- (vm_object_offset_t) 0,
- TRUE);
- if (!object->pager_initialized)
- vm_object_pager_create(object);
- if (!object->pager_initialized) {
- /*
- * Still no pager for the object.
- * Reactivate the page.
- *
- * Should only happen if there is no
- * default pager.
- */
- m->pageout = FALSE;
-
- vm_page_lockspin_queues();
-
- vm_pageout_throttle_up(m);
- vm_page_activate(m);
- vm_pageout_dirty_no_pager++;
-
- vm_page_unlock_queues();
-
- /*
- * And we are done with it.
- */
- vm_object_activity_end(object);
- vm_object_unlock(object);
-
- vm_page_lockspin_queues();
- continue;
- }
- }
- pager = object->pager;
-
- if (pager == MEMORY_OBJECT_NULL) {
- /*
- * This pager has been destroyed by either
- * memory_object_destroy or vm_object_destroy, and
- * so there is nowhere for the page to go.
- */
- if (m->pageout) {
- /*
- * Just free the page... VM_PAGE_FREE takes
- * care of cleaning up all the state...
- * including doing the vm_pageout_throttle_up
- */
- VM_PAGE_FREE(m);
- } else {
- vm_page_lockspin_queues();
-
- vm_pageout_throttle_up(m);
- vm_page_activate(m);
-
- vm_page_unlock_queues();
-
- /*
- * And we are done with it.
- */
- }
- vm_object_activity_end(object);
- vm_object_unlock(object);
-
- vm_page_lockspin_queues();
- continue;
- }
-#if 0
- /*
- * we don't hold the page queue lock
- * so this check isn't safe to make
- */
- VM_PAGE_CHECK(m);
-#endif
- /*
- * give back the activity_in_progress reference we
- * took when we queued up this page and replace it
- * it with a paging_in_progress reference that will
- * also hold the paging offset from changing and
- * prevent the object from terminating
- */
- vm_object_activity_end(object);
- vm_object_paging_begin(object);
- vm_object_unlock(object);
+ vm_page_free_target = VM_PAGE_FREE_TARGET_LIMIT;
- /*
- * Send the data to the pager.
- * any pageout clustering happens there
- */
- memory_object_data_return(pager,
- m->offset + object->paging_offset,
- PAGE_SIZE,
- NULL,
- NULL,
- FALSE,
- FALSE,
- 0);
+ if (vm_page_free_target < vm_page_free_min + 5)
+ vm_page_free_target = vm_page_free_min + 5;
- vm_object_lock(object);
- vm_object_paging_end(object);
- vm_object_unlock(object);
+ vm_page_throttle_limit = vm_page_free_target - (vm_page_free_target / 2);
+}
- vm_pageout_io_throttle();
+/*
+ * vm_pageout is the high level pageout daemon.
+ */
- vm_page_lockspin_queues();
+void
+vm_pageout_continue(void)
+{
+ DTRACE_VM2(pgrrun, int, 1, (uint64_t *), NULL);
+ vm_pageout_scan_event_counter++;
+
+#if !CONFIG_EMBEDDED
+ lck_mtx_lock(&vm_page_queue_free_lock);
+ vm_pageout_running = TRUE;
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+#endif /* CONFIG_EMBEDDED */
+
+ vm_pageout_scan();
+ /*
+ * we hold both the vm_page_queue_free_lock
+ * and the vm_page_queues_lock at this point
+ */
+ assert(vm_page_free_wanted == 0);
+ assert(vm_page_free_wanted_privileged == 0);
+ assert_wait((event_t) &vm_page_free_wanted, THREAD_UNINT);
+
+#if !CONFIG_EMBEDDED
+ vm_pageout_running = FALSE;
+ if (vm_pageout_waiter) {
+ vm_pageout_waiter = FALSE;
+ thread_wakeup((event_t)&vm_pageout_waiter);
}
- q->pgo_busy = FALSE;
- q->pgo_idle = TRUE;
+#endif /* !CONFIG_EMBEDDED */
- assert_wait((event_t) &q->pgo_pending, THREAD_UNINT);
+ lck_mtx_unlock(&vm_page_queue_free_lock);
vm_page_unlock_queues();
- thread_block_parameter((thread_continue_t)vm_pageout_iothread_continue, (void *) q);
+ counter(c_vm_pageout_block++);
+ thread_block((thread_continue_t)vm_pageout_continue);
/*NOTREACHED*/
}
+#if !CONFIG_EMBEDDED
+kern_return_t
+vm_pageout_wait(uint64_t deadline)
+{
+ kern_return_t kr;
+
+ lck_mtx_lock(&vm_page_queue_free_lock);
+ for (kr = KERN_SUCCESS; vm_pageout_running && (KERN_SUCCESS == kr); ) {
+ vm_pageout_waiter = TRUE;
+ if (THREAD_AWAKENED != lck_mtx_sleep_deadline(
+ &vm_page_queue_free_lock, LCK_SLEEP_DEFAULT,
+ (event_t) &vm_pageout_waiter, THREAD_UNINT, deadline)) {
+ kr = KERN_OPERATION_TIMED_OUT;
+ }
+ }
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+
+ return (kr);
+}
+#endif /* !CONFIG_EMBEDDED */
+
static void
vm_pageout_iothread_external_continue(struct vm_pageout_queue *q)
vm_object_offset_t offset;
memory_object_t pager;
-
+ /* On systems without a compressor, the external IO thread clears its
+ * VM privileged bit to accommodate large allocations (e.g. bulk UPL
+ * creation)
+ */
if (vm_pageout_internal_iothread != THREAD_NULL)
current_thread()->options &= ~TH_OPT_VMPRIV;
vm_page_lockspin_queues();
- while ( !queue_empty(&q->pgo_pending) ) {
+ while ( !vm_page_queue_empty(&q->pgo_pending) ) {
q->pgo_busy = TRUE;
- queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq);
- if (m->object->object_slid) {
- panic("slid page %p not allowed on this path\n", m);
- }
- VM_PAGE_CHECK(m);
- m->pageout_queue = FALSE;
- m->pageq.next = NULL;
- m->pageq.prev = NULL;
+ vm_page_queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq);
+ assert(m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q);
+ VM_PAGE_CHECK(m);
/*
* grab a snapshot of the object and offset this
* page is tabled in so that we can relookup this
* this page in this object... we hold an activity_in_progress
* on this object which will keep it from terminating
*/
- object = m->object;
+ object = VM_PAGE_OBJECT(m);
offset = m->offset;
+ if (object->object_slid) {
+ panic("slid page %p not allowed on this path\n", m);
+ }
+ m->vm_page_q_state = VM_PAGE_NOT_ON_Q;
+ VM_PAGE_ZERO_PAGEQ_ENTRY(m);
+
vm_page_unlock_queues();
vm_object_lock(object);
m = vm_page_lookup(object, offset);
if (m == NULL ||
- m->busy || m->cleaning || m->pageout_queue || !m->laundry) {
+ m->busy || m->cleaning || !m->laundry || (m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q)) {
/*
* it's either the same page that someone else has
* started cleaning (or it's finished cleaning or
* memory_object_destroy or vm_object_destroy, and
* so there is nowhere for the page to go.
*/
- if (m->pageout) {
+ if (m->free_when_done) {
/*
* Just free the page... VM_PAGE_FREE takes
* care of cleaning up all the state...
}
-uint32_t vm_compressor_failed;
+#define MAX_FREE_BATCH 32
+uint32_t vm_compressor_time_thread; /* Set via sysctl to record time accrued by
+ * this thread.
+ */
-static void
+
+#if DEVELOPMENT || DEBUG
+uint64_t compressor_epoch_start, compressor_epoch_stop, compressor_threads_runtime;
+#endif
+
+void
+vm_pageout_iothread_internal_continue(struct cq *);
+void
vm_pageout_iothread_internal_continue(struct cq *cq)
{
struct vm_pageout_queue *q;
vm_page_t m = NULL;
- vm_object_t object;
- memory_object_t pager;
boolean_t pgo_draining;
vm_page_t local_q;
int local_cnt;
vm_page_t local_freeq = NULL;
int local_freed = 0;
int local_batch_size;
- kern_return_t retval;
- int compressed_count_delta;
-
-
+ int ncomps = 0;
+#if DEVELOPMENT || DEBUG
+ boolean_t marked_active = FALSE;
+#endif
KERNEL_DEBUG(0xe040000c | DBG_FUNC_END, 0, 0, 0, 0, 0);
q = cq->q;
- local_batch_size = q->pgo_maxlaundry / (vm_compressor_thread_count * 4);
+ local_batch_size = q->pgo_maxlaundry / (vm_compressor_thread_count * 2);
+#if RECORD_THE_COMPRESSED_DATA
+ if (q->pgo_laundry)
+ c_compressed_record_init();
+#endif
while (TRUE) {
+ int pages_left_on_q = 0;
local_cnt = 0;
local_q = NULL;
KERNEL_DEBUG(0xe0400014 | DBG_FUNC_START, 0, 0, 0, 0, 0);
-
- vm_page_lock_queues();
+ vm_page_lock_queues();
+#if DEVELOPMENT || DEBUG
+ if (marked_active == FALSE) {
+ vmct_active++;
+ vmct_state[cq->id] = VMCT_ACTIVE;
+ marked_active = TRUE;
+ if (vmct_active == 1) {
+ compressor_epoch_start = mach_absolute_time();
+ }
+ }
+#endif
KERNEL_DEBUG(0xe0400014 | DBG_FUNC_END, 0, 0, 0, 0, 0);
- KERNEL_DEBUG(0xe0400018 | DBG_FUNC_START, 0, 0, 0, 0, 0);
+ KERNEL_DEBUG(0xe0400018 | DBG_FUNC_START, q->pgo_laundry, 0, 0, 0, 0);
- while ( !queue_empty(&q->pgo_pending) && local_cnt < local_batch_size) {
-
- queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq);
+ while ( !vm_page_queue_empty(&q->pgo_pending) && local_cnt < local_batch_size) {
+ vm_page_queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq);
+ assert(m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q);
VM_PAGE_CHECK(m);
+
+ m->vm_page_q_state = VM_PAGE_NOT_ON_Q;
+ VM_PAGE_ZERO_PAGEQ_ENTRY(m);
+ m->laundry = FALSE;
- m->pageout_queue = FALSE;
- m->pageq.prev = NULL;
-
- m->pageq.next = (queue_entry_t)local_q;
+ m->snext = local_q;
local_q = m;
local_cnt++;
}
q->pgo_busy = TRUE;
- if ((pgo_draining = q->pgo_draining) == FALSE)
+ if ((pgo_draining = q->pgo_draining) == FALSE) {
vm_pageout_throttle_up_batch(q, local_cnt);
+ pages_left_on_q = q->pgo_laundry;
+ } else
+ pages_left_on_q = q->pgo_laundry - local_cnt;
vm_page_unlock_queues();
- KERNEL_DEBUG(0xe0400018 | DBG_FUNC_END, 0, 0, 0, 0, 0);
+#if !RECORD_THE_COMPRESSED_DATA
+ if (pages_left_on_q >= local_batch_size && cq->id < (vm_compressor_thread_count - 1)) {
+ thread_wakeup((event_t) ((uintptr_t)&q->pgo_pending + cq->id + 1));
+ }
+#endif
+ KERNEL_DEBUG(0xe0400018 | DBG_FUNC_END, q->pgo_laundry, 0, 0, 0, 0);
while (local_q) {
-
- m = local_q;
- local_q = (vm_page_t)m->pageq.next;
- m->pageq.next = NULL;
- if (m->object->object_slid) {
- panic("slid page %p not allowed on this path\n", m);
- }
-
- object = m->object;
- pager = object->pager;
-
- if (!object->pager_initialized || pager == MEMORY_OBJECT_NULL) {
-
- KERNEL_DEBUG(0xe0400010 | DBG_FUNC_START, object, pager, 0, 0, 0);
-
- vm_object_lock(object);
-
- /*
- * If there is no memory object for the page, create
- * one and hand it to the compression pager.
- */
-
- if (!object->pager_initialized)
- vm_object_collapse(object, (vm_object_offset_t) 0, TRUE);
- if (!object->pager_initialized)
- vm_object_compressor_pager_create(object);
-
- if (!object->pager_initialized) {
- /*
- * Still no pager for the object.
- * Reactivate the page.
- *
- * Should only happen if there is no
- * compression pager
- */
- m->pageout = FALSE;
- m->laundry = FALSE;
- PAGE_WAKEUP_DONE(m);
-
- vm_page_lockspin_queues();
- vm_page_activate(m);
- vm_pageout_dirty_no_pager++;
- vm_page_unlock_queues();
-
- /*
- * And we are done with it.
- */
- vm_object_activity_end(object);
- vm_object_unlock(object);
-
- continue;
- }
- pager = object->pager;
+ KERNEL_DEBUG(0xe0400024 | DBG_FUNC_START, local_cnt, 0, 0, 0, 0);
- if (pager == MEMORY_OBJECT_NULL) {
- /*
- * This pager has been destroyed by either
- * memory_object_destroy or vm_object_destroy, and
- * so there is nowhere for the page to go.
- */
- if (m->pageout) {
- /*
- * Just free the page... VM_PAGE_FREE takes
- * care of cleaning up all the state...
- * including doing the vm_pageout_throttle_up
- */
- VM_PAGE_FREE(m);
- } else {
- m->laundry = FALSE;
- PAGE_WAKEUP_DONE(m);
+ m = local_q;
+ local_q = m->snext;
+ m->snext = NULL;
- vm_page_lockspin_queues();
- vm_page_activate(m);
- vm_page_unlock_queues();
+ if (vm_pageout_compress_page(&cq->current_chead, cq->scratch_buf, m, FALSE) == KERN_SUCCESS) {
+ ncomps++;
+ m->snext = local_freeq;
+ local_freeq = m;
+ local_freed++;
- /*
- * And we are done with it.
- */
- }
- vm_object_activity_end(object);
- vm_object_unlock(object);
+ if (local_freed >= MAX_FREE_BATCH) {
+ vm_pageout_freed_after_compression += local_freed;
- continue;
+ vm_page_free_list(local_freeq, TRUE);
+ local_freeq = NULL;
+ local_freed = 0;
}
- vm_object_unlock(object);
-
- KERNEL_DEBUG(0xe0400010 | DBG_FUNC_END, object, pager, 0, 0, 0);
}
- while (vm_page_free_count < (vm_page_free_reserved - COMPRESSOR_FREE_RESERVED_LIMIT)) {
+#if !CONFIG_JETSAM
+ while (vm_page_free_count < COMPRESSOR_FREE_RESERVED_LIMIT) {
kern_return_t wait_result;
int need_wakeup = 0;
if (local_freeq) {
- vm_page_free_list(local_freeq, TRUE);
+ vm_pageout_freed_after_compression += local_freed;
+ vm_page_free_list(local_freeq, TRUE);
local_freeq = NULL;
local_freed = 0;
}
lck_mtx_lock_spin(&vm_page_queue_free_lock);
- if (vm_page_free_count < (vm_page_free_reserved - COMPRESSOR_FREE_RESERVED_LIMIT)) {
-
+ if (vm_page_free_count < COMPRESSOR_FREE_RESERVED_LIMIT) {
+
if (vm_page_free_wanted_privileged++ == 0)
need_wakeup = 1;
wait_result = assert_wait((event_t)&vm_page_free_wanted_privileged, THREAD_UNINT);
thread_wakeup((event_t)&vm_page_free_wanted);
if (wait_result == THREAD_WAITING)
+
thread_block(THREAD_CONTINUE_NULL);
} else
lck_mtx_unlock(&vm_page_queue_free_lock);
}
-
- assert(object->activity_in_progress > 0);
-
- retval = vm_compressor_pager_put(
- pager,
- m->offset + object->paging_offset,
- m->phys_page,
- &cq->current_chead,
- cq->scratch_buf,
- &compressed_count_delta);
-
- vm_object_lock(object);
- assert(object->activity_in_progress > 0);
-
- assert(m->object == object);
-
- vm_compressor_pager_count(pager,
- compressed_count_delta,
- FALSE, /* shared_lock */
- object);
-
- m->laundry = FALSE;
- m->pageout = FALSE;
-
- if (retval == KERN_SUCCESS) {
- /*
- * If the object is purgeable, its owner's
- * purgeable ledgers will be updated in
- * vm_page_remove() but the page still
- * contributes to the owner's memory footprint,
- * so account for it as such.
- */
- if (object->purgable != VM_PURGABLE_DENY &&
- object->vo_purgeable_owner != NULL) {
- /* one more compressed purgeable page */
- vm_purgeable_compressed_update(object,
- +1);
- }
-
- vm_page_compressions_failing = FALSE;
-
- VM_STAT_INCR(compressions);
-
- if (m->tabled)
- vm_page_remove(m, TRUE);
- vm_object_activity_end(object);
- vm_object_unlock(object);
-
- m->pageq.next = (queue_entry_t)local_freeq;
- local_freeq = m;
- local_freed++;
-
- } else {
- PAGE_WAKEUP_DONE(m);
-
- vm_page_lockspin_queues();
-
- vm_page_activate(m);
- vm_compressor_failed++;
-
- vm_page_compressions_failing = TRUE;
-
- vm_page_unlock_queues();
-
- vm_object_activity_end(object);
- vm_object_unlock(object);
- }
+#endif
}
if (local_freeq) {
+ vm_pageout_freed_after_compression += local_freed;
+
vm_page_free_list(local_freeq, TRUE);
-
local_freeq = NULL;
local_freed = 0;
}
q->pgo_busy = FALSE;
q->pgo_idle = TRUE;
- assert_wait((event_t) &q->pgo_pending, THREAD_UNINT);
+ assert_wait((event_t) ((uintptr_t)&q->pgo_pending + cq->id), THREAD_UNINT);
+#if DEVELOPMENT || DEBUG
+ if (marked_active == TRUE) {
+ vmct_active--;
+ vmct_state[cq->id] = VMCT_IDLE;
+
+ if (vmct_active == 0) {
+ compressor_epoch_stop = mach_absolute_time();
+ assert(compressor_epoch_stop > compressor_epoch_start);
+ /* This interval includes intervals where one or more
+ * compressor threads were pre-empted
+ */
+ vmct_stats.vmct_cthreads_total += compressor_epoch_stop - compressor_epoch_start;
+ }
+
+ }
+#endif
vm_page_unlock_queues();
+#if DEVELOPMENT || DEBUG
+ if (__improbable(vm_compressor_time_thread)) {
+ vmct_stats.vmct_runtimes[cq->id] = thread_get_runtime_self();
+ vmct_stats.vmct_pages[cq->id] += ncomps;
+ vmct_stats.vmct_iterations[cq->id]++;
+ if (ncomps > vmct_stats.vmct_maxpages[cq->id]) {
+ vmct_stats.vmct_maxpages[cq->id] = ncomps;
+ }
+ if (ncomps < vmct_stats.vmct_minpages[cq->id]) {
+ vmct_stats.vmct_minpages[cq->id] = ncomps;
+ }
+ }
+#endif
KERNEL_DEBUG(0xe0400018 | DBG_FUNC_END, 0, 0, 0, 0, 0);
}
-
+kern_return_t
+vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m, boolean_t object_locked_by_caller)
+{
+ vm_object_t object;
+ memory_object_t pager;
+ int compressed_count_delta;
+ kern_return_t retval;
+
+ object = VM_PAGE_OBJECT(m);
+
+ if (object->object_slid) {
+ panic("slid page %p not allowed on this path\n", m);
+ }
+ assert(!m->free_when_done);
+ assert(!m->laundry);
+
+ pager = object->pager;
+
+ if (object_locked_by_caller == FALSE && (!object->pager_initialized || pager == MEMORY_OBJECT_NULL)) {
+
+ KERNEL_DEBUG(0xe0400010 | DBG_FUNC_START, object, pager, 0, 0, 0);
+
+ vm_object_lock(object);
+
+ /*
+ * If there is no memory object for the page, create
+ * one and hand it to the compression pager.
+ */
+
+ if (!object->pager_initialized)
+ vm_object_collapse(object, (vm_object_offset_t) 0, TRUE);
+ if (!object->pager_initialized)
+ vm_object_compressor_pager_create(object);
+
+ pager = object->pager;
+
+ if (!object->pager_initialized || pager == MEMORY_OBJECT_NULL) {
+ /*
+ * Still no pager for the object,
+ * or the pager has been destroyed.
+ * Reactivate the page.
+ *
+ * Should only happen if there is no
+ * compression pager
+ */
+ PAGE_WAKEUP_DONE(m);
+
+ vm_page_lockspin_queues();
+ vm_page_activate(m);
+ vm_pageout_dirty_no_pager++;
+ vm_page_unlock_queues();
+
+ /*
+ * And we are done with it.
+ */
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+
+ return KERN_FAILURE;
+ }
+ vm_object_unlock(object);
+
+ KERNEL_DEBUG(0xe0400010 | DBG_FUNC_END, object, pager, 0, 0, 0);
+ }
+ assert(object->pager_initialized && pager != MEMORY_OBJECT_NULL);
+
+ if (object_locked_by_caller == FALSE)
+ assert(object->activity_in_progress > 0);
+
+ retval = vm_compressor_pager_put(
+ pager,
+ m->offset + object->paging_offset,
+ VM_PAGE_GET_PHYS_PAGE(m),
+ current_chead,
+ scratch_buf,
+ &compressed_count_delta);
+
+ if (object_locked_by_caller == FALSE) {
+ vm_object_lock(object);
+
+ assert(object->activity_in_progress > 0);
+ assert(VM_PAGE_OBJECT(m) == object);
+ }
+
+ vm_compressor_pager_count(pager,
+ compressed_count_delta,
+ FALSE, /* shared_lock */
+ object);
+
+ assert( !VM_PAGE_WIRED(m));
+
+ if (retval == KERN_SUCCESS) {
+ /*
+ * If the object is purgeable, its owner's
+ * purgeable ledgers will be updated in
+ * vm_page_remove() but the page still
+ * contributes to the owner's memory footprint,
+ * so account for it as such.
+ */
+ if (object->purgable != VM_PURGABLE_DENY &&
+ object->vo_purgeable_owner != NULL) {
+ /* one more compressed purgeable page */
+ vm_purgeable_compressed_update(object,
+ +1);
+ }
+ VM_STAT_INCR(compressions);
+
+ if (m->tabled)
+ vm_page_remove(m, TRUE);
+
+ } else {
+ PAGE_WAKEUP_DONE(m);
+
+ vm_page_lockspin_queues();
+
+ vm_page_activate(m);
+ vm_compressor_failed++;
+
+ vm_page_unlock_queues();
+ }
+ if (object_locked_by_caller == FALSE) {
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+ }
+ return retval;
+}
+
+
static void
-vm_pageout_adjust_io_throttles(struct vm_pageout_queue *iq, struct vm_pageout_queue *eq, boolean_t req_lowpriority)
+vm_pageout_adjust_eq_iothrottle(struct vm_pageout_queue *eq, boolean_t req_lowpriority)
{
uint32_t policy;
- boolean_t set_iq = FALSE;
- boolean_t set_eq = FALSE;
if (hibernate_cleaning_in_progress == TRUE)
req_lowpriority = FALSE;
- if ((DEFAULT_PAGER_IS_ACTIVE || DEFAULT_FREEZER_IS_ACTIVE) && iq->pgo_inited == TRUE && iq->pgo_lowpriority != req_lowpriority)
- set_iq = TRUE;
-
- if (eq->pgo_inited == TRUE && eq->pgo_lowpriority != req_lowpriority)
- set_eq = TRUE;
-
- if (set_iq == TRUE || set_eq == TRUE) {
-
+ if (eq->pgo_inited == TRUE && eq->pgo_lowpriority != req_lowpriority) {
+
vm_page_unlock_queues();
if (req_lowpriority == TRUE) {
policy = THROTTLE_LEVEL_PAGEOUT_UNTHROTTLED;
DTRACE_VM(laundryunthrottle);
}
- if (set_iq == TRUE) {
- proc_set_task_policy_thread(kernel_task, iq->pgo_tid, TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy);
+ proc_set_thread_policy_with_tid(kernel_task, eq->pgo_tid,
+ TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy);
- iq->pgo_lowpriority = req_lowpriority;
- }
- if (set_eq == TRUE) {
- proc_set_task_policy_thread(kernel_task, eq->pgo_tid, TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy);
+ eq->pgo_lowpriority = req_lowpriority;
- eq->pgo_lowpriority = req_lowpriority;
- }
vm_page_lock_queues();
}
}
self->options |= TH_OPT_VMPRIV;
- DTRACE_VM2(laundrythrottle, int, 1, (uint64_t *), NULL);
+ DTRACE_VM2(laundrythrottle, int, 1, (uint64_t *), NULL);
- proc_set_task_policy_thread(kernel_task, self->thread_id, TASK_POLICY_EXTERNAL,
- TASK_POLICY_IO, THROTTLE_LEVEL_PAGEOUT_THROTTLED);
+ proc_set_thread_policy(self, TASK_POLICY_EXTERNAL,
+ TASK_POLICY_IO, THROTTLE_LEVEL_PAGEOUT_THROTTLED);
vm_page_lock_queues();
vm_page_unlock_queues();
- if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE)
- vm_pageout_iothread_external_continue(&vm_pageout_queue_external);
- else
- vm_pageout_iothread_continue(&vm_pageout_queue_external);
+ vm_pageout_iothread_external_continue(&vm_pageout_queue_external);
/*NOTREACHED*/
}
self->options |= TH_OPT_VMPRIV;
- if (DEFAULT_PAGER_IS_ACTIVE || DEFAULT_FREEZER_IS_ACTIVE) {
- DTRACE_VM2(laundrythrottle, int, 1, (uint64_t *), NULL);
-
- proc_set_task_policy_thread(kernel_task, self->thread_id, TASK_POLICY_EXTERNAL,
- TASK_POLICY_IO, THROTTLE_LEVEL_PAGEOUT_THROTTLED);
- }
vm_page_lock_queues();
vm_pageout_queue_internal.pgo_tid = self->thread_id;
vm_page_unlock_queues();
- if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) {
- cq->q = &vm_pageout_queue_internal;
- cq->current_chead = NULL;
- cq->scratch_buf = kalloc(COMPRESSOR_SCRATCH_BUF_SIZE);
+ if (vm_restricted_to_single_processor == TRUE)
+ thread_vm_bind_group_add();
- vm_pageout_iothread_internal_continue(cq);
- } else
- vm_pageout_iothread_continue(&vm_pageout_queue_internal);
+
+ thread_set_thread_name(current_thread(), "VM_compressor");
+#if DEVELOPMENT || DEBUG
+ vmct_stats.vmct_minpages[cq->id] = INT32_MAX;
+#endif
+ vm_pageout_iothread_internal_continue(cq);
/*NOTREACHED*/
}
vm_pressure_level_t old_level = kVMPressureNormal;
int new_level = -1;
-
+ unsigned int total_pages;
uint64_t available_memory = 0;
if (vm_pressure_events_enabled == FALSE)
return;
+#if CONFIG_EMBEDDED
+
+ available_memory = (uint64_t) memorystatus_available_pages;
- available_memory = (((uint64_t) AVAILABLE_NON_COMPRESSED_MEMORY) * 100);
+#else /* CONFIG_EMBEDDED */
+ available_memory = (uint64_t) AVAILABLE_NON_COMPRESSED_MEMORY;
+ memorystatus_available_pages = (uint64_t) AVAILABLE_NON_COMPRESSED_MEMORY;
- memorystatus_level = (unsigned int) (available_memory / atop_64(max_mem));
+#endif /* CONFIG_EMBEDDED */
+
+ total_pages = (unsigned int) atop_64(max_mem);
+#if CONFIG_SECLUDED_MEMORY
+ total_pages -= vm_page_secluded_count;
+#endif /* CONFIG_SECLUDED_MEMORY */
+ memorystatus_level = (unsigned int) ((available_memory * 100) / total_pages);
if (memorystatus_manual_testing_on) {
return;
kern_return_t
mach_vm_pressure_level_monitor(__unused boolean_t wait_for_pressure, __unused unsigned int *pressure_level) {
-#if !VM_PRESSURE_EVENTS
+#if CONFIG_EMBEDDED
+
+ return KERN_FAILURE;
+
+#elif !VM_PRESSURE_EVENTS
return KERN_FAILURE;
* called once per-second via "compute_averages"
*/
void
-compute_pageout_gc_throttle()
+compute_pageout_gc_throttle(__unused void *arg)
{
if (vm_pageout_considered_page != vm_pageout_considered_page_last) {
}
}
+/*
+ * vm_pageout_garbage_collect can also be called when the zone allocator needs
+ * to call zone_gc on a different thread in order to trigger zone-map-exhaustion
+ * jetsams. We need to check if the zone map size is above its jetsam limit to
+ * decide if this was indeed the case.
+ *
+ * We need to do this on a different thread because of the following reasons:
+ *
+ * 1. In the case of synchronous jetsams, the leaking process can try to jetsam
+ * itself causing the system to hang. We perform synchronous jetsams if we're
+ * leaking in the VM map entries zone, so the leaking process could be doing a
+ * zalloc for a VM map entry while holding its vm_map lock, when it decides to
+ * jetsam itself. We also need the vm_map lock on the process termination path,
+ * which would now lead the dying process to deadlock against itself.
+ *
+ * 2. The jetsam path might need to allocate zone memory itself. We could try
+ * using the non-blocking variant of zalloc for this path, but we can still
+ * end up trying to do a kernel_memory_allocate when the zone_map is almost
+ * full.
+ */
+
+extern boolean_t is_zone_map_nearing_exhaustion(void);
-static void
+void
vm_pageout_garbage_collect(int collect)
{
-
if (collect) {
- boolean_t buf_large_zfree = FALSE;
- boolean_t first_try = TRUE;
+ if (is_zone_map_nearing_exhaustion()) {
+ /*
+ * Woken up by the zone allocator for zone-map-exhaustion jetsams.
+ *
+ * Bail out after calling zone_gc (which triggers the
+ * zone-map-exhaustion jetsams). If we fall through, the subsequent
+ * operations that clear out a bunch of caches might allocate zone
+ * memory themselves (for eg. vm_map operations would need VM map
+ * entries). Since the zone map is almost full at this point, we
+ * could end up with a panic. We just need to quickly jetsam a
+ * process and exit here.
+ *
+ * It could so happen that we were woken up to relieve memory
+ * pressure and the zone map also happened to be near its limit at
+ * the time, in which case we'll skip out early. But that should be
+ * ok; if memory pressure persists, the thread will simply be woken
+ * up again.
+ */
+ consider_zone_gc(TRUE);
- stack_collect();
+ } else {
+ /* Woken up by vm_pageout_scan or compute_pageout_gc_throttle. */
+ boolean_t buf_large_zfree = FALSE;
+ boolean_t first_try = TRUE;
- consider_machine_collect();
- m_drain();
+ stack_collect();
- do {
- if (consider_buffer_cache_collect != NULL) {
- buf_large_zfree = (*consider_buffer_cache_collect)(0);
- }
- if (first_try == TRUE || buf_large_zfree == TRUE) {
- /*
- * consider_zone_gc should be last, because the other operations
- * might return memory to zones.
- */
- consider_zone_gc(buf_large_zfree);
- }
- first_try = FALSE;
+ consider_machine_collect();
+ m_drain();
- } while (buf_large_zfree == TRUE && vm_page_free_count < vm_page_free_target);
+ do {
+ if (consider_buffer_cache_collect != NULL) {
+ buf_large_zfree = (*consider_buffer_cache_collect)(0);
+ }
+ if (first_try == TRUE || buf_large_zfree == TRUE) {
+ /*
+ * consider_zone_gc should be last, because the other operations
+ * might return memory to zones.
+ */
+ consider_zone_gc(FALSE);
+ }
+ first_try = FALSE;
- consider_machine_adjust();
+ } while (buf_large_zfree == TRUE && vm_page_free_count < vm_page_free_target);
+
+ consider_machine_adjust();
+ }
}
+
assert_wait((event_t) &vm_pageout_garbage_collect, THREAD_UNINT);
thread_block_parameter((thread_continue_t) vm_pageout_garbage_collect, (void *)1);
}
-void vm_pageout_reinit_tuneables(void);
-
-void
-vm_pageout_reinit_tuneables(void)
-{
- vm_compressor_minorcompact_threshold_divisor = 18;
- vm_compressor_majorcompact_threshold_divisor = 22;
- vm_compressor_unthrottle_threshold_divisor = 32;
-}
-
-
#if VM_PAGE_BUCKETS_CHECK
#if VM_PAGE_FAKE_BUCKETS
extern vm_map_offset_t vm_page_fake_buckets_start, vm_page_fake_buckets_end;
#endif /* VM_PAGE_FAKE_BUCKETS */
#endif /* VM_PAGE_BUCKETS_CHECK */
-#define FBDP_TEST_COLLAPSE_COMPRESSOR 0
-#if FBDP_TEST_COLLAPSE_COMPRESSOR
-extern boolean_t vm_object_collapse_compressor_allowed;
-#include <IOKit/IOLib.h>
-#endif /* FBDP_TEST_COLLAPSE_COMPRESSOR */
-#define FBDP_TEST_WIRE_AND_EXTRACT 0
-#if FBDP_TEST_WIRE_AND_EXTRACT
-extern ledger_template_t task_ledger_template;
-#include <mach/mach_vm.h>
-extern ppnum_t vm_map_get_phys_page(vm_map_t map,
- vm_offset_t offset);
-#endif /* FBDP_TEST_WIRE_AND_EXTRACT */
+
+void
+vm_set_restrictions()
+{
+ host_basic_info_data_t hinfo;
+ mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
+
+#define BSD_HOST 1
+ host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
+
+ assert(hinfo.max_cpus > 0);
+
+ if (hinfo.max_cpus <= 3) {
+ /*
+ * on systems with a limited number of CPUS, bind the
+ * 4 major threads that can free memory and that tend to use
+ * a fair bit of CPU under pressured conditions to a single processor.
+ * This insures that these threads don't hog all of the available CPUs
+ * (important for camera launch), while allowing them to run independently
+ * w/r to locks... the 4 threads are
+ * vm_pageout_scan, vm_pageout_iothread_internal (compressor),
+ * vm_compressor_swap_trigger_thread (minor and major compactions),
+ * memorystatus_thread (jetsams).
+ *
+ * the first time the thread is run, it is responsible for checking the
+ * state of vm_restricted_to_single_processor, and if TRUE it calls
+ * thread_bind_master... someday this should be replaced with a group
+ * scheduling mechanism and KPI.
+ */
+ vm_restricted_to_single_processor = TRUE;
+ }
+}
void
vm_pageout(void)
* Set thread privileges.
*/
s = splsched();
+
thread_lock(self);
- self->priority = BASEPRI_PREEMPT - 1;
- set_sched_pri(self, self->priority);
+ self->options |= TH_OPT_VMPRIV;
+ sched_set_thread_base_priority(self, BASEPRI_VM);
thread_unlock(self);
if (!self->reserved_stack)
self->reserved_stack = self->kernel_stack;
+ if (vm_restricted_to_single_processor == TRUE)
+ thread_vm_bind_group_add();
+
splx(s);
+ thread_set_thread_name(current_thread(), "VM_pageout_scan");
+
/*
* Initialize some paging parameters.
*/
vm_page_free_reserve(0);
- queue_init(&vm_pageout_queue_external.pgo_pending);
+ vm_page_queue_init(&vm_pageout_queue_external.pgo_pending);
vm_pageout_queue_external.pgo_maxlaundry = VM_PAGE_LAUNDRY_MAX;
vm_pageout_queue_external.pgo_laundry = 0;
vm_pageout_queue_external.pgo_idle = FALSE;
vm_pageout_queue_external.pgo_tid = -1;
vm_pageout_queue_external.pgo_inited = FALSE;
-
- queue_init(&vm_pageout_queue_internal.pgo_pending);
+ vm_page_queue_init(&vm_pageout_queue_internal.pgo_pending);
vm_pageout_queue_internal.pgo_maxlaundry = 0;
vm_pageout_queue_internal.pgo_laundry = 0;
vm_pageout_queue_internal.pgo_idle = FALSE;
/* external pageout and garbage collection threads started here */
result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_external, NULL,
- BASEPRI_PREEMPT - 1,
+ BASEPRI_VM,
&vm_pageout_external_iothread);
if (result != KERN_SUCCESS)
panic("vm_pageout_iothread_external: create failed");
if (result != KERN_SUCCESS)
panic("vm_pageout_garbage_collect: create failed");
- thread_deallocate(thread);
-
-#if VM_PRESSURE_EVENTS
- result = kernel_thread_start_priority((thread_continue_t)vm_pressure_thread, NULL,
- BASEPRI_DEFAULT,
- &thread);
-
- if (result != KERN_SUCCESS)
- panic("vm_pressure_thread: create failed");
-
- thread_deallocate(thread);
-#endif
-
- vm_object_reaper_init();
-
- if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE)
- vm_compressor_pager_init();
-
-#if VM_PRESSURE_EVENTS
- vm_pressure_events_enabled = TRUE;
-#endif /* VM_PRESSURE_EVENTS */
-
-#if CONFIG_PHANTOM_CACHE
- vm_phantom_cache_init();
-#endif
-#if VM_PAGE_BUCKETS_CHECK
-#if VM_PAGE_FAKE_BUCKETS
- printf("**** DEBUG: protecting fake buckets [0x%llx:0x%llx]\n",
- (uint64_t) vm_page_fake_buckets_start,
- (uint64_t) vm_page_fake_buckets_end);
- pmap_protect(kernel_pmap,
- vm_page_fake_buckets_start,
- vm_page_fake_buckets_end,
- VM_PROT_READ);
-// *(char *) vm_page_fake_buckets_start = 'x'; /* panic! */
-#endif /* VM_PAGE_FAKE_BUCKETS */
-#endif /* VM_PAGE_BUCKETS_CHECK */
-
-#if VM_OBJECT_TRACKING
- vm_object_tracking_init();
-#endif /* VM_OBJECT_TRACKING */
-
-
-#if FBDP_TEST_COLLAPSE_COMPRESSOR
- vm_object_size_t backing_size, top_size;
- vm_object_t backing_object, top_object;
- vm_map_offset_t backing_offset, top_offset;
- unsigned char *backing_address, *top_address;
- kern_return_t kr;
-
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR:\n");
-
- /* create backing object */
- backing_size = 15 * PAGE_SIZE;
- backing_object = vm_object_allocate(backing_size);
- assert(backing_object != VM_OBJECT_NULL);
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: created backing object %p\n",
- backing_object);
- /* map backing object */
- backing_offset = 0;
- kr = vm_map_enter(kernel_map, &backing_offset, backing_size, 0,
- VM_FLAGS_ANYWHERE, backing_object, 0, FALSE,
- VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
- assert(kr == KERN_SUCCESS);
- backing_address = (unsigned char *) backing_offset;
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: "
- "mapped backing object %p at 0x%llx\n",
- backing_object, (uint64_t) backing_offset);
- /* populate with pages to be compressed in backing object */
- backing_address[0x1*PAGE_SIZE] = 0xB1;
- backing_address[0x4*PAGE_SIZE] = 0xB4;
- backing_address[0x7*PAGE_SIZE] = 0xB7;
- backing_address[0xa*PAGE_SIZE] = 0xBA;
- backing_address[0xd*PAGE_SIZE] = 0xBD;
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: "
- "populated pages to be compressed in "
- "backing_object %p\n", backing_object);
- /* compress backing object */
- vm_object_pageout(backing_object);
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: compressing backing_object %p\n",
- backing_object);
- /* wait for all the pages to be gone */
- while (*(volatile int *)&backing_object->resident_page_count != 0)
- IODelay(10);
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: backing_object %p compressed\n",
- backing_object);
- /* populate with pages to be resident in backing object */
- backing_address[0x0*PAGE_SIZE] = 0xB0;
- backing_address[0x3*PAGE_SIZE] = 0xB3;
- backing_address[0x6*PAGE_SIZE] = 0xB6;
- backing_address[0x9*PAGE_SIZE] = 0xB9;
- backing_address[0xc*PAGE_SIZE] = 0xBC;
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: "
- "populated pages to be resident in "
- "backing_object %p\n", backing_object);
- /* leave the other pages absent */
- /* mess with the paging_offset of the backing_object */
- assert(backing_object->paging_offset == 0);
- backing_object->paging_offset = 0x3000;
-
- /* create top object */
- top_size = 9 * PAGE_SIZE;
- top_object = vm_object_allocate(top_size);
- assert(top_object != VM_OBJECT_NULL);
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: created top object %p\n",
- top_object);
- /* map top object */
- top_offset = 0;
- kr = vm_map_enter(kernel_map, &top_offset, top_size, 0,
- VM_FLAGS_ANYWHERE, top_object, 0, FALSE,
- VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
- assert(kr == KERN_SUCCESS);
- top_address = (unsigned char *) top_offset;
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: "
- "mapped top object %p at 0x%llx\n",
- top_object, (uint64_t) top_offset);
- /* populate with pages to be compressed in top object */
- top_address[0x3*PAGE_SIZE] = 0xA3;
- top_address[0x4*PAGE_SIZE] = 0xA4;
- top_address[0x5*PAGE_SIZE] = 0xA5;
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: "
- "populated pages to be compressed in "
- "top_object %p\n", top_object);
- /* compress top object */
- vm_object_pageout(top_object);
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: compressing top_object %p\n",
- top_object);
- /* wait for all the pages to be gone */
- while (top_object->resident_page_count != 0);
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: top_object %p compressed\n",
- top_object);
- /* populate with pages to be resident in top object */
- top_address[0x0*PAGE_SIZE] = 0xA0;
- top_address[0x1*PAGE_SIZE] = 0xA1;
- top_address[0x2*PAGE_SIZE] = 0xA2;
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: "
- "populated pages to be resident in "
- "top_object %p\n", top_object);
- /* leave the other pages absent */
-
- /* link the 2 objects */
- vm_object_reference(backing_object);
- top_object->shadow = backing_object;
- top_object->vo_shadow_offset = 0x3000;
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: linked %p and %p\n",
- top_object, backing_object);
-
- /* unmap backing object */
- vm_map_remove(kernel_map,
- backing_offset,
- backing_offset + backing_size,
- 0);
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: "
- "unmapped backing_object %p [0x%llx:0x%llx]\n",
- backing_object,
- (uint64_t) backing_offset,
- (uint64_t) (backing_offset + backing_size));
-
- /* collapse */
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: collapsing %p\n", top_object);
- vm_object_lock(top_object);
- vm_object_collapse(top_object, 0, FALSE);
- vm_object_unlock(top_object);
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: collapsed %p\n", top_object);
-
- /* did it work? */
- if (top_object->shadow != VM_OBJECT_NULL) {
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: not collapsed\n");
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: FAIL\n");
- if (vm_object_collapse_compressor_allowed) {
- panic("FBDP_TEST_COLLAPSE_COMPRESSOR: FAIL\n");
- }
- } else {
- /* check the contents of the mapping */
- unsigned char expect[9] =
- { 0xA0, 0xA1, 0xA2, /* resident in top */
- 0xA3, 0xA4, 0xA5, /* compressed in top */
- 0xB9, /* resident in backing + shadow_offset */
- 0xBD, /* compressed in backing + shadow_offset + paging_offset */
- 0x00 }; /* absent in both */
- unsigned char actual[9];
- unsigned int i, errors;
-
- errors = 0;
- for (i = 0; i < sizeof (actual); i++) {
- actual[i] = (unsigned char) top_address[i*PAGE_SIZE];
- if (actual[i] != expect[i]) {
- errors++;
- }
- }
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: "
- "actual [%x %x %x %x %x %x %x %x %x] "
- "expect [%x %x %x %x %x %x %x %x %x] "
- "%d errors\n",
- actual[0], actual[1], actual[2], actual[3],
- actual[4], actual[5], actual[6], actual[7],
- actual[8],
- expect[0], expect[1], expect[2], expect[3],
- expect[4], expect[5], expect[6], expect[7],
- expect[8],
- errors);
- if (errors) {
- panic("FBDP_TEST_COLLAPSE_COMPRESSOR: FAIL\n");
- } else {
- printf("FBDP_TEST_COLLAPSE_COMPRESSOR: PASS\n");
- }
- }
-#endif /* FBDP_TEST_COLLAPSE_COMPRESSOR */
+ thread_deallocate(thread);
-#if FBDP_TEST_WIRE_AND_EXTRACT
- ledger_t ledger;
- vm_map_t user_map, wire_map;
- mach_vm_address_t user_addr, wire_addr;
- mach_vm_size_t user_size, wire_size;
- mach_vm_offset_t cur_offset;
- vm_prot_t cur_prot, max_prot;
- ppnum_t user_ppnum, wire_ppnum;
- kern_return_t kr;
+#if VM_PRESSURE_EVENTS
+ result = kernel_thread_start_priority((thread_continue_t)vm_pressure_thread, NULL,
+ BASEPRI_DEFAULT,
+ &thread);
- ledger = ledger_instantiate(task_ledger_template,
- LEDGER_CREATE_ACTIVE_ENTRIES);
- user_map = vm_map_create(pmap_create(ledger, 0, TRUE),
- 0x100000000ULL,
- 0x200000000ULL,
- TRUE);
- wire_map = vm_map_create(NULL,
- 0x100000000ULL,
- 0x200000000ULL,
- TRUE);
- user_addr = 0;
- user_size = 0x10000;
- kr = mach_vm_allocate(user_map,
- &user_addr,
- user_size,
- VM_FLAGS_ANYWHERE);
- assert(kr == KERN_SUCCESS);
- wire_addr = 0;
- wire_size = user_size;
- kr = mach_vm_remap(wire_map,
- &wire_addr,
- wire_size,
- 0,
- VM_FLAGS_ANYWHERE,
- user_map,
- user_addr,
- FALSE,
- &cur_prot,
- &max_prot,
- VM_INHERIT_NONE);
- assert(kr == KERN_SUCCESS);
- for (cur_offset = 0;
- cur_offset < wire_size;
- cur_offset += PAGE_SIZE) {
- kr = vm_map_wire_and_extract(wire_map,
- wire_addr + cur_offset,
- VM_PROT_DEFAULT,
- TRUE,
- &wire_ppnum);
- assert(kr == KERN_SUCCESS);
- user_ppnum = vm_map_get_phys_page(user_map,
- user_addr + cur_offset);
- printf("FBDP_TEST_WIRE_AND_EXTRACT: kr=0x%x "
- "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n",
- kr,
- user_map, user_addr + cur_offset, user_ppnum,
- wire_map, wire_addr + cur_offset, wire_ppnum);
- if (kr != KERN_SUCCESS ||
- wire_ppnum == 0 ||
- wire_ppnum != user_ppnum) {
- panic("FBDP_TEST_WIRE_AND_EXTRACT: FAIL\n");
- }
- }
- cur_offset -= PAGE_SIZE;
- kr = vm_map_wire_and_extract(wire_map,
- wire_addr + cur_offset,
- VM_PROT_DEFAULT,
- TRUE,
- &wire_ppnum);
- assert(kr == KERN_SUCCESS);
- printf("FBDP_TEST_WIRE_AND_EXTRACT: re-wire kr=0x%x "
- "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n",
- kr,
- user_map, user_addr + cur_offset, user_ppnum,
- wire_map, wire_addr + cur_offset, wire_ppnum);
- if (kr != KERN_SUCCESS ||
- wire_ppnum == 0 ||
- wire_ppnum != user_ppnum) {
- panic("FBDP_TEST_WIRE_AND_EXTRACT: FAIL\n");
- }
-
- printf("FBDP_TEST_WIRE_AND_EXTRACT: PASS\n");
-#endif /* FBDP_TEST_WIRE_AND_EXTRACT */
+ if (result != KERN_SUCCESS)
+ panic("vm_pressure_thread: create failed");
+
+ thread_deallocate(thread);
+#endif
+
+ vm_object_reaper_init();
+
+
+ bzero(&vm_config, sizeof(vm_config));
+
+ switch(vm_compressor_mode) {
+
+ case VM_PAGER_DEFAULT:
+ printf("mapping deprecated VM_PAGER_DEFAULT to VM_PAGER_COMPRESSOR_WITH_SWAP\n");
+
+ case VM_PAGER_COMPRESSOR_WITH_SWAP:
+ vm_config.compressor_is_present = TRUE;
+ vm_config.swap_is_present = TRUE;
+ vm_config.compressor_is_active = TRUE;
+ vm_config.swap_is_active = TRUE;
+ break;
+
+ case VM_PAGER_COMPRESSOR_NO_SWAP:
+ vm_config.compressor_is_present = TRUE;
+ vm_config.swap_is_present = TRUE;
+ vm_config.compressor_is_active = TRUE;
+ break;
+
+ case VM_PAGER_FREEZER_DEFAULT:
+ printf("mapping deprecated VM_PAGER_FREEZER_DEFAULT to VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP\n");
+
+ case VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP:
+ vm_config.compressor_is_present = TRUE;
+ vm_config.swap_is_present = TRUE;
+ break;
+
+ case VM_PAGER_COMPRESSOR_NO_SWAP_PLUS_FREEZER_COMPRESSOR_WITH_SWAP:
+ vm_config.compressor_is_present = TRUE;
+ vm_config.swap_is_present = TRUE;
+ vm_config.compressor_is_active = TRUE;
+ vm_config.freezer_swap_is_active = TRUE;
+ break;
+
+ case VM_PAGER_NOT_CONFIGURED:
+ break;
+
+ default:
+ printf("unknown compressor mode - %x\n", vm_compressor_mode);
+ break;
+ }
+ if (VM_CONFIG_COMPRESSOR_IS_PRESENT)
+ vm_compressor_pager_init();
+
+#if VM_PRESSURE_EVENTS
+ vm_pressure_events_enabled = TRUE;
+#endif /* VM_PRESSURE_EVENTS */
+
+#if CONFIG_PHANTOM_CACHE
+ vm_phantom_cache_init();
+#endif
+#if VM_PAGE_BUCKETS_CHECK
+#if VM_PAGE_FAKE_BUCKETS
+ printf("**** DEBUG: protecting fake buckets [0x%llx:0x%llx]\n",
+ (uint64_t) vm_page_fake_buckets_start,
+ (uint64_t) vm_page_fake_buckets_end);
+ pmap_protect(kernel_pmap,
+ vm_page_fake_buckets_start,
+ vm_page_fake_buckets_end,
+ VM_PROT_READ);
+// *(char *) vm_page_fake_buckets_start = 'x'; /* panic! */
+#endif /* VM_PAGE_FAKE_BUCKETS */
+#endif /* VM_PAGE_BUCKETS_CHECK */
+
+#if VM_OBJECT_TRACKING
+ vm_object_tracking_init();
+#endif /* VM_OBJECT_TRACKING */
+ vm_tests();
vm_pageout_continue();
-#define MAX_COMRPESSOR_THREAD_COUNT 8
-
-struct cq ciq[MAX_COMRPESSOR_THREAD_COUNT];
-
+#if CONFIG_EMBEDDED
+int vm_compressor_thread_count = 1;
+#else
int vm_compressor_thread_count = 2;
+#endif
kern_return_t
vm_pageout_internal_start(void)
int i;
host_basic_info_data_t hinfo;
- if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) {
- mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
-#define BSD_HOST 1
- host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
+ assert (VM_CONFIG_COMPRESSOR_IS_PRESENT);
- assert(hinfo.max_cpus > 0);
+ mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
+#define BSD_HOST 1
+ host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
- if (vm_compressor_thread_count >= hinfo.max_cpus)
- vm_compressor_thread_count = hinfo.max_cpus - 1;
- if (vm_compressor_thread_count <= 0)
- vm_compressor_thread_count = 1;
- else if (vm_compressor_thread_count > MAX_COMRPESSOR_THREAD_COUNT)
- vm_compressor_thread_count = MAX_COMRPESSOR_THREAD_COUNT;
+ assert(hinfo.max_cpus > 0);
- vm_pageout_queue_internal.pgo_maxlaundry = (vm_compressor_thread_count * 4) * VM_PAGE_LAUNDRY_MAX;
- } else {
+ PE_parse_boot_argn("vmcomp_threads", &vm_compressor_thread_count, sizeof(vm_compressor_thread_count));
+ if (vm_compressor_thread_count >= hinfo.max_cpus)
+ vm_compressor_thread_count = hinfo.max_cpus - 1;
+ if (vm_compressor_thread_count <= 0)
vm_compressor_thread_count = 1;
- vm_pageout_queue_internal.pgo_maxlaundry = VM_PAGE_LAUNDRY_MAX;
- }
+ else if (vm_compressor_thread_count > MAX_COMPRESSOR_THREAD_COUNT)
+ vm_compressor_thread_count = MAX_COMPRESSOR_THREAD_COUNT;
+
+ vm_pageout_queue_internal.pgo_maxlaundry = (vm_compressor_thread_count * 4) * VM_PAGE_LAUNDRY_MAX;
+
+ PE_parse_boot_argn("vmpgoi_maxlaundry", &vm_pageout_queue_internal.pgo_maxlaundry, sizeof(vm_pageout_queue_internal.pgo_maxlaundry));
for (i = 0; i < vm_compressor_thread_count; i++) {
+ ciq[i].id = i;
+ ciq[i].q = &vm_pageout_queue_internal;
+ ciq[i].current_chead = NULL;
+ ciq[i].scratch_buf = kalloc(COMPRESSOR_SCRATCH_BUF_SIZE);
+
+ result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal, (void *)&ciq[i], BASEPRI_VM, &vm_pageout_internal_iothread);
- result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal, (void *)&ciq[i], BASEPRI_PREEMPT - 1, &vm_pageout_internal_iothread);
if (result == KERN_SUCCESS)
thread_deallocate(vm_pageout_internal_iothread);
else
bzero((char *)upl + upl_size, page_field_size);
upl->flags = upl_flags | flags;
- upl->src_object = NULL;
upl->kaddr = (vm_offset_t)0;
upl->size = 0;
upl->map_object = NULL;
upl->highest_page = 0;
upl_lock_init(upl);
upl->vector_upl = NULL;
+ upl->associated_upl = NULL;
#if CONFIG_IOSCHED
if (type & UPL_CREATE_IO_TRACKING) {
upl->upl_priority = proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO);
upl_t *upl_ptr,
upl_page_info_array_t user_page_list,
unsigned int *page_list_count,
- int cntrl_flags)
+ upl_control_flags_t cntrl_flags,
+ vm_tag_t tag)
{
vm_page_t dst_page = VM_PAGE_NULL;
vm_object_offset_t dst_offset;
int dw_count;
int dw_limit;
int io_tracking_flag = 0;
+ int grab_options;
+ ppnum_t phys_page;
if (cntrl_flags & ~UPL_VALID_FLAGS) {
/*
upl->flags |= UPL_SHADOWED;
}
- /*
- * ENCRYPTED SWAP:
- * Just mark the UPL as "encrypted" here.
- * We'll actually encrypt the pages later,
- * in upl_encrypt(), when the caller has
- * selected which pages need to go to swap.
- */
- if (cntrl_flags & UPL_ENCRYPT)
- upl->flags |= UPL_ENCRYPTED;
-
if (cntrl_flags & UPL_FOR_PAGEOUT)
upl->flags |= UPL_PAGEOUT;
vm_object_lock(object);
vm_object_activity_begin(object);
+ grab_options = 0;
+#if CONFIG_SECLUDED_MEMORY
+ if (object->can_grab_secluded) {
+ grab_options |= VM_PAGE_GRAB_SECLUDED;
+ }
+#endif /* CONFIG_SECLUDED_MEMORY */
+
/*
* we can lock in the paging_offset once paging_in_progress is set
*/
if ((cntrl_flags & UPL_WILL_MODIFY) && must_throttle_writes() == TRUE) {
boolean_t isSSD = FALSE;
+#if CONFIG_EMBEDDED
+ isSSD = TRUE;
+#else
vnode_pager_get_isSSD(object->pager, &isSSD);
+#endif
vm_object_unlock(object);
OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages);
goto try_next_page;
}
+ phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
+
/*
* grab this up front...
* a high percentange of the time we're going to
* the pmap layer by grabbing it here and recording it
*/
if (dst_page->pmapped)
- refmod_state = pmap_get_refmod(dst_page->phys_page);
+ refmod_state = pmap_get_refmod(phys_page);
else
refmod_state = 0;
- if ( (refmod_state & VM_MEM_REFERENCED) && dst_page->inactive ) {
+ if ( (refmod_state & VM_MEM_REFERENCED) && VM_PAGE_INACTIVE(dst_page)) {
/*
* page is on inactive list and referenced...
* reactivate it now... this gets it out of the
* can't have been referenced recently...
*/
if ( (hibernate_cleaning_in_progress == TRUE ||
- (!((refmod_state & VM_MEM_REFERENCED) || dst_page->reference) || dst_page->throttled)) &&
- ((refmod_state & VM_MEM_MODIFIED) || dst_page->dirty || dst_page->precious) ) {
+ (!((refmod_state & VM_MEM_REFERENCED) || dst_page->reference) ||
+ (dst_page->vm_page_q_state == VM_PAGE_ON_THROTTLED_Q))) &&
+ ((refmod_state & VM_MEM_MODIFIED) || dst_page->dirty || dst_page->precious) ) {
goto check_busy;
}
dont_return:
check_busy:
if (dst_page->busy) {
if (cntrl_flags & UPL_NOBLOCK) {
- if (user_page_list)
+ if (user_page_list)
user_page_list[entry].phys_addr = 0;
+ dwp->dw_mask = 0;
goto try_next_page;
}
continue;
}
- /*
- * ENCRYPTED SWAP:
- * The caller is gathering this page and might
- * access its contents later on. Decrypt the
- * page before adding it to the UPL, so that
- * the caller never sees encrypted data.
- */
- if (! (cntrl_flags & UPL_ENCRYPT) && dst_page->encrypted) {
- int was_busy;
-
- /*
- * save the current state of busy
- * mark page as busy while decrypt
- * is in progress since it will drop
- * the object lock...
- */
- was_busy = dst_page->busy;
- dst_page->busy = TRUE;
-
- vm_page_decrypt(dst_page, 0);
- vm_page_decrypt_for_upl_counter++;
- /*
- * restore to original busy state
- */
- dst_page->busy = was_busy;
- }
- if (dst_page->pageout_queue == TRUE) {
+ if (dst_page->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) {
vm_page_lockspin_queues();
- if (dst_page->pageout_queue == TRUE) {
+ if (dst_page->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) {
/*
* we've buddied up a page for a clustered pageout
* that has already been moved to the pageout
*/
if (dst_page->pageout)
encountered_lrp = TRUE;
- if ((dst_page->dirty || (dst_page->object->internal && dst_page->precious))) {
+ if ((dst_page->dirty || (object->internal && dst_page->precious))) {
if (encountered_lrp)
CLUSTER_STAT(pages_at_higher_offsets++;)
else
hw_dirty = refmod_state & VM_MEM_MODIFIED;
dirty = hw_dirty ? TRUE : dst_page->dirty;
- if (dst_page->phys_page > upl->highest_page)
- upl->highest_page = dst_page->phys_page;
+ if (phys_page > upl->highest_page)
+ upl->highest_page = phys_page;
+
+ assert (!pmap_is_noencrypt(phys_page));
if (cntrl_flags & UPL_SET_LITE) {
unsigned int pg_num;
lite_list[pg_num>>5] |= 1 << (pg_num & 31);
if (hw_dirty)
- pmap_clear_modify(dst_page->phys_page);
+ pmap_clear_modify(phys_page);
/*
* Mark original page as cleaning
alias_page->absent = FALSE;
alias_page = NULL;
}
-#if MACH_PAGEMAP
- /*
- * Record that this page has been
- * written out
- */
- vm_external_state_set(object->existence_map, dst_page->offset);
-#endif /*MACH_PAGEMAP*/
if (dirty) {
SET_PAGE_DIRTY(dst_page, FALSE);
} else {
if (!dirty)
dst_page->precious = TRUE;
- if ( (cntrl_flags & UPL_ENCRYPT) ) {
- /*
- * ENCRYPTED SWAP:
- * We want to deny access to the target page
- * because its contents are about to be
- * encrypted and the user would be very
- * confused to see encrypted data instead
- * of their data.
- * We also set "encrypted_cleaning" to allow
- * vm_pageout_scan() to demote that page
- * from "adjacent/clean-in-place" to
- * "target/clean-and-free" if it bumps into
- * this page during its scanning while we're
- * still processing this cluster.
- */
- dst_page->busy = TRUE;
- dst_page->encrypted_cleaning = TRUE;
- }
if ( !(cntrl_flags & UPL_CLEAN_IN_PLACE) ) {
if ( !VM_PAGE_WIRED(dst_page))
- dst_page->pageout = TRUE;
+ dst_page->free_when_done = TRUE;
}
} else {
if ((cntrl_flags & UPL_WILL_MODIFY) && object->copy != last_copy_object) {
continue;
}
- if (dst_page->laundry) {
- dst_page->pageout = FALSE;
-
+ if (dst_page->laundry)
vm_pageout_steal_laundry(dst_page, FALSE);
- }
} else {
if (object->private) {
/*
dst_page = vm_object_page_grab(object);
if (dst_page != VM_PAGE_NULL)
- vm_page_release(dst_page);
+ vm_page_release(dst_page,
+ FALSE);
dst_page = vm_object_page_grab(object);
}
/*
* need to allocate a page
*/
- dst_page = vm_page_grab();
+ dst_page = vm_page_grab_options(grab_options);
}
if (dst_page == VM_PAGE_NULL) {
if ( (cntrl_flags & (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) == (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) {
VM_STAT_INCR(pageins);
}
}
- /*
- * ENCRYPTED SWAP:
- */
- if (cntrl_flags & UPL_ENCRYPT) {
- /*
- * The page is going to be encrypted when we
- * get it from the pager, so mark it so.
- */
- dst_page->encrypted = TRUE;
- } else {
- /*
- * Otherwise, the page will not contain
- * encrypted data.
- */
- dst_page->encrypted = FALSE;
- }
+ phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
+
dst_page->overwriting = TRUE;
if (dst_page->pmapped) {
* eliminate all mappings from the
* original object and its prodigy
*/
- refmod_state = pmap_disconnect(dst_page->phys_page);
+ refmod_state = pmap_disconnect(phys_page);
else
- refmod_state = pmap_get_refmod(dst_page->phys_page);
+ refmod_state = pmap_get_refmod(phys_page);
} else
refmod_state = 0;
lite_list[pg_num>>5] |= 1 << (pg_num & 31);
if (hw_dirty)
- pmap_clear_modify(dst_page->phys_page);
+ pmap_clear_modify(phys_page);
/*
* Mark original page as cleaning
dwp->dw_mask |= DW_set_reference;
}
if (cntrl_flags & UPL_PRECIOUS) {
- if (dst_page->object->internal) {
+ if (object->internal) {
SET_PAGE_DIRTY(dst_page, FALSE);
dst_page->precious = FALSE;
} else {
if (dst_page->busy)
upl->flags |= UPL_HAS_BUSY;
- if (dst_page->phys_page > upl->highest_page)
- upl->highest_page = dst_page->phys_page;
+ if (phys_page > upl->highest_page)
+ upl->highest_page = phys_page;
+ assert (!pmap_is_noencrypt(phys_page));
if (user_page_list) {
- user_page_list[entry].phys_addr = dst_page->phys_page;
- user_page_list[entry].pageout = dst_page->pageout;
+ user_page_list[entry].phys_addr = phys_page;
+ user_page_list[entry].free_when_done = dst_page->free_when_done;
user_page_list[entry].absent = dst_page->absent;
user_page_list[entry].dirty = dst_page->dirty;
user_page_list[entry].precious = dst_page->precious;
user_page_list[entry].device = FALSE;
user_page_list[entry].needed = FALSE;
if (dst_page->clustered == TRUE)
- user_page_list[entry].speculative = dst_page->speculative;
+ user_page_list[entry].speculative = (dst_page->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE;
else
user_page_list[entry].speculative = FALSE;
user_page_list[entry].cs_validated = dst_page->cs_validated;
user_page_list[entry].cs_tainted = dst_page->cs_tainted;
user_page_list[entry].cs_nx = dst_page->cs_nx;
+ user_page_list[entry].mark = FALSE;
}
/*
* if UPL_RET_ONLY_ABSENT is set, then
VM_PAGE_ADD_DELAYED_WORK(dwp, dst_page, dw_count);
if (dw_count >= dw_limit) {
- vm_page_do_delayed_work(object, &dw_array[0], dw_count);
+ vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count);
dwp = &dw_array[0];
dw_count = 0;
xfer_size -= PAGE_SIZE;
}
if (dw_count)
- vm_page_do_delayed_work(object, &dw_array[0], dw_count);
+ vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count);
if (alias_page != NULL) {
VM_PAGE_FREE(alias_page);
return KERN_SUCCESS;
}
-/* JMM - Backward compatability for now */
-kern_return_t
-vm_fault_list_request( /* forward */
- memory_object_control_t control,
- vm_object_offset_t offset,
- upl_size_t size,
- upl_t *upl_ptr,
- upl_page_info_t **user_page_list_ptr,
- unsigned int page_list_count,
- int cntrl_flags);
-kern_return_t
-vm_fault_list_request(
- memory_object_control_t control,
- vm_object_offset_t offset,
- upl_size_t size,
- upl_t *upl_ptr,
- upl_page_info_t **user_page_list_ptr,
- unsigned int page_list_count,
- int cntrl_flags)
-{
- unsigned int local_list_count;
- upl_page_info_t *user_page_list;
- kern_return_t kr;
-
- if((cntrl_flags & UPL_VECTOR)==UPL_VECTOR)
- return KERN_INVALID_ARGUMENT;
-
- if (user_page_list_ptr != NULL) {
- local_list_count = page_list_count;
- user_page_list = *user_page_list_ptr;
- } else {
- local_list_count = 0;
- user_page_list = NULL;
- }
- kr = memory_object_upl_request(control,
- offset,
- size,
- upl_ptr,
- user_page_list,
- &local_list_count,
- cntrl_flags);
-
- if(kr != KERN_SUCCESS)
- return kr;
-
- if ((user_page_list_ptr != NULL) && (cntrl_flags & UPL_INTERNAL)) {
- *user_page_list_ptr = UPL_GET_INTERNAL_PAGE_LIST(*upl_ptr);
- }
-
- return KERN_SUCCESS;
-}
-
-
-
/*
* Routine: vm_object_super_upl_request
* Purpose:
upl_t *upl,
upl_page_info_t *user_page_list,
unsigned int *page_list_count,
- int cntrl_flags)
+ upl_control_flags_t cntrl_flags,
+ vm_tag_t tag)
{
if (object->paging_offset > offset || ((cntrl_flags & UPL_VECTOR)==UPL_VECTOR))
return KERN_FAILURE;
offset = base_offset;
size = super_size;
}
- return vm_object_upl_request(object, offset, size, upl, user_page_list, page_list_count, cntrl_flags);
+ return vm_object_upl_request(object, offset, size, upl, user_page_list, page_list_count, cntrl_flags, tag);
}
+#if CONFIG_EMBEDDED
+int cs_executable_create_upl = 0;
+extern int proc_selfpid(void);
+extern char *proc_name_address(void *p);
+#endif /* CONFIG_EMBEDDED */
kern_return_t
vm_map_create_upl(
upl_t *upl,
upl_page_info_array_t page_list,
unsigned int *count,
- int *flags)
+ upl_control_flags_t *flags,
+ vm_tag_t tag)
{
- vm_map_entry_t entry;
- int caller_flags;
- int force_data_sync;
- int sync_cow_data;
- vm_object_t local_object;
- vm_map_offset_t local_offset;
- vm_map_offset_t local_start;
- kern_return_t ret;
+ vm_map_entry_t entry;
+ upl_control_flags_t caller_flags;
+ int force_data_sync;
+ int sync_cow_data;
+ vm_object_t local_object;
+ vm_map_offset_t local_offset;
+ vm_map_offset_t local_start;
+ kern_return_t ret;
+
+ assert(page_aligned(offset));
caller_flags = *flags;
REDISCOVER_ENTRY:
vm_map_lock_read(map);
- if (vm_map_lookup_entry(map, offset, &entry)) {
-
- if ((entry->vme_end - offset) < *upl_size) {
- *upl_size = (upl_size_t) (entry->vme_end - offset);
- assert(*upl_size == entry->vme_end - offset);
- }
+ if (!vm_map_lookup_entry(map, offset, &entry)) {
+ vm_map_unlock_read(map);
+ return KERN_FAILURE;
+ }
- if (caller_flags & UPL_QUERY_OBJECT_TYPE) {
- *flags = 0;
+ if ((entry->vme_end - offset) < *upl_size) {
+ *upl_size = (upl_size_t) (entry->vme_end - offset);
+ assert(*upl_size == entry->vme_end - offset);
+ }
- if ( !entry->is_sub_map && entry->object.vm_object != VM_OBJECT_NULL) {
- if (entry->object.vm_object->private)
- *flags = UPL_DEV_MEMORY;
+ if (caller_flags & UPL_QUERY_OBJECT_TYPE) {
+ *flags = 0;
- if (entry->object.vm_object->phys_contiguous)
- *flags |= UPL_PHYS_CONTIG;
- }
- vm_map_unlock_read(map);
+ if (!entry->is_sub_map &&
+ VME_OBJECT(entry) != VM_OBJECT_NULL) {
+ if (VME_OBJECT(entry)->private)
+ *flags = UPL_DEV_MEMORY;
- return KERN_SUCCESS;
+ if (VME_OBJECT(entry)->phys_contiguous)
+ *flags |= UPL_PHYS_CONTIG;
}
+ vm_map_unlock_read(map);
+ return KERN_SUCCESS;
+ }
- if (entry->is_sub_map) {
- vm_map_t submap;
+ if (VME_OBJECT(entry) == VM_OBJECT_NULL ||
+ !VME_OBJECT(entry)->phys_contiguous) {
+ if (*upl_size > MAX_UPL_SIZE_BYTES)
+ *upl_size = MAX_UPL_SIZE_BYTES;
+ }
- submap = entry->object.sub_map;
- local_start = entry->vme_start;
- local_offset = entry->offset;
+ /*
+ * Create an object if necessary.
+ */
+ if (VME_OBJECT(entry) == VM_OBJECT_NULL) {
- vm_map_reference(submap);
- vm_map_unlock_read(map);
+ if (vm_map_lock_read_to_write(map))
+ goto REDISCOVER_ENTRY;
- ret = vm_map_create_upl(submap,
- local_offset + (offset - local_start),
- upl_size, upl, page_list, count, flags);
- vm_map_deallocate(submap);
+ VME_OBJECT_SET(entry,
+ vm_object_allocate((vm_size_t)
+ (entry->vme_end -
+ entry->vme_start)));
+ VME_OFFSET_SET(entry, 0);
+ assert(entry->use_pmap);
- return ret;
- }
+ vm_map_lock_write_to_read(map);
+ }
+
+ if (!(caller_flags & UPL_COPYOUT_FROM) &&
+ !(entry->protection & VM_PROT_WRITE)) {
+ vm_map_unlock_read(map);
+ return KERN_PROTECTION_FAILURE;
+ }
+
+#if CONFIG_EMBEDDED
+ if (map->pmap != kernel_pmap &&
+ (caller_flags & UPL_COPYOUT_FROM) &&
+ (entry->protection & VM_PROT_EXECUTE) &&
+ !(entry->protection & VM_PROT_WRITE)) {
+ vm_offset_t kaddr;
+ vm_size_t ksize;
- if (entry->object.vm_object == VM_OBJECT_NULL || !entry->object.vm_object->phys_contiguous) {
- if (*upl_size > MAX_UPL_SIZE_BYTES)
- *upl_size = MAX_UPL_SIZE_BYTES;
- }
/*
- * Create an object if necessary.
+ * We're about to create a read-only UPL backed by
+ * memory from an executable mapping.
+ * Wiring the pages would result in the pages being copied
+ * (due to the "MAP_PRIVATE" mapping) and no longer
+ * code-signed, so no longer eligible for execution.
+ * Instead, let's copy the data into a kernel buffer and
+ * create the UPL from this kernel buffer.
+ * The kernel buffer is then freed, leaving the UPL holding
+ * the last reference on the VM object, so the memory will
+ * be released when the UPL is committed.
*/
- if (entry->object.vm_object == VM_OBJECT_NULL) {
- if (vm_map_lock_read_to_write(map))
- goto REDISCOVER_ENTRY;
+ vm_map_unlock_read(map);
+ /* allocate kernel buffer */
+ ksize = round_page(*upl_size);
+ kaddr = 0;
+ ret = kmem_alloc_pageable(kernel_map,
+ &kaddr,
+ ksize,
+ tag);
+ if (ret == KERN_SUCCESS) {
+ /* copyin the user data */
+ assert(page_aligned(offset));
+ ret = copyinmap(map, offset, (void *)kaddr, *upl_size);
+ }
+ if (ret == KERN_SUCCESS) {
+ if (ksize > *upl_size) {
+ /* zero out the extra space in kernel buffer */
+ memset((void *)(kaddr + *upl_size),
+ 0,
+ ksize - *upl_size);
+ }
+ /* create the UPL from the kernel buffer */
+ ret = vm_map_create_upl(kernel_map, kaddr, upl_size,
+ upl, page_list, count, flags, tag);
+ }
+ if (kaddr != 0) {
+ /* free the kernel buffer */
+ kmem_free(kernel_map, kaddr, ksize);
+ kaddr = 0;
+ ksize = 0;
+ }
+#if DEVELOPMENT || DEBUG
+ DTRACE_VM4(create_upl_from_executable,
+ vm_map_t, map,
+ vm_map_address_t, offset,
+ upl_size_t, *upl_size,
+ kern_return_t, ret);
+#endif /* DEVELOPMENT || DEBUG */
+ return ret;
+ }
+#endif /* CONFIG_EMBEDDED */
+
+ local_object = VME_OBJECT(entry);
+ assert(local_object != VM_OBJECT_NULL);
+
+ if (!entry->is_sub_map &&
+ !entry->needs_copy &&
+ *upl_size != 0 &&
+ local_object->vo_size > *upl_size && /* partial UPL */
+ entry->wired_count == 0 && /* No COW for entries that are wired */
+ (map->pmap != kernel_pmap) && /* alias checks */
+ (vm_map_entry_should_cow_for_true_share(entry) /* case 1 */
+ ||
+ (/* case 2 */
+ local_object->internal &&
+ (local_object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) &&
+ local_object->ref_count > 1))) {
+ vm_prot_t prot;
- entry->object.vm_object = vm_object_allocate((vm_size_t)(entry->vme_end - entry->vme_start));
- entry->offset = 0;
+ /*
+ * Case 1:
+ * Set up the targeted range for copy-on-write to avoid
+ * applying true_share/copy_delay to the entire object.
+ *
+ * Case 2:
+ * This map entry covers only part of an internal
+ * object. There could be other map entries covering
+ * other areas of this object and some of these map
+ * entries could be marked as "needs_copy", which
+ * assumes that the object is COPY_SYMMETRIC.
+ * To avoid marking this object as COPY_DELAY and
+ * "true_share", let's shadow it and mark the new
+ * (smaller) object as "true_share" and COPY_DELAY.
+ */
- vm_map_lock_write_to_read(map);
+ if (vm_map_lock_read_to_write(map)) {
+ goto REDISCOVER_ENTRY;
}
- if (!(caller_flags & UPL_COPYOUT_FROM)) {
- if (!(entry->protection & VM_PROT_WRITE)) {
- vm_map_unlock_read(map);
- return KERN_PROTECTION_FAILURE;
- }
+ vm_map_lock_assert_exclusive(map);
+ assert(VME_OBJECT(entry) == local_object);
+
+ vm_map_clip_start(map,
+ entry,
+ vm_map_trunc_page(offset,
+ VM_MAP_PAGE_MASK(map)));
+ vm_map_clip_end(map,
+ entry,
+ vm_map_round_page(offset + *upl_size,
+ VM_MAP_PAGE_MASK(map)));
+ if ((entry->vme_end - offset) < *upl_size) {
+ *upl_size = (upl_size_t) (entry->vme_end - offset);
+ assert(*upl_size == entry->vme_end - offset);
}
- local_object = entry->object.vm_object;
- if (vm_map_entry_should_cow_for_true_share(entry) &&
- local_object->vo_size > *upl_size &&
- *upl_size != 0) {
- vm_prot_t prot;
+ prot = entry->protection & ~VM_PROT_WRITE;
+ if (override_nx(map, VME_ALIAS(entry)) && prot)
+ prot |= VM_PROT_EXECUTE;
+ vm_object_pmap_protect(local_object,
+ VME_OFFSET(entry),
+ entry->vme_end - entry->vme_start,
+ ((entry->is_shared ||
+ map->mapped_in_other_pmaps)
+ ? PMAP_NULL
+ : map->pmap),
+ entry->vme_start,
+ prot);
- /*
- * Set up the targeted range for copy-on-write to avoid
- * applying true_share/copy_delay to the entire object.
- */
+ assert(entry->wired_count == 0);
- if (vm_map_lock_read_to_write(map)) {
- goto REDISCOVER_ENTRY;
- }
-
- vm_map_clip_start(map,
- entry,
- vm_map_trunc_page(offset,
- VM_MAP_PAGE_MASK(map)));
- vm_map_clip_end(map,
- entry,
- vm_map_round_page(offset + *upl_size,
- VM_MAP_PAGE_MASK(map)));
- if ((entry->vme_end - offset) < *upl_size) {
- *upl_size = (upl_size_t) (entry->vme_end - offset);
- assert(*upl_size == entry->vme_end - offset);
- }
-
- prot = entry->protection & ~VM_PROT_WRITE;
- if (override_nx(map, entry->alias) && prot)
- prot |= VM_PROT_EXECUTE;
- vm_object_pmap_protect(local_object,
- entry->offset,
- entry->vme_end - entry->vme_start,
- ((entry->is_shared || map->mapped_in_other_pmaps)
- ? PMAP_NULL
- : map->pmap),
- entry->vme_start,
- prot);
+ /*
+ * Lock the VM object and re-check its status: if it's mapped
+ * in another address space, we could still be racing with
+ * another thread holding that other VM map exclusively.
+ */
+ vm_object_lock(local_object);
+ if (local_object->true_share) {
+ /* object is already in proper state: no COW needed */
+ assert(local_object->copy_strategy !=
+ MEMORY_OBJECT_COPY_SYMMETRIC);
+ } else {
+ /* not true_share: ask for copy-on-write below */
+ assert(local_object->copy_strategy ==
+ MEMORY_OBJECT_COPY_SYMMETRIC);
entry->needs_copy = TRUE;
-
- vm_map_lock_write_to_read(map);
}
+ vm_object_unlock(local_object);
- if (entry->needs_copy) {
- /*
- * Honor copy-on-write for COPY_SYMMETRIC
- * strategy.
- */
- vm_map_t local_map;
- vm_object_t object;
- vm_object_offset_t new_offset;
- vm_prot_t prot;
- boolean_t wired;
- vm_map_version_t version;
- vm_map_t real_map;
- vm_prot_t fault_type;
-
- local_map = map;
-
- if (caller_flags & UPL_COPYOUT_FROM) {
- fault_type = VM_PROT_READ | VM_PROT_COPY;
- vm_counters.create_upl_extra_cow++;
- vm_counters.create_upl_extra_cow_pages += (entry->vme_end - entry->vme_start) / PAGE_SIZE;
+ vm_map_lock_write_to_read(map);
+ }
+
+ if (entry->needs_copy) {
+ /*
+ * Honor copy-on-write for COPY_SYMMETRIC
+ * strategy.
+ */
+ vm_map_t local_map;
+ vm_object_t object;
+ vm_object_offset_t new_offset;
+ vm_prot_t prot;
+ boolean_t wired;
+ vm_map_version_t version;
+ vm_map_t real_map;
+ vm_prot_t fault_type;
+
+ local_map = map;
+
+ if (caller_flags & UPL_COPYOUT_FROM) {
+ fault_type = VM_PROT_READ | VM_PROT_COPY;
+ vm_counters.create_upl_extra_cow++;
+ vm_counters.create_upl_extra_cow_pages +=
+ (entry->vme_end - entry->vme_start) / PAGE_SIZE;
+ } else {
+ fault_type = VM_PROT_WRITE;
+ }
+ if (vm_map_lookup_locked(&local_map,
+ offset, fault_type,
+ OBJECT_LOCK_EXCLUSIVE,
+ &version, &object,
+ &new_offset, &prot, &wired,
+ NULL,
+ &real_map) != KERN_SUCCESS) {
+ if (fault_type == VM_PROT_WRITE) {
+ vm_counters.create_upl_lookup_failure_write++;
} else {
- fault_type = VM_PROT_WRITE;
- }
- if (vm_map_lookup_locked(&local_map,
- offset, fault_type,
- OBJECT_LOCK_EXCLUSIVE,
- &version, &object,
- &new_offset, &prot, &wired,
- NULL,
- &real_map) != KERN_SUCCESS) {
- if (fault_type == VM_PROT_WRITE) {
- vm_counters.create_upl_lookup_failure_write++;
- } else {
- vm_counters.create_upl_lookup_failure_copy++;
- }
- vm_map_unlock_read(local_map);
- return KERN_FAILURE;
+ vm_counters.create_upl_lookup_failure_copy++;
}
- if (real_map != map)
- vm_map_unlock(real_map);
vm_map_unlock_read(local_map);
+ return KERN_FAILURE;
+ }
+ if (real_map != map)
+ vm_map_unlock(real_map);
+ vm_map_unlock_read(local_map);
- vm_object_unlock(object);
+ vm_object_unlock(object);
- goto REDISCOVER_ENTRY;
- }
+ goto REDISCOVER_ENTRY;
+ }
- if (sync_cow_data) {
- if (entry->object.vm_object->shadow || entry->object.vm_object->copy) {
- local_object = entry->object.vm_object;
- local_start = entry->vme_start;
- local_offset = entry->offset;
-
- vm_object_reference(local_object);
- vm_map_unlock_read(map);
-
- if (local_object->shadow && local_object->copy) {
- vm_object_lock_request(
- local_object->shadow,
- (vm_object_offset_t)
- ((offset - local_start) +
- local_offset) +
- local_object->vo_shadow_offset,
- *upl_size, FALSE,
- MEMORY_OBJECT_DATA_SYNC,
- VM_PROT_NO_CHANGE);
- }
- sync_cow_data = FALSE;
- vm_object_deallocate(local_object);
+ if (entry->is_sub_map) {
+ vm_map_t submap;
- goto REDISCOVER_ENTRY;
- }
- }
- if (force_data_sync) {
- local_object = entry->object.vm_object;
- local_start = entry->vme_start;
- local_offset = entry->offset;
+ submap = VME_SUBMAP(entry);
+ local_start = entry->vme_start;
+ local_offset = VME_OFFSET(entry);
+
+ vm_map_reference(submap);
+ vm_map_unlock_read(map);
+
+ ret = vm_map_create_upl(submap,
+ local_offset + (offset - local_start),
+ upl_size, upl, page_list, count, flags, tag);
+ vm_map_deallocate(submap);
+
+ return ret;
+ }
+
+ if (sync_cow_data &&
+ (VME_OBJECT(entry)->shadow ||
+ VME_OBJECT(entry)->copy)) {
+ local_object = VME_OBJECT(entry);
+ local_start = entry->vme_start;
+ local_offset = VME_OFFSET(entry);
- vm_object_reference(local_object);
- vm_map_unlock_read(map);
+ vm_object_reference(local_object);
+ vm_map_unlock_read(map);
- vm_object_lock_request(
- local_object,
- (vm_object_offset_t)
- ((offset - local_start) + local_offset),
- (vm_object_size_t)*upl_size, FALSE,
+ if (local_object->shadow && local_object->copy) {
+ vm_object_lock_request(local_object->shadow,
+ ((vm_object_offset_t)
+ ((offset - local_start) +
+ local_offset) +
+ local_object->vo_shadow_offset),
+ *upl_size, FALSE,
MEMORY_OBJECT_DATA_SYNC,
VM_PROT_NO_CHANGE);
-
- force_data_sync = FALSE;
- vm_object_deallocate(local_object);
-
- goto REDISCOVER_ENTRY;
}
- if (entry->object.vm_object->private)
- *flags = UPL_DEV_MEMORY;
- else
- *flags = 0;
-
- if (entry->object.vm_object->phys_contiguous)
- *flags |= UPL_PHYS_CONTIG;
+ sync_cow_data = FALSE;
+ vm_object_deallocate(local_object);
- local_object = entry->object.vm_object;
- local_offset = entry->offset;
+ goto REDISCOVER_ENTRY;
+ }
+ if (force_data_sync) {
+ local_object = VME_OBJECT(entry);
local_start = entry->vme_start;
+ local_offset = VME_OFFSET(entry);
vm_object_reference(local_object);
vm_map_unlock_read(map);
- ret = vm_object_iopl_request(local_object,
- (vm_object_offset_t) ((offset - local_start) + local_offset),
- *upl_size,
- upl,
- page_list,
- count,
- caller_flags);
+ vm_object_lock_request(local_object,
+ ((vm_object_offset_t)
+ ((offset - local_start) +
+ local_offset)),
+ (vm_object_size_t)*upl_size,
+ FALSE,
+ MEMORY_OBJECT_DATA_SYNC,
+ VM_PROT_NO_CHANGE);
+
+ force_data_sync = FALSE;
vm_object_deallocate(local_object);
- return(ret);
- }
+ goto REDISCOVER_ENTRY;
+ }
+ if (VME_OBJECT(entry)->private)
+ *flags = UPL_DEV_MEMORY;
+ else
+ *flags = 0;
+
+ if (VME_OBJECT(entry)->phys_contiguous)
+ *flags |= UPL_PHYS_CONTIG;
+
+ local_object = VME_OBJECT(entry);
+ local_offset = VME_OFFSET(entry);
+ local_start = entry->vme_start;
+
+#if CONFIG_EMBEDDED
+ /*
+ * Wiring will copy the pages to the shadow object.
+ * The shadow object will not be code-signed so
+ * attempting to execute code from these copied pages
+ * would trigger a code-signing violation.
+ */
+ if (entry->protection & VM_PROT_EXECUTE) {
+#if MACH_ASSERT
+ printf("pid %d[%s] create_upl out of executable range from "
+ "0x%llx to 0x%llx: side effects may include "
+ "code-signing violations later on\n",
+ proc_selfpid(),
+ (current_task()->bsd_info
+ ? proc_name_address(current_task()->bsd_info)
+ : "?"),
+ (uint64_t) entry->vme_start,
+ (uint64_t) entry->vme_end);
+#endif /* MACH_ASSERT */
+ DTRACE_VM2(cs_executable_create_upl,
+ uint64_t, (uint64_t)entry->vme_start,
+ uint64_t, (uint64_t)entry->vme_end);
+ cs_executable_create_upl++;
+ }
+#endif /* CONFIG_EMBEDDED */
+
+ vm_object_lock(local_object);
+
+ /*
+ * Ensure that this object is "true_share" and "copy_delay" now,
+ * while we're still holding the VM map lock. After we unlock the map,
+ * anything could happen to that mapping, including some copy-on-write
+ * activity. We need to make sure that the IOPL will point at the
+ * same memory as the mapping.
+ */
+ if (local_object->true_share) {
+ assert(local_object->copy_strategy !=
+ MEMORY_OBJECT_COPY_SYMMETRIC);
+ } else if (local_object != kernel_object &&
+ local_object != compressor_object &&
+ !local_object->phys_contiguous) {
+#if VM_OBJECT_TRACKING_OP_TRUESHARE
+ if (!local_object->true_share &&
+ vm_object_tracking_inited) {
+ void *bt[VM_OBJECT_TRACKING_BTDEPTH];
+ int num = 0;
+ num = OSBacktrace(bt,
+ VM_OBJECT_TRACKING_BTDEPTH);
+ btlog_add_entry(vm_object_tracking_btlog,
+ local_object,
+ VM_OBJECT_TRACKING_OP_TRUESHARE,
+ bt,
+ num);
+ }
+#endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
+ local_object->true_share = TRUE;
+ if (local_object->copy_strategy ==
+ MEMORY_OBJECT_COPY_SYMMETRIC) {
+ local_object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
+ }
+ }
+
+ vm_object_reference_locked(local_object);
+ vm_object_unlock(local_object);
+
vm_map_unlock_read(map);
- return(KERN_FAILURE);
+ ret = vm_object_iopl_request(local_object,
+ ((vm_object_offset_t)
+ ((offset - local_start) + local_offset)),
+ *upl_size,
+ upl,
+ page_list,
+ count,
+ caller_flags,
+ tag);
+ vm_object_deallocate(local_object);
+
+ return ret;
}
/*
}
}
- kr = kmem_suballoc(map, &vector_upl_dst_addr, vector_upl->size, FALSE, VM_FLAGS_ANYWHERE, &vector_upl_submap);
+ kr = kmem_suballoc(map, &vector_upl_dst_addr, vector_upl->size, FALSE,
+ VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_NONE,
+ &vector_upl_submap);
if( kr != KERN_SUCCESS )
panic("Vector UPL submap allocation failed\n");
map = vector_upl_submap;
assert(alias_page->fictitious);
alias_page->fictitious = FALSE;
alias_page->private = TRUE;
- alias_page->pageout = TRUE;
+ alias_page->free_when_done = TRUE;
/*
* since m is a page in the upl it must
* already be wired or BUSY, so it's
* safe to assign the underlying physical
* page to the alias
*/
- alias_page->phys_page = m->phys_page;
+ VM_PAGE_SET_PHYS_PAGE(alias_page, VM_PAGE_GET_PHYS_PAGE(m));
vm_object_unlock(object);
vm_page_lockspin_queues();
- vm_page_wire(alias_page);
+ vm_page_wire(alias_page, VM_KERN_MEMORY_NONE, TRUE);
vm_page_unlock_queues();
- /*
- * ENCRYPTED SWAP:
- * The virtual page ("m") has to be wired in some way
- * here or its physical page ("m->phys_page") could
- * be recycled at any time.
- * Assuming this is enforced by the caller, we can't
- * get an encrypted page here. Since the encryption
- * key depends on the VM page's "pager" object and
- * the "paging_offset", we couldn't handle 2 pageable
- * VM pages (with different pagers and paging_offsets)
- * sharing the same physical page: we could end up
- * encrypting with one key (via one VM page) and
- * decrypting with another key (via the alias VM page).
- */
- ASSERT_PAGE_DECRYPTED(m);
-
- vm_page_insert(alias_page, upl->map_object, new_offset);
+ vm_page_insert_wired(alias_page, upl->map_object, new_offset, VM_KERN_MEMORY_NONE);
assert(!alias_page->wanted);
alias_page->busy = FALSE;
* NEED A UPL_MAP ALIAS
*/
kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,
- VM_FLAGS_ANYWHERE, upl->map_object, offset, FALSE,
+ VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_OSFMK,
+ upl->map_object, offset, FALSE,
VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
if (kr != KERN_SUCCESS) {
+ vm_object_deallocate(upl->map_object);
upl_unlock(upl);
return(kr);
}
}
else {
kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,
- VM_FLAGS_FIXED, upl->map_object, offset, FALSE,
+ VM_FLAGS_FIXED, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_OSFMK,
+ upl->map_object, offset, FALSE,
VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
if(kr)
panic("vm_map_enter failed for a Vector UPL\n");
/* m->wpmapped = TRUE; */
assert(map->pmap == kernel_pmap);
- PMAP_ENTER(map->pmap, addr, m, VM_PROT_DEFAULT, VM_PROT_NONE, 0, TRUE);
+ PMAP_ENTER(map->pmap, addr, m, VM_PROT_DEFAULT, VM_PROT_NONE, 0, TRUE, kr);
+
+ assert(kr == KERN_SUCCESS);
+#if KASAN
+ kasan_notify_address(addr, PAGE_SIZE_64);
+#endif
}
offset += PAGE_SIZE_64;
}
return KERN_FAILURE;
}
+
kern_return_t
upl_commit_range(
upl_t upl,
upl_size_t xfer_size, subupl_size = size;
vm_object_t shadow_object;
vm_object_t object;
+ vm_object_t m_object;
vm_object_offset_t target_offset;
upl_offset_t subupl_offset = offset;
int entry;
int throttle_page = 0;
int unwired_count = 0;
int local_queue_count = 0;
- queue_head_t local_queue;
+ vm_page_t first_local, last_local;
*empty = FALSE;
entry = offset/PAGE_SIZE;
target_offset = (vm_object_offset_t)offset;
+ assert(!(target_offset & PAGE_MASK));
+ assert(!(xfer_size & PAGE_MASK));
+
if (upl->flags & UPL_KERNEL_OBJECT)
vm_object_lock_shared(shadow_object);
else
vm_object_lock(shadow_object);
+ VM_OBJECT_WIRED_PAGE_UPDATE_START(shadow_object);
+
if (upl->flags & UPL_ACCESS_BLOCKED) {
assert(shadow_object->blocked_access);
shadow_object->blocked_access = FALSE;
*/
flags &= ~UPL_COMMIT_CS_VALIDATED;
}
- if (!VM_DYNAMIC_PAGING_ENABLED(memory_manager_default) && shadow_object->internal)
+ if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal)
should_be_throttled = TRUE;
dwp = &dw_array[0];
shadow_object->purgable != VM_PURGABLE_VOLATILE &&
shadow_object->purgable != VM_PURGABLE_EMPTY) {
- if (!queue_empty(&shadow_object->memq)) {
- queue_init(&local_queue);
+ if (!vm_page_queue_empty(&shadow_object->memq)) {
+
if (size == shadow_object->vo_size) {
- nxt_page = (vm_page_t)queue_first(&shadow_object->memq);
+ nxt_page = (vm_page_t)vm_page_queue_first(&shadow_object->memq);
fast_path_full_commit = 1;
}
fast_path_possible = 1;
- if (!VM_DYNAMIC_PAGING_ENABLED(memory_manager_default) && shadow_object->internal &&
+ if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal &&
(shadow_object->purgable == VM_PURGABLE_DENY ||
shadow_object->purgable == VM_PURGABLE_NONVOLATILE ||
shadow_object->purgable == VM_PURGABLE_VOLATILE)) {
}
}
}
+ first_local = VM_PAGE_NULL;
+ last_local = VM_PAGE_NULL;
while (xfer_size) {
vm_page_t t, m;
if (nxt_page != VM_PAGE_NULL) {
m = nxt_page;
- nxt_page = (vm_page_t)queue_next(&nxt_page->listq);
+ nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->listq);
target_offset = m->offset;
}
pg_num = (unsigned int) (target_offset/PAGE_SIZE);
if (upl->flags & UPL_SHADOWED) {
if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) {
- t->pageout = FALSE;
+ t->free_when_done = FALSE;
VM_PAGE_FREE(t);
if (m == VM_PAGE_NULL)
goto commit_next_page;
- if (m->compressor) {
+ m_object = VM_PAGE_OBJECT(m);
+
+ if (m->vm_page_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
assert(m->busy);
dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
#if DEVELOPMENT || DEBUG
vm_cs_validated_resets++;
#endif
- pmap_disconnect(m->phys_page);
+ pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
}
clear_refmod |= VM_MEM_MODIFIED;
}
dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
}
if (fast_path_possible) {
- assert(m->object->purgable != VM_PURGABLE_EMPTY);
- assert(m->object->purgable != VM_PURGABLE_VOLATILE);
+ assert(m_object->purgable != VM_PURGABLE_EMPTY);
+ assert(m_object->purgable != VM_PURGABLE_VOLATILE);
if (m->absent) {
+ assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q);
assert(m->wire_count == 0);
assert(m->busy);
} else {
if (m->wire_count == 0)
panic("wire_count == 0, m = %p, obj = %p\n", m, shadow_object);
+ assert(m->vm_page_q_state == VM_PAGE_IS_WIRED);
/*
* XXX FBDP need to update some other
* counters here (purgeable_wired_count)
* (ledgers), ...
*/
- assert(m->wire_count);
+ assert(m->wire_count > 0);
m->wire_count--;
- if (m->wire_count == 0)
+ if (m->wire_count == 0) {
+ m->vm_page_q_state = VM_PAGE_NOT_ON_Q;
unwired_count++;
+ }
}
if (m->wire_count == 0) {
- queue_enter(&local_queue, m, vm_page_t, pageq);
+ assert(m->pageq.next == 0 && m->pageq.prev == 0);
+
+ if (last_local == VM_PAGE_NULL) {
+ assert(first_local == VM_PAGE_NULL);
+
+ last_local = m;
+ first_local = m;
+ } else {
+ assert(first_local != VM_PAGE_NULL);
+
+ m->pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local);
+ first_local->pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(m);
+ first_local = m;
+ }
local_queue_count++;
if (throttle_page) {
- m->throttled = TRUE;
+ m->vm_page_q_state = VM_PAGE_ON_THROTTLED_Q;
} else {
- if (flags & UPL_COMMIT_INACTIVATE)
- m->inactive = TRUE;
- else
- m->active = TRUE;
+ if (flags & UPL_COMMIT_INACTIVATE) {
+ if (shadow_object->internal)
+ m->vm_page_q_state = VM_PAGE_ON_INACTIVE_INTERNAL_Q;
+ else
+ m->vm_page_q_state = VM_PAGE_ON_INACTIVE_EXTERNAL_Q;
+ } else
+ m->vm_page_q_state = VM_PAGE_ON_ACTIVE_Q;
}
}
} else {
}
goto commit_next_page;
}
- assert(!m->compressor);
+ assert(m->vm_page_q_state != VM_PAGE_USED_BY_COMPRESSOR);
if (page_list)
page_list[entry].phys_addr = 0;
dwp->dw_mask |= DW_vm_pageout_throttle_up;
if (VM_PAGE_WIRED(m))
- m->pageout = FALSE;
+ m->free_when_done = FALSE;
if (! (flags & UPL_COMMIT_CS_VALIDATED) &&
m->cs_validated && !m->cs_tainted) {
#if DEVELOPMENT || DEBUG
vm_cs_validated_resets++;
#endif
- pmap_disconnect(m->phys_page);
+ pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
}
if (m->overwriting) {
/*
*/
if (m->busy) {
#if CONFIG_PHANTOM_CACHE
- if (m->absent && !m->object->internal)
+ if (m->absent && !m_object->internal)
dwp->dw_mask |= DW_vm_phantom_cache_update;
#endif
m->absent = FALSE;
}
m->overwriting = FALSE;
}
- if (m->encrypted_cleaning == TRUE) {
- m->encrypted_cleaning = FALSE;
-
- dwp->dw_mask |= DW_clear_busy | DW_PAGE_WAKEUP;
- }
m->cleaning = FALSE;
- if (m->pageout) {
+ if (m->free_when_done) {
/*
* With the clean queue enabled, UPL_PAGEOUT should
* no longer set the pageout bit. It's pages now go
* to the clean queue.
*/
assert(!(flags & UPL_PAGEOUT));
+ assert(!m_object->internal);
- m->pageout = FALSE;
+ m->free_when_done = FALSE;
#if MACH_CLUSTER_STATS
if (m->wanted) vm_pageout_target_collisions++;
#endif
if ((flags & UPL_COMMIT_SET_DIRTY) ||
- (m->pmapped && (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED))) {
+ (m->pmapped && (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED))) {
/*
* page was re-dirtied after we started
* the pageout... reactivate it since
* page has been successfully cleaned
* go ahead and free it for other use
*/
- if (m->object->internal) {
+ if (m_object->internal) {
DTRACE_VM2(anonpgout, int, 1, (uint64_t *), NULL);
} else {
DTRACE_VM2(fspgout, int, 1, (uint64_t *), NULL);
}
#if MACH_CLUSTER_STATS
if (m->wpmapped)
- m->dirty = pmap_is_modified(m->phys_page);
+ m->dirty = pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m));
if (m->dirty) vm_pageout_cluster_dirtied++;
else vm_pageout_cluster_cleaned++;
dwp->dw_mask |= DW_enqueue_cleaned;
vm_pageout_enqueued_cleaned_from_inactive_dirty++;
- } else if (should_be_throttled == TRUE && !m->active && !m->inactive && !m->speculative && !m->throttled) {
+ } else if (should_be_throttled == TRUE && (m->vm_page_q_state == VM_PAGE_NOT_ON_Q)) {
/*
* page coming back in from being 'frozen'...
* it was dirty before it was frozen, so keep it so
dwp->dw_mask |= DW_vm_page_activate;
} else {
- if ((flags & UPL_COMMIT_INACTIVATE) && !m->clustered && !m->speculative) {
+ if ((flags & UPL_COMMIT_INACTIVATE) && !m->clustered && (m->vm_page_q_state != VM_PAGE_ON_SPECULATIVE_Q)) {
dwp->dw_mask |= DW_vm_page_deactivate_internal;
clear_refmod |= VM_MEM_REFERENCED;
- } else if (!m->active && !m->inactive && !m->speculative) {
+ } else if ( !VM_PAGE_PAGEABLE(m)) {
if (m->clustered || (flags & UPL_COMMIT_SPECULATE))
dwp->dw_mask |= DW_vm_page_speculate;
commit_next_page:
if (clear_refmod)
- pmap_clear_refmod(m->phys_page, clear_refmod);
+ pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(m), clear_refmod);
target_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count);
if (dw_count >= dw_limit) {
- vm_page_do_delayed_work(shadow_object, &dw_array[0], dw_count);
+ vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);
dwp = &dw_array[0];
dw_count = 0;
}
}
if (dw_count)
- vm_page_do_delayed_work(shadow_object, &dw_array[0], dw_count);
+ vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);
if (fast_path_possible) {
if (local_queue_count || unwired_count) {
if (local_queue_count) {
- vm_page_t first_local, last_local;
vm_page_t first_target;
- queue_head_t *target_queue;
+ vm_page_queue_head_t *target_queue;
if (throttle_page)
target_queue = &vm_page_queue_throttled;
/*
* Transfer the entire local queue to a regular LRU page queues.
*/
- first_local = (vm_page_t) queue_first(&local_queue);
- last_local = (vm_page_t) queue_last(&local_queue);
-
vm_page_lockspin_queues();
- first_target = (vm_page_t) queue_first(target_queue);
+ first_target = (vm_page_t) vm_page_queue_first(target_queue);
- if (queue_empty(target_queue))
- queue_last(target_queue) = (queue_entry_t) last_local;
+ if (vm_page_queue_empty(target_queue))
+ target_queue->prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local);
else
- queue_prev(&first_target->pageq) = (queue_entry_t) last_local;
+ first_target->pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local);
- queue_first(target_queue) = (queue_entry_t) first_local;
- queue_prev(&first_local->pageq) = (queue_entry_t) target_queue;
- queue_next(&last_local->pageq) = (queue_entry_t) first_target;
+ target_queue->next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local);
+ first_local->pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(target_queue);
+ last_local->pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_target);
/*
* Adjust the global page counts.
}
vm_page_unlock_queues();
- shadow_object->wired_page_count -= unwired_count;
+ VM_OBJECT_WIRED_PAGE_COUNT(shadow_object, -unwired_count);
}
}
occupied = 1;
}
}
} else {
- if (queue_empty(&upl->map_object->memq))
+ if (vm_page_queue_empty(&upl->map_object->memq))
occupied = 0;
}
if (occupied == 0) {
*/
}
}
+ VM_OBJECT_WIRED_PAGE_UPDATE_END(shadow_object, shadow_object->wire_tag);
vm_object_unlock(shadow_object);
if (object != shadow_object)
vm_object_unlock(object);
}
goto process_upl_to_commit;
}
-
if (pgpgout_count) {
DTRACE_VM2(pgpgout, int, pgpgout_count, (uint64_t *), NULL);
}
entry = offset/PAGE_SIZE;
target_offset = (vm_object_offset_t)offset;
+ assert(!(target_offset & PAGE_MASK));
+ assert(!(xfer_size & PAGE_MASK));
+
if (upl->flags & UPL_KERNEL_OBJECT)
vm_object_lock_shared(shadow_object);
else
}
if (upl->flags & UPL_SHADOWED) {
if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) {
- t->pageout = FALSE;
+ t->free_when_done = FALSE;
VM_PAGE_FREE(t);
if (m != VM_PAGE_NULL) {
- assert(!m->compressor);
+ assert(m->vm_page_q_state != VM_PAGE_USED_BY_COMPRESSOR);
if (m->absent) {
boolean_t must_free = TRUE;
*/
must_free = TRUE;
}
-
- /*
- * ENCRYPTED SWAP:
- * If the page was already encrypted,
- * we don't really need to decrypt it
- * now. It will get decrypted later,
- * on demand, as soon as someone needs
- * to access its contents.
- */
-
m->cleaning = FALSE;
- m->encrypted_cleaning = FALSE;
if (m->overwriting && !m->busy) {
/*
}
m->overwriting = FALSE;
}
- if (m->encrypted_cleaning == TRUE) {
- m->encrypted_cleaning = FALSE;
-
- dwp->dw_mask |= DW_clear_busy;
- }
- m->pageout = FALSE;
+ m->free_when_done = FALSE;
m->cleaning = FALSE;
-#if MACH_PAGEMAP
- vm_external_state_clr(m->object->existence_map, m->offset);
-#endif /* MACH_PAGEMAP */
+
if (error & UPL_ABORT_DUMP_PAGES) {
- pmap_disconnect(m->phys_page);
+ pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
dwp->dw_mask |= DW_vm_page_free;
} else {
*/
dwp->dw_mask |= DW_vm_page_lru;
- } else if (!m->active && !m->inactive && !m->speculative)
+ } else if ( !VM_PAGE_PAGEABLE(m))
dwp->dw_mask |= DW_vm_page_deactivate_internal;
}
dwp->dw_mask |= DW_PAGE_WAKEUP;
VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count);
if (dw_count >= dw_limit) {
- vm_page_do_delayed_work(shadow_object, &dw_array[0], dw_count);
+ vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);
dwp = &dw_array[0];
dw_count = 0;
}
}
if (dw_count)
- vm_page_do_delayed_work(shadow_object, &dw_array[0], dw_count);
+ vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);
occupied = 1;
}
}
} else {
- if (queue_empty(&upl->map_object->memq))
+ if (vm_page_queue_empty(&upl->map_object->memq))
occupied = 0;
}
if (occupied == 0) {
{
boolean_t empty;
+ if (upl == UPL_NULL)
+ return KERN_INVALID_ARGUMENT;
+
return upl_abort_range(upl, 0, upl->size, error, &empty);
}
{
boolean_t empty;
+ if (upl == UPL_NULL)
+ return KERN_INVALID_ARGUMENT;
+
return upl_commit_range(upl, 0, upl->size, 0, page_list, count, &empty);
}
void
iopl_valid_data(
- upl_t upl)
+ upl_t upl,
+ vm_tag_t tag)
{
vm_object_t object;
vm_offset_t offset;
if (object == kernel_object || object == compressor_object)
panic("iopl_valid_data: object == kernel or compressor");
- if (object->purgable == VM_PURGABLE_VOLATILE)
- panic("iopl_valid_data: object == VM_PURGABLE_VOLATILE");
+ if (object->purgable == VM_PURGABLE_VOLATILE ||
+ object->purgable == VM_PURGABLE_EMPTY)
+ panic("iopl_valid_data: object %p purgable %d",
+ object, object->purgable);
size = upl->size;
vm_object_lock(object);
+ VM_OBJECT_WIRED_PAGE_UPDATE_START(object);
if (object->vo_size == size && object->resident_page_count == (size / PAGE_SIZE))
- nxt_page = (vm_page_t)queue_first(&object->memq);
+ nxt_page = (vm_page_t)vm_page_queue_first(&object->memq);
else
offset = 0 + upl->offset - object->paging_offset;
if (nxt_page != VM_PAGE_NULL) {
m = nxt_page;
- nxt_page = (vm_page_t)queue_next(&nxt_page->listq);
+ nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->listq);
} else {
m = vm_page_lookup(object, offset);
offset += PAGE_SIZE;
if (m->pageq.next || m->pageq.prev)
panic("iopl_valid_data: busy+absent page on page queue");
+ if (m->reusable) {
+ panic("iopl_valid_data: %p is reusable", m);
+ }
m->absent = FALSE;
m->dirty = TRUE;
+ assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q);
+ assert(m->wire_count == 0);
m->wire_count++;
- wired_count++;
+ assert(m->wire_count);
+ if (m->wire_count == 1) {
+ m->vm_page_q_state = VM_PAGE_IS_WIRED;
+ wired_count++;
+ } else {
+ panic("iopl_valid_data: %p already wired\n", m);
+ }
PAGE_WAKEUP_DONE(m);
}
size -= PAGE_SIZE;
}
if (wired_count) {
- object->wired_page_count += wired_count;
+
+ VM_OBJECT_WIRED_PAGE_COUNT(object, wired_count);
+ assert(object->resident_page_count >= object->wired_page_count);
+
+ /* no need to adjust purgeable accounting for this object: */
+ assert(object->purgable != VM_PURGABLE_VOLATILE);
+ assert(object->purgable != VM_PURGABLE_EMPTY);
vm_page_lockspin_queues();
vm_page_wire_count += wired_count;
vm_page_unlock_queues();
}
+ VM_OBJECT_WIRED_PAGE_UPDATE_END(object, tag);
vm_object_unlock(object);
}
-
-
void
vm_object_set_pmap_cache_attr(
vm_object_t object,
}
}
+
+boolean_t vm_object_iopl_wire_full(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t, vm_tag_t);
+kern_return_t vm_object_iopl_wire_empty(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t, vm_tag_t, vm_object_offset_t *, int);
+
+
+
+boolean_t
+vm_object_iopl_wire_full(vm_object_t object, upl_t upl, upl_page_info_array_t user_page_list,
+ wpl_array_t lite_list, upl_control_flags_t cntrl_flags, vm_tag_t tag)
+{
+ vm_page_t dst_page;
+ unsigned int entry;
+ int page_count;
+ int delayed_unlock = 0;
+ boolean_t retval = TRUE;
+ ppnum_t phys_page;
+
+ vm_object_lock_assert_exclusive(object);
+ assert(object->purgable != VM_PURGABLE_VOLATILE);
+ assert(object->purgable != VM_PURGABLE_EMPTY);
+ assert(object->pager == NULL);
+ assert(object->copy == NULL);
+ assert(object->shadow == NULL);
+
+ page_count = object->resident_page_count;
+ dst_page = (vm_page_t)vm_page_queue_first(&object->memq);
+
+ vm_page_lock_queues();
+
+ while (page_count--) {
+
+ if (dst_page->busy ||
+ dst_page->fictitious ||
+ dst_page->absent ||
+ dst_page->error ||
+ dst_page->cleaning ||
+ dst_page->restart ||
+ dst_page->laundry) {
+ retval = FALSE;
+ goto done;
+ }
+ if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->written_by_kernel == TRUE) {
+ retval = FALSE;
+ goto done;
+ }
+ dst_page->reference = TRUE;
+
+ vm_page_wire(dst_page, tag, FALSE);
+
+ if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
+ SET_PAGE_DIRTY(dst_page, FALSE);
+ }
+ entry = (unsigned int)(dst_page->offset / PAGE_SIZE);
+ assert(entry >= 0 && entry < object->resident_page_count);
+ lite_list[entry>>5] |= 1 << (entry & 31);
+
+ phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
+
+ if (phys_page > upl->highest_page)
+ upl->highest_page = phys_page;
+
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = phys_page;
+ user_page_list[entry].absent = dst_page->absent;
+ user_page_list[entry].dirty = dst_page->dirty;
+ user_page_list[entry].free_when_done = dst_page->free_when_done;
+ user_page_list[entry].precious = dst_page->precious;
+ user_page_list[entry].device = FALSE;
+ user_page_list[entry].speculative = FALSE;
+ user_page_list[entry].cs_validated = FALSE;
+ user_page_list[entry].cs_tainted = FALSE;
+ user_page_list[entry].cs_nx = FALSE;
+ user_page_list[entry].needed = FALSE;
+ user_page_list[entry].mark = FALSE;
+ }
+ if (delayed_unlock++ > 256) {
+ delayed_unlock = 0;
+ lck_mtx_yield(&vm_page_queue_lock);
+
+ VM_CHECK_MEMORYSTATUS;
+ }
+ dst_page = (vm_page_t)vm_page_queue_next(&dst_page->listq);
+ }
+done:
+ vm_page_unlock_queues();
+
+ VM_CHECK_MEMORYSTATUS;
+
+ return (retval);
+}
+
+
+kern_return_t
+vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t user_page_list,
+ wpl_array_t lite_list, upl_control_flags_t cntrl_flags, vm_tag_t tag, vm_object_offset_t *dst_offset, int page_count)
+{
+ vm_page_t dst_page;
+ boolean_t no_zero_fill = FALSE;
+ int interruptible;
+ int pages_wired = 0;
+ int pages_inserted = 0;
+ int entry = 0;
+ uint64_t delayed_ledger_update = 0;
+ kern_return_t ret = KERN_SUCCESS;
+ int grab_options;
+ ppnum_t phys_page;
+
+ vm_object_lock_assert_exclusive(object);
+ assert(object->purgable != VM_PURGABLE_VOLATILE);
+ assert(object->purgable != VM_PURGABLE_EMPTY);
+ assert(object->pager == NULL);
+ assert(object->copy == NULL);
+ assert(object->shadow == NULL);
+
+ if (cntrl_flags & UPL_SET_INTERRUPTIBLE)
+ interruptible = THREAD_ABORTSAFE;
+ else
+ interruptible = THREAD_UNINT;
+
+ if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO))
+ no_zero_fill = TRUE;
+
+ grab_options = 0;
+#if CONFIG_SECLUDED_MEMORY
+ if (object->can_grab_secluded) {
+ grab_options |= VM_PAGE_GRAB_SECLUDED;
+ }
+#endif /* CONFIG_SECLUDED_MEMORY */
+
+ while (page_count--) {
+
+ while ((dst_page = vm_page_grab_options(grab_options))
+ == VM_PAGE_NULL) {
+
+ OSAddAtomic(page_count, &vm_upl_wait_for_pages);
+
+ VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);
+
+ if (vm_page_wait(interruptible) == FALSE) {
+ /*
+ * interrupted case
+ */
+ OSAddAtomic(-page_count, &vm_upl_wait_for_pages);
+
+ VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1);
+
+ ret = MACH_SEND_INTERRUPTED;
+ goto done;
+ }
+ OSAddAtomic(-page_count, &vm_upl_wait_for_pages);
+
+ VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
+ }
+ if (no_zero_fill == FALSE)
+ vm_page_zero_fill(dst_page);
+ else
+ dst_page->absent = TRUE;
+
+ dst_page->reference = TRUE;
+
+ if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
+ SET_PAGE_DIRTY(dst_page, FALSE);
+ }
+ if (dst_page->absent == FALSE) {
+ assert(dst_page->vm_page_q_state == VM_PAGE_NOT_ON_Q);
+ assert(dst_page->wire_count == 0);
+ dst_page->wire_count++;
+ dst_page->vm_page_q_state = VM_PAGE_IS_WIRED;
+ assert(dst_page->wire_count);
+ pages_wired++;
+ PAGE_WAKEUP_DONE(dst_page);
+ }
+ pages_inserted++;
+
+ vm_page_insert_internal(dst_page, object, *dst_offset, tag, FALSE, TRUE, TRUE, TRUE, &delayed_ledger_update);
+
+ lite_list[entry>>5] |= 1 << (entry & 31);
+
+ phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
+
+ if (phys_page > upl->highest_page)
+ upl->highest_page = phys_page;
+
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = phys_page;
+ user_page_list[entry].absent = dst_page->absent;
+ user_page_list[entry].dirty = dst_page->dirty;
+ user_page_list[entry].free_when_done = FALSE;
+ user_page_list[entry].precious = FALSE;
+ user_page_list[entry].device = FALSE;
+ user_page_list[entry].speculative = FALSE;
+ user_page_list[entry].cs_validated = FALSE;
+ user_page_list[entry].cs_tainted = FALSE;
+ user_page_list[entry].cs_nx = FALSE;
+ user_page_list[entry].needed = FALSE;
+ user_page_list[entry].mark = FALSE;
+ }
+ entry++;
+ *dst_offset += PAGE_SIZE_64;
+ }
+done:
+ if (pages_wired) {
+ vm_page_lockspin_queues();
+ vm_page_wire_count += pages_wired;
+ vm_page_unlock_queues();
+ }
+ if (pages_inserted) {
+ if (object->internal) {
+ OSAddAtomic(pages_inserted, &vm_page_internal_count);
+ } else {
+ OSAddAtomic(pages_inserted, &vm_page_external_count);
+ }
+ }
+ if (delayed_ledger_update) {
+ task_t owner;
+
+ owner = object->vo_purgeable_owner;
+ assert(owner);
+
+ /* more non-volatile bytes */
+ ledger_credit(owner->ledger,
+ task_ledgers.purgeable_nonvolatile,
+ delayed_ledger_update);
+ /* more footprint */
+ ledger_credit(owner->ledger,
+ task_ledgers.phys_footprint,
+ delayed_ledger_update);
+ }
+ return (ret);
+}
+
+
unsigned int vm_object_iopl_request_sleep_for_cleaning = 0;
+
kern_return_t
vm_object_iopl_request(
vm_object_t object,
upl_t *upl_ptr,
upl_page_info_array_t user_page_list,
unsigned int *page_list_count,
- int cntrl_flags)
+ upl_control_flags_t cntrl_flags,
+ vm_tag_t tag)
{
vm_page_t dst_page;
vm_object_offset_t dst_offset;
boolean_t caller_lookup;
int io_tracking_flag = 0;
int interruptible;
+ ppnum_t phys_page;
boolean_t set_cache_attr_needed = FALSE;
boolean_t free_wired_pages = FALSE;
- int fast_path_possible = 0;
-
+ boolean_t fast_path_empty_req = FALSE;
+ boolean_t fast_path_full_req = FALSE;
if (cntrl_flags & ~UPL_VALID_FLAGS) {
/*
return KERN_INVALID_ADDRESS;
}
}
-
- if (cntrl_flags & UPL_ENCRYPT) {
- /*
- * ENCRYPTED SWAP:
- * The paging path doesn't use this interface,
- * so we don't support the UPL_ENCRYPT flag
- * here. We won't encrypt the pages.
- */
- assert(! (cntrl_flags & UPL_ENCRYPT));
- }
if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO))
no_zero_fill = TRUE;
upl->flags |= UPL_ACCESS_BLOCKED;
}
- if (!(cntrl_flags & (UPL_NEED_32BIT_ADDR | UPL_BLOCK_ACCESS)) &&
- object->purgable != VM_PURGABLE_VOLATILE &&
- object->purgable != VM_PURGABLE_EMPTY &&
- object->copy == NULL &&
- size == object->vo_size &&
- offset == 0 &&
- object->resident_page_count == 0 &&
- object->shadow == NULL &&
- object->pager == NULL)
- {
- fast_path_possible = 1;
- set_cache_attr_needed = TRUE;
- }
-
#if CONFIG_IOSCHED || UPL_DEBUG
if (upl->flags & UPL_TRACKED_BY_OBJECT) {
vm_object_activity_begin(object);
}
#endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
+ vm_object_lock_assert_exclusive(object);
object->true_share = TRUE;
if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
iopl_cow_pages += size >> PAGE_SHIFT;
#endif
}
+ if (!(cntrl_flags & (UPL_NEED_32BIT_ADDR | UPL_BLOCK_ACCESS)) &&
+ object->purgable != VM_PURGABLE_VOLATILE &&
+ object->purgable != VM_PURGABLE_EMPTY &&
+ object->copy == NULL &&
+ size == object->vo_size &&
+ offset == 0 &&
+ object->shadow == NULL &&
+ object->pager == NULL)
+ {
+ if (object->resident_page_count == size_in_pages)
+ {
+ assert(object != compressor_object);
+ assert(object != kernel_object);
+ fast_path_full_req = TRUE;
+ }
+ else if (object->resident_page_count == 0)
+ {
+ assert(object != compressor_object);
+ assert(object != kernel_object);
+ fast_path_empty_req = TRUE;
+ set_cache_attr_needed = TRUE;
+ }
+ }
+
if (cntrl_flags & UPL_SET_INTERRUPTIBLE)
interruptible = THREAD_ABORTSAFE;
else
dst_offset = offset;
dw_count = 0;
- if (fast_path_possible) {
- int wired_count = 0;
-
- while (xfer_size) {
-
- while ( (dst_page = vm_page_grab()) == VM_PAGE_NULL) {
- OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages);
-
- VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);
-
- if (vm_page_wait(interruptible) == FALSE) {
- /*
- * interrupted case
- */
- OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
-
- VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1);
-
- if (wired_count) {
- vm_page_lockspin_queues();
- vm_page_wire_count += wired_count;
- vm_page_unlock_queues();
-
- free_wired_pages = TRUE;
- }
- ret = MACH_SEND_INTERRUPTED;
-
- goto return_err;
- }
- OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
-
- VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
- }
- if (no_zero_fill == FALSE)
- vm_page_zero_fill(dst_page);
- else
- dst_page->absent = TRUE;
+ if (fast_path_full_req) {
- dst_page->reference = TRUE;
+ if (vm_object_iopl_wire_full(object, upl, user_page_list, lite_list, cntrl_flags, tag) == TRUE)
+ goto finish;
+ /*
+ * we couldn't complete the processing of this request on the fast path
+ * so fall through to the slow path and finish up
+ */
- if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
- SET_PAGE_DIRTY(dst_page, FALSE);
- }
- if (dst_page->absent == FALSE) {
- assert(object->purgable != VM_PURGABLE_VOLATILE);
- assert(object->purgable != VM_PURGABLE_EMPTY);
- dst_page->wire_count++;
- wired_count++;
+ } else if (fast_path_empty_req) {
- PAGE_WAKEUP_DONE(dst_page);
- }
- vm_page_insert_internal(dst_page, object, dst_offset, FALSE, TRUE, TRUE);
-
- lite_list[entry>>5] |= 1 << (entry & 31);
-
- if (dst_page->phys_page > upl->highest_page)
- upl->highest_page = dst_page->phys_page;
-
- if (user_page_list) {
- user_page_list[entry].phys_addr = dst_page->phys_page;
- user_page_list[entry].absent = dst_page->absent;
- user_page_list[entry].dirty = dst_page->dirty;
- user_page_list[entry].precious = FALSE;
- user_page_list[entry].pageout = FALSE;
- user_page_list[entry].device = FALSE;
- user_page_list[entry].needed = FALSE;
- user_page_list[entry].speculative = FALSE;
- user_page_list[entry].cs_validated = FALSE;
- user_page_list[entry].cs_tainted = FALSE;
- }
- entry++;
- dst_offset += PAGE_SIZE_64;
- xfer_size -= PAGE_SIZE;
- size_in_pages--;
+ if (cntrl_flags & UPL_REQUEST_NO_FAULT) {
+ ret = KERN_MEMORY_ERROR;
+ goto return_err;
}
- if (wired_count) {
- vm_page_lockspin_queues();
- vm_page_wire_count += wired_count;
- vm_page_unlock_queues();
+ ret = vm_object_iopl_wire_empty(object, upl, user_page_list, lite_list, cntrl_flags, tag, &dst_offset, size_in_pages);
+
+ if (ret) {
+ free_wired_pages = TRUE;
+ goto return_err;
}
goto finish;
}
while (xfer_size) {
vm_fault_return_t result;
- unsigned int pg_num;
dwp->dw_mask = 0;
+ if (fast_path_full_req) {
+ /*
+ * if we get here, it means that we ran into a page
+ * state we couldn't handle in the fast path and
+ * bailed out to the slow path... since the order
+ * we look at pages is different between the 2 paths,
+ * the following check is needed to determine whether
+ * this page was already processed in the fast path
+ */
+ if (lite_list[entry>>5] & (1 << (entry & 31)))
+ goto skip_page;
+ }
dst_page = vm_page_lookup(object, dst_offset);
- /*
- * ENCRYPTED SWAP:
- * If the page is encrypted, we need to decrypt it,
- * so force a soft page fault.
- */
if (dst_page == VM_PAGE_NULL ||
dst_page->busy ||
- dst_page->encrypted ||
dst_page->error ||
dst_page->restart ||
dst_page->absent ||
if (top_page != VM_PAGE_NULL) {
vm_object_t local_object;
- local_object = top_page->object;
-
- if (top_page->object != dst_page->object) {
+ local_object = VM_PAGE_OBJECT(top_page);
+
+ /*
+ * comparing 2 packed pointers
+ */
+ if (top_page->vm_page_object != dst_page->vm_page_object) {
vm_object_lock(local_object);
VM_PAGE_FREE(top_page);
vm_object_paging_end(local_object);
break;
case VM_FAULT_MEMORY_SHORTAGE:
- OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages);
+ OSAddAtomic((size_in_pages - entry), &vm_upl_wait_for_pages);
VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);
if (vm_page_wait(interruptible)) {
- OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
+ OSAddAtomic(-(size_in_pages - entry), &vm_upl_wait_for_pages);
VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
vm_object_lock(object);
break;
}
- OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
+ OSAddAtomic(-(size_in_pages - entry), &vm_upl_wait_for_pages);
VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1);
} while (result != VM_FAULT_SUCCESS);
}
+ phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
+
if (upl->flags & UPL_KERNEL_OBJECT)
goto record_phys_addr;
- if (dst_page->compressor) {
+ if (dst_page->vm_page_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
dst_page->busy = TRUE;
goto record_phys_addr;
}
PAGE_SLEEP(object, dst_page, THREAD_UNINT);
continue;
}
- if (dst_page->laundry) {
- dst_page->pageout = FALSE;
-
+ if (dst_page->laundry)
vm_pageout_steal_laundry(dst_page, FALSE);
- }
+
if ( (cntrl_flags & UPL_NEED_32BIT_ADDR) &&
- dst_page->phys_page >= (max_valid_dma_address >> PAGE_SHIFT) ) {
+ phys_page >= (max_valid_dma_address >> PAGE_SHIFT) ) {
vm_page_t low_page;
int refmod;
* to find the new page being substituted.
*/
if (dst_page->pmapped)
- refmod = pmap_disconnect(dst_page->phys_page);
+ refmod = pmap_disconnect(phys_page);
else
refmod = 0;
*/
if ( !dst_page->absent)
dst_page->busy = FALSE;
+
+ phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
}
if ( !dst_page->busy)
dwp->dw_mask |= DW_vm_page_wire;
SET_PAGE_DIRTY(dst_page, TRUE);
}
if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->written_by_kernel == TRUE) {
- pmap_sync_page_attributes_phys(dst_page->phys_page);
+ pmap_sync_page_attributes_phys(phys_page);
dst_page->written_by_kernel = FALSE;
}
if (dst_page->busy)
upl->flags |= UPL_HAS_BUSY;
- pg_num = (unsigned int) ((dst_offset-offset)/PAGE_SIZE);
- assert(pg_num == (dst_offset-offset)/PAGE_SIZE);
- lite_list[pg_num>>5] |= 1 << (pg_num & 31);
+ lite_list[entry>>5] |= 1 << (entry & 31);
- if (dst_page->phys_page > upl->highest_page)
- upl->highest_page = dst_page->phys_page;
+ if (phys_page > upl->highest_page)
+ upl->highest_page = phys_page;
if (user_page_list) {
- user_page_list[entry].phys_addr = dst_page->phys_page;
- user_page_list[entry].pageout = dst_page->pageout;
+ user_page_list[entry].phys_addr = phys_page;
+ user_page_list[entry].free_when_done = dst_page->free_when_done;
user_page_list[entry].absent = dst_page->absent;
user_page_list[entry].dirty = dst_page->dirty;
user_page_list[entry].precious = dst_page->precious;
user_page_list[entry].device = FALSE;
user_page_list[entry].needed = FALSE;
if (dst_page->clustered == TRUE)
- user_page_list[entry].speculative = dst_page->speculative;
+ user_page_list[entry].speculative = (dst_page->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE;
else
user_page_list[entry].speculative = FALSE;
user_page_list[entry].cs_validated = dst_page->cs_validated;
user_page_list[entry].cs_tainted = dst_page->cs_tainted;
user_page_list[entry].cs_nx = dst_page->cs_nx;
+ user_page_list[entry].mark = FALSE;
}
if (object != kernel_object && object != compressor_object) {
/*
if (dst_page->clustered)
VM_PAGE_CONSUME_CLUSTERED(dst_page);
}
+skip_page:
entry++;
dst_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
- size_in_pages--;
if (dwp->dw_mask) {
VM_PAGE_ADD_DELAYED_WORK(dwp, dst_page, dw_count);
if (dw_count >= dw_limit) {
- vm_page_do_delayed_work(object, &dw_array[0], dw_count);
+ vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count);
dwp = &dw_array[0];
dw_count = 0;
}
}
}
- if (dw_count)
- vm_page_do_delayed_work(object, &dw_array[0], dw_count);
+ assert(entry == size_in_pages);
+ if (dw_count)
+ vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count);
finish:
if (user_page_list && set_cache_attr_needed == TRUE)
- vm_object_set_pmap_cache_attr(object, user_page_list, entry, TRUE);
+ vm_object_set_pmap_cache_attr(object, user_page_list, size_in_pages, TRUE);
if (page_list_count != NULL) {
if (upl->flags & UPL_INTERNAL)
*page_list_count = 0;
- else if (*page_list_count > entry)
- *page_list_count = entry;
+ else if (*page_list_count > size_in_pages)
+ *page_list_count = size_in_pages;
}
vm_object_unlock(object);
assert(!object->blocked_access);
object->blocked_access = TRUE;
}
+
return KERN_SUCCESS;
return_err:
/*
- * ENCRYPTED SWAP:
- *
- * Rationale: the user might have some encrypted data on disk (via
- * FileVault or any other mechanism). That data is then decrypted in
- * memory, which is safe as long as the machine is secure. But that
- * decrypted data in memory could be paged out to disk by the default
- * pager. The data would then be stored on disk in clear (not encrypted)
- * and it could be accessed by anyone who gets physical access to the
- * disk (if the laptop or the disk gets stolen for example). This weakens
- * the security offered by FileVault.
- *
- * Solution: the default pager will optionally request that all the
- * pages it gathers for pageout be encrypted, via the UPL interfaces,
- * before it sends this UPL to disk via the vnode_pageout() path.
- *
- * Notes:
- *
- * To avoid disrupting the VM LRU algorithms, we want to keep the
- * clean-in-place mechanisms, which allow us to send some extra pages to
- * swap (clustering) without actually removing them from the user's
- * address space. We don't want the user to unknowingly access encrypted
- * data, so we have to actually remove the encrypted pages from the page
- * table. When the user accesses the data, the hardware will fail to
- * locate the virtual page in its page table and will trigger a page
- * fault. We can then decrypt the page and enter it in the page table
- * again. Whenever we allow the user to access the contents of a page,
- * we have to make sure it's not encrypted.
- *
- *
- */
-/*
- * ENCRYPTED SWAP:
* Reserve of virtual addresses in the kernel address space.
* We need to map the physical pages in the kernel, so that we
- * can call the encryption/decryption routines with a kernel
+ * can call the code-signing or slide routines with a kernel
* virtual address. We keep this pool of pre-allocated kernel
* virtual addresses so that we don't have to scan the kernel's
- * virtaul address space each time we need to encrypt or decrypt
+ * virtaul address space each time we need to work with
* a physical page.
- * It would be nice to be able to encrypt and decrypt in physical
- * mode but that might not always be more efficient...
*/
decl_simple_lock_data(,vm_paging_lock)
#define VM_PAGING_NUM_PAGES 64
VM_PAGING_NUM_PAGES * PAGE_SIZE,
0,
0,
+ VM_MAP_KERNEL_FLAGS_NONE,
+ VM_KERN_MEMORY_NONE,
&map_entry);
if (kr != KERN_SUCCESS) {
panic("vm_paging_map_init: kernel_map full\n");
}
- map_entry->object.vm_object = kernel_object;
- map_entry->offset = page_map_offset;
+ VME_OBJECT_SET(map_entry, kernel_object);
+ VME_OFFSET_SET(map_entry, page_map_offset);
map_entry->protection = VM_PROT_NONE;
map_entry->max_protection = VM_PROT_NONE;
map_entry->permanent = TRUE;
}
/*
- * ENCRYPTED SWAP:
* vm_paging_map_object:
* Maps part of a VM object's pages in the kernel
* virtual address space, using the pre-allocated
/* use permanent 1-to-1 kernel mapping of physical memory ? */
#if __x86_64__
*address = (vm_map_offset_t)
- PHYSMAP_PTOV((pmap_paddr_t)page->phys_page <<
+ PHYSMAP_PTOV((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(page) <<
PAGE_SHIFT);
*need_unmap = FALSE;
return KERN_SUCCESS;
+#elif __arm__ || __arm64__
+ *address = (vm_map_offset_t)
+ phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(page) << PAGE_SHIFT);
+ *need_unmap = FALSE;
+ return KERN_SUCCESS;
#else
#warn "vm_paging_map_object: no 1-to-1 kernel mapping of physical memory..."
#endif
protection,
VM_PROT_NONE,
0,
- TRUE);
+ TRUE,
+ kr);
+ assert(kr == KERN_SUCCESS);
vm_paging_objects_mapped++;
vm_paging_pages_mapped++;
*address = page_map_offset;
*need_unmap = TRUE;
+#if KASAN
+ kasan_notify_address(page_map_offset, PAGE_SIZE);
+#endif
+
/* all done and mapped, ready to use ! */
return KERN_SUCCESS;
}
map_size,
0,
VM_FLAGS_ANYWHERE,
+ VM_MAP_KERNEL_FLAGS_NONE,
+ VM_KERN_MEMORY_NONE,
object,
object_offset,
FALSE,
}
page->pmapped = TRUE;
- //assert(pmap_verify_free(page->phys_page));
+ //assert(pmap_verify_free(VM_PAGE_GET_PHYS_PAGE(page)));
PMAP_ENTER(kernel_pmap,
*address + page_map_offset,
page,
protection,
VM_PROT_NONE,
0,
- TRUE);
+ TRUE,
+ kr);
+ assert(kr == KERN_SUCCESS);
+#if KASAN
+ kasan_notify_address(*address + page_map_offset, PAGE_SIZE);
+#endif
}
vm_paging_objects_mapped_slow++;
}
/*
- * ENCRYPTED SWAP:
* vm_paging_unmap_object:
* Unmaps part of a VM object's pages from the kernel
* virtual address space.
* kernel virtual address. Deallocate the
* virtual memory.
*/
- if (object != VM_OBJECT_NULL) {
- vm_object_unlock(object);
- }
- kr = vm_map_remove(kernel_map, start, end, VM_MAP_NO_FLAGS);
- if (object != VM_OBJECT_NULL) {
- vm_object_lock(object);
- }
- assert(kr == KERN_SUCCESS);
- } else {
- /*
- * We used a kernel virtual address from our
- * pre-allocated pool. Put it back in the pool
- * for next time.
- */
- assert(end - start == PAGE_SIZE);
- i = (int) ((start - vm_paging_base_address) >> PAGE_SHIFT);
- assert(i >= 0 && i < VM_PAGING_NUM_PAGES);
-
- /* undo the pmap mapping */
- pmap_remove(kernel_pmap, start, end);
-
- simple_lock(&vm_paging_lock);
- vm_paging_page_inuse[i] = FALSE;
- if (vm_paging_page_waiter) {
- thread_wakeup(&vm_paging_page_waiter);
- }
- simple_unlock(&vm_paging_lock);
- }
-}
-
-#if ENCRYPTED_SWAP
-/*
- * Encryption data.
- * "iv" is the "initial vector". Ideally, we want to
- * have a different one for each page we encrypt, so that
- * crackers can't find encryption patterns too easily.
- */
-#define SWAP_CRYPT_AES_KEY_SIZE 128 /* XXX 192 and 256 don't work ! */
-boolean_t swap_crypt_ctx_initialized = FALSE;
-uint32_t swap_crypt_key[8]; /* big enough for a 256 key */
-aes_ctx swap_crypt_ctx;
-const unsigned char swap_crypt_null_iv[AES_BLOCK_SIZE] = {0xa, };
-
-#if DEBUG
-boolean_t swap_crypt_ctx_tested = FALSE;
-unsigned char swap_crypt_test_page_ref[4096] __attribute__((aligned(4096)));
-unsigned char swap_crypt_test_page_encrypt[4096] __attribute__((aligned(4096)));
-unsigned char swap_crypt_test_page_decrypt[4096] __attribute__((aligned(4096)));
-#endif /* DEBUG */
-
-/*
- * Initialize the encryption context: key and key size.
- */
-void swap_crypt_ctx_initialize(void); /* forward */
-void
-swap_crypt_ctx_initialize(void)
-{
- unsigned int i;
-
- /*
- * No need for locking to protect swap_crypt_ctx_initialized
- * because the first use of encryption will come from the
- * pageout thread (we won't pagein before there's been a pageout)
- * and there's only one pageout thread.
- */
- if (swap_crypt_ctx_initialized == FALSE) {
- for (i = 0;
- i < (sizeof (swap_crypt_key) /
- sizeof (swap_crypt_key[0]));
- i++) {
- swap_crypt_key[i] = random();
- }
- aes_encrypt_key((const unsigned char *) swap_crypt_key,
- SWAP_CRYPT_AES_KEY_SIZE,
- &swap_crypt_ctx.encrypt);
- aes_decrypt_key((const unsigned char *) swap_crypt_key,
- SWAP_CRYPT_AES_KEY_SIZE,
- &swap_crypt_ctx.decrypt);
- swap_crypt_ctx_initialized = TRUE;
- }
-
-#if DEBUG
- /*
- * Validate the encryption algorithms.
- */
- if (swap_crypt_ctx_tested == FALSE) {
- /* initialize */
- for (i = 0; i < 4096; i++) {
- swap_crypt_test_page_ref[i] = (char) i;
- }
- /* encrypt */
- aes_encrypt_cbc(swap_crypt_test_page_ref,
- swap_crypt_null_iv,
- PAGE_SIZE / AES_BLOCK_SIZE,
- swap_crypt_test_page_encrypt,
- &swap_crypt_ctx.encrypt);
- /* decrypt */
- aes_decrypt_cbc(swap_crypt_test_page_encrypt,
- swap_crypt_null_iv,
- PAGE_SIZE / AES_BLOCK_SIZE,
- swap_crypt_test_page_decrypt,
- &swap_crypt_ctx.decrypt);
- /* compare result with original */
- for (i = 0; i < 4096; i ++) {
- if (swap_crypt_test_page_decrypt[i] !=
- swap_crypt_test_page_ref[i]) {
- panic("encryption test failed");
- }
- }
-
- /* encrypt again */
- aes_encrypt_cbc(swap_crypt_test_page_decrypt,
- swap_crypt_null_iv,
- PAGE_SIZE / AES_BLOCK_SIZE,
- swap_crypt_test_page_decrypt,
- &swap_crypt_ctx.encrypt);
- /* decrypt in place */
- aes_decrypt_cbc(swap_crypt_test_page_decrypt,
- swap_crypt_null_iv,
- PAGE_SIZE / AES_BLOCK_SIZE,
- swap_crypt_test_page_decrypt,
- &swap_crypt_ctx.decrypt);
- for (i = 0; i < 4096; i ++) {
- if (swap_crypt_test_page_decrypt[i] !=
- swap_crypt_test_page_ref[i]) {
- panic("in place encryption test failed");
- }
- }
-
- swap_crypt_ctx_tested = TRUE;
- }
-#endif /* DEBUG */
-}
-
-/*
- * ENCRYPTED SWAP:
- * vm_page_encrypt:
- * Encrypt the given page, for secure paging.
- * The page might already be mapped at kernel virtual
- * address "kernel_mapping_offset". Otherwise, we need
- * to map it.
- *
- * Context:
- * The page's object is locked, but this lock will be released
- * and re-acquired.
- * The page is busy and not accessible by users (not entered in any pmap).
- */
-void
-vm_page_encrypt(
- vm_page_t page,
- vm_map_offset_t kernel_mapping_offset)
-{
- kern_return_t kr;
- vm_map_size_t kernel_mapping_size;
- boolean_t kernel_mapping_needs_unmap;
- vm_offset_t kernel_vaddr;
- union {
- unsigned char aes_iv[AES_BLOCK_SIZE];
- struct {
- memory_object_t pager_object;
- vm_object_offset_t paging_offset;
- } vm;
- } encrypt_iv;
-
- if (! vm_pages_encrypted) {
- vm_pages_encrypted = TRUE;
- }
-
- assert(page->busy);
-
- if (page->encrypted) {
- /*
- * Already encrypted: no need to do it again.
- */
- vm_page_encrypt_already_encrypted_counter++;
- return;
- }
- assert(page->dirty || page->precious);
-
- ASSERT_PAGE_DECRYPTED(page);
-
- /*
- * Take a paging-in-progress reference to keep the object
- * alive even if we have to unlock it (in vm_paging_map_object()
- * for example)...
- */
- vm_object_paging_begin(page->object);
-
- if (kernel_mapping_offset == 0) {
- /*
- * The page hasn't already been mapped in kernel space
- * by the caller. Map it now, so that we can access
- * its contents and encrypt them.
- */
- kernel_mapping_size = PAGE_SIZE;
- kernel_mapping_needs_unmap = FALSE;
- kr = vm_paging_map_object(page,
- page->object,
- page->offset,
- VM_PROT_READ | VM_PROT_WRITE,
- FALSE,
- &kernel_mapping_size,
- &kernel_mapping_offset,
- &kernel_mapping_needs_unmap);
- if (kr != KERN_SUCCESS) {
- panic("vm_page_encrypt: "
- "could not map page in kernel: 0x%x\n",
- kr);
- }
- } else {
- kernel_mapping_size = 0;
- kernel_mapping_needs_unmap = FALSE;
- }
- kernel_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping_offset);
-
- if (swap_crypt_ctx_initialized == FALSE) {
- swap_crypt_ctx_initialize();
- }
- assert(swap_crypt_ctx_initialized);
-
- /*
- * Prepare an "initial vector" for the encryption.
- * We use the "pager" and the "paging_offset" for that
- * page to obfuscate the encrypted data a bit more and
- * prevent crackers from finding patterns that they could
- * use to break the key.
- */
- bzero(&encrypt_iv.aes_iv[0], sizeof (encrypt_iv.aes_iv));
- encrypt_iv.vm.pager_object = page->object->pager;
- encrypt_iv.vm.paging_offset =
- page->object->paging_offset + page->offset;
-
- /* encrypt the "initial vector" */
- aes_encrypt_cbc((const unsigned char *) &encrypt_iv.aes_iv[0],
- swap_crypt_null_iv,
- 1,
- &encrypt_iv.aes_iv[0],
- &swap_crypt_ctx.encrypt);
-
- /*
- * Encrypt the page.
- */
- aes_encrypt_cbc((const unsigned char *) kernel_vaddr,
- &encrypt_iv.aes_iv[0],
- PAGE_SIZE / AES_BLOCK_SIZE,
- (unsigned char *) kernel_vaddr,
- &swap_crypt_ctx.encrypt);
-
- vm_page_encrypt_counter++;
-
- /*
- * Unmap the page from the kernel's address space,
- * if we had to map it ourselves. Otherwise, let
- * the caller undo the mapping if needed.
- */
- if (kernel_mapping_needs_unmap) {
- vm_paging_unmap_object(page->object,
- kernel_mapping_offset,
- kernel_mapping_offset + kernel_mapping_size);
- }
-
- /*
- * Clear the "reference" and "modified" bits.
- * This should clean up any impact the encryption had
- * on them.
- * The page was kept busy and disconnected from all pmaps,
- * so it can't have been referenced or modified from user
- * space.
- * The software bits will be reset later after the I/O
- * has completed (in upl_commit_range()).
- */
- pmap_clear_refmod(page->phys_page, VM_MEM_REFERENCED | VM_MEM_MODIFIED);
-
- page->encrypted = TRUE;
-
- vm_object_paging_end(page->object);
-}
-
-/*
- * ENCRYPTED SWAP:
- * vm_page_decrypt:
- * Decrypt the given page.
- * The page might already be mapped at kernel virtual
- * address "kernel_mapping_offset". Otherwise, we need
- * to map it.
- *
- * Context:
- * The page's VM object is locked but will be unlocked and relocked.
- * The page is busy and not accessible by users (not entered in any pmap).
- */
-void
-vm_page_decrypt(
- vm_page_t page,
- vm_map_offset_t kernel_mapping_offset)
-{
- kern_return_t kr;
- vm_map_size_t kernel_mapping_size;
- vm_offset_t kernel_vaddr;
- boolean_t kernel_mapping_needs_unmap;
- union {
- unsigned char aes_iv[AES_BLOCK_SIZE];
- struct {
- memory_object_t pager_object;
- vm_object_offset_t paging_offset;
- } vm;
- } decrypt_iv;
- boolean_t was_dirty;
-
- assert(page->busy);
- assert(page->encrypted);
-
- was_dirty = page->dirty;
-
- /*
- * Take a paging-in-progress reference to keep the object
- * alive even if we have to unlock it (in vm_paging_map_object()
- * for example)...
- */
- vm_object_paging_begin(page->object);
-
- if (kernel_mapping_offset == 0) {
- /*
- * The page hasn't already been mapped in kernel space
- * by the caller. Map it now, so that we can access
- * its contents and decrypt them.
- */
- kernel_mapping_size = PAGE_SIZE;
- kernel_mapping_needs_unmap = FALSE;
- kr = vm_paging_map_object(page,
- page->object,
- page->offset,
- VM_PROT_READ | VM_PROT_WRITE,
- FALSE,
- &kernel_mapping_size,
- &kernel_mapping_offset,
- &kernel_mapping_needs_unmap);
- if (kr != KERN_SUCCESS) {
- panic("vm_page_decrypt: "
- "could not map page in kernel: 0x%x\n",
- kr);
- }
- } else {
- kernel_mapping_size = 0;
- kernel_mapping_needs_unmap = FALSE;
- }
- kernel_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping_offset);
-
- assert(swap_crypt_ctx_initialized);
-
- /*
- * Prepare an "initial vector" for the decryption.
- * It has to be the same as the "initial vector" we
- * used to encrypt that page.
- */
- bzero(&decrypt_iv.aes_iv[0], sizeof (decrypt_iv.aes_iv));
- decrypt_iv.vm.pager_object = page->object->pager;
- decrypt_iv.vm.paging_offset =
- page->object->paging_offset + page->offset;
-
- /* encrypt the "initial vector" */
- aes_encrypt_cbc((const unsigned char *) &decrypt_iv.aes_iv[0],
- swap_crypt_null_iv,
- 1,
- &decrypt_iv.aes_iv[0],
- &swap_crypt_ctx.encrypt);
-
- /*
- * Decrypt the page.
- */
- aes_decrypt_cbc((const unsigned char *) kernel_vaddr,
- &decrypt_iv.aes_iv[0],
- PAGE_SIZE / AES_BLOCK_SIZE,
- (unsigned char *) kernel_vaddr,
- &swap_crypt_ctx.decrypt);
- vm_page_decrypt_counter++;
-
- /*
- * Unmap the page from the kernel's address space,
- * if we had to map it ourselves. Otherwise, let
- * the caller undo the mapping if needed.
- */
- if (kernel_mapping_needs_unmap) {
- vm_paging_unmap_object(page->object,
- kernel_vaddr,
- kernel_vaddr + PAGE_SIZE);
- }
-
- if (was_dirty) {
- /*
- * The pager did not specify that the page would be
- * clean when it got paged in, so let's not clean it here
- * either.
- */
- } else {
- /*
- * After decryption, the page is actually still clean.
- * It was encrypted as part of paging, which "cleans"
- * the "dirty" pages.
- * Noone could access it after it was encrypted
- * and the decryption doesn't count.
- */
- page->dirty = FALSE;
- assert (page->cs_validated == FALSE);
- pmap_clear_refmod(page->phys_page, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
- }
- page->encrypted = FALSE;
-
- /*
- * We've just modified the page's contents via the data cache and part
- * of the new contents might still be in the cache and not yet in RAM.
- * Since the page is now available and might get gathered in a UPL to
- * be part of a DMA transfer from a driver that expects the memory to
- * be coherent at this point, we have to flush the data cache.
- */
- pmap_sync_page_attributes_phys(page->phys_page);
- /*
- * Since the page is not mapped yet, some code might assume that it
- * doesn't need to invalidate the instruction cache when writing to
- * that page. That code relies on "pmapped" being FALSE, so that the
- * caches get synchronized when the page is first mapped.
- */
- assert(pmap_verify_free(page->phys_page));
- page->pmapped = FALSE;
- page->wpmapped = FALSE;
-
- vm_object_paging_end(page->object);
-}
-
-#if DEVELOPMENT || DEBUG
-unsigned long upl_encrypt_upls = 0;
-unsigned long upl_encrypt_pages = 0;
-#endif
-
-/*
- * ENCRYPTED SWAP:
- *
- * upl_encrypt:
- * Encrypts all the pages in the UPL, within the specified range.
- *
- */
-void
-upl_encrypt(
- upl_t upl,
- upl_offset_t crypt_offset,
- upl_size_t crypt_size)
-{
- upl_size_t upl_size, subupl_size=crypt_size;
- upl_offset_t offset_in_upl, subupl_offset=crypt_offset;
- vm_object_t upl_object;
- vm_object_offset_t upl_offset;
- vm_page_t page;
- vm_object_t shadow_object;
- vm_object_offset_t shadow_offset;
- vm_object_offset_t paging_offset;
- vm_object_offset_t base_offset;
- int isVectorUPL = 0;
- upl_t vector_upl = NULL;
-
- if((isVectorUPL = vector_upl_is_valid(upl)))
- vector_upl = upl;
-
-process_upl_to_encrypt:
- if(isVectorUPL) {
- crypt_size = subupl_size;
- crypt_offset = subupl_offset;
- upl = vector_upl_subupl_byoffset(vector_upl, &crypt_offset, &crypt_size);
- if(upl == NULL)
- panic("upl_encrypt: Accessing a sub-upl that doesn't exist\n");
- subupl_size -= crypt_size;
- subupl_offset += crypt_size;
- }
-
-#if DEVELOPMENT || DEBUG
- upl_encrypt_upls++;
- upl_encrypt_pages += crypt_size / PAGE_SIZE;
-#endif
- upl_object = upl->map_object;
- upl_offset = upl->offset;
- upl_size = upl->size;
-
- vm_object_lock(upl_object);
-
- /*
- * Find the VM object that contains the actual pages.
- */
- if (upl_object->pageout) {
- shadow_object = upl_object->shadow;
- /*
- * The offset in the shadow object is actually also
- * accounted for in upl->offset. It possibly shouldn't be
- * this way, but for now don't account for it twice.
- */
- shadow_offset = 0;
- assert(upl_object->paging_offset == 0); /* XXX ? */
- vm_object_lock(shadow_object);
- } else {
- shadow_object = upl_object;
- shadow_offset = 0;
- }
-
- paging_offset = shadow_object->paging_offset;
- vm_object_paging_begin(shadow_object);
-
- if (shadow_object != upl_object)
- vm_object_unlock(upl_object);
-
-
- base_offset = shadow_offset;
- base_offset += upl_offset;
- base_offset += crypt_offset;
- base_offset -= paging_offset;
-
- assert(crypt_offset + crypt_size <= upl_size);
-
- for (offset_in_upl = 0;
- offset_in_upl < crypt_size;
- offset_in_upl += PAGE_SIZE) {
- page = vm_page_lookup(shadow_object,
- base_offset + offset_in_upl);
- if (page == VM_PAGE_NULL) {
- panic("upl_encrypt: "
- "no page for (obj=%p,off=0x%llx+0x%x)!\n",
- shadow_object,
- base_offset,
- offset_in_upl);
+ if (object != VM_OBJECT_NULL) {
+ vm_object_unlock(object);
+ }
+ kr = vm_map_remove(kernel_map, start, end, VM_MAP_NO_FLAGS);
+ if (object != VM_OBJECT_NULL) {
+ vm_object_lock(object);
}
+ assert(kr == KERN_SUCCESS);
+ } else {
/*
- * Disconnect the page from all pmaps, so that nobody can
- * access it while it's encrypted. After that point, all
- * accesses to this page will cause a page fault and block
- * while the page is busy being encrypted. After the
- * encryption completes, any access will cause a
- * page fault and the page gets decrypted at that time.
+ * We used a kernel virtual address from our
+ * pre-allocated pool. Put it back in the pool
+ * for next time.
*/
- pmap_disconnect(page->phys_page);
- vm_page_encrypt(page, 0);
+ assert(end - start == PAGE_SIZE);
+ i = (int) ((start - vm_paging_base_address) >> PAGE_SHIFT);
+ assert(i >= 0 && i < VM_PAGING_NUM_PAGES);
- if (vm_object_lock_avoid(shadow_object)) {
- /*
- * Give vm_pageout_scan() a chance to convert more
- * pages from "clean-in-place" to "clean-and-free",
- * if it's interested in the same pages we selected
- * in this cluster.
- */
- vm_object_unlock(shadow_object);
- mutex_pause(2);
- vm_object_lock(shadow_object);
+ /* undo the pmap mapping */
+ pmap_remove(kernel_pmap, start, end);
+
+ simple_lock(&vm_paging_lock);
+ vm_paging_page_inuse[i] = FALSE;
+ if (vm_paging_page_waiter) {
+ thread_wakeup(&vm_paging_page_waiter);
}
+ simple_unlock(&vm_paging_lock);
}
-
- vm_object_paging_end(shadow_object);
- vm_object_unlock(shadow_object);
-
- if(isVectorUPL && subupl_size)
- goto process_upl_to_encrypt;
-}
-
-#else /* ENCRYPTED_SWAP */
-void
-upl_encrypt(
- __unused upl_t upl,
- __unused upl_offset_t crypt_offset,
- __unused upl_size_t crypt_size)
-{
-}
-
-void
-vm_page_encrypt(
- __unused vm_page_t page,
- __unused vm_map_offset_t kernel_mapping_offset)
-{
-}
-
-void
-vm_page_decrypt(
- __unused vm_page_t page,
- __unused vm_map_offset_t kernel_mapping_offset)
-{
}
-#endif /* ENCRYPTED_SWAP */
/*
* page->object must be locked
vm_page_lockspin_queues();
}
+ page->free_when_done = FALSE;
/*
* need to drop the laundry count...
* we may also need to remove it
{
boolean_t result = FALSE;
vm_shared_region_slide_info_t si;
+ vm_object_t m_object;
+
+ m_object = VM_PAGE_OBJECT(m);
- vm_object_lock_assert_held(m->object);
+ vm_object_lock_assert_held(m_object);
/* make sure our page belongs to the one object allowed to do this */
- if (!m->object->object_slid) {
+ if (!m_object->object_slid) {
goto done;
}
- si = m->object->vo_slide_info;
+ si = m_object->vo_slide_info;
if (si == NULL) {
goto done;
}
vm_map_size_t kernel_mapping_size;
boolean_t kernel_mapping_needs_unmap;
vm_offset_t kernel_vaddr;
- uint32_t pageIndex = 0;
+ uint32_t pageIndex;
+ uint32_t slide_chunk;
+ vm_object_t page_object;
+
+ page_object = VM_PAGE_OBJECT(page);
assert(!page->slid);
- assert(page->object->object_slid);
- vm_object_lock_assert_exclusive(page->object);
+ assert(page_object->object_slid);
+ vm_object_lock_assert_exclusive(page_object);
if (page->error)
return KERN_FAILURE;
* alive even if we have to unlock it (in vm_paging_map_object()
* for example)...
*/
- vm_object_paging_begin(page->object);
+ vm_object_paging_begin(page_object);
if (kernel_mapping_offset == 0) {
/*
kernel_mapping_size = PAGE_SIZE;
kernel_mapping_needs_unmap = FALSE;
kr = vm_paging_map_object(page,
- page->object,
+ page_object,
page->offset,
VM_PROT_READ | VM_PROT_WRITE,
FALSE,
/*assert that slide_file_info.start/end are page-aligned?*/
assert(!page->slid);
- assert(page->object->object_slid);
-
- /* on some platforms this is an extern int, on others it's a cpp macro */
- __unreachable_ok_push
- /* TODO: Consider this */
- if (!TEST_PAGE_SIZE_4K) {
- for (int i = 0; i < 4; i++) {
- pageIndex = (uint32_t)((page->offset - page->object->vo_slide_info->start)/0x1000);
- kr = vm_shared_region_slide_page(page->object->vo_slide_info, kernel_vaddr + (0x1000*i), pageIndex + i);
+ assert(page_object->object_slid);
+
+ pageIndex = (uint32_t)((page->offset -
+ page_object->vo_slide_info->start) /
+ PAGE_SIZE_FOR_SR_SLIDE);
+ for (slide_chunk = 0;
+ slide_chunk < PAGE_SIZE / PAGE_SIZE_FOR_SR_SLIDE;
+ slide_chunk++) {
+ kr = vm_shared_region_slide_page(page_object->vo_slide_info,
+ (kernel_vaddr +
+ (slide_chunk *
+ PAGE_SIZE_FOR_SR_SLIDE)),
+ (pageIndex + slide_chunk));
+ if (kr != KERN_SUCCESS) {
+ break;
}
- } else {
- pageIndex = (uint32_t)((page->offset - page->object->vo_slide_info->start)/PAGE_SIZE);
- kr = vm_shared_region_slide_page(page->object->vo_slide_info, kernel_vaddr, pageIndex);
}
- __unreachable_ok_pop
vm_page_slide_counter++;
* Unmap the page from the kernel's address space,
*/
if (kernel_mapping_needs_unmap) {
- vm_paging_unmap_object(page->object,
+ vm_paging_unmap_object(page_object,
kernel_vaddr,
kernel_vaddr + PAGE_SIZE);
}
page->dirty = FALSE;
- pmap_clear_refmod(page->phys_page, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
+ pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(page), VM_MEM_MODIFIED | VM_MEM_REFERENCED);
if (kr != KERN_SUCCESS || cs_debug > 1) {
printf("vm_page_slide(%p): "
"obj %p off 0x%llx mobj %p moff 0x%llx\n",
page,
- page->object, page->offset,
- page->object->pager,
- page->offset + page->object->paging_offset);
+ page_object, page->offset,
+ page_object->pager,
+ page->offset + page_object->paging_offset);
}
if (kr == KERN_SUCCESS) {
vm_page_slide_errors++;
}
- vm_object_paging_end(page->object);
+ vm_object_paging_end(page_object);
return kr;
}
return(UPL_PHYS_PAGE(upl, index));
}
+void upl_page_set_mark(upl_page_info_t *upl, int index, boolean_t v)
+{
+ upl[index].mark = v;
+}
+
+boolean_t upl_page_get_mark(upl_page_info_t *upl, int index)
+{
+ return upl[index].mark;
+}
+
void
vm_countdirtypages(void)
{
precpages=0;
vm_page_lock_queues();
- m = (vm_page_t) queue_first(&vm_page_queue_inactive);
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
do {
if (m ==(vm_page_t )0) break;
if(m->dirty) dpages++;
- if(m->pageout) pgopages++;
+ if(m->free_when_done) pgopages++;
if(m->precious) precpages++;
- assert(m->object != kernel_object);
- m = (vm_page_t) queue_next(&m->pageq);
+ assert(VM_PAGE_OBJECT(m) != kernel_object);
+ m = (vm_page_t) vm_page_queue_next(&m->pageq);
if (m ==(vm_page_t )0) break;
- } while (!queue_end(&vm_page_queue_inactive,(queue_entry_t) m));
+ } while (!vm_page_queue_end(&vm_page_queue_inactive, (vm_page_queue_entry_t) m));
vm_page_unlock_queues();
vm_page_lock_queues();
- m = (vm_page_t) queue_first(&vm_page_queue_throttled);
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_throttled);
do {
if (m ==(vm_page_t )0) break;
dpages++;
assert(m->dirty);
- assert(!m->pageout);
- assert(m->object != kernel_object);
- m = (vm_page_t) queue_next(&m->pageq);
+ assert(!m->free_when_done);
+ assert(VM_PAGE_OBJECT(m) != kernel_object);
+ m = (vm_page_t) vm_page_queue_next(&m->pageq);
if (m ==(vm_page_t )0) break;
- } while (!queue_end(&vm_page_queue_throttled,(queue_entry_t) m));
+ } while (!vm_page_queue_end(&vm_page_queue_throttled, (vm_page_queue_entry_t) m));
vm_page_unlock_queues();
vm_page_lock_queues();
- m = (vm_page_t) queue_first(&vm_page_queue_anonymous);
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
do {
if (m ==(vm_page_t )0) break;
if(m->dirty) dpages++;
- if(m->pageout) pgopages++;
+ if(m->free_when_done) pgopages++;
if(m->precious) precpages++;
- assert(m->object != kernel_object);
- m = (vm_page_t) queue_next(&m->pageq);
+ assert(VM_PAGE_OBJECT(m) != kernel_object);
+ m = (vm_page_t) vm_page_queue_next(&m->pageq);
if (m ==(vm_page_t )0) break;
- } while (!queue_end(&vm_page_queue_anonymous,(queue_entry_t) m));
+ } while (!vm_page_queue_end(&vm_page_queue_anonymous, (vm_page_queue_entry_t) m));
vm_page_unlock_queues();
printf("IN Q: %d : %d : %d\n", dpages, pgopages, precpages);
precpages=0;
vm_page_lock_queues();
- m = (vm_page_t) queue_first(&vm_page_queue_active);
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
+
+ do {
+ if(m == (vm_page_t )0) break;
+ if(m->dirty) dpages++;
+ if(m->free_when_done) pgopages++;
+ if(m->precious) precpages++;
+
+ assert(VM_PAGE_OBJECT(m) != kernel_object);
+ m = (vm_page_t) vm_page_queue_next(&m->pageq);
+ if(m == (vm_page_t )0) break;
+
+ } while (!vm_page_queue_end(&vm_page_queue_active, (vm_page_queue_entry_t) m));
+ vm_page_unlock_queues();
+
+ printf("AC Q: %d : %d : %d\n", dpages, pgopages, precpages);
+
+}
+#endif /* MACH_BSD */
+
+
+#if CONFIG_IOSCHED
+int upl_get_cached_tier(upl_t upl)
+{
+ assert(upl);
+ if (upl->flags & UPL_TRACKED_BY_OBJECT)
+ return (upl->upl_priority);
+ return (-1);
+}
+#endif /* CONFIG_IOSCHED */
+
+ppnum_t upl_get_highest_page(
+ upl_t upl)
+{
+ return upl->highest_page;
+}
+
+upl_size_t upl_get_size(
+ upl_t upl)
+{
+ return upl->size;
+}
+
+upl_t upl_associated_upl(upl_t upl)
+{
+ return upl->associated_upl;
+}
+
+void upl_set_associated_upl(upl_t upl, upl_t associated_upl)
+{
+ upl->associated_upl = associated_upl;
+}
+
+struct vnode * upl_lookup_vnode(upl_t upl)
+{
+ if (!upl->map_object->internal)
+ return vnode_pager_lookup_vnode(upl->map_object->pager);
+ else
+ return NULL;
+}
+
+#if UPL_DEBUG
+kern_return_t upl_ubc_alias_set(upl_t upl, uintptr_t alias1, uintptr_t alias2)
+{
+ upl->ubc_alias1 = alias1;
+ upl->ubc_alias2 = alias2;
+ return KERN_SUCCESS;
+}
+int upl_ubc_alias_get(upl_t upl, uintptr_t * al, uintptr_t * al2)
+{
+ if(al)
+ *al = upl->ubc_alias1;
+ if(al2)
+ *al2 = upl->ubc_alias2;
+ return KERN_SUCCESS;
+}
+#endif /* UPL_DEBUG */
+
+#if VM_PRESSURE_EVENTS
+/*
+ * Upward trajectory.
+ */
+extern boolean_t vm_compressor_low_on_space(void);
+
+boolean_t
+VM_PRESSURE_NORMAL_TO_WARNING(void) {
+
+ if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
+
+ /* Available pages below our threshold */
+ if (memorystatus_available_pages < memorystatus_available_pages_pressure) {
+ /* No frozen processes to kill */
+ if (memorystatus_frozen_count == 0) {
+ /* Not enough suspended processes available. */
+ if (memorystatus_suspended_count < MEMORYSTATUS_SUSPENDED_THRESHOLD) {
+ return TRUE;
+ }
+ }
+ }
+ return FALSE;
+
+ } else {
+ return ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0);
+ }
+}
+
+boolean_t
+VM_PRESSURE_WARNING_TO_CRITICAL(void) {
+
+ if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
+
+ /* Available pages below our threshold */
+ if (memorystatus_available_pages < memorystatus_available_pages_critical) {
+ return TRUE;
+ }
+ return FALSE;
+ } else {
+ return (vm_compressor_low_on_space() || (AVAILABLE_NON_COMPRESSED_MEMORY < ((12 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0);
+ }
+}
+
+/*
+ * Downward trajectory.
+ */
+boolean_t
+VM_PRESSURE_WARNING_TO_NORMAL(void) {
+
+ if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
+
+ /* Available pages above our threshold */
+ unsigned int target_threshold = (unsigned int) (memorystatus_available_pages_pressure + ((15 * memorystatus_available_pages_pressure) / 100));
+ if (memorystatus_available_pages > target_threshold) {
+ return TRUE;
+ }
+ return FALSE;
+ } else {
+ return ((AVAILABLE_NON_COMPRESSED_MEMORY > ((12 * VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) / 10)) ? 1 : 0);
+ }
+}
+
+boolean_t
+VM_PRESSURE_CRITICAL_TO_WARNING(void) {
+
+ if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
+
+ /* Available pages above our threshold */
+ unsigned int target_threshold = (unsigned int)(memorystatus_available_pages_critical + ((15 * memorystatus_available_pages_critical) / 100));
+ if (memorystatus_available_pages > target_threshold) {
+ return TRUE;
+ }
+ return FALSE;
+ } else {
+ return ((AVAILABLE_NON_COMPRESSED_MEMORY > ((14 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0);
+ }
+}
+#endif /* VM_PRESSURE_EVENTS */
+
+
+
+#define VM_TEST_COLLAPSE_COMPRESSOR 0
+#define VM_TEST_WIRE_AND_EXTRACT 0
+#define VM_TEST_PAGE_WIRE_OVERFLOW_PANIC 0
+#if __arm64__
+#define VM_TEST_KERNEL_OBJECT_FAULT 0
+#endif /* __arm64__ */
+#define VM_TEST_DEVICE_PAGER_TRANSPOSE (DEVELOPMENT || DEBUG)
+
+#if VM_TEST_COLLAPSE_COMPRESSOR
+extern boolean_t vm_object_collapse_compressor_allowed;
+#include <IOKit/IOLib.h>
+static void
+vm_test_collapse_compressor(void)
+{
+ vm_object_size_t backing_size, top_size;
+ vm_object_t backing_object, top_object;
+ vm_map_offset_t backing_offset, top_offset;
+ unsigned char *backing_address, *top_address;
+ kern_return_t kr;
+
+ printf("VM_TEST_COLLAPSE_COMPRESSOR:\n");
+
+ /* create backing object */
+ backing_size = 15 * PAGE_SIZE;
+ backing_object = vm_object_allocate(backing_size);
+ assert(backing_object != VM_OBJECT_NULL);
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: created backing object %p\n",
+ backing_object);
+ /* map backing object */
+ backing_offset = 0;
+ kr = vm_map_enter(kernel_map, &backing_offset, backing_size, 0,
+ VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE,
+ backing_object, 0, FALSE,
+ VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
+ assert(kr == KERN_SUCCESS);
+ backing_address = (unsigned char *) backing_offset;
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: "
+ "mapped backing object %p at 0x%llx\n",
+ backing_object, (uint64_t) backing_offset);
+ /* populate with pages to be compressed in backing object */
+ backing_address[0x1*PAGE_SIZE] = 0xB1;
+ backing_address[0x4*PAGE_SIZE] = 0xB4;
+ backing_address[0x7*PAGE_SIZE] = 0xB7;
+ backing_address[0xa*PAGE_SIZE] = 0xBA;
+ backing_address[0xd*PAGE_SIZE] = 0xBD;
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: "
+ "populated pages to be compressed in "
+ "backing_object %p\n", backing_object);
+ /* compress backing object */
+ vm_object_pageout(backing_object);
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: compressing backing_object %p\n",
+ backing_object);
+ /* wait for all the pages to be gone */
+ while (*(volatile int *)&backing_object->resident_page_count != 0)
+ IODelay(10);
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: backing_object %p compressed\n",
+ backing_object);
+ /* populate with pages to be resident in backing object */
+ backing_address[0x0*PAGE_SIZE] = 0xB0;
+ backing_address[0x3*PAGE_SIZE] = 0xB3;
+ backing_address[0x6*PAGE_SIZE] = 0xB6;
+ backing_address[0x9*PAGE_SIZE] = 0xB9;
+ backing_address[0xc*PAGE_SIZE] = 0xBC;
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: "
+ "populated pages to be resident in "
+ "backing_object %p\n", backing_object);
+ /* leave the other pages absent */
+ /* mess with the paging_offset of the backing_object */
+ assert(backing_object->paging_offset == 0);
+ backing_object->paging_offset = 0x3000;
+
+ /* create top object */
+ top_size = 9 * PAGE_SIZE;
+ top_object = vm_object_allocate(top_size);
+ assert(top_object != VM_OBJECT_NULL);
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: created top object %p\n",
+ top_object);
+ /* map top object */
+ top_offset = 0;
+ kr = vm_map_enter(kernel_map, &top_offset, top_size, 0,
+ VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE,
+ top_object, 0, FALSE,
+ VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
+ assert(kr == KERN_SUCCESS);
+ top_address = (unsigned char *) top_offset;
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: "
+ "mapped top object %p at 0x%llx\n",
+ top_object, (uint64_t) top_offset);
+ /* populate with pages to be compressed in top object */
+ top_address[0x3*PAGE_SIZE] = 0xA3;
+ top_address[0x4*PAGE_SIZE] = 0xA4;
+ top_address[0x5*PAGE_SIZE] = 0xA5;
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: "
+ "populated pages to be compressed in "
+ "top_object %p\n", top_object);
+ /* compress top object */
+ vm_object_pageout(top_object);
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: compressing top_object %p\n",
+ top_object);
+ /* wait for all the pages to be gone */
+ while (top_object->resident_page_count != 0)
+ IODelay(10);
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: top_object %p compressed\n",
+ top_object);
+ /* populate with pages to be resident in top object */
+ top_address[0x0*PAGE_SIZE] = 0xA0;
+ top_address[0x1*PAGE_SIZE] = 0xA1;
+ top_address[0x2*PAGE_SIZE] = 0xA2;
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: "
+ "populated pages to be resident in "
+ "top_object %p\n", top_object);
+ /* leave the other pages absent */
- do {
- if(m == (vm_page_t )0) break;
- if(m->dirty) dpages++;
- if(m->pageout) pgopages++;
- if(m->precious) precpages++;
+ /* link the 2 objects */
+ vm_object_reference(backing_object);
+ top_object->shadow = backing_object;
+ top_object->vo_shadow_offset = 0x3000;
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: linked %p and %p\n",
+ top_object, backing_object);
- assert(m->object != kernel_object);
- m = (vm_page_t) queue_next(&m->pageq);
- if(m == (vm_page_t )0) break;
+ /* unmap backing object */
+ vm_map_remove(kernel_map,
+ backing_offset,
+ backing_offset + backing_size,
+ 0);
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: "
+ "unmapped backing_object %p [0x%llx:0x%llx]\n",
+ backing_object,
+ (uint64_t) backing_offset,
+ (uint64_t) (backing_offset + backing_size));
- } while (!queue_end(&vm_page_queue_active,(queue_entry_t) m));
- vm_page_unlock_queues();
+ /* collapse */
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: collapsing %p\n", top_object);
+ vm_object_lock(top_object);
+ vm_object_collapse(top_object, 0, FALSE);
+ vm_object_unlock(top_object);
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: collapsed %p\n", top_object);
- printf("AC Q: %d : %d : %d\n", dpages, pgopages, precpages);
+ /* did it work? */
+ if (top_object->shadow != VM_OBJECT_NULL) {
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: not collapsed\n");
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: FAIL\n");
+ if (vm_object_collapse_compressor_allowed) {
+ panic("VM_TEST_COLLAPSE_COMPRESSOR: FAIL\n");
+ }
+ } else {
+ /* check the contents of the mapping */
+ unsigned char expect[9] =
+ { 0xA0, 0xA1, 0xA2, /* resident in top */
+ 0xA3, 0xA4, 0xA5, /* compressed in top */
+ 0xB9, /* resident in backing + shadow_offset */
+ 0xBD, /* compressed in backing + shadow_offset + paging_offset */
+ 0x00 }; /* absent in both */
+ unsigned char actual[9];
+ unsigned int i, errors;
+ errors = 0;
+ for (i = 0; i < sizeof (actual); i++) {
+ actual[i] = (unsigned char) top_address[i*PAGE_SIZE];
+ if (actual[i] != expect[i]) {
+ errors++;
+ }
+ }
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: "
+ "actual [%x %x %x %x %x %x %x %x %x] "
+ "expect [%x %x %x %x %x %x %x %x %x] "
+ "%d errors\n",
+ actual[0], actual[1], actual[2], actual[3],
+ actual[4], actual[5], actual[6], actual[7],
+ actual[8],
+ expect[0], expect[1], expect[2], expect[3],
+ expect[4], expect[5], expect[6], expect[7],
+ expect[8],
+ errors);
+ if (errors) {
+ panic("VM_TEST_COLLAPSE_COMPRESSOR: FAIL\n");
+ } else {
+ printf("VM_TEST_COLLAPSE_COMPRESSOR: PASS\n");
+ }
+ }
}
-#endif /* MACH_BSD */
+#else /* VM_TEST_COLLAPSE_COMPRESSOR */
+#define vm_test_collapse_compressor()
+#endif /* VM_TEST_COLLAPSE_COMPRESSOR */
-ppnum_t upl_get_highest_page(
- upl_t upl)
+#if VM_TEST_WIRE_AND_EXTRACT
+extern ledger_template_t task_ledger_template;
+#include <mach/mach_vm.h>
+extern ppnum_t vm_map_get_phys_page(vm_map_t map,
+ vm_offset_t offset);
+static void
+vm_test_wire_and_extract(void)
{
- return upl->highest_page;
-}
+ ledger_t ledger;
+ vm_map_t user_map, wire_map;
+ mach_vm_address_t user_addr, wire_addr;
+ mach_vm_size_t user_size, wire_size;
+ mach_vm_offset_t cur_offset;
+ vm_prot_t cur_prot, max_prot;
+ ppnum_t user_ppnum, wire_ppnum;
+ kern_return_t kr;
-upl_size_t upl_get_size(
- upl_t upl)
-{
- return upl->size;
-}
+ ledger = ledger_instantiate(task_ledger_template,
+ LEDGER_CREATE_ACTIVE_ENTRIES);
+ user_map = vm_map_create(pmap_create(ledger, 0, PMAP_CREATE_64BIT),
+ 0x100000000ULL,
+ 0x200000000ULL,
+ TRUE);
+ wire_map = vm_map_create(NULL,
+ 0x100000000ULL,
+ 0x200000000ULL,
+ TRUE);
+ user_addr = 0;
+ user_size = 0x10000;
+ kr = mach_vm_allocate(user_map,
+ &user_addr,
+ user_size,
+ VM_FLAGS_ANYWHERE);
+ assert(kr == KERN_SUCCESS);
+ wire_addr = 0;
+ wire_size = user_size;
+ kr = mach_vm_remap(wire_map,
+ &wire_addr,
+ wire_size,
+ 0,
+ VM_FLAGS_ANYWHERE,
+ user_map,
+ user_addr,
+ FALSE,
+ &cur_prot,
+ &max_prot,
+ VM_INHERIT_NONE);
+ assert(kr == KERN_SUCCESS);
+ for (cur_offset = 0;
+ cur_offset < wire_size;
+ cur_offset += PAGE_SIZE) {
+ kr = vm_map_wire_and_extract(wire_map,
+ wire_addr + cur_offset,
+ VM_PROT_DEFAULT | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK),
+ TRUE,
+ &wire_ppnum);
+ assert(kr == KERN_SUCCESS);
+ user_ppnum = vm_map_get_phys_page(user_map,
+ user_addr + cur_offset);
+ printf("VM_TEST_WIRE_AND_EXTRACT: kr=0x%x "
+ "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n",
+ kr,
+ user_map, user_addr + cur_offset, user_ppnum,
+ wire_map, wire_addr + cur_offset, wire_ppnum);
+ if (kr != KERN_SUCCESS ||
+ wire_ppnum == 0 ||
+ wire_ppnum != user_ppnum) {
+ panic("VM_TEST_WIRE_AND_EXTRACT: FAIL\n");
+ }
+ }
+ cur_offset -= PAGE_SIZE;
+ kr = vm_map_wire_and_extract(wire_map,
+ wire_addr + cur_offset,
+ VM_PROT_DEFAULT,
+ TRUE,
+ &wire_ppnum);
+ assert(kr == KERN_SUCCESS);
+ printf("VM_TEST_WIRE_AND_EXTRACT: re-wire kr=0x%x "
+ "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n",
+ kr,
+ user_map, user_addr + cur_offset, user_ppnum,
+ wire_map, wire_addr + cur_offset, wire_ppnum);
+ if (kr != KERN_SUCCESS ||
+ wire_ppnum == 0 ||
+ wire_ppnum != user_ppnum) {
+ panic("VM_TEST_WIRE_AND_EXTRACT: FAIL\n");
+ }
-#if UPL_DEBUG
-kern_return_t upl_ubc_alias_set(upl_t upl, uintptr_t alias1, uintptr_t alias2)
-{
- upl->ubc_alias1 = alias1;
- upl->ubc_alias2 = alias2;
- return KERN_SUCCESS;
-}
-int upl_ubc_alias_get(upl_t upl, uintptr_t * al, uintptr_t * al2)
-{
- if(al)
- *al = upl->ubc_alias1;
- if(al2)
- *al2 = upl->ubc_alias2;
- return KERN_SUCCESS;
+ printf("VM_TEST_WIRE_AND_EXTRACT: PASS\n");
}
-#endif /* UPL_DEBUG */
+#else /* VM_TEST_WIRE_AND_EXTRACT */
+#define vm_test_wire_and_extract()
+#endif /* VM_TEST_WIRE_AND_EXTRACT */
-#if VM_PRESSURE_EVENTS
-/*
- * Upward trajectory.
- */
-extern boolean_t vm_compressor_low_on_space(void);
-
-boolean_t
-VM_PRESSURE_NORMAL_TO_WARNING(void) {
+#if VM_TEST_PAGE_WIRE_OVERFLOW_PANIC
+static void
+vm_test_page_wire_overflow_panic(void)
+{
+ vm_object_t object;
+ vm_page_t page;
- if (DEFAULT_PAGER_IS_ACTIVE || DEFAULT_FREEZER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPLESS) {
-
- /* Available pages below our threshold */
- if (memorystatus_available_pages < memorystatus_available_pages_pressure) {
- /* No frozen processes to kill */
- if (memorystatus_frozen_count == 0) {
- /* Not enough suspended processes available. */
- if (memorystatus_suspended_count < MEMORYSTATUS_SUSPENDED_THRESHOLD) {
- return TRUE;
- }
- }
- }
- return FALSE;
+ printf("VM_TEST_PAGE_WIRE_OVERFLOW_PANIC: starting...\n");
- } else {
- return ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0);
- }
+ object = vm_object_allocate(PAGE_SIZE);
+ vm_object_lock(object);
+ page = vm_page_alloc(object, 0x0);
+ vm_page_lock_queues();
+ do {
+ vm_page_wire(page, 1, FALSE);
+ } while (page->wire_count != 0);
+ vm_page_unlock_queues();
+ vm_object_unlock(object);
+ panic("FBDP(%p,%p): wire_count overflow not detected\n",
+ object, page);
}
+#else /* VM_TEST_PAGE_WIRE_OVERFLOW_PANIC */
+#define vm_test_page_wire_overflow_panic()
+#endif /* VM_TEST_PAGE_WIRE_OVERFLOW_PANIC */
-boolean_t
-VM_PRESSURE_WARNING_TO_CRITICAL(void) {
-
- if (DEFAULT_PAGER_IS_ACTIVE || DEFAULT_FREEZER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPLESS) {
- /* Available pages below our threshold */
- if (memorystatus_available_pages < memorystatus_available_pages_critical) {
- return TRUE;
- }
- return FALSE;
+#if __arm64__ && VM_TEST_KERNEL_OBJECT_FAULT
+extern int copyinframe(vm_address_t fp, char *frame, boolean_t is64bit);
+static void
+vm_test_kernel_object_fault(void)
+{
+ kern_return_t kr;
+ vm_offset_t stack;
+ uintptr_t frameb[2];
+ int ret;
+
+ kr = kernel_memory_allocate(kernel_map, &stack,
+ kernel_stack_size + (2*PAGE_SIZE),
+ 0,
+ (KMA_KSTACK | KMA_KOBJECT |
+ KMA_GUARD_FIRST | KMA_GUARD_LAST),
+ VM_KERN_MEMORY_STACK);
+ if (kr != KERN_SUCCESS) {
+ panic("VM_TEST_KERNEL_OBJECT_FAULT: kernel_memory_allocate kr 0x%x\n", kr);
+ }
+ ret = copyinframe((uintptr_t)stack, (char *)frameb, TRUE);
+ if (ret != 0) {
+ printf("VM_TEST_KERNEL_OBJECT_FAULT: PASS\n");
} else {
- return (vm_compressor_low_on_space() || (AVAILABLE_NON_COMPRESSED_MEMORY < ((12 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0);
+ printf("VM_TEST_KERNEL_OBJECT_FAULT: FAIL\n");
}
+ vm_map_remove(kernel_map,
+ stack,
+ stack + kernel_stack_size + (2*PAGE_SIZE),
+ VM_MAP_REMOVE_KUNWIRE);
+ stack = 0;
}
+#else /* __arm64__ && VM_TEST_KERNEL_OBJECT_FAULT */
+#define vm_test_kernel_object_fault()
+#endif /* __arm64__ && VM_TEST_KERNEL_OBJECT_FAULT */
-/*
- * Downward trajectory.
- */
-boolean_t
-VM_PRESSURE_WARNING_TO_NORMAL(void) {
+#if VM_TEST_DEVICE_PAGER_TRANSPOSE
+static void
+vm_test_device_pager_transpose(void)
+{
+ memory_object_t device_pager;
+ vm_object_t anon_object, device_object;
+ vm_size_t size;
+ vm_map_offset_t anon_mapping, device_mapping;
+ kern_return_t kr;
- if (DEFAULT_PAGER_IS_ACTIVE || DEFAULT_FREEZER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPLESS) {
- /* Available pages above our threshold */
- unsigned int target_threshold = memorystatus_available_pages_pressure + ((15 * memorystatus_available_pages_pressure) / 100);
- if (memorystatus_available_pages > target_threshold) {
- return TRUE;
- }
- return FALSE;
- } else {
- return ((AVAILABLE_NON_COMPRESSED_MEMORY > ((12 * VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) / 10)) ? 1 : 0);
- }
-}
+ size = 3 * PAGE_SIZE;
+ anon_object = vm_object_allocate(size);
+ assert(anon_object != VM_OBJECT_NULL);
+ device_pager = device_pager_setup(NULL, 0, size, 0);
+ assert(device_pager != NULL);
+ device_object = memory_object_to_vm_object(device_pager);
+ assert(device_object != VM_OBJECT_NULL);
+ anon_mapping = 0;
+ kr = vm_map_enter(kernel_map, &anon_mapping, size, 0,
+ VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_NONE,
+ anon_object, 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL,
+ VM_INHERIT_DEFAULT);
+ assert(kr == KERN_SUCCESS);
+ device_mapping = 0;
+ kr = vm_map_enter_mem_object(kernel_map, &device_mapping, size, 0,
+ VM_FLAGS_ANYWHERE,
+ VM_MAP_KERNEL_FLAGS_NONE,
+ VM_KERN_MEMORY_NONE,
+ (void *)device_pager, 0, FALSE,
+ VM_PROT_DEFAULT, VM_PROT_ALL,
+ VM_INHERIT_DEFAULT);
+ assert(kr == KERN_SUCCESS);
+ memory_object_deallocate(device_pager);
+
+ vm_object_lock(anon_object);
+ vm_object_activity_begin(anon_object);
+ anon_object->blocked_access = TRUE;
+ vm_object_unlock(anon_object);
+ vm_object_lock(device_object);
+ vm_object_activity_begin(device_object);
+ device_object->blocked_access = TRUE;
+ vm_object_unlock(device_object);
+
+ assert(anon_object->ref_count == 1);
+ assert(!anon_object->named);
+ assert(device_object->ref_count == 2);
+ assert(device_object->named);
+
+ kr = vm_object_transpose(device_object, anon_object, size);
+ assert(kr == KERN_SUCCESS);
-boolean_t
-VM_PRESSURE_CRITICAL_TO_WARNING(void) {
+ vm_object_lock(anon_object);
+ vm_object_activity_end(anon_object);
+ anon_object->blocked_access = FALSE;
+ vm_object_unlock(anon_object);
+ vm_object_lock(device_object);
+ vm_object_activity_end(device_object);
+ device_object->blocked_access = FALSE;
+ vm_object_unlock(device_object);
+
+ assert(anon_object->ref_count == 2);
+ assert(anon_object->named);
+ kr = vm_deallocate(kernel_map, anon_mapping, size);
+ assert(kr == KERN_SUCCESS);
+ assert(device_object->ref_count == 1);
+ assert(!device_object->named);
+ kr = vm_deallocate(kernel_map, device_mapping, size);
+ assert(kr == KERN_SUCCESS);
- if (DEFAULT_PAGER_IS_ACTIVE || DEFAULT_FREEZER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPLESS) {
- /* Available pages above our threshold */
- unsigned int target_threshold = memorystatus_available_pages_critical + ((15 * memorystatus_available_pages_critical) / 100);
- if (memorystatus_available_pages > target_threshold) {
- return TRUE;
- }
- return FALSE;
- } else {
- return ((AVAILABLE_NON_COMPRESSED_MEMORY > ((14 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0);
- }
+ printf("VM_TEST_DEVICE_PAGER_TRANSPOSE: PASS\n");
}
-#endif /* VM_PRESSURE_EVENTS */
+#else /* VM_TEST_DEVICE_PAGER_TRANSPOSE */
+#define vm_test_device_pager_transpose()
+#endif /* VM_TEST_DEVICE_PAGER_TRANSPOSE */
+void
+vm_tests(void)
+{
+ vm_test_collapse_compressor();
+ vm_test_wire_and_extract();
+ vm_test_page_wire_overflow_panic();
+ vm_test_kernel_object_fault();
+ vm_test_device_pager_transpose();
+}
projects / apple / xnu.git / blobdiff