X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/d41d1dae2cd00cc08c7982087d1c445180cad9f5..eee3565979933af707c711411001ba11fe406a3c:/osfmk/vm/vm_pageout.c diff --git a/osfmk/vm/vm_pageout.c b/osfmk/vm/vm_pageout.c index 4098fb8bc..01a9f4506 100644 --- a/osfmk/vm/vm_pageout.c +++ b/osfmk/vm/vm_pageout.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2009 Apple Inc. All rights reserved. + * Copyright (c) 2000-2014 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -68,8 +68,6 @@ #include #include #include -#include -#include #include #include @@ -91,15 +89,13 @@ #include #include #include +#include #include #include -#if CONFIG_EMBEDDED -#include -#endif - #include +#include #include #include #include @@ -108,28 +104,54 @@ #include /* must be last */ #include #include +#include +#include +#if CONFIG_PHANTOM_CACHE +#include +#endif /* * ENCRYPTED SWAP: */ -#include <../bsd/crypto/aes/aes.h> +#include extern u_int32_t random(void); /* from */ +extern int cs_debug; + #if UPL_DEBUG #include #endif +extern void m_drain(void); + +#if VM_PRESSURE_EVENTS +extern unsigned int memorystatus_available_pages; +extern unsigned int memorystatus_available_pages_pressure; +extern unsigned int memorystatus_available_pages_critical; +extern unsigned int memorystatus_frozen_count; +extern unsigned int memorystatus_suspended_count; + +extern vm_pressure_level_t memorystatus_vm_pressure_level; +int memorystatus_purge_on_warning = 2; +int memorystatus_purge_on_urgent = 5; +int memorystatus_purge_on_critical = 8; + +void vm_pressure_response(void); +boolean_t vm_pressure_thread_running = FALSE; +extern void consider_vm_pressure_events(void); + +#define MEMORYSTATUS_SUSPENDED_THRESHOLD 4 +#endif /* VM_PRESSURE_EVENTS */ + +boolean_t vm_pressure_changed = FALSE; + #ifndef VM_PAGEOUT_BURST_ACTIVE_THROTTLE /* maximum iterations of the active queue to move pages to inactive */ #define VM_PAGEOUT_BURST_ACTIVE_THROTTLE 100 #endif #ifndef VM_PAGEOUT_BURST_INACTIVE_THROTTLE /* maximum iterations of the inactive queue w/o stealing/cleaning a page */ -#ifdef CONFIG_EMBEDDED -#define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 1024 -#else #define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 4096 #endif -#endif #ifndef VM_PAGEOUT_DEADLOCK_RELIEF #define VM_PAGEOUT_DEADLOCK_RELIEF 100 /* number of pages to move to break deadlock */ @@ -140,11 +162,11 @@ extern u_int32_t random(void); /* from */ #endif #ifndef VM_PAGE_LAUNDRY_MAX -#define VM_PAGE_LAUNDRY_MAX 16UL /* maximum pageouts on a given pageout queue */ +#define VM_PAGE_LAUNDRY_MAX 128UL /* maximum pageouts on a given pageout queue */ #endif /* VM_PAGEOUT_LAUNDRY_MAX */ #ifndef VM_PAGEOUT_BURST_WAIT -#define VM_PAGEOUT_BURST_WAIT 30 /* milliseconds per page */ +#define VM_PAGEOUT_BURST_WAIT 10 /* milliseconds */ #endif /* VM_PAGEOUT_BURST_WAIT */ #ifndef VM_PAGEOUT_EMPTY_WAIT @@ -159,10 +181,26 @@ extern u_int32_t random(void); /* from */ #define VM_PAGEOUT_IDLE_WAIT 10 /* milliseconds */ #endif /* VM_PAGEOUT_IDLE_WAIT */ +#ifndef VM_PAGEOUT_SWAP_WAIT +#define VM_PAGEOUT_SWAP_WAIT 50 /* milliseconds */ +#endif /* VM_PAGEOUT_SWAP_WAIT */ + +#ifndef VM_PAGEOUT_PRESSURE_PAGES_CONSIDERED +#define VM_PAGEOUT_PRESSURE_PAGES_CONSIDERED 1000 /* maximum pages considered before we issue a pressure event */ +#endif /* VM_PAGEOUT_PRESSURE_PAGES_CONSIDERED */ + +#ifndef VM_PAGEOUT_PRESSURE_EVENT_MONITOR_SECS +#define VM_PAGEOUT_PRESSURE_EVENT_MONITOR_SECS 5 /* seconds */ +#endif /* VM_PAGEOUT_PRESSURE_EVENT_MONITOR_SECS */ + +unsigned int vm_page_speculative_q_age_ms = VM_PAGE_SPECULATIVE_Q_AGE_MS; +unsigned int vm_page_speculative_percentage = 5; + #ifndef VM_PAGE_SPECULATIVE_TARGET -#define VM_PAGE_SPECULATIVE_TARGET(total) ((total) * 1 / 20) +#define VM_PAGE_SPECULATIVE_TARGET(total) ((total) * 1 / (100 / vm_page_speculative_percentage)) #endif /* VM_PAGE_SPECULATIVE_TARGET */ + #ifndef VM_PAGE_INACTIVE_HEALTHY_LIMIT #define VM_PAGE_INACTIVE_HEALTHY_LIMIT(total) ((total) * 1 / 200) #endif /* VM_PAGE_INACTIVE_HEALTHY_LIMIT */ @@ -181,7 +219,7 @@ extern u_int32_t random(void); /* from */ */ #ifndef VM_PAGE_INACTIVE_TARGET -#define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 1 / 3) +#define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 1 / 2) #endif /* VM_PAGE_INACTIVE_TARGET */ /* @@ -190,29 +228,22 @@ extern u_int32_t random(void); /* from */ */ #ifndef VM_PAGE_FREE_TARGET -#ifdef CONFIG_EMBEDDED -#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 100) -#else #define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80) -#endif #endif /* VM_PAGE_FREE_TARGET */ + /* * The pageout daemon always starts running once vm_page_free_count * falls below vm_page_free_min. */ #ifndef VM_PAGE_FREE_MIN -#ifdef CONFIG_EMBEDDED -#define VM_PAGE_FREE_MIN(free) (10 + (free) / 200) -#else #define VM_PAGE_FREE_MIN(free) (10 + (free) / 100) -#endif #endif /* VM_PAGE_FREE_MIN */ -#define VM_PAGE_FREE_MIN_LIMIT 1500 -#define VM_PAGE_FREE_TARGET_LIMIT 2000 - +#define VM_PAGE_FREE_RESERVED_LIMIT 1700 +#define VM_PAGE_FREE_MIN_LIMIT 3500 +#define VM_PAGE_FREE_TARGET_LIMIT 4000 /* * When vm_page_free_count falls below vm_page_free_reserved, @@ -237,14 +268,12 @@ extern u_int32_t random(void); /* from */ */ #define VM_PAGE_REACTIVATE_LIMIT_MAX 20000 #ifndef VM_PAGE_REACTIVATE_LIMIT -#ifdef CONFIG_EMBEDDED -#define VM_PAGE_REACTIVATE_LIMIT(avail) (VM_PAGE_INACTIVE_TARGET(avail) / 2) -#else #define VM_PAGE_REACTIVATE_LIMIT(avail) (MAX((avail) * 1 / 20,VM_PAGE_REACTIVATE_LIMIT_MAX)) -#endif #endif /* VM_PAGE_REACTIVATE_LIMIT */ -#define VM_PAGEOUT_INACTIVE_FORCE_RECLAIM 100 +#define VM_PAGEOUT_INACTIVE_FORCE_RECLAIM 1000 + +extern boolean_t hibernate_cleaning_in_progress; /* * Exported variable used to broadcast the activation of the pageout scan @@ -258,21 +287,52 @@ unsigned int vm_pageout_scan_event_counter = 0; /* * Forward declarations for internal routines. */ +struct cq { + struct vm_pageout_queue *q; + void *current_chead; + char *scratch_buf; + int id; +}; +#define MAX_COMPRESSOR_THREAD_COUNT 8 + +struct cq ciq[MAX_COMPRESSOR_THREAD_COUNT]; + +void *vm_pageout_immediate_chead; +char *vm_pageout_immediate_scratch_buf; + + +#if VM_PRESSURE_EVENTS +void vm_pressure_thread(void); + +boolean_t VM_PRESSURE_NORMAL_TO_WARNING(void); +boolean_t VM_PRESSURE_WARNING_TO_CRITICAL(void); +boolean_t VM_PRESSURE_WARNING_TO_NORMAL(void); +boolean_t VM_PRESSURE_CRITICAL_TO_WARNING(void); +#endif static void vm_pageout_garbage_collect(int); -static void vm_pageout_iothread_continue(struct vm_pageout_queue *); static void vm_pageout_iothread_external(void); -static void vm_pageout_iothread_internal(void); +static void vm_pageout_iothread_internal(struct cq *cq); +static void vm_pageout_adjust_io_throttles(struct vm_pageout_queue *, struct vm_pageout_queue *, boolean_t); extern void vm_pageout_continue(void); extern void vm_pageout_scan(void); +static void vm_pageout_immediate(vm_page_t, boolean_t); +boolean_t vm_compressor_immediate_preferred = FALSE; +boolean_t vm_compressor_immediate_preferred_override = FALSE; +boolean_t vm_restricted_to_single_processor = FALSE; +static boolean_t vm_pageout_waiter = FALSE; +static boolean_t vm_pageout_running = FALSE; + + static thread_t vm_pageout_external_iothread = THREAD_NULL; static thread_t vm_pageout_internal_iothread = THREAD_NULL; unsigned int vm_pageout_reserved_internal = 0; unsigned int vm_pageout_reserved_really = 0; +unsigned int vm_pageout_swap_wait = 0; unsigned int vm_pageout_idle_wait = 0; /* milliseconds */ unsigned int vm_pageout_empty_wait = 0; /* milliseconds */ unsigned int vm_pageout_burst_wait = 0; /* milliseconds */ @@ -282,19 +342,8 @@ unsigned int vm_pageout_inactive_relief = 0; unsigned int vm_pageout_burst_active_throttle = 0; unsigned int vm_pageout_burst_inactive_throttle = 0; -/* - * Protection against zero fill flushing live working sets derived - * from existing backing store and files - */ -unsigned int vm_accellerate_zf_pageout_trigger = 400; -unsigned int zf_queue_min_count = 100; -unsigned int vm_zf_queue_count = 0; +int vm_upl_wait_for_pages = 0; -#if defined(__ppc__) /* On ppc, vm statistics are still 32-bit */ -unsigned int vm_zf_count = 0; -#else -uint64_t vm_zf_count __attribute__((aligned(8))) = 0; -#endif /* * These variables record the pageout daemon's actions: @@ -309,14 +358,36 @@ unsigned int vm_pageout_inactive_forced = 0; /* debugging */ unsigned int vm_pageout_inactive_nolock = 0; /* debugging */ unsigned int vm_pageout_inactive_avoid = 0; /* debugging */ unsigned int vm_pageout_inactive_busy = 0; /* debugging */ +unsigned int vm_pageout_inactive_error = 0; /* debugging */ unsigned int vm_pageout_inactive_absent = 0; /* debugging */ +unsigned int vm_pageout_inactive_notalive = 0; /* debugging */ unsigned int vm_pageout_inactive_used = 0; /* debugging */ +unsigned int vm_pageout_cache_evicted = 0; /* debugging */ unsigned int vm_pageout_inactive_clean = 0; /* debugging */ -unsigned int vm_pageout_inactive_dirty = 0; /* debugging */ +unsigned int vm_pageout_speculative_clean = 0; /* debugging */ + +unsigned int vm_pageout_freed_from_cleaned = 0; +unsigned int vm_pageout_freed_from_speculative = 0; +unsigned int vm_pageout_freed_from_inactive_clean = 0; + +unsigned int vm_pageout_enqueued_cleaned_from_inactive_clean = 0; +unsigned int vm_pageout_enqueued_cleaned_from_inactive_dirty = 0; + +unsigned int vm_pageout_cleaned_reclaimed = 0; /* debugging; how many cleaned pages are reclaimed by the pageout scan */ +unsigned int vm_pageout_cleaned_reactivated = 0; /* debugging; how many cleaned pages are found to be referenced on pageout (and are therefore reactivated) */ +unsigned int vm_pageout_cleaned_reference_reactivated = 0; +unsigned int vm_pageout_cleaned_volatile_reactivated = 0; +unsigned int vm_pageout_cleaned_fault_reactivated = 0; +unsigned int vm_pageout_cleaned_commit_reactivated = 0; /* debugging; how many cleaned pages are found to be referenced on commit (and are therefore reactivated) */ +unsigned int vm_pageout_cleaned_busy = 0; +unsigned int vm_pageout_cleaned_nolock = 0; + +unsigned int vm_pageout_inactive_dirty_internal = 0; /* debugging */ +unsigned int vm_pageout_inactive_dirty_external = 0; /* debugging */ unsigned int vm_pageout_inactive_deactivated = 0; /* debugging */ -unsigned int vm_pageout_inactive_zf = 0; /* debugging */ +unsigned int vm_pageout_inactive_anonymous = 0; /* debugging */ unsigned int vm_pageout_dirty_no_pager = 0; /* debugging */ -unsigned int vm_pageout_purged_objects = 0; /* debugging */ +unsigned int vm_pageout_purged_objects = 0; /* used for sysctl vm stats */ unsigned int vm_stat_discard = 0; /* debugging */ unsigned int vm_stat_discard_sent = 0; /* debugging */ unsigned int vm_stat_discard_failure = 0; /* debugging */ @@ -325,19 +396,24 @@ unsigned int vm_pageout_reactivation_limit_exceeded = 0; /* debugging */ unsigned int vm_pageout_catch_ups = 0; /* debugging */ unsigned int vm_pageout_inactive_force_reclaim = 0; /* debugging */ +unsigned int vm_pageout_scan_reclaimed_throttled = 0; unsigned int vm_pageout_scan_active_throttled = 0; -unsigned int vm_pageout_scan_inactive_throttled = 0; +unsigned int vm_pageout_scan_inactive_throttled_internal = 0; +unsigned int vm_pageout_scan_inactive_throttled_external = 0; unsigned int vm_pageout_scan_throttle = 0; /* debugging */ -unsigned int vm_pageout_scan_throttle_aborted = 0; /* debugging */ unsigned int vm_pageout_scan_burst_throttle = 0; /* debugging */ unsigned int vm_pageout_scan_empty_throttle = 0; /* debugging */ +unsigned int vm_pageout_scan_swap_throttle = 0; /* debugging */ unsigned int vm_pageout_scan_deadlock_detected = 0; /* debugging */ unsigned int vm_pageout_scan_active_throttle_success = 0; /* debugging */ unsigned int vm_pageout_scan_inactive_throttle_success = 0; /* debugging */ - +unsigned int vm_pageout_inactive_external_forced_jetsam_count = 0; /* debugging */ +unsigned int vm_pageout_scan_throttle_deferred = 0; /* debugging */ +unsigned int vm_pageout_scan_yield_unthrottled = 0; /* debugging */ unsigned int vm_page_speculative_count_drifts = 0; unsigned int vm_page_speculative_count_drift_max = 0; + /* * Backing store throttle when BS is exhausted */ @@ -348,6 +424,8 @@ unsigned int vm_pageout_in_place = 0; unsigned int vm_page_steal_pageout_page = 0; +struct vm_config vm_config; + /* * ENCRYPTED SWAP: * counters and statistics... @@ -359,8 +437,8 @@ unsigned long vm_page_encrypt_abort_counter = 0; unsigned long vm_page_encrypt_already_encrypted_counter = 0; boolean_t vm_pages_encrypted = FALSE; /* are there encrypted pages ? */ -struct vm_pageout_queue vm_pageout_queue_internal; -struct vm_pageout_queue vm_pageout_queue_external; +struct vm_pageout_queue vm_pageout_queue_internal __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT))); +struct vm_pageout_queue vm_pageout_queue_external __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT))); unsigned int vm_page_speculative_target = 0; @@ -372,6 +450,24 @@ boolean_t (* volatile consider_buffer_cache_collect)(int) = NULL; unsigned long vm_cs_validated_resets = 0; #endif +int vm_debug_events = 0; + +#if CONFIG_MEMORYSTATUS +#if !CONFIG_JETSAM +extern boolean_t memorystatus_idle_exit_from_VM(void); +#endif +extern boolean_t memorystatus_kill_on_VM_page_shortage(boolean_t async); +extern void memorystatus_on_pageout_scan_end(void); + +uint32_t vm_pageout_memorystatus_fb_factor_nr = 5; +uint32_t vm_pageout_memorystatus_fb_factor_dr = 2; +#if DEVELOPMENT || DEBUG +uint32_t vm_grab_anon_overrides = 0; +uint32_t vm_grab_anon_nops = 0; +#endif + +#endif + /* * Routine: vm_backing_store_disable * Purpose: @@ -433,33 +529,27 @@ vm_pageout_object_terminate( shadow_object = object->shadow; vm_object_lock(shadow_object); - while (!queue_empty(&object->memq)) { + while (!vm_page_queue_empty(&object->memq)) { vm_page_t p, m; vm_object_offset_t offset; - p = (vm_page_t) queue_first(&object->memq); + p = (vm_page_t) vm_page_queue_first(&object->memq); assert(p->private); - assert(p->pageout); - p->pageout = FALSE; + assert(p->free_when_done); + p->free_when_done = FALSE; assert(!p->cleaning); + assert(!p->laundry); offset = p->offset; VM_PAGE_FREE(p); p = VM_PAGE_NULL; m = vm_page_lookup(shadow_object, - offset + object->shadow_offset); + offset + object->vo_shadow_offset); if(m == VM_PAGE_NULL) continue; - assert(m->cleaning); - /* used as a trigger on upl_commit etc to recognize the */ - /* pageout daemon's subseqent desire to pageout a cleaning */ - /* page. When the bit is on the upl commit code will */ - /* respect the pageout bit in the target page over the */ - /* caller's page list indication */ - m->dump_cleaning = FALSE; assert((m->dirty) || (m->precious) || (m->busy && m->cleaning)); @@ -469,9 +559,8 @@ vm_pageout_object_terminate( * Also decrement the burst throttle (if external). */ vm_page_lock_queues(); - if (m->laundry) { + if (m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) vm_pageout_throttle_up(m); - } /* * Handle the "target" page(s). These pages are to be freed if @@ -481,12 +570,13 @@ vm_pageout_object_terminate( * pages may have been modified between the selection as an * adjacent page and conversion to a target. */ - if (m->pageout) { + if (m->free_when_done) { assert(m->busy); + assert(m->vm_page_q_state == VM_PAGE_IS_WIRED); assert(m->wire_count == 1); m->cleaning = FALSE; m->encrypted_cleaning = FALSE; - m->pageout = FALSE; + m->free_when_done = FALSE; #if MACH_CLUSTER_STATS if (m->wanted) vm_pageout_target_collisions++; #endif @@ -500,10 +590,11 @@ vm_pageout_object_terminate( * can detect whether the page was redirtied during * pageout by checking the modify state. */ - if (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED) - m->dirty = TRUE; - else - m->dirty = FALSE; + if (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED) { + SET_PAGE_DIRTY(m, FALSE); + } else { + m->dirty = FALSE; + } if (m->dirty) { CLUSTER_STAT(vm_pageout_target_page_dirtied++;) @@ -523,57 +614,65 @@ vm_pageout_object_terminate( * If prep_pin_count is nonzero, then someone is using the * page, so make it active. */ - if (!m->active && !m->inactive && !m->throttled && !m->private) { + if ((m->vm_page_q_state == VM_PAGE_NOT_ON_Q) && !m->private) { if (m->reference) vm_page_activate(m); else vm_page_deactivate(m); } - if((m->busy) && (m->cleaning)) { - - /* the request_page_list case, (COPY_OUT_FROM FALSE) */ - m->busy = FALSE; - - /* We do not re-set m->dirty ! */ - /* The page was busy so no extraneous activity */ - /* could have occurred. COPY_INTO is a read into the */ - /* new pages. CLEAN_IN_PLACE does actually write */ - /* out the pages but handling outside of this code */ - /* will take care of resetting dirty. We clear the */ - /* modify however for the Programmed I/O case. */ - pmap_clear_modify(m->phys_page); + if (m->overwriting) { + /* + * the (COPY_OUT_FROM == FALSE) request_page_list case + */ + if (m->busy) { + /* + * We do not re-set m->dirty ! + * The page was busy so no extraneous activity + * could have occurred. COPY_INTO is a read into the + * new pages. CLEAN_IN_PLACE does actually write + * out the pages but handling outside of this code + * will take care of resetting dirty. We clear the + * modify however for the Programmed I/O case. + */ + pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m)); - m->absent = FALSE; - m->overwriting = FALSE; - } else if (m->overwriting) { - /* alternate request page list, write to page_list */ - /* case. Occurs when the original page was wired */ - /* at the time of the list request */ - assert(VM_PAGE_WIRED(m)); - vm_page_unwire(m, TRUE); /* reactivates */ + m->busy = FALSE; + m->absent = FALSE; + } else { + /* + * alternate (COPY_OUT_FROM == FALSE) request_page_list case + * Occurs when the original page was wired + * at the time of the list request + */ + assert(VM_PAGE_WIRED(m)); + vm_page_unwire(m, TRUE); /* reactivates */ + } m->overwriting = FALSE; } else { - /* - * Set the dirty state according to whether or not the page was - * modified during the pageout. Note that we purposefully do - * NOT call pmap_clear_modify since the page is still mapped. - * If the page were to be dirtied between the 2 calls, this - * this fact would be lost. This code is only necessary to - * maintain statistics, since the pmap module is always - * consulted if m->dirty is false. - */ + /* + * Set the dirty state according to whether or not the page was + * modified during the pageout. Note that we purposefully do + * NOT call pmap_clear_modify since the page is still mapped. + * If the page were to be dirtied between the 2 calls, this + * this fact would be lost. This code is only necessary to + * maintain statistics, since the pmap module is always + * consulted if m->dirty is false. + */ #if MACH_CLUSTER_STATS - m->dirty = pmap_is_modified(m->phys_page); + m->dirty = pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m)); if (m->dirty) vm_pageout_cluster_dirtied++; else vm_pageout_cluster_cleaned++; if (m->wanted) vm_pageout_cluster_collisions++; #else - m->dirty = 0; + m->dirty = FALSE; #endif } + if (m->encrypted_cleaning == TRUE) { + m->encrypted_cleaning = FALSE; + m->busy = FALSE; + } m->cleaning = FALSE; - m->encrypted_cleaning = FALSE; /* * Wakeup any thread waiting for the page to be un-cleaning. @@ -605,7 +704,7 @@ vm_pageout_object_terminate( * must be locked. * */ -void +static void vm_pageclean_setup( vm_page_t m, vm_page_t new_m, @@ -618,17 +717,17 @@ vm_pageclean_setup( #endif XPR(XPR_VM_PAGEOUT, - "vm_pageclean_setup, obj 0x%X off 0x%X page 0x%X new 0x%X new_off 0x%X\n", - m->object, m->offset, m, + "vm_pageclean_setup, obj 0x%X off 0x%X page 0x%X new 0x%X new_off 0x%X\n", + VM_PAGE_OBJECT(m), m->offset, m, new_m, new_offset); - pmap_clear_modify(m->phys_page); + pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m)); /* * Mark original page as cleaning in place. */ m->cleaning = TRUE; - m->dirty = TRUE; + SET_PAGE_DIRTY(m, FALSE); m->precious = FALSE; /* @@ -636,17 +735,17 @@ vm_pageclean_setup( * the real page. */ assert(new_m->fictitious); - assert(new_m->phys_page == vm_page_fictitious_addr); + assert(VM_PAGE_GET_PHYS_PAGE(new_m) == vm_page_fictitious_addr); new_m->fictitious = FALSE; new_m->private = TRUE; - new_m->pageout = TRUE; - new_m->phys_page = m->phys_page; + new_m->free_when_done = TRUE; + VM_PAGE_SET_PHYS_PAGE(new_m, VM_PAGE_GET_PHYS_PAGE(m)); vm_page_lockspin_queues(); - vm_page_wire(new_m); + vm_page_wire(new_m, VM_KERN_MEMORY_NONE, TRUE); vm_page_unlock_queues(); - vm_page_insert(new_m, new_object, new_offset); + vm_page_insert_wired(new_m, new_object, new_offset, VM_KERN_MEMORY_NONE); assert(!new_m->wanted); new_m->busy = FALSE; } @@ -675,13 +774,18 @@ vm_pageout_initialize_page( { vm_object_t object; vm_object_offset_t paging_offset; - vm_page_t holding_page; memory_object_t pager; XPR(XPR_VM_PAGEOUT, "vm_pageout_initialize_page, page 0x%X\n", m, 0, 0, 0, 0); + + assert(VM_CONFIG_COMPRESSOR_IS_PRESENT); + + object = VM_PAGE_OBJECT(m); + assert(m->busy); + assert(object->internal); /* * Verify that we really want to clean this page @@ -693,12 +797,12 @@ vm_pageout_initialize_page( /* * Create a paging reference to let us play with the object. */ - object = m->object; paging_offset = m->offset + object->paging_offset; if (m->absent || m->error || m->restart || (!m->dirty && !m->precious)) { - VM_PAGE_FREE(m); panic("reservation without pageout?"); /* alan */ + + VM_PAGE_FREE(m); vm_object_unlock(object); return; @@ -713,26 +817,22 @@ vm_pageout_initialize_page( pager = object->pager; if (pager == MEMORY_OBJECT_NULL) { - VM_PAGE_FREE(m); panic("missing pager for copy object"); + + VM_PAGE_FREE(m); return; } - /* set the page for future call to vm_fault_list_request */ - vm_object_paging_begin(object); - holding_page = NULL; - - pmap_clear_modify(m->phys_page); - m->dirty = TRUE; - m->busy = TRUE; - m->list_req_pending = TRUE; - m->cleaning = TRUE; - m->pageout = TRUE; - - vm_page_lockspin_queues(); - vm_page_wire(m); - vm_page_unlock_queues(); + /* + * set the page for future call to vm_fault_list_request + */ + pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m)); + SET_PAGE_DIRTY(m, FALSE); + /* + * keep the object from collapsing or terminating + */ + vm_object_paging_begin(object); vm_object_unlock(object); /* @@ -766,7 +866,7 @@ struct { * which will page it out and attempt to clean adjacent pages * in the same operation. * - * The page must be busy, and the object and queues locked. We will take a + * The object and queues must be locked. We will take a * paging reference to prevent deallocation or collapse when we * release the object lock back at the call site. The I/O thread * is responsible for consuming this reference @@ -774,10 +874,10 @@ struct { * The page must not be on any pageout queue. */ -void -vm_pageout_cluster(vm_page_t m) +int +vm_pageout_cluster(vm_page_t m, boolean_t immediate_ok, boolean_t keep_object_locked) { - vm_object_t object = m->object; + vm_object_t object = VM_PAGE_OBJECT(m); struct vm_pageout_queue *q; @@ -786,49 +886,57 @@ vm_pageout_cluster(vm_page_t m) object, m->offset, m, 0, 0); VM_PAGE_CHECK(m); + LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); + vm_object_lock_assert_exclusive(object); /* * Only a certain kind of page is appreciated here. */ - assert(m->busy && (m->dirty || m->precious) && (!VM_PAGE_WIRED(m))); - assert(!m->cleaning && !m->pageout && !m->inactive && !m->active); - assert(!m->throttled); + assert((m->dirty || m->precious) && (!VM_PAGE_WIRED(m))); + assert(!m->cleaning && !m->laundry); + assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q); /* - * protect the object from collapse - - * locking in the object's paging_offset. + * protect the object from collapse or termination */ - vm_object_paging_begin(object); + vm_object_activity_begin(object); - /* - * set the page for future call to vm_fault_list_request - * page should already be marked busy - */ - vm_page_wire(m); - m->list_req_pending = TRUE; - m->cleaning = TRUE; - m->pageout = TRUE; + if (object->internal == TRUE) { + assert(VM_CONFIG_COMPRESSOR_IS_PRESENT); + + m->busy = TRUE; + + if (vm_compressor_immediate_preferred == TRUE && immediate_ok == TRUE) { + panic("immediate compressor mode no longer supported\n"); + + if (keep_object_locked == FALSE) + vm_object_unlock(object); + vm_page_unlock_queues(); - if (object->internal == TRUE) + vm_pageout_immediate(m, keep_object_locked); + + return (1); + } q = &vm_pageout_queue_internal; - else + } else q = &vm_pageout_queue_external; - /* + /* * pgo_laundry count is tied to the laundry bit */ - m->laundry = TRUE; + m->laundry = TRUE; q->pgo_laundry++; - m->pageout_queue = TRUE; - queue_enter(&q->pgo_pending, m, vm_page_t, pageq); + m->vm_page_q_state = VM_PAGE_ON_PAGEOUT_Q; + vm_page_queue_enter(&q->pgo_pending, m, vm_page_t, pageq); if (q->pgo_idle == TRUE) { - q->pgo_idle = FALSE; - thread_wakeup((event_t) &q->pgo_pending); + q->pgo_idle = FALSE; + thread_wakeup((event_t) &q->pgo_pending); } - VM_PAGE_CHECK(m); + + return (0); } @@ -843,62 +951,73 @@ unsigned long vm_pageout_throttle_up_count = 0; */ void vm_pageout_throttle_up( - vm_page_t m) + vm_page_t m) { - struct vm_pageout_queue *q; + struct vm_pageout_queue *q; + vm_object_t m_object; - assert(m->object != VM_OBJECT_NULL); - assert(m->object != kernel_object); + m_object = VM_PAGE_OBJECT(m); - vm_pageout_throttle_up_count++; + assert(m_object != VM_OBJECT_NULL); + assert(m_object != kernel_object); - if (m->object->internal == TRUE) - q = &vm_pageout_queue_internal; - else - q = &vm_pageout_queue_external; + LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); + vm_object_lock_assert_exclusive(m_object); - if (m->pageout_queue == TRUE) { + vm_pageout_throttle_up_count++; - queue_remove(&q->pgo_pending, m, vm_page_t, pageq); - m->pageout_queue = FALSE; + if (m_object->internal == TRUE) + q = &vm_pageout_queue_internal; + else + q = &vm_pageout_queue_external; - m->pageq.next = NULL; - m->pageq.prev = NULL; + if (m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) { - vm_object_paging_end(m->object); - } - if (m->laundry == TRUE) { - m->laundry = FALSE; - q->pgo_laundry--; + vm_page_queue_remove(&q->pgo_pending, m, vm_page_t, pageq); + m->vm_page_q_state = VM_PAGE_NOT_ON_Q; - if (q->pgo_throttled == TRUE) { - q->pgo_throttled = FALSE; - thread_wakeup((event_t) &q->pgo_laundry); - } - if (q->pgo_draining == TRUE && q->pgo_laundry == 0) { - q->pgo_draining = FALSE; - thread_wakeup((event_t) (&q->pgo_laundry+1)); - } + VM_PAGE_ZERO_PAGEQ_ENTRY(m); + + vm_object_activity_end(m_object); + } + if (m->laundry == TRUE) { + + m->laundry = FALSE; + q->pgo_laundry--; + + if (q->pgo_throttled == TRUE) { + q->pgo_throttled = FALSE; + thread_wakeup((event_t) &q->pgo_laundry); + } + if (q->pgo_draining == TRUE && q->pgo_laundry == 0) { + q->pgo_draining = FALSE; + thread_wakeup((event_t) (&q->pgo_laundry+1)); + } } } -/* - * vm_pageout_scan does the dirty work for the pageout daemon. - * It returns with vm_page_queue_free_lock held and - * vm_page_free_wanted == 0. - */ +static void +vm_pageout_throttle_up_batch( + struct vm_pageout_queue *q, + int batch_cnt) +{ + LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); -#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT (3 * MAX_UPL_TRANSFER) + vm_pageout_throttle_up_count += batch_cnt; -#define FCS_IDLE 0 -#define FCS_DELAYED 1 -#define FCS_DEADLOCK_DETECTED 2 + q->pgo_laundry -= batch_cnt; + + if (q->pgo_throttled == TRUE) { + q->pgo_throttled = FALSE; + thread_wakeup((event_t) &q->pgo_laundry); + } + if (q->pgo_draining == TRUE && q->pgo_laundry == 0) { + q->pgo_draining = FALSE; + thread_wakeup((event_t) (&q->pgo_laundry+1)); + } +} -struct flow_control { - int state; - mach_timespec_t ts; -}; /* @@ -931,6 +1050,10 @@ unsigned int vm_memory_pressure = 0; #define VM_PAGEOUT_STAT_AFTER(i) \ (((i) == VM_PAGEOUT_STAT_SIZE - 1) ? 0 : (i) + 1) +#if VM_PAGE_BUCKETS_CHECK +int vm_page_buckets_check_interval = 10; /* in seconds */ +#endif /* VM_PAGE_BUCKETS_CHECK */ + /* * Called from compute_averages(). */ @@ -940,6 +1063,14 @@ compute_memory_pressure( { unsigned int vm_pageout_next; +#if VM_PAGE_BUCKETS_CHECK + /* check the consistency of VM page buckets at regular interval */ + static int counter = 0; + if ((++counter % vm_page_buckets_check_interval) == 0) { + vm_page_buckets_check(); + } +#endif /* VM_PAGE_BUCKETS_CHECK */ + vm_memory_pressure = vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].reclaimed; @@ -952,6 +1083,14 @@ compute_memory_pressure( vm_pageout_stat_now = vm_pageout_next; } + +/* + * IMPORTANT + * mach_vm_ctl_page_free_wanted() is called indirectly, via + * mach_vm_pressure_monitor(), when taking a stackshot. Therefore, + * it must be safe in the restricted stackshot context. Locks and/or + * blocking are not allowable. + */ unsigned int mach_vm_ctl_page_free_wanted(void) { @@ -968,6 +1107,15 @@ mach_vm_ctl_page_free_wanted(void) return page_free_wanted; } + +/* + * IMPORTANT: + * mach_vm_pressure_monitor() is called when taking a stackshot, with + * wait_for_pressure FALSE, so that code path must remain safe in the + * restricted stackshot context. No blocking or locks are allowable. + * on that code path. + */ + kern_return_t mach_vm_pressure_monitor( boolean_t wait_for_pressure, @@ -1019,1548 +1167,3368 @@ mach_vm_pressure_monitor( } /* provide number of pages reclaimed in the last "nsecs_monitored" */ - do { - vm_pageout_now = vm_pageout_stat_now; - pages_reclaimed = 0; - for (vm_pageout_then = - VM_PAGEOUT_STAT_BEFORE(vm_pageout_now); - vm_pageout_then != vm_pageout_now && - nsecs_monitored-- != 0; - vm_pageout_then = - VM_PAGEOUT_STAT_BEFORE(vm_pageout_then)) { - pages_reclaimed += vm_pageout_stats[vm_pageout_then].reclaimed; - } - } while (vm_pageout_now != vm_pageout_stat_now); + vm_pageout_now = vm_pageout_stat_now; + pages_reclaimed = 0; + for (vm_pageout_then = + VM_PAGEOUT_STAT_BEFORE(vm_pageout_now); + vm_pageout_then != vm_pageout_now && + nsecs_monitored-- != 0; + vm_pageout_then = + VM_PAGEOUT_STAT_BEFORE(vm_pageout_then)) { + pages_reclaimed += vm_pageout_stats[vm_pageout_then].reclaimed; + } *pages_reclaimed_p = pages_reclaimed; return KERN_SUCCESS; } -/* Page States: Used below to maintain the page state - before it's removed from it's Q. This saved state - helps us do the right accounting in certain cases -*/ - -#define PAGE_STATE_SPECULATIVE 1 -#define PAGE_STATE_THROTTLED 2 -#define PAGE_STATE_ZEROFILL 3 -#define PAGE_STATE_INACTIVE 4 - -#define VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m) \ - MACRO_BEGIN \ - /* \ - * If a "reusable" page somehow made it back into \ - * the active queue, it's been re-used and is not \ - * quite re-usable. \ - * If the VM object was "all_reusable", consider it \ - * as "all re-used" instead of converting it to \ - * "partially re-used", which could be expensive. \ - */ \ - if ((m)->reusable || \ - (m)->object->all_reusable) { \ - vm_object_reuse_pages((m)->object, \ - (m)->offset, \ - (m)->offset + PAGE_SIZE_64, \ - FALSE); \ - } \ - MACRO_END -void -vm_pageout_scan(void) -{ - unsigned int loop_count = 0; - unsigned int inactive_burst_count = 0; - unsigned int active_burst_count = 0; - unsigned int reactivated_this_call; - unsigned int reactivate_limit; - vm_page_t local_freeq = NULL; - int local_freed = 0; - int delayed_unlock; - int refmod_state = 0; - int vm_pageout_deadlock_target = 0; - struct vm_pageout_queue *iq; - struct vm_pageout_queue *eq; - struct vm_speculative_age_q *sq; - struct flow_control flow_control = { 0, { 0, 0 } }; - boolean_t inactive_throttled = FALSE; - boolean_t try_failed; - mach_timespec_t ts; - unsigned int msecs = 0; - vm_object_t object; - vm_object_t last_object_tried; -#if defined(__ppc__) /* On ppc, vm statistics are still 32-bit */ - unsigned int zf_ratio; - unsigned int zf_run_count; -#else - uint64_t zf_ratio; - uint64_t zf_run_count; -#endif - uint32_t catch_up_count = 0; - uint32_t inactive_reclaim_run; - boolean_t forced_reclaim; - int page_prev_state = 0; - flow_control.state = FCS_IDLE; - iq = &vm_pageout_queue_internal; - eq = &vm_pageout_queue_external; - sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q]; +#if DEVELOPMENT || DEBUG +static void +vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *, int); - XPR(XPR_VM_PAGEOUT, "vm_pageout_scan\n", 0, 0, 0, 0, 0); +/* + * condition variable used to make sure there is + * only a single sweep going on at a time + */ +boolean_t vm_pageout_disconnect_all_pages_active = FALSE; - + +void +vm_pageout_disconnect_all_pages() +{ vm_page_lock_queues(); - delayed_unlock = 1; /* must be nonzero if Qs are locked, 0 if unlocked */ - /* - * Calculate the max number of referenced pages on the inactive - * queue that we will reactivate. - */ - reactivated_this_call = 0; - reactivate_limit = VM_PAGE_REACTIVATE_LIMIT(vm_page_active_count + - vm_page_inactive_count); - inactive_reclaim_run = 0; + if (vm_pageout_disconnect_all_pages_active == TRUE) { + vm_page_unlock_queues(); + return; + } + vm_pageout_disconnect_all_pages_active = TRUE; + vm_page_unlock_queues(); + vm_pageout_disconnect_all_pages_in_queue(&vm_page_queue_throttled, vm_page_throttled_count); + vm_pageout_disconnect_all_pages_in_queue(&vm_page_queue_anonymous, vm_page_anonymous_count); + vm_pageout_disconnect_all_pages_in_queue(&vm_page_queue_active, vm_page_active_count); -/*???*/ /* - * We want to gradually dribble pages from the active queue - * to the inactive queue. If we let the inactive queue get - * very small, and then suddenly dump many pages into it, - * those pages won't get a sufficient chance to be referenced - * before we start taking them from the inactive queue. - * - * We must limit the rate at which we send pages to the pagers. - * data_write messages consume memory, for message buffers and - * for map-copy objects. If we get too far ahead of the pagers, - * we can potentially run out of memory. - * - * We can use the laundry count to limit directly the number - * of pages outstanding to the default pager. A similar - * strategy for external pagers doesn't work, because - * external pagers don't have to deallocate the pages sent them, - * and because we might have to send pages to external pagers - * even if they aren't processing writes. So we also - * use a burst count to limit writes to external pagers. - * - * When memory is very tight, we can't rely on external pagers to - * clean pages. They probably aren't running, because they - * aren't vm-privileged. If we kept sending dirty pages to them, - * we could exhaust the free list. - */ + vm_pageout_disconnect_all_pages_active = FALSE; +} -Restart: - assert(delayed_unlock!=0); - - /* - * A page is "zero-filled" if it was not paged in from somewhere, - * and it belongs to an object at least VM_ZF_OBJECT_SIZE_THRESHOLD big. - * Recalculate the zero-filled page ratio. We use this to apportion - * victimized pages between the normal and zero-filled inactive - * queues according to their relative abundance in memory. Thus if a task - * is flooding memory with zf pages, we begin to hunt them down. - * It would be better to throttle greedy tasks at a higher level, - * but at the moment mach vm cannot do this. - */ - { -#if defined(__ppc__) /* On ppc, vm statistics are still 32-bit */ - uint32_t total = vm_page_active_count + vm_page_inactive_count; - uint32_t normal = total - vm_zf_count; -#else - uint64_t total = vm_page_active_count + vm_page_inactive_count; - uint64_t normal = total - vm_zf_count; -#endif +void +vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *q, int qcount) +{ + vm_page_t m; + vm_object_t t_object = NULL; + vm_object_t l_object = NULL; + vm_object_t m_object = NULL; + int delayed_unlock = 0; + int try_failed_count = 0; + int disconnected_count = 0; + int paused_count = 0; + int object_locked_count = 0; + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_DISCONNECT_ALL_PAGE_MAPPINGS)) | DBG_FUNC_START, + q, qcount, 0, 0, 0); - /* zf_ratio is the number of zf pages we victimize per normal page */ - - if (vm_zf_count < vm_accellerate_zf_pageout_trigger) - zf_ratio = 0; - else if ((vm_zf_count <= normal) || (normal == 0)) - zf_ratio = 1; - else - zf_ratio = vm_zf_count / normal; - - zf_run_count = 0; - } - - /* - * Recalculate vm_page_inactivate_target. - */ - vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count + - vm_page_inactive_count + - vm_page_speculative_count); - /* - * don't want to wake the pageout_scan thread up everytime we fall below - * the targets... set a low water mark at 0.25% below the target - */ - vm_page_inactive_min = vm_page_inactive_target - (vm_page_inactive_target / 400); + vm_page_lock_queues(); - vm_page_speculative_target = VM_PAGE_SPECULATIVE_TARGET(vm_page_active_count + - vm_page_inactive_count); - object = NULL; - last_object_tried = NULL; - try_failed = FALSE; - - if ((vm_page_inactive_count + vm_page_speculative_count) < VM_PAGE_INACTIVE_HEALTHY_LIMIT(vm_page_active_count)) - catch_up_count = vm_page_inactive_count + vm_page_speculative_count; - else - catch_up_count = 0; - - for (;;) { - vm_page_t m; + while (qcount && !vm_page_queue_empty(q)) { - DTRACE_VM2(rev, int, 1, (uint64_t *), NULL); - - if (delayed_unlock == 0) { - vm_page_lock_queues(); - delayed_unlock = 1; - } + LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); - /* - * Don't sweep through active queue more than the throttle - * which should be kept relatively low - */ - active_burst_count = MIN(vm_pageout_burst_active_throttle, - vm_page_active_count); + m = (vm_page_t) vm_page_queue_first(q); + m_object = VM_PAGE_OBJECT(m); /* - * Move pages from active to inactive. + * check to see if we currently are working + * with the same object... if so, we've + * already got the lock */ - if ((vm_page_inactive_count + vm_page_speculative_count) >= vm_page_inactive_target) - goto done_moving_active_pages; - - while (!queue_empty(&vm_page_queue_active) && active_burst_count) { - - if (active_burst_count) - active_burst_count--; - - vm_pageout_active++; - - m = (vm_page_t) queue_first(&vm_page_queue_active); - - assert(m->active && !m->inactive); - assert(!m->laundry); - assert(m->object != kernel_object); - assert(m->phys_page != vm_page_guard_addr); - - DTRACE_VM2(scan, int, 1, (uint64_t *), NULL); + if (m_object != l_object) { + /* + * the object associated with candidate page is + * different from the one we were just working + * with... dump the lock if we still own it + */ + if (l_object != NULL) { + vm_object_unlock(l_object); + l_object = NULL; + } + if (m_object != t_object) + try_failed_count = 0; /* - * Try to lock object; since we've already got the + * Try to lock object; since we've alread got the * page queues lock, we can only 'try' for this one. * if the 'try' fails, we need to do a mutex_pause * to allow the owner of the object lock a chance to - * run... otherwise, we're likely to trip over this - * object in the same state as we work our way through - * the queue... clumps of pages associated with the same - * object are fairly typical on the inactive and active queues + * run... */ - if (m->object != object) { - if (object != NULL) { - vm_object_unlock(object); - object = NULL; - vm_pageout_scan_wants_object = VM_OBJECT_NULL; - } - if (!vm_object_lock_try_scan(m->object)) { - /* - * move page to end of active queue and continue - */ - queue_remove(&vm_page_queue_active, m, - vm_page_t, pageq); - queue_enter(&vm_page_queue_active, m, - vm_page_t, pageq); - - try_failed = TRUE; - - m = (vm_page_t) queue_first(&vm_page_queue_active); - /* - * this is the next object we're going to be interested in - * try to make sure it's available after the mutex_yield - * returns control - */ - vm_pageout_scan_wants_object = m->object; + if ( !vm_object_lock_try_scan(m_object)) { - goto done_with_activepage; + if (try_failed_count > 20) { + goto reenter_pg_on_q; } - object = m->object; + vm_page_unlock_queues(); + mutex_pause(try_failed_count++); + vm_page_lock_queues(); + delayed_unlock = 0; - try_failed = FALSE; + paused_count++; + + t_object = m_object; + continue; } + object_locked_count++; + l_object = m_object; + } + if ( !m_object->alive || m->encrypted_cleaning || m->cleaning || m->laundry || m->busy || m->absent || m->error || m->free_when_done) { /* - * if the page is BUSY, then we pull it - * off the active queue and leave it alone. - * when BUSY is cleared, it will get stuck - * back on the appropriate queue + * put it back on the head of its queue */ - if (m->busy) { - queue_remove(&vm_page_queue_active, m, - vm_page_t, pageq); - m->pageq.next = NULL; - m->pageq.prev = NULL; + goto reenter_pg_on_q; + } + if (m->pmapped == TRUE) { + + pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)); + + disconnected_count++; + } +reenter_pg_on_q: + vm_page_queue_remove(q, m, vm_page_t, pageq); + vm_page_queue_enter(q, m, vm_page_t, pageq); + + qcount--; + try_failed_count = 0; - if (!m->fictitious) - vm_page_active_count--; - m->active = FALSE; + if (delayed_unlock++ > 128) { - goto done_with_activepage; + if (l_object != NULL) { + vm_object_unlock(l_object); + l_object = NULL; } + lck_mtx_yield(&vm_page_queue_lock); + delayed_unlock = 0; + } + } + if (l_object != NULL) { + vm_object_unlock(l_object); + l_object = NULL; + } + vm_page_unlock_queues(); - /* deal with a rogue "reusable" page */ - VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m); + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_DISCONNECT_ALL_PAGE_MAPPINGS)) | DBG_FUNC_END, + q, disconnected_count, object_locked_count, paused_count, 0); +} - /* - * Deactivate the page while holding the object - * locked, so we know the page is still not busy. - * This should prevent races between pmap_enter - * and pmap_clear_reference. The page might be - * absent or fictitious, but vm_page_deactivate - * can handle that. - */ - vm_page_deactivate(m); +#endif -done_with_activepage: - if (delayed_unlock++ > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT || try_failed == TRUE) { - if (object != NULL) { - vm_pageout_scan_wants_object = VM_OBJECT_NULL; - vm_object_unlock(object); - object = NULL; - } - if (local_freeq) { - vm_page_unlock_queues(); - vm_page_free_list(local_freeq, TRUE); - - local_freeq = NULL; - local_freed = 0; - vm_page_lock_queues(); - } else - lck_mtx_yield(&vm_page_queue_lock); +static void +vm_pageout_page_queue(vm_page_queue_head_t *, int); - delayed_unlock = 1; +/* + * condition variable used to make sure there is + * only a single sweep going on at a time + */ +boolean_t vm_pageout_anonymous_pages_active = FALSE; - /* - * continue the while loop processing - * the active queue... need to hold - * the page queues lock - */ - } - } +void +vm_pageout_anonymous_pages() +{ + if (VM_CONFIG_COMPRESSOR_IS_PRESENT) { + vm_page_lock_queues(); - /********************************************************************** - * above this point we're playing with the active queue - * below this point we're playing with the throttling mechanisms - * and the inactive queue - **********************************************************************/ + if (vm_pageout_anonymous_pages_active == TRUE) { + vm_page_unlock_queues(); + return; + } + vm_pageout_anonymous_pages_active = TRUE; + vm_page_unlock_queues(); -done_moving_active_pages: + vm_pageout_page_queue(&vm_page_queue_throttled, vm_page_throttled_count); + vm_pageout_page_queue(&vm_page_queue_anonymous, vm_page_anonymous_count); + vm_pageout_page_queue(&vm_page_queue_active, vm_page_active_count); - /* - * We are done if we have met our target *and* - * nobody is still waiting for a page. - */ - if (vm_page_free_count + local_freed >= vm_page_free_target) { - if (object != NULL) { - vm_object_unlock(object); - object = NULL; - } - vm_pageout_scan_wants_object = VM_OBJECT_NULL; + if (VM_CONFIG_SWAP_IS_PRESENT) + vm_consider_swapping(); - if (local_freeq) { - vm_page_unlock_queues(); - vm_page_free_list(local_freeq, TRUE); - - local_freeq = NULL; - local_freed = 0; - vm_page_lock_queues(); - } - /* - * inactive target still not met... keep going - * until we get the queues balanced - */ + vm_page_lock_queues(); + vm_pageout_anonymous_pages_active = FALSE; + vm_page_unlock_queues(); + } +} - /* - * Recalculate vm_page_inactivate_target. - */ - vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count + - vm_page_inactive_count + - vm_page_speculative_count); -#ifndef CONFIG_EMBEDDED - /* - * XXX: if no active pages can be reclaimed, pageout scan can be stuck trying - * to balance the queues - */ - if (((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) && - !queue_empty(&vm_page_queue_active)) - continue; -#endif +void +vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount) +{ + vm_page_t m; + vm_object_t t_object = NULL; + vm_object_t l_object = NULL; + vm_object_t m_object = NULL; + int delayed_unlock = 0; + int try_failed_count = 0; + int refmod_state; + int pmap_options; + struct vm_pageout_queue *iq; + ppnum_t phys_page; - lck_mtx_lock(&vm_page_queue_free_lock); - if ((vm_page_free_count >= vm_page_free_target) && - (vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) { + iq = &vm_pageout_queue_internal; + + vm_page_lock_queues(); - vm_page_unlock_queues(); + while (qcount && !vm_page_queue_empty(q)) { - thread_wakeup((event_t) &vm_pageout_garbage_collect); + LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); - assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL); + if (VM_PAGE_Q_THROTTLED(iq)) { - return; + if (l_object != NULL) { + vm_object_unlock(l_object); + l_object = NULL; } - lck_mtx_unlock(&vm_page_queue_free_lock); + iq->pgo_draining = TRUE; + + assert_wait((event_t) (&iq->pgo_laundry + 1), THREAD_INTERRUPTIBLE); + vm_page_unlock_queues(); + + thread_block(THREAD_CONTINUE_NULL); + + vm_page_lock_queues(); + delayed_unlock = 0; + continue; } - + m = (vm_page_t) vm_page_queue_first(q); + m_object = VM_PAGE_OBJECT(m); + /* - * Before anything, we check if we have any ripe volatile - * objects around. If so, try to purge the first object. - * If the purge fails, fall through to reclaim a page instead. - * If the purge succeeds, go back to the top and reevalute - * the new memory situation. + * check to see if we currently are working + * with the same object... if so, we've + * already got the lock */ - assert (available_for_purge>=0); - if (available_for_purge) - { - if (object != NULL) { - vm_object_unlock(object); - object = NULL; - } - if(TRUE == vm_purgeable_object_purge_one()) { - continue; - } - } - - if (queue_empty(&sq->age_q) && vm_page_speculative_count) { + if (m_object != l_object) { + if ( !m_object->internal) + goto reenter_pg_on_q; + /* - * try to pull pages from the aging bins - * see vm_page.h for an explanation of how - * this mechanism works + * the object associated with candidate page is + * different from the one we were just working + * with... dump the lock if we still own it */ - struct vm_speculative_age_q *aq; - mach_timespec_t ts_fully_aged; - boolean_t can_steal = FALSE; - int num_scanned_queues; - - aq = &vm_page_queue_speculative[speculative_steal_index]; + if (l_object != NULL) { + vm_object_unlock(l_object); + l_object = NULL; + } + if (m_object != t_object) + try_failed_count = 0; - num_scanned_queues = 0; - while (queue_empty(&aq->age_q) && - num_scanned_queues++ != VM_PAGE_MAX_SPECULATIVE_AGE_Q) { + /* + * Try to lock object; since we've alread got the + * page queues lock, we can only 'try' for this one. + * if the 'try' fails, we need to do a mutex_pause + * to allow the owner of the object lock a chance to + * run... + */ + if ( !vm_object_lock_try_scan(m_object)) { - speculative_steal_index++; + if (try_failed_count > 20) { + goto reenter_pg_on_q; + } + vm_page_unlock_queues(); + mutex_pause(try_failed_count++); + vm_page_lock_queues(); + delayed_unlock = 0; - if (speculative_steal_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q) - speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q; - - aq = &vm_page_queue_speculative[speculative_steal_index]; + t_object = m_object; + continue; } + l_object = m_object; + } + if ( !m_object->alive || m->encrypted_cleaning || m->cleaning || m->laundry || m->busy || m->absent || m->error || m->free_when_done) { + /* + * page is not to be cleaned + * put it back on the head of its queue + */ + goto reenter_pg_on_q; + } + phys_page = VM_PAGE_GET_PHYS_PAGE(m); - if (num_scanned_queues == - VM_PAGE_MAX_SPECULATIVE_AGE_Q + 1) { - /* - * XXX We've scanned all the speculative - * queues but still haven't found one - * that is not empty, even though - * vm_page_speculative_count is not 0. - */ - /* report the anomaly... */ - printf("vm_pageout_scan: " - "all speculative queues empty " - "but count=%d. Re-adjusting.\n", - vm_page_speculative_count); - if (vm_page_speculative_count > - vm_page_speculative_count_drift_max) - vm_page_speculative_count_drift_max = vm_page_speculative_count; - vm_page_speculative_count_drifts++; -#if 6553678 - Debugger("vm_pageout_scan: no speculative pages"); -#endif - /* readjust... */ - vm_page_speculative_count = 0; - /* ... and continue */ - continue; + if (m->reference == FALSE && m->pmapped == TRUE) { + refmod_state = pmap_get_refmod(phys_page); + + if (refmod_state & VM_MEM_REFERENCED) + m->reference = TRUE; + if (refmod_state & VM_MEM_MODIFIED) { + SET_PAGE_DIRTY(m, FALSE); + } + } + if (m->reference == TRUE) { + m->reference = FALSE; + pmap_clear_refmod_options(phys_page, VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL); + goto reenter_pg_on_q; + } + if (m->pmapped == TRUE) { + if (m->dirty || m->precious) { + pmap_options = PMAP_OPTIONS_COMPRESSOR; + } else { + pmap_options = PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED; } + refmod_state = pmap_disconnect_options(phys_page, pmap_options, NULL); + if (refmod_state & VM_MEM_MODIFIED) { + SET_PAGE_DIRTY(m, FALSE); + } + } + if ( !m->dirty && !m->precious) { + vm_page_unlock_queues(); + VM_PAGE_FREE(m); + vm_page_lock_queues(); + delayed_unlock = 0; - if (vm_page_speculative_count > vm_page_speculative_target) - can_steal = TRUE; - else { - ts_fully_aged.tv_sec = (VM_PAGE_MAX_SPECULATIVE_AGE_Q * VM_PAGE_SPECULATIVE_Q_AGE_MS) / 1000; - ts_fully_aged.tv_nsec = ((VM_PAGE_MAX_SPECULATIVE_AGE_Q * VM_PAGE_SPECULATIVE_Q_AGE_MS) % 1000) - * 1000 * NSEC_PER_USEC; + goto next_pg; + } + if (!m_object->pager_initialized || m_object->pager == MEMORY_OBJECT_NULL) { + + if (!m_object->pager_initialized) { - ADD_MACH_TIMESPEC(&ts_fully_aged, &aq->age_ts); + vm_page_unlock_queues(); - clock_sec_t sec; - clock_nsec_t nsec; - clock_get_system_nanotime(&sec, &nsec); - ts.tv_sec = (unsigned int) sec; - ts.tv_nsec = nsec; + vm_object_collapse(m_object, (vm_object_offset_t) 0, TRUE); + + if (!m_object->pager_initialized) + vm_object_compressor_pager_create(m_object); - if (CMP_MACH_TIMESPEC(&ts, &ts_fully_aged) >= 0) - can_steal = TRUE; + vm_page_lock_queues(); + delayed_unlock = 0; } - if (can_steal == TRUE) - vm_page_speculate_ageit(aq); + if (!m_object->pager_initialized || m_object->pager == MEMORY_OBJECT_NULL) + goto reenter_pg_on_q; + /* + * vm_object_compressor_pager_create will drop the object lock + * which means 'm' may no longer be valid to use + */ + continue; } - /* - * Sometimes we have to pause: - * 1) No inactive pages - nothing to do. - * 2) Flow control - default pageout queue is full - * 3) Loop control - no acceptable pages found on the inactive queue - * within the last vm_pageout_burst_inactive_throttle iterations + * we've already factored out pages in the laundry which + * means this page can't be on the pageout queue so it's + * safe to do the vm_page_queues_remove */ - if (queue_empty(&vm_page_queue_inactive) && queue_empty(&vm_page_queue_zf) && queue_empty(&sq->age_q) && - (VM_PAGE_Q_THROTTLED(iq) || queue_empty(&vm_page_queue_throttled))) { - vm_pageout_scan_empty_throttle++; - msecs = vm_pageout_empty_wait; - goto vm_pageout_scan_delay; + vm_page_queues_remove(m, TRUE); - } else if (inactive_burst_count >= - MIN(vm_pageout_burst_inactive_throttle, - (vm_page_inactive_count + - vm_page_speculative_count))) { - vm_pageout_scan_burst_throttle++; - msecs = vm_pageout_burst_wait; - goto vm_pageout_scan_delay; + LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); - } else if (VM_PAGE_Q_THROTTLED(iq) && IP_VALID(memory_manager_default)) { - clock_sec_t sec; - clock_nsec_t nsec; + vm_pageout_cluster(m, FALSE, FALSE); - switch (flow_control.state) { + goto next_pg; - case FCS_IDLE: -reset_deadlock_timer: - ts.tv_sec = vm_pageout_deadlock_wait / 1000; - ts.tv_nsec = (vm_pageout_deadlock_wait % 1000) * 1000 * NSEC_PER_USEC; +reenter_pg_on_q: + vm_page_queue_remove(q, m, vm_page_t, pageq); + vm_page_queue_enter(q, m, vm_page_t, pageq); +next_pg: + qcount--; + try_failed_count = 0; + + if (delayed_unlock++ > 128) { + + if (l_object != NULL) { + vm_object_unlock(l_object); + l_object = NULL; + } + lck_mtx_yield(&vm_page_queue_lock); + delayed_unlock = 0; + } + } + if (l_object != NULL) { + vm_object_unlock(l_object); + l_object = NULL; + } + vm_page_unlock_queues(); +} + + + +/* + * function in BSD to apply I/O throttle to the pageout thread + */ +extern void vm_pageout_io_throttle(void); + +#define VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, obj) \ + MACRO_BEGIN \ + /* \ + * If a "reusable" page somehow made it back into \ + * the active queue, it's been re-used and is not \ + * quite re-usable. \ + * If the VM object was "all_reusable", consider it \ + * as "all re-used" instead of converting it to \ + * "partially re-used", which could be expensive. \ + */ \ + assert(VM_PAGE_OBJECT((m)) == (obj)); \ + if ((m)->reusable || \ + (obj)->all_reusable) { \ + vm_object_reuse_pages((obj), \ + (m)->offset, \ + (m)->offset + PAGE_SIZE_64, \ + FALSE); \ + } \ + MACRO_END + + +#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT 64 +#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX 1024 + +#define FCS_IDLE 0 +#define FCS_DELAYED 1 +#define FCS_DEADLOCK_DETECTED 2 + +struct flow_control { + int state; + mach_timespec_t ts; +}; + +#if CONFIG_BACKGROUND_QUEUE +uint64_t vm_pageout_considered_bq_internal = 0; +uint64_t vm_pageout_considered_bq_external = 0; +uint64_t vm_pageout_rejected_bq_internal = 0; +uint64_t vm_pageout_rejected_bq_external = 0; +#endif +uint32_t vm_pageout_considered_page = 0; +uint32_t vm_page_filecache_min = 0; + +#define ANONS_GRABBED_LIMIT 2 + +#if CONFIG_SECLUDED_MEMORY +extern vm_page_t vm_page_grab_secluded(void); +uint64_t vm_pageout_freed_from_secluded = 0; +uint64_t vm_pageout_secluded_reactivated = 0; /* debugging; how many secluded pages are found to be referenced on pageout (and are therefore reactivated) */ +uint64_t vm_pageout_secluded_burst_count = 0; +#endif /* CONFIG_SECLUDED_MEMORY */ + +/* + * vm_pageout_scan does the dirty work for the pageout daemon. + * It returns with both vm_page_queue_free_lock and vm_page_queue_lock + * held and vm_page_free_wanted == 0. + */ +void +vm_pageout_scan(void) +{ + unsigned int loop_count = 0; + unsigned int inactive_burst_count = 0; + unsigned int active_burst_count = 0; + unsigned int reactivated_this_call; + unsigned int reactivate_limit; + vm_page_t local_freeq = NULL; + int local_freed = 0; + int delayed_unlock; + int delayed_unlock_limit = 0; + int refmod_state = 0; + int vm_pageout_deadlock_target = 0; + struct vm_pageout_queue *iq; + struct vm_pageout_queue *eq; + struct vm_speculative_age_q *sq; + struct flow_control flow_control = { 0, { 0, 0 } }; + boolean_t inactive_throttled = FALSE; + boolean_t try_failed; + mach_timespec_t ts; + unsigned int msecs = 0; + vm_object_t object; + vm_object_t last_object_tried; + uint32_t catch_up_count = 0; + uint32_t inactive_reclaim_run; + boolean_t exceeded_burst_throttle; + boolean_t grab_anonymous = FALSE; + boolean_t force_anonymous = FALSE; + int anons_grabbed = 0; + int page_prev_q_state = 0; + boolean_t requeue_insert_first = FALSE; +#if CONFIG_BACKGROUND_QUEUE + boolean_t ignore_reference = FALSE; +#endif +#if CONFIG_SECLUDED_MEMORY + boolean_t ignore_reference_secluded; +#endif /* CONFIG_SECLUDED_MEMORY */ + int cache_evict_throttle = 0; + uint32_t vm_pageout_inactive_external_forced_reactivate_limit = 0; + int force_purge = 0; +#define DELAY_SPECULATIVE_AGE 1000 + int delay_speculative_age = 0; + vm_object_t m_object = VM_OBJECT_NULL; + +#if VM_PRESSURE_EVENTS + vm_pressure_level_t pressure_level; +#endif /* VM_PRESSURE_EVENTS */ + + VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_START, + vm_pageout_speculative_clean, vm_pageout_inactive_clean, + vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external); + + flow_control.state = FCS_IDLE; + iq = &vm_pageout_queue_internal; + eq = &vm_pageout_queue_external; + sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q]; + + + XPR(XPR_VM_PAGEOUT, "vm_pageout_scan\n", 0, 0, 0, 0, 0); + + + vm_page_lock_queues(); + delayed_unlock = 1; /* must be nonzero if Qs are locked, 0 if unlocked */ + + /* + * Calculate the max number of referenced pages on the inactive + * queue that we will reactivate. + */ + reactivated_this_call = 0; + reactivate_limit = VM_PAGE_REACTIVATE_LIMIT(vm_page_active_count + + vm_page_inactive_count); + inactive_reclaim_run = 0; + + vm_pageout_inactive_external_forced_reactivate_limit = vm_page_active_count + vm_page_inactive_count; + + /* + * We want to gradually dribble pages from the active queue + * to the inactive queue. If we let the inactive queue get + * very small, and then suddenly dump many pages into it, + * those pages won't get a sufficient chance to be referenced + * before we start taking them from the inactive queue. + * + * We must limit the rate at which we send pages to the pagers + * so that we don't tie up too many pages in the I/O queues. + * We implement a throttling mechanism using the laundry count + * to limit the number of pages outstanding to the default + * and external pagers. We can bypass the throttles and look + * for clean pages if the pageout queues don't drain in a timely + * fashion since this may indicate that the pageout paths are + * stalled waiting for memory, which only we can provide. + */ + + +Restart: + + + assert(delayed_unlock!=0); + + /* + * Recalculate vm_page_inactivate_target. + */ + vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count + + vm_page_inactive_count + + vm_page_speculative_count); + + vm_page_anonymous_min = vm_page_inactive_target / 20; + + + /* + * don't want to wake the pageout_scan thread up everytime we fall below + * the targets... set a low water mark at 0.25% below the target + */ + vm_page_inactive_min = vm_page_inactive_target - (vm_page_inactive_target / 400); + + if (vm_page_speculative_percentage > 50) + vm_page_speculative_percentage = 50; + else if (vm_page_speculative_percentage <= 0) + vm_page_speculative_percentage = 1; + + vm_page_speculative_target = VM_PAGE_SPECULATIVE_TARGET(vm_page_active_count + + vm_page_inactive_count); + + object = NULL; + last_object_tried = NULL; + try_failed = FALSE; + + if ((vm_page_inactive_count + vm_page_speculative_count) < VM_PAGE_INACTIVE_HEALTHY_LIMIT(vm_page_active_count)) + catch_up_count = vm_page_inactive_count + vm_page_speculative_count; + else + catch_up_count = 0; + + for (;;) { + vm_page_t m; + + DTRACE_VM2(rev, int, 1, (uint64_t *), NULL); + +#if CONFIG_SECLUDED_MEMORY + if (vm_page_secluded_count > vm_page_secluded_target && + object != NULL) { + vm_object_unlock(object); + object = NULL; + vm_pageout_scan_wants_object = VM_OBJECT_NULL; + } + + /* + * Deal with secluded_q overflow. + */ + if (vm_page_secluded_count > vm_page_secluded_target && + secluded_aging_policy == SECLUDED_AGING_FIFO) { + unsigned int secluded_overflow; + vm_page_t secluded_page; + + /* + * SECLUDED_AGING_FIFO: + * No aging, just reclaim the excess pages + * at the tail of the secluded queue. + * We're reclaiming pages and we're not hogging + * any global lock, so no need for throttling. + */ + + secluded_overflow = (vm_page_secluded_count - + vm_page_secluded_target); + /* transfer to free queue */ + vm_page_unlock_queues(); + while (secluded_overflow--) { + secluded_page = vm_page_grab_secluded(); + if (secluded_page == VM_PAGE_NULL) { + break; + } + assert(secluded_page->busy); + assert(secluded_page->pageq.next == 0 && + secluded_page->pageq.prev == 0); + + secluded_page->snext = local_freeq; + local_freeq = secluded_page; + local_freed++; + secluded_page = VM_PAGE_NULL; + } + } else if (vm_page_secluded_count > vm_page_secluded_target && + secluded_aging_policy == SECLUDED_AGING_ALONG_ACTIVE) { + unsigned int secluded_overflow; + vm_page_t secluded_page; + + /* + * SECLUDED_AGING_ALONG_ACTIVE: + * There might be free pages at the tail of the + * secluded queue: + * just move them to the free queue (in batches). + * There can also be an excessive number of "inuse" + * pages: + * we age them by resetting their "referenced" bit and + * moving them to the inactive queue. Their trip + * through the secluded queue was equivalent to a trip + * through the active queue. + * + * We're holding the page queue lock, so we need + * to throttle and give someone else a chance to + * grab that lock if needed. + * + * We're also limiting the number of secluded "inuse" + * pages that get moved to the inactive queue, using + * the same "active_bust_count" method we use when + * balancing the active and inactive queues, because + * there can be a large number + * of extra "inuse" pages and handling them gets in the + * way of actually reclaiming memory. + */ + + active_burst_count = MIN(vm_pageout_burst_active_throttle, + vm_page_secluded_count_inuse); + delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT; + delayed_unlock = 1; + secluded_overflow = (vm_page_secluded_count - + vm_page_secluded_target); + while (secluded_overflow-- > 0 && + vm_page_secluded_count > vm_page_secluded_target) { + assert((vm_page_secluded_count_free + + vm_page_secluded_count_inuse) == + vm_page_secluded_count); + secluded_page = vm_page_queue_first(&vm_page_queue_secluded); + assert(secluded_page->vm_page_q_state == + VM_PAGE_ON_SECLUDED_Q); + vm_page_queues_remove(secluded_page, FALSE); + assert(!secluded_page->fictitious); + assert(!VM_PAGE_WIRED(secluded_page)); + if (secluded_page->vm_page_object == 0) { + /* transfer to free queue */ + assert(secluded_page->busy); + secluded_page->snext = local_freeq; + local_freeq = secluded_page; + local_freed++; + } else { + /* transfer to head of inactive queue */ + pmap_clear_refmod_options( + VM_PAGE_GET_PHYS_PAGE(secluded_page), + VM_MEM_REFERENCED, + PMAP_OPTIONS_NOFLUSH, + (void *)NULL); + vm_page_enqueue_inactive(secluded_page, + FALSE); + if (active_burst_count-- == 0) { + vm_pageout_secluded_burst_count++; + break; + } + } + secluded_page = VM_PAGE_NULL; + if (delayed_unlock++ > delayed_unlock_limit) { + if (local_freeq) { + vm_page_unlock_queues(); + VM_DEBUG_EVENT( + vm_pageout_freelist, + VM_PAGEOUT_FREELIST, + DBG_FUNC_START, + vm_page_free_count, + local_freed, + delayed_unlock_limit, + 1); + vm_page_free_list(local_freeq, + TRUE); + VM_DEBUG_EVENT( + vm_pageout_freelist, + VM_PAGEOUT_FREELIST, + DBG_FUNC_END, + vm_page_free_count, + 0, 0, 1); + local_freeq = NULL; + local_freed = 0; + vm_page_lock_queues(); + } else { + lck_mtx_yield(&vm_page_queue_lock); + } + delayed_unlock = 1; + } + } + delayed_unlock = 1; + } else if (vm_page_secluded_count > vm_page_secluded_target && + secluded_aging_policy == SECLUDED_AGING_AFTER_INACTIVE) { + /* + * SECLUDED_AGING_AFTER_INACTIVE: + * No balancing needed at this point: when we get to + * the "choose a victim" part below, we'll consider the + * extra secluded pages before any inactive page. + */ + } else if (vm_page_secluded_count > vm_page_secluded_target && + secluded_aging_policy == SECLUDED_AGING_BEFORE_ACTIVE) { + unsigned int secluded_overflow; + vm_page_t secluded_page; + + /* + * SECLUDED_AGING_BEFORE_ACTIVE: + * Excess secluded pages go to the active queue and + * will later go to the inactive queue. + */ + active_burst_count = MIN(vm_pageout_burst_active_throttle, + vm_page_secluded_count_inuse); + delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT; + delayed_unlock = 1; + secluded_overflow = (vm_page_secluded_count - + vm_page_secluded_target); + while (secluded_overflow-- > 0 && + vm_page_secluded_count > vm_page_secluded_target) { + assert((vm_page_secluded_count_free + + vm_page_secluded_count_inuse) == + vm_page_secluded_count); + secluded_page = vm_page_queue_first(&vm_page_queue_secluded); + assert(secluded_page->vm_page_q_state == + VM_PAGE_ON_SECLUDED_Q); + vm_page_queues_remove(secluded_page, FALSE); + assert(!secluded_page->fictitious); + assert(!VM_PAGE_WIRED(secluded_page)); + if (secluded_page->vm_page_object == 0) { + /* transfer to free queue */ + assert(secluded_page->busy); + secluded_page->snext = local_freeq; + local_freeq = secluded_page; + local_freed++; + } else { + /* transfer to head of active queue */ + vm_page_enqueue_active(secluded_page, + FALSE); + if (active_burst_count-- == 0) { + vm_pageout_secluded_burst_count++; + break; + } + } + secluded_page = VM_PAGE_NULL; + if (delayed_unlock++ > delayed_unlock_limit) { + if (local_freeq) { + vm_page_unlock_queues(); + VM_DEBUG_EVENT( + vm_pageout_freelist, + VM_PAGEOUT_FREELIST, + DBG_FUNC_START, + vm_page_free_count, + local_freed, + delayed_unlock_limit, + 1); + vm_page_free_list(local_freeq, + TRUE); + VM_DEBUG_EVENT( + vm_pageout_freelist, + VM_PAGEOUT_FREELIST, + DBG_FUNC_END, + vm_page_free_count, + 0, 0, 1); + local_freeq = NULL; + local_freed = 0; + vm_page_lock_queues(); + } else { + lck_mtx_yield(&vm_page_queue_lock); + } + delayed_unlock = 1; + } + } + delayed_unlock = 1; + } else if (vm_page_secluded_count > vm_page_secluded_target) { + panic("unsupported secluded_aging_policy %d\n", + secluded_aging_policy); + } + if (local_freeq) { + vm_page_unlock_queues(); + VM_DEBUG_EVENT(vm_pageout_freelist, + VM_PAGEOUT_FREELIST, + DBG_FUNC_START, + vm_page_free_count, + local_freed, + 0, + 0); + vm_page_free_list(local_freeq, TRUE); + VM_DEBUG_EVENT(vm_pageout_freelist, + VM_PAGEOUT_FREELIST, + DBG_FUNC_END, + vm_page_free_count, 0, 0, 0); + local_freeq = NULL; + local_freed = 0; + vm_page_lock_queues(); + } +#endif /* CONFIG_SECLUDED_MEMORY */ + + assert(delayed_unlock); + + if (vm_upl_wait_for_pages < 0) + vm_upl_wait_for_pages = 0; + + delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT + vm_upl_wait_for_pages; + + if (delayed_unlock_limit > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX) + delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX; + + /* + * Move pages from active to inactive if we're below the target + */ + /* if we are trying to make clean, we need to make sure we actually have inactive - mj */ + if ((vm_page_inactive_count + vm_page_speculative_count) >= vm_page_inactive_target) + goto done_moving_active_pages; + + if (object != NULL) { + vm_object_unlock(object); + object = NULL; + vm_pageout_scan_wants_object = VM_OBJECT_NULL; + } + /* + * Don't sweep through active queue more than the throttle + * which should be kept relatively low + */ + active_burst_count = MIN(vm_pageout_burst_active_throttle, vm_page_active_count); + + VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_START, + vm_pageout_inactive, vm_pageout_inactive_used, vm_page_free_count, local_freed); + + VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_NONE, + vm_pageout_speculative_clean, vm_pageout_inactive_clean, + vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external); + memoryshot(VM_PAGEOUT_BALANCE, DBG_FUNC_START); + + + while (!vm_page_queue_empty(&vm_page_queue_active) && active_burst_count--) { + + vm_pageout_active++; + + m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active); + + assert(m->vm_page_q_state == VM_PAGE_ON_ACTIVE_Q); + assert(!m->laundry); + assert(VM_PAGE_OBJECT(m) != kernel_object); + assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr); + + DTRACE_VM2(scan, int, 1, (uint64_t *), NULL); + + /* + * by not passing in a pmap_flush_context we will forgo any TLB flushing, local or otherwise... + * + * a TLB flush isn't really needed here since at worst we'll miss the reference bit being + * updated in the PTE if a remote processor still has this mapping cached in its TLB when the + * new reference happens. If no futher references happen on the page after that remote TLB flushes + * we'll see a clean, non-referenced page when it eventually gets pulled out of the inactive queue + * by pageout_scan, which is just fine since the last reference would have happened quite far + * in the past (TLB caches don't hang around for very long), and of course could just as easily + * have happened before we moved the page + */ + pmap_clear_refmod_options(VM_PAGE_GET_PHYS_PAGE(m), VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL); + + /* + * The page might be absent or busy, + * but vm_page_deactivate can handle that. + * FALSE indicates that we don't want a H/W clear reference + */ + vm_page_deactivate_internal(m, FALSE); + + if (delayed_unlock++ > delayed_unlock_limit) { + + if (local_freeq) { + vm_page_unlock_queues(); + + VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START, + vm_page_free_count, local_freed, delayed_unlock_limit, 1); + + vm_page_free_list(local_freeq, TRUE); + + VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END, + vm_page_free_count, 0, 0, 1); + + local_freeq = NULL; + local_freed = 0; + vm_page_lock_queues(); + } else { + lck_mtx_yield(&vm_page_queue_lock); + } + + delayed_unlock = 1; + + /* + * continue the while loop processing + * the active queue... need to hold + * the page queues lock + */ + } + } + + VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_END, + vm_page_active_count, vm_page_inactive_count, vm_page_speculative_count, vm_page_inactive_target); + memoryshot(VM_PAGEOUT_BALANCE, DBG_FUNC_END); + + /********************************************************************** + * above this point we're playing with the active queue + * below this point we're playing with the throttling mechanisms + * and the inactive queue + **********************************************************************/ + +done_moving_active_pages: + +#if CONFIG_BACKGROUND_QUEUE + if ((vm_page_free_count + local_freed >= vm_page_free_target) && + ((vm_page_background_mode < VM_PAGE_BG_LEVEL_2) || (vm_page_background_count <= vm_page_background_target))) +#else + if (vm_page_free_count + local_freed >= vm_page_free_target) +#endif + { + if (object != NULL) { + vm_object_unlock(object); + object = NULL; + } + vm_pageout_scan_wants_object = VM_OBJECT_NULL; + + vm_page_unlock_queues(); + + if (local_freeq) { + + VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START, + vm_page_free_count, local_freed, delayed_unlock_limit, 2); + + vm_page_free_list(local_freeq, TRUE); + + VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END, + vm_page_free_count, local_freed, 0, 2); + + local_freeq = NULL; + local_freed = 0; + } + vm_consider_waking_compactor_swapper(); + + vm_page_lock_queues(); + + /* + * make sure the pageout I/O threads are running + * throttled in case there are still requests + * in the laundry... since we have met our targets + * we don't need the laundry to be cleaned in a timely + * fashion... so let's avoid interfering with foreground + * activity + */ + vm_pageout_adjust_io_throttles(iq, eq, TRUE); + + /* + * recalculate vm_page_inactivate_target + */ + vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count + + vm_page_inactive_count + + vm_page_speculative_count); + if (((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) && + !vm_page_queue_empty(&vm_page_queue_active)) { + /* + * inactive target still not met... keep going + * until we get the queues balanced... + */ + continue; + } + lck_mtx_lock(&vm_page_queue_free_lock); + + if ((vm_page_free_count >= vm_page_free_target) && + (vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) { + /* + * done - we have met our target *and* + * there is no one waiting for a page. + */ +return_from_scan: + assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL); + + VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_NONE, + vm_pageout_inactive, vm_pageout_inactive_used, 0, 0); + VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_END, + vm_pageout_speculative_clean, vm_pageout_inactive_clean, + vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external); + + return; + } + lck_mtx_unlock(&vm_page_queue_free_lock); + } + + /* + * Before anything, we check if we have any ripe volatile + * objects around. If so, try to purge the first object. + * If the purge fails, fall through to reclaim a page instead. + * If the purge succeeds, go back to the top and reevalute + * the new memory situation. + */ + + assert (available_for_purge>=0); + force_purge = 0; /* no force-purging */ + +#if VM_PRESSURE_EVENTS + pressure_level = memorystatus_vm_pressure_level; + + if (pressure_level > kVMPressureNormal) { + + if (pressure_level >= kVMPressureCritical) { + force_purge = memorystatus_purge_on_critical; + } else if (pressure_level >= kVMPressureUrgent) { + force_purge = memorystatus_purge_on_urgent; + } else if (pressure_level >= kVMPressureWarning) { + force_purge = memorystatus_purge_on_warning; + } + } +#endif /* VM_PRESSURE_EVENTS */ + + if (available_for_purge || force_purge) { + + if (object != NULL) { + vm_object_unlock(object); + object = NULL; + } + + memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_START); + + VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_START, vm_page_free_count, 0, 0, 0); + if (vm_purgeable_object_purge_one(force_purge, C_DONT_BLOCK)) { + vm_pageout_purged_objects++; + VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_END, vm_page_free_count, 0, 0, 0); + memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_END); + continue; + } + VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_END, 0, 0, 0, -1); + memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_END); + } + + if (vm_page_queue_empty(&sq->age_q) && vm_page_speculative_count) { + /* + * try to pull pages from the aging bins... + * see vm_page.h for an explanation of how + * this mechanism works + */ + struct vm_speculative_age_q *aq; + boolean_t can_steal = FALSE; + int num_scanned_queues; + + aq = &vm_page_queue_speculative[speculative_steal_index]; + + num_scanned_queues = 0; + while (vm_page_queue_empty(&aq->age_q) && + num_scanned_queues++ != VM_PAGE_MAX_SPECULATIVE_AGE_Q) { + + speculative_steal_index++; + + if (speculative_steal_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q) + speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q; + + aq = &vm_page_queue_speculative[speculative_steal_index]; + } + + if (num_scanned_queues == VM_PAGE_MAX_SPECULATIVE_AGE_Q + 1) { + /* + * XXX We've scanned all the speculative + * queues but still haven't found one + * that is not empty, even though + * vm_page_speculative_count is not 0. + * + * report the anomaly... + */ + printf("vm_pageout_scan: " + "all speculative queues empty " + "but count=%d. Re-adjusting.\n", + vm_page_speculative_count); + if (vm_page_speculative_count > vm_page_speculative_count_drift_max) + vm_page_speculative_count_drift_max = vm_page_speculative_count; + vm_page_speculative_count_drifts++; +#if DEVELOPMENT || DEBUG + panic("vm_pageout_scan: vm_page_speculative_count=%d but queues are empty", vm_page_speculative_count); +#endif /* DEVELOPMENT || DEBUG */ + /* readjust... */ + vm_page_speculative_count = 0; + /* ... and continue */ + continue; + } + + if (vm_page_speculative_count > vm_page_speculative_target) + can_steal = TRUE; + else { + if (!delay_speculative_age) { + mach_timespec_t ts_fully_aged; + + ts_fully_aged.tv_sec = (VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_page_speculative_q_age_ms) / 1000; + ts_fully_aged.tv_nsec = ((VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_page_speculative_q_age_ms) % 1000) + * 1000 * NSEC_PER_USEC; + + ADD_MACH_TIMESPEC(&ts_fully_aged, &aq->age_ts); + + clock_sec_t sec; + clock_nsec_t nsec; + clock_get_system_nanotime(&sec, &nsec); + ts.tv_sec = (unsigned int) sec; + ts.tv_nsec = nsec; + + if (CMP_MACH_TIMESPEC(&ts, &ts_fully_aged) >= 0) + can_steal = TRUE; + else + delay_speculative_age++; + } else { + delay_speculative_age++; + if (delay_speculative_age == DELAY_SPECULATIVE_AGE) + delay_speculative_age = 0; + } + } + if (can_steal == TRUE) + vm_page_speculate_ageit(aq); + } +#if CONFIG_BACKGROUND_QUEUE + if (vm_page_queue_empty(&sq->age_q) && cache_evict_throttle == 0 && + ((vm_page_background_mode == VM_PAGE_BG_DISABLED) || (vm_page_background_count <= vm_page_background_target))) +#else + if (vm_page_queue_empty(&sq->age_q) && cache_evict_throttle == 0) +#endif + { + int pages_evicted; + + if (object != NULL) { + vm_object_unlock(object); + object = NULL; + } + pages_evicted = vm_object_cache_evict(100, 10); + + if (pages_evicted) { + + vm_pageout_cache_evicted += pages_evicted; + + VM_DEBUG_EVENT(vm_pageout_cache_evict, VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE, + vm_page_free_count, pages_evicted, vm_pageout_cache_evicted, 0); + memoryshot(VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE); + + /* + * we just freed up to 100 pages, + * so go back to the top of the main loop + * and re-evaulate the memory situation + */ + continue; + } else + cache_evict_throttle = 100; + } + if (cache_evict_throttle) + cache_evict_throttle--; + +#if CONFIG_JETSAM + /* + * don't let the filecache_min fall below 15% of available memory + * on systems with an active compressor that isn't nearing its + * limits w/r to accepting new data + * + * on systems w/o the compressor/swapper, the filecache is always + * a very large percentage of the AVAILABLE_NON_COMPRESSED_MEMORY + * since most (if not all) of the anonymous pages are in the + * throttled queue (which isn't counted as available) which + * effectively disables this filter + */ + if (vm_compressor_low_on_space()) + vm_page_filecache_min = 0; + else + vm_page_filecache_min = (AVAILABLE_NON_COMPRESSED_MEMORY / 7); +#else + /* + * don't let the filecache_min fall below 33% of available memory... + */ + vm_page_filecache_min = (AVAILABLE_NON_COMPRESSED_MEMORY / 3); +#endif + if (vm_page_free_count < (vm_page_free_reserved / 4)) + vm_page_filecache_min = 0; + + exceeded_burst_throttle = FALSE; + /* + * Sometimes we have to pause: + * 1) No inactive pages - nothing to do. + * 2) Loop control - no acceptable pages found on the inactive queue + * within the last vm_pageout_burst_inactive_throttle iterations + * 3) Flow control - default pageout queue is full + */ + if (vm_page_queue_empty(&vm_page_queue_inactive) && + vm_page_queue_empty(&vm_page_queue_anonymous) && + vm_page_queue_empty(&sq->age_q)) { + vm_pageout_scan_empty_throttle++; + msecs = vm_pageout_empty_wait; + goto vm_pageout_scan_delay; + + } else if (inactive_burst_count >= + MIN(vm_pageout_burst_inactive_throttle, + (vm_page_inactive_count + + vm_page_speculative_count))) { + vm_pageout_scan_burst_throttle++; + msecs = vm_pageout_burst_wait; + + exceeded_burst_throttle = TRUE; + goto vm_pageout_scan_delay; + + } else if (vm_page_free_count > (vm_page_free_reserved / 4) && + VM_PAGEOUT_SCAN_NEEDS_TO_THROTTLE()) { + vm_pageout_scan_swap_throttle++; + msecs = vm_pageout_swap_wait; + goto vm_pageout_scan_delay; + + } else if (VM_PAGE_Q_THROTTLED(iq) && + VM_DYNAMIC_PAGING_ENABLED()) { + clock_sec_t sec; + clock_nsec_t nsec; + + switch (flow_control.state) { + + case FCS_IDLE: + if ((vm_page_free_count + local_freed) < vm_page_free_target) { + + if (object != NULL) { + vm_object_unlock(object); + object = NULL; + } + vm_pageout_scan_wants_object = VM_OBJECT_NULL; + + vm_page_unlock_queues(); + + if (local_freeq) { + + VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START, + vm_page_free_count, local_freed, delayed_unlock_limit, 3); + + vm_page_free_list(local_freeq, TRUE); + + VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END, + vm_page_free_count, local_freed, 0, 3); + + local_freeq = NULL; + local_freed = 0; + } + thread_yield_internal(1); + + vm_page_lock_queues(); + + if (!VM_PAGE_Q_THROTTLED(iq)) { + vm_pageout_scan_yield_unthrottled++; + continue; + } + if (vm_page_pageable_external_count > vm_page_filecache_min && + !vm_page_queue_empty(&vm_page_queue_inactive)) { + anons_grabbed = ANONS_GRABBED_LIMIT; + vm_pageout_scan_throttle_deferred++; + goto consider_inactive; + } + if (((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) && vm_page_active_count) + continue; + } +reset_deadlock_timer: + ts.tv_sec = vm_pageout_deadlock_wait / 1000; + ts.tv_nsec = (vm_pageout_deadlock_wait % 1000) * 1000 * NSEC_PER_USEC; clock_get_system_nanotime(&sec, &nsec); flow_control.ts.tv_sec = (unsigned int) sec; flow_control.ts.tv_nsec = nsec; ADD_MACH_TIMESPEC(&flow_control.ts, &ts); - flow_control.state = FCS_DELAYED; - msecs = vm_pageout_deadlock_wait; + flow_control.state = FCS_DELAYED; + msecs = vm_pageout_deadlock_wait; + + break; + + case FCS_DELAYED: + clock_get_system_nanotime(&sec, &nsec); + ts.tv_sec = (unsigned int) sec; + ts.tv_nsec = nsec; + + if (CMP_MACH_TIMESPEC(&ts, &flow_control.ts) >= 0) { + /* + * the pageout thread for the default pager is potentially + * deadlocked since the + * default pager queue has been throttled for more than the + * allowable time... we need to move some clean pages or dirty + * pages belonging to the external pagers if they aren't throttled + * vm_page_free_wanted represents the number of threads currently + * blocked waiting for pages... we'll move one page for each of + * these plus a fixed amount to break the logjam... once we're done + * moving this number of pages, we'll re-enter the FSC_DELAYED state + * with a new timeout target since we have no way of knowing + * whether we've broken the deadlock except through observation + * of the queue associated with the default pager... we need to + * stop moving pages and allow the system to run to see what + * state it settles into. + */ + vm_pageout_deadlock_target = vm_pageout_deadlock_relief + vm_page_free_wanted + vm_page_free_wanted_privileged; + vm_pageout_scan_deadlock_detected++; + flow_control.state = FCS_DEADLOCK_DETECTED; + thread_wakeup((event_t) &vm_pageout_garbage_collect); + goto consider_inactive; + } + /* + * just resniff instead of trying + * to compute a new delay time... we're going to be + * awakened immediately upon a laundry completion, + * so we won't wait any longer than necessary + */ + msecs = vm_pageout_idle_wait; + break; + + case FCS_DEADLOCK_DETECTED: + if (vm_pageout_deadlock_target) + goto consider_inactive; + goto reset_deadlock_timer; + + } +vm_pageout_scan_delay: + if (object != NULL) { + vm_object_unlock(object); + object = NULL; + } + vm_pageout_scan_wants_object = VM_OBJECT_NULL; + + vm_page_unlock_queues(); + + if (local_freeq) { + + VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START, + vm_page_free_count, local_freed, delayed_unlock_limit, 3); + + vm_page_free_list(local_freeq, TRUE); + + VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END, + vm_page_free_count, local_freed, 0, 3); + + local_freeq = NULL; + local_freed = 0; + } + vm_consider_waking_compactor_swapper(); + + vm_page_lock_queues(); + + if (flow_control.state == FCS_DELAYED && + !VM_PAGE_Q_THROTTLED(iq)) { + flow_control.state = FCS_IDLE; + goto consider_inactive; + } + + if (vm_page_free_count >= vm_page_free_target) { + /* + * we're here because + * 1) someone else freed up some pages while we had + * the queues unlocked above + * and we've hit one of the 3 conditions that + * cause us to pause the pageout scan thread + * + * since we already have enough free pages, + * let's avoid stalling and return normally + * + * before we return, make sure the pageout I/O threads + * are running throttled in case there are still requests + * in the laundry... since we have enough free pages + * we don't need the laundry to be cleaned in a timely + * fashion... so let's avoid interfering with foreground + * activity + * + * we don't want to hold vm_page_queue_free_lock when + * calling vm_pageout_adjust_io_throttles (since it + * may cause other locks to be taken), we do the intitial + * check outside of the lock. Once we take the lock, + * we recheck the condition since it may have changed. + * if it has, no problem, we will make the threads + * non-throttled before actually blocking + */ + vm_pageout_adjust_io_throttles(iq, eq, TRUE); + } + lck_mtx_lock(&vm_page_queue_free_lock); + + if (vm_page_free_count >= vm_page_free_target && + (vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) { + goto return_from_scan; + } + lck_mtx_unlock(&vm_page_queue_free_lock); + + if ((vm_page_free_count + vm_page_cleaned_count) < vm_page_free_target) { + /* + * we're most likely about to block due to one of + * the 3 conditions that cause vm_pageout_scan to + * not be able to make forward progress w/r + * to providing new pages to the free queue, + * so unthrottle the I/O threads in case we + * have laundry to be cleaned... it needs + * to be completed ASAP. + * + * even if we don't block, we want the io threads + * running unthrottled since the sum of free + + * clean pages is still under our free target + */ + vm_pageout_adjust_io_throttles(iq, eq, FALSE); + } + if (vm_page_cleaned_count > 0 && exceeded_burst_throttle == FALSE) { + /* + * if we get here we're below our free target and + * we're stalling due to a full laundry queue or + * we don't have any inactive pages other then + * those in the clean queue... + * however, we have pages on the clean queue that + * can be moved to the free queue, so let's not + * stall the pageout scan + */ + flow_control.state = FCS_IDLE; + goto consider_inactive; + } + VM_CHECK_MEMORYSTATUS; + + if (flow_control.state != FCS_IDLE) + vm_pageout_scan_throttle++; + iq->pgo_throttled = TRUE; + + assert_wait_timeout((event_t) &iq->pgo_laundry, THREAD_INTERRUPTIBLE, msecs, 1000*NSEC_PER_USEC); + counter(c_vm_pageout_scan_block++); + + vm_page_unlock_queues(); + + assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL); + + VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START, + iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0); + memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START); + + thread_block(THREAD_CONTINUE_NULL); + + VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END, + iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0); + memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END); + + vm_page_lock_queues(); + delayed_unlock = 1; + + iq->pgo_throttled = FALSE; + + if (loop_count >= vm_page_inactive_count) + loop_count = 0; + inactive_burst_count = 0; + + goto Restart; + /*NOTREACHED*/ + } + + + flow_control.state = FCS_IDLE; +consider_inactive: + vm_pageout_inactive_external_forced_reactivate_limit = MIN((vm_page_active_count + vm_page_inactive_count), + vm_pageout_inactive_external_forced_reactivate_limit); + loop_count++; + inactive_burst_count++; + vm_pageout_inactive++; + + + /* + * Choose a victim. + */ + while (1) { + uint32_t inactive_external_count; + +#if CONFIG_BACKGROUND_QUEUE + ignore_reference = FALSE; +#endif /* CONFIG_BACKGROUND_QUEUE */ + + m = NULL; + m_object = VM_OBJECT_NULL; + + if (VM_DYNAMIC_PAGING_ENABLED()) { + assert(vm_page_throttled_count == 0); + assert(vm_page_queue_empty(&vm_page_queue_throttled)); + } + + +#if CONFIG_SECLUDED_MEMORY + if ((secluded_aging_policy == + SECLUDED_AGING_AFTER_INACTIVE) && + vm_page_secluded_count > vm_page_secluded_target) { + /* + * SECLUDED_AGING_AFTER_INACTIVE: + * Secluded pages have already been aged + * through the active and inactive queues, and + * we now have too many of them, so let's + * balance that queue by considering reclaiming + * the oldest page in the secluded queue. + */ + assert(!vm_page_queue_empty(&vm_page_queue_secluded)); + m = (vm_page_t) vm_page_queue_first(&vm_page_queue_secluded); + if (m->vm_page_object == 0) { + /* + * It's already a free page: + * just move it to a free queue. + */ + vm_page_queues_remove(m, TRUE); + assert(m->busy); + assert(m->pageq.next == 0); + assert(m->pageq.prev == 0); + m->snext = local_freeq; + local_freeq = m; + local_freed++; + goto done_with_inactivepage; + } + /* + * Not a free page: we've found our next + * "victim". + */ + break; + } +#endif /* CONFIG_SECLUDED_MEMORY */ + +#if CONFIG_BACKGROUND_QUEUE + if (vm_page_background_mode != VM_PAGE_BG_DISABLED && (vm_page_background_count > vm_page_background_target)) { + vm_object_t bg_m_object = NULL; + + m = (vm_page_t) vm_page_queue_first(&vm_page_queue_background); + + bg_m_object = VM_PAGE_OBJECT(m); + + if (!VM_PAGE_PAGEABLE(m)) { + /* + * This page is on the background queue + * but not on a pageable queue. This is + * likely a transient state and whoever + * took it out of its pageable queue + * will likely put it back on a pageable + * queue soon but we can't deal with it + * at this point, so let's ignore this + * page. + */ + } else if (force_anonymous == FALSE || bg_m_object->internal) { + ignore_reference = TRUE; + + if (bg_m_object->internal) + vm_pageout_considered_bq_internal++; + else + vm_pageout_considered_bq_external++; + + break; + } + } +#endif + + /* + * The most eligible pages are ones we paged in speculatively, + * but which have not yet been touched. + */ + if (!vm_page_queue_empty(&sq->age_q) && force_anonymous == FALSE) { + m = (vm_page_t) vm_page_queue_first(&sq->age_q); + + assert(m->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q); + + break; + } + /* + * Try a clean-queue inactive page. + */ + if (!vm_page_queue_empty(&vm_page_queue_cleaned)) { + m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned); + + assert(m->vm_page_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q); + + break; + } + + grab_anonymous = (vm_page_anonymous_count > vm_page_anonymous_min); + inactive_external_count = vm_page_inactive_count - vm_page_anonymous_count; + + if ((vm_page_pageable_external_count < vm_page_filecache_min || force_anonymous == TRUE) || + ((inactive_external_count < vm_page_anonymous_count) && (inactive_external_count < (vm_page_pageable_external_count / 3)))) { + grab_anonymous = TRUE; + anons_grabbed = 0; + } +#if CONFIG_JETSAM + /* If the file-backed pool has accumulated + * significantly more pages than the jetsam + * threshold, prefer to reclaim those + * inline to minimise compute overhead of reclaiming + * anonymous pages. + * This calculation does not account for the CPU local + * external page queues, as those are expected to be + * much smaller relative to the global pools. + */ + if (grab_anonymous) { + if (vm_page_pageable_external_count > + vm_page_filecache_min) { + if ((vm_page_pageable_external_count * + vm_pageout_memorystatus_fb_factor_dr) > + (memorystatus_available_pages_critical * + vm_pageout_memorystatus_fb_factor_nr)) { + grab_anonymous = FALSE; +#if DEVELOPMENT || DEBUG + vm_grab_anon_overrides++; +#endif + } + } +#if DEVELOPMENT || DEBUG + if (grab_anonymous) { + vm_grab_anon_nops++; + + } +#endif + } +#endif /* CONFIG_JETSAM */ + + if (grab_anonymous == FALSE || anons_grabbed >= ANONS_GRABBED_LIMIT || vm_page_queue_empty(&vm_page_queue_anonymous)) { + + if ( !vm_page_queue_empty(&vm_page_queue_inactive) ) { + m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive); + + assert(m->vm_page_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q); + anons_grabbed = 0; + + if (vm_page_pageable_external_count < vm_page_filecache_min) { + if ((++reactivated_this_call % 100)) + goto must_activate_page; + /* + * steal 1% of the file backed pages even if + * we are under the limit that has been set + * for a healthy filecache + */ + } + break; + } + } + if ( !vm_page_queue_empty(&vm_page_queue_anonymous) ) { + m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous); + + assert(m->vm_page_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q); + anons_grabbed++; + + break; + } + + /* + * if we've gotten here, we have no victim page. + * if making clean, free the local freed list and return. + * if making free, check to see if we've finished balancing the queues + * yet, if we haven't just continue, else panic + */ + vm_page_unlock_queues(); + + if (object != NULL) { + vm_object_unlock(object); + object = NULL; + } + vm_pageout_scan_wants_object = VM_OBJECT_NULL; + + if (local_freeq) { + VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START, + vm_page_free_count, local_freed, delayed_unlock_limit, 5); + + vm_page_free_list(local_freeq, TRUE); + + VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END, + vm_page_free_count, local_freed, 0, 5); + + local_freeq = NULL; + local_freed = 0; + } + vm_page_lock_queues(); + delayed_unlock = 1; + + force_anonymous = FALSE; + + if ((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) + goto Restart; + + if (!vm_page_queue_empty(&sq->age_q)) + goto Restart; + + panic("vm_pageout: no victim"); + + /* NOTREACHED */ + } + assert(VM_PAGE_PAGEABLE(m)); + m_object = VM_PAGE_OBJECT(m); + force_anonymous = FALSE; + + page_prev_q_state = m->vm_page_q_state; + requeue_insert_first = FALSE; + /* + * we just found this page on one of our queues... + * it can't also be on the pageout queue, so safe + * to call vm_page_queues_remove + */ + vm_page_queues_remove(m, TRUE); + + assert(!m->laundry); + assert(!m->private); + assert(!m->fictitious); + assert(m_object != kernel_object); + assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr); + + + if (page_prev_q_state != VM_PAGE_ON_SPECULATIVE_Q && + page_prev_q_state != VM_PAGE_ON_SECLUDED_Q) + vm_pageout_stats[vm_pageout_stat_now].considered++; + + DTRACE_VM2(scan, int, 1, (uint64_t *), NULL); + + /* + * check to see if we currently are working + * with the same object... if so, we've + * already got the lock + */ + if (m_object != object) { + /* + * the object associated with candidate page is + * different from the one we were just working + * with... dump the lock if we still own it + */ + if (object != NULL) { + vm_object_unlock(object); + object = NULL; + vm_pageout_scan_wants_object = VM_OBJECT_NULL; + } + /* + * Try to lock object; since we've alread got the + * page queues lock, we can only 'try' for this one. + * if the 'try' fails, we need to do a mutex_pause + * to allow the owner of the object lock a chance to + * run... otherwise, we're likely to trip over this + * object in the same state as we work our way through + * the queue... clumps of pages associated with the same + * object are fairly typical on the inactive and active queues + */ + if (!vm_object_lock_try_scan(m_object)) { + vm_page_t m_want = NULL; + + vm_pageout_inactive_nolock++; + + if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) + vm_pageout_cleaned_nolock++; + + if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) + requeue_insert_first = TRUE; + + pmap_clear_reference(VM_PAGE_GET_PHYS_PAGE(m)); + m->reference = FALSE; + + /* + * m->object must be stable since we hold the page queues lock... + * we can update the scan_collisions field sans the object lock + * since it is a separate field and this is the only spot that does + * a read-modify-write operation and it is never executed concurrently... + * we can asynchronously set this field to 0 when creating a UPL, so it + * is possible for the value to be a bit non-determistic, but that's ok + * since it's only used as a hint + */ + m_object->scan_collisions = 1; + + if ( !vm_page_queue_empty(&sq->age_q) ) + m_want = (vm_page_t) vm_page_queue_first(&sq->age_q); + else if ( !vm_page_queue_empty(&vm_page_queue_cleaned)) + m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned); + else if ( !vm_page_queue_empty(&vm_page_queue_inactive) && + (anons_grabbed >= ANONS_GRABBED_LIMIT || vm_page_queue_empty(&vm_page_queue_anonymous))) + m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive); + else if ( !vm_page_queue_empty(&vm_page_queue_anonymous)) + m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous); + + /* + * this is the next object we're going to be interested in + * try to make sure its available after the mutex_yield + * returns control + */ + if (m_want) + vm_pageout_scan_wants_object = VM_PAGE_OBJECT(m_want); + + /* + * force us to dump any collected free pages + * and to pause before moving on + */ + try_failed = TRUE; + + goto requeue_page; + } + object = m_object; + vm_pageout_scan_wants_object = VM_OBJECT_NULL; + + try_failed = FALSE; + } + assert(m_object == object); + assert(VM_PAGE_OBJECT(m) == m_object); + + if (catch_up_count) + catch_up_count--; + + if (m->busy) { + if (m->encrypted_cleaning) { + /* + * ENCRYPTED SWAP: + * if this page has already been picked up as + * part of a page-out cluster, it will be busy + * because it is being encrypted (see + * vm_object_upl_request()). But we still + * want to demote it from "clean-in-place" + * (aka "adjacent") to "clean-and-free" (aka + * "target"), so let's ignore its "busy" bit + * here and proceed to check for "cleaning" a + * little bit below... + * + * CAUTION CAUTION: + * A "busy" page should still be left alone for + * most purposes, so we have to be very careful + * not to process that page too much. + */ + assert(m->cleaning); + goto consider_inactive_page; + } + + /* + * Somebody is already playing with this page. + * Put it back on the appropriate queue + * + */ + vm_pageout_inactive_busy++; + + if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) + vm_pageout_cleaned_busy++; + +requeue_page: + if (requeue_insert_first) + vm_page_enqueue_inactive(m, TRUE); + else + vm_page_enqueue_inactive(m, FALSE); +#if CONFIG_BACKGROUND_QUEUE + if (ignore_reference == TRUE) { + if (m_object->internal) + vm_pageout_rejected_bq_internal++; + else + vm_pageout_rejected_bq_external++; + } +#endif + goto done_with_inactivepage; + } + + + /* + * If it's absent, in error or the object is no longer alive, + * we can reclaim the page... in the no longer alive case, + * there are 2 states the page can be in that preclude us + * from reclaiming it - busy or cleaning - that we've already + * dealt with + */ + if (m->absent || m->error || !object->alive) { + + if (m->absent) + vm_pageout_inactive_absent++; + else if (!object->alive) + vm_pageout_inactive_notalive++; + else + vm_pageout_inactive_error++; +reclaim_page: + if (vm_pageout_deadlock_target) { + vm_pageout_scan_inactive_throttle_success++; + vm_pageout_deadlock_target--; + } + + DTRACE_VM2(dfree, int, 1, (uint64_t *), NULL); + + if (object->internal) { + DTRACE_VM2(anonfree, int, 1, (uint64_t *), NULL); + } else { + DTRACE_VM2(fsfree, int, 1, (uint64_t *), NULL); + } + assert(!m->cleaning); + assert(!m->laundry); + + m->busy = TRUE; + + /* + * remove page from object here since we're already + * behind the object lock... defer the rest of the work + * we'd normally do in vm_page_free_prepare_object + * until 'vm_page_free_list' is called + */ + if (m->tabled) + vm_page_remove(m, TRUE); + + assert(m->pageq.next == 0 && m->pageq.prev == 0); + m->snext = local_freeq; + local_freeq = m; + local_freed++; + +#if CONFIG_SECLUDED_MEMORY + if (page_prev_q_state == VM_PAGE_ON_SECLUDED_Q) + vm_pageout_freed_from_secluded++; +#endif /* CONFIG_SECLUDED_MEMORY */ + if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) + vm_pageout_freed_from_speculative++; + else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) + vm_pageout_freed_from_cleaned++; + else + vm_pageout_freed_from_inactive_clean++; + + if (page_prev_q_state != VM_PAGE_ON_SPECULATIVE_Q && + page_prev_q_state != VM_PAGE_ON_SECLUDED_Q) + vm_pageout_stats[vm_pageout_stat_now].reclaimed++; + + inactive_burst_count = 0; + goto done_with_inactivepage; + } + /* + * If the object is empty, the page must be reclaimed even + * if dirty or used. + * If the page belongs to a volatile object, we stick it back + * on. + */ + if (object->copy == VM_OBJECT_NULL) { + if (object->purgable == VM_PURGABLE_EMPTY) { + if (m->pmapped == TRUE) { + /* unmap the page */ + refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)); + if (refmod_state & VM_MEM_MODIFIED) { + SET_PAGE_DIRTY(m, FALSE); + } + } + if (m->dirty || m->precious) { + /* we saved the cost of cleaning this page ! */ + vm_page_purged_count++; + } + goto reclaim_page; + } + + if (VM_CONFIG_COMPRESSOR_IS_ACTIVE) { + /* + * With the VM compressor, the cost of + * reclaiming a page is much lower (no I/O), + * so if we find a "volatile" page, it's better + * to let it get compressed rather than letting + * it occupy a full page until it gets purged. + * So no need to check for "volatile" here. + */ + } else if (object->purgable == VM_PURGABLE_VOLATILE) { + /* + * Avoid cleaning a "volatile" page which might + * be purged soon. + */ + + /* if it's wired, we can't put it on our queue */ + assert(!VM_PAGE_WIRED(m)); + + /* just stick it back on! */ + reactivated_this_call++; + + if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) + vm_pageout_cleaned_volatile_reactivated++; + + goto reactivate_page; + } + } + +consider_inactive_page: + if (m->busy) { + /* + * CAUTION CAUTION: + * A "busy" page should always be left alone, except... + */ + if (m->cleaning && m->encrypted_cleaning) { + /* + * ENCRYPTED_SWAP: + * We could get here with a "busy" page + * if it's being encrypted during a + * "clean-in-place" operation. We'll deal + * with it right away by testing if it has been + * referenced and either reactivating it or + * promoting it from "clean-in-place" to + * "clean-and-free". + */ + } else { + panic("\"busy\" page considered for pageout\n"); + } + } + + /* + * If it's being used, reactivate. + * (Fictitious pages are either busy or absent.) + * First, update the reference and dirty bits + * to make sure the page is unreferenced. + */ + refmod_state = -1; + + if (m->reference == FALSE && m->pmapped == TRUE) { + refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m)); + + if (refmod_state & VM_MEM_REFERENCED) + m->reference = TRUE; + if (refmod_state & VM_MEM_MODIFIED) { + SET_PAGE_DIRTY(m, FALSE); + } + } + + /* + * if (m->cleaning && !m->free_when_done) + * If already cleaning this page in place and it hasn't + * been recently referenced, just pull off the queue. + * We can leave the page mapped, and upl_commit_range + * will put it on the clean queue. + * + * note: if m->encrypted_cleaning == TRUE, then + * m->cleaning == TRUE + * and we'll handle it here + * + * if (m->free_when_done && !m->cleaning) + * an msync INVALIDATE is in progress... + * this page has been marked for destruction + * after it has been cleaned, + * but not yet gathered into a UPL + * where 'cleaning' will be set... + * just leave it off the paging queues + * + * if (m->free_when_done && m->clenaing) + * an msync INVALIDATE is in progress + * and the UPL has already gathered this page... + * just leave it off the paging queues + */ + + /* + * page with m->free_when_done and still on the queues means that an + * MS_INVALIDATE is in progress on this page... leave it alone + */ + if (m->free_when_done) { + goto done_with_inactivepage; + } + + /* if cleaning, reactivate if referenced. otherwise, just pull off queue */ + if (m->cleaning) { + if (m->reference == TRUE) { + reactivated_this_call++; + goto reactivate_page; + } else { + goto done_with_inactivepage; + } + } + + if (m->reference || m->dirty) { + /* deal with a rogue "reusable" page */ + VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, m_object); + } + +#if CONFIG_SECLUDED_MEMORY + if (secluded_for_filecache && + vm_page_secluded_target > 0 && + m_object->eligible_for_secluded && + secluded_aging_policy == SECLUDED_AGING_FIFO) { + /* + * SECLUDED_AGING_FIFO: + * This victim page is eligible for the secluded pool + * and we're not aging secluded pages, so let's not + * reactivate it if it's been re-referenced. + * Later on, we'll move it to the secluded queue + * instead of freeing it. + */ + ignore_reference_secluded = TRUE; + } else { + ignore_reference_secluded = FALSE; + } +#endif /* CONFIG_SECLUDED_MEMORY */ + + if (!m->no_cache && +#if CONFIG_BACKGROUND_QUEUE + ignore_reference == FALSE && +#endif +#if CONFIG_SECLUDED_MEMORY + ignore_reference_secluded == FALSE && +#endif /* CONFIG_SECLUDED_MEMORY */ + (m->reference || + (m->xpmapped && !object->internal && (vm_page_xpmapped_external_count < (vm_page_external_count / 4))))) { + /* + * The page we pulled off the inactive list has + * been referenced. It is possible for other + * processors to be touching pages faster than we + * can clear the referenced bit and traverse the + * inactive queue, so we limit the number of + * reactivations. + */ + if (++reactivated_this_call >= reactivate_limit) { + vm_pageout_reactivation_limit_exceeded++; + } else if (catch_up_count) { + vm_pageout_catch_ups++; + } else if (++inactive_reclaim_run >= VM_PAGEOUT_INACTIVE_FORCE_RECLAIM) { + vm_pageout_inactive_force_reclaim++; + } else { + uint32_t isinuse; + + if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) + vm_pageout_cleaned_reference_reactivated++; + +reactivate_page: + if ( !object->internal && object->pager != MEMORY_OBJECT_NULL && + vnode_pager_get_isinuse(object->pager, &isinuse) == KERN_SUCCESS && !isinuse) { + /* + * no explict mappings of this object exist + * and it's not open via the filesystem + */ + vm_page_deactivate(m); + vm_pageout_inactive_deactivated++; + } else { +must_activate_page: + /* + * The page was/is being used, so put back on active list. + */ + vm_page_activate(m); + VM_STAT_INCR(reactivations); + inactive_burst_count = 0; + } +#if CONFIG_BACKGROUND_QUEUE + if (ignore_reference == TRUE) { + if (m_object->internal) + vm_pageout_rejected_bq_internal++; + else + vm_pageout_rejected_bq_external++; + } +#endif + if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) + vm_pageout_cleaned_reactivated++; +#if CONFIG_SECLUDED_MEMORY + if (page_prev_q_state == VM_PAGE_ON_SECLUDED_Q) + vm_pageout_secluded_reactivated++; +#endif /* CONFIG_SECLUDED_MEMORY */ + + vm_pageout_inactive_used++; + + goto done_with_inactivepage; + } + /* + * Make sure we call pmap_get_refmod() if it + * wasn't already called just above, to update + * the dirty bit. + */ + if ((refmod_state == -1) && !m->dirty && m->pmapped) { + refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m)); + if (refmod_state & VM_MEM_MODIFIED) { + SET_PAGE_DIRTY(m, FALSE); + } + } + } + + XPR(XPR_VM_PAGEOUT, + "vm_pageout_scan, replace object 0x%X offset 0x%X page 0x%X\n", + object, m->offset, m, 0,0); + + /* + * we've got a candidate page to steal... + * + * m->dirty is up to date courtesy of the + * preceding check for m->reference... if + * we get here, then m->reference had to be + * FALSE (or possibly "reactivate_limit" was + * exceeded), but in either case we called + * pmap_get_refmod() and updated both + * m->reference and m->dirty + * + * if it's dirty or precious we need to + * see if the target queue is throtttled + * it if is, we need to skip over it by moving it back + * to the end of the inactive queue + */ + + inactive_throttled = FALSE; + + if (m->dirty || m->precious) { + if (object->internal) { + if (VM_PAGE_Q_THROTTLED(iq)) + inactive_throttled = TRUE; + } else if (VM_PAGE_Q_THROTTLED(eq)) { + inactive_throttled = TRUE; + } + } +throttle_inactive: + if (!VM_DYNAMIC_PAGING_ENABLED() && + object->internal && m->dirty && + (object->purgable == VM_PURGABLE_DENY || + object->purgable == VM_PURGABLE_NONVOLATILE || + object->purgable == VM_PURGABLE_VOLATILE)) { + vm_page_check_pageable_safe(m); + assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q); + vm_page_queue_enter(&vm_page_queue_throttled, m, + vm_page_t, pageq); + m->vm_page_q_state = VM_PAGE_ON_THROTTLED_Q; + vm_page_throttled_count++; + + vm_pageout_scan_reclaimed_throttled++; + + inactive_burst_count = 0; + goto done_with_inactivepage; + } + if (inactive_throttled == TRUE) { + + if (object->internal == FALSE) { + /* + * we need to break up the following potential deadlock case... + * a) The external pageout thread is stuck on the truncate lock for a file that is being extended i.e. written. + * b) The thread doing the writing is waiting for pages while holding the truncate lock + * c) Most of the pages in the inactive queue belong to this file. + * + * we are potentially in this deadlock because... + * a) the external pageout queue is throttled + * b) we're done with the active queue and moved on to the inactive queue + * c) we've got a dirty external page + * + * since we don't know the reason for the external pageout queue being throttled we + * must suspect that we are deadlocked, so move the current page onto the active queue + * in an effort to cause a page from the active queue to 'age' to the inactive queue + * + * if we don't have jetsam configured (i.e. we have a dynamic pager), set + * 'force_anonymous' to TRUE to cause us to grab a page from the cleaned/anonymous + * pool the next time we select a victim page... if we can make enough new free pages, + * the deadlock will break, the external pageout queue will empty and it will no longer + * be throttled + * + * if we have jestam configured, keep a count of the pages reactivated this way so + * that we can try to find clean pages in the active/inactive queues before + * deciding to jetsam a process + */ + vm_pageout_scan_inactive_throttled_external++; + + vm_page_check_pageable_safe(m); + assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q); + vm_page_queue_enter(&vm_page_queue_active, m, vm_page_t, pageq); + m->vm_page_q_state = VM_PAGE_ON_ACTIVE_Q; + vm_page_active_count++; + vm_page_pageable_external_count++; + + vm_pageout_adjust_io_throttles(iq, eq, FALSE); + +#if CONFIG_MEMORYSTATUS && CONFIG_JETSAM + vm_pageout_inactive_external_forced_reactivate_limit--; + + if (vm_pageout_inactive_external_forced_reactivate_limit <= 0) { + vm_pageout_inactive_external_forced_reactivate_limit = vm_page_active_count + vm_page_inactive_count; + /* + * Possible deadlock scenario so request jetsam action + */ + assert(object); + vm_object_unlock(object); + object = VM_OBJECT_NULL; + vm_page_unlock_queues(); + + VM_DEBUG_CONSTANT_EVENT(vm_pageout_jetsam, VM_PAGEOUT_JETSAM, DBG_FUNC_START, + vm_page_active_count, vm_page_inactive_count, vm_page_free_count, vm_page_free_count); + + /* Kill first suitable process */ + if (memorystatus_kill_on_VM_page_shortage(FALSE) == FALSE) { + panic("vm_pageout_scan: Jetsam request failed\n"); + } + + VM_DEBUG_CONSTANT_EVENT(vm_pageout_jetsam, VM_PAGEOUT_JETSAM, DBG_FUNC_END, 0, 0, 0, 0); + + vm_pageout_inactive_external_forced_jetsam_count++; + vm_page_lock_queues(); + delayed_unlock = 1; + } +#else /* CONFIG_MEMORYSTATUS && CONFIG_JETSAM */ + force_anonymous = TRUE; +#endif + inactive_burst_count = 0; + goto done_with_inactivepage; + } else { + vm_pageout_scan_inactive_throttled_internal++; + + goto must_activate_page; + } + } + + /* + * we've got a page that we can steal... + * eliminate all mappings and make sure + * we have the up-to-date modified state + * + * if we need to do a pmap_disconnect then we + * need to re-evaluate m->dirty since the pmap_disconnect + * provides the true state atomically... the + * page was still mapped up to the pmap_disconnect + * and may have been dirtied at the last microsecond + * + * Note that if 'pmapped' is FALSE then the page is not + * and has not been in any map, so there is no point calling + * pmap_disconnect(). m->dirty could have been set in anticipation + * of likely usage of the page. + */ + if (m->pmapped == TRUE) { + int pmap_options; + + /* + * Don't count this page as going into the compressor + * if any of these are true: + * 1) compressed pager isn't enabled + * 2) Freezer enabled device with compressed pager + * backend (exclusive use) i.e. most of the VM system + * (including vm_pageout_scan) has no knowledge of + * the compressor + * 3) This page belongs to a file and hence will not be + * sent into the compressor + */ + if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE || + object->internal == FALSE) { + pmap_options = 0; + } else if (m->dirty || m->precious) { + /* + * VM knows that this page is dirty (or + * precious) and needs to be compressed + * rather than freed. + * Tell the pmap layer to count this page + * as "compressed". + */ + pmap_options = PMAP_OPTIONS_COMPRESSOR; + } else { + /* + * VM does not know if the page needs to + * be preserved but the pmap layer might tell + * us if any mapping has "modified" it. + * Let's the pmap layer to count this page + * as compressed if and only if it has been + * modified. + */ + pmap_options = + PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED; + } + refmod_state = pmap_disconnect_options(VM_PAGE_GET_PHYS_PAGE(m), + pmap_options, + NULL); + if (refmod_state & VM_MEM_MODIFIED) { + SET_PAGE_DIRTY(m, FALSE); + } + } + /* + * reset our count of pages that have been reclaimed + * since the last page was 'stolen' + */ + inactive_reclaim_run = 0; + + /* + * If it's clean and not precious, we can free the page. + */ + if (!m->dirty && !m->precious) { + + if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) + vm_pageout_speculative_clean++; + else { + if (page_prev_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q) + vm_pageout_inactive_anonymous++; + else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) + vm_pageout_cleaned_reclaimed++; + + vm_pageout_inactive_clean++; + } + +#if CONFIG_SECLUDED_MEMORY + if (secluded_for_filecache && + vm_page_secluded_target > 0 && + !m->fictitious && + m_object->eligible_for_secluded && + num_tasks_can_use_secluded_mem == 0 && + (secluded_aging_policy == SECLUDED_AGING_FIFO || + ((secluded_aging_policy == + SECLUDED_AGING_AFTER_INACTIVE) && + (page_prev_q_state != VM_PAGE_ON_SECLUDED_Q)))) { + assert(page_prev_q_state != VM_PAGE_ON_SECLUDED_Q); + assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q); + LCK_MTX_ASSERT(&vm_page_queue_lock, + LCK_MTX_ASSERT_OWNED); + vm_page_queue_enter(&vm_page_queue_secluded, + m, + vm_page_t, + pageq); + m->vm_page_q_state = VM_PAGE_ON_SECLUDED_Q; + vm_object_unlock(m_object); + object = VM_OBJECT_NULL; + vm_page_secluded_count++; + vm_page_secluded_count_inuse++; + assert(!m_object->internal); +// vm_page_pageable_external_count++; + m = VM_PAGE_NULL; + goto done_with_inactivepage; + } +#endif /* CONFIG_SECLUDED_MEMORY */ + + /* + * OK, at this point we have found a page we are going to free. + */ +#if CONFIG_PHANTOM_CACHE + if (!object->internal) + vm_phantom_cache_add_ghost(m); +#endif + goto reclaim_page; + } + + /* + * The page may have been dirtied since the last check + * for a throttled target queue (which may have been skipped + * if the page was clean then). With the dirty page + * disconnected here, we can make one final check. + */ + if (object->internal) { + if (VM_PAGE_Q_THROTTLED(iq)) + inactive_throttled = TRUE; + } else if (VM_PAGE_Q_THROTTLED(eq)) { + inactive_throttled = TRUE; + } + + if (inactive_throttled == TRUE) + goto throttle_inactive; + +#if VM_PRESSURE_EVENTS +#if CONFIG_JETSAM + + /* + * If Jetsam is enabled, then the sending + * of memory pressure notifications is handled + * from the same thread that takes care of high-water + * and other jetsams i.e. the memorystatus_thread. + */ + +#else /* CONFIG_JETSAM */ + + vm_pressure_response(); + +#endif /* CONFIG_JETSAM */ +#endif /* VM_PRESSURE_EVENTS */ + + if (page_prev_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q) + vm_pageout_inactive_anonymous++; + if (object->internal) + vm_pageout_inactive_dirty_internal++; + else + vm_pageout_inactive_dirty_external++; + + /* + * do NOT set the pageout bit! + * sure, we might need free pages, but this page is going to take time to become free + * anyway, so we may as well put it on the clean queue first and take it from there later + * if necessary. that way, we'll ensure we don't free up too much. -mj + */ + vm_pageout_cluster(m, FALSE, FALSE); + +done_with_inactivepage: + + if (delayed_unlock++ > delayed_unlock_limit || try_failed == TRUE) { + boolean_t need_delay = TRUE; + + if (object != NULL) { + vm_pageout_scan_wants_object = VM_OBJECT_NULL; + vm_object_unlock(object); + object = NULL; + } + vm_page_unlock_queues(); + + if (local_freeq) { + + VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START, + vm_page_free_count, local_freed, delayed_unlock_limit, 4); + + vm_page_free_list(local_freeq, TRUE); + + VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END, + vm_page_free_count, local_freed, 0, 4); + + local_freeq = NULL; + local_freed = 0; + need_delay = FALSE; + } + vm_consider_waking_compactor_swapper(); + + vm_page_lock_queues(); + + if (need_delay == TRUE) + lck_mtx_yield(&vm_page_queue_lock); + + delayed_unlock = 1; + } + vm_pageout_considered_page++; + + /* + * back to top of pageout scan loop + */ + } +} + + +int vm_page_free_count_init; + +void +vm_page_free_reserve( + int pages) +{ + int free_after_reserve; + + if (VM_CONFIG_COMPRESSOR_IS_PRESENT) { + + if ((vm_page_free_reserved + pages + COMPRESSOR_FREE_RESERVED_LIMIT) >= (VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT)) + vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT; + else + vm_page_free_reserved += (pages + COMPRESSOR_FREE_RESERVED_LIMIT); + + } else { + if ((vm_page_free_reserved + pages) >= VM_PAGE_FREE_RESERVED_LIMIT) + vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT; + else + vm_page_free_reserved += pages; + } + free_after_reserve = vm_page_free_count_init - vm_page_free_reserved; + + vm_page_free_min = vm_page_free_reserved + + VM_PAGE_FREE_MIN(free_after_reserve); + + if (vm_page_free_min > VM_PAGE_FREE_MIN_LIMIT) + vm_page_free_min = VM_PAGE_FREE_MIN_LIMIT; + + vm_page_free_target = vm_page_free_reserved + + VM_PAGE_FREE_TARGET(free_after_reserve); + + if (vm_page_free_target > VM_PAGE_FREE_TARGET_LIMIT) + vm_page_free_target = VM_PAGE_FREE_TARGET_LIMIT; + + if (vm_page_free_target < vm_page_free_min + 5) + vm_page_free_target = vm_page_free_min + 5; + + vm_page_throttle_limit = vm_page_free_target - (vm_page_free_target / 2); +} + +/* + * vm_pageout is the high level pageout daemon. + */ + +void +vm_pageout_continue(void) +{ + DTRACE_VM2(pgrrun, int, 1, (uint64_t *), NULL); + vm_pageout_scan_event_counter++; + + lck_mtx_lock(&vm_page_queue_free_lock); + vm_pageout_running = TRUE; + lck_mtx_unlock(&vm_page_queue_free_lock); + + vm_pageout_scan(); + /* + * we hold both the vm_page_queue_free_lock + * and the vm_page_queues_lock at this point + */ + assert(vm_page_free_wanted == 0); + assert(vm_page_free_wanted_privileged == 0); + assert_wait((event_t) &vm_page_free_wanted, THREAD_UNINT); + + vm_pageout_running = FALSE; + if (vm_pageout_waiter) { + vm_pageout_waiter = FALSE; + thread_wakeup((event_t)&vm_pageout_waiter); + } + + lck_mtx_unlock(&vm_page_queue_free_lock); + vm_page_unlock_queues(); + + counter(c_vm_pageout_block++); + thread_block((thread_continue_t)vm_pageout_continue); + /*NOTREACHED*/ +} + +kern_return_t +vm_pageout_wait(uint64_t deadline) +{ + kern_return_t kr; + + lck_mtx_lock(&vm_page_queue_free_lock); + for (kr = KERN_SUCCESS; vm_pageout_running && (KERN_SUCCESS == kr); ) { + vm_pageout_waiter = TRUE; + if (THREAD_AWAKENED != lck_mtx_sleep_deadline( + &vm_page_queue_free_lock, LCK_SLEEP_DEFAULT, + (event_t) &vm_pageout_waiter, THREAD_UNINT, deadline)) { + kr = KERN_OPERATION_TIMED_OUT; + } + } + lck_mtx_unlock(&vm_page_queue_free_lock); + + return (kr); +} + + +static void +vm_pageout_iothread_external_continue(struct vm_pageout_queue *q) +{ + vm_page_t m = NULL; + vm_object_t object; + vm_object_offset_t offset; + memory_object_t pager; + + + if (vm_pageout_internal_iothread != THREAD_NULL) + current_thread()->options &= ~TH_OPT_VMPRIV; + + vm_page_lockspin_queues(); + + while ( !vm_page_queue_empty(&q->pgo_pending) ) { + + q->pgo_busy = TRUE; + vm_page_queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq); + + assert(m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q); + VM_PAGE_CHECK(m); + /* + * grab a snapshot of the object and offset this + * page is tabled in so that we can relookup this + * page after we've taken the object lock - these + * fields are stable while we hold the page queues lock + * but as soon as we drop it, there is nothing to keep + * this page in this object... we hold an activity_in_progress + * on this object which will keep it from terminating + */ + object = VM_PAGE_OBJECT(m); + offset = m->offset; + + if (object->object_slid) { + panic("slid page %p not allowed on this path\n", m); + } + m->vm_page_q_state = VM_PAGE_NOT_ON_Q; + VM_PAGE_ZERO_PAGEQ_ENTRY(m); + + vm_page_unlock_queues(); + + vm_object_lock(object); + + m = vm_page_lookup(object, offset); + + if (m == NULL || + m->busy || m->cleaning || !m->laundry || (m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q)) { + /* + * it's either the same page that someone else has + * started cleaning (or it's finished cleaning or + * been put back on the pageout queue), or + * the page has been freed or we have found a + * new page at this offset... in all of these cases + * we merely need to release the activity_in_progress + * we took when we put the page on the pageout queue + */ + vm_object_activity_end(object); + vm_object_unlock(object); + + vm_page_lockspin_queues(); + continue; + } + pager = object->pager; + + if (pager == MEMORY_OBJECT_NULL) { + /* + * This pager has been destroyed by either + * memory_object_destroy or vm_object_destroy, and + * so there is nowhere for the page to go. + */ + if (m->free_when_done) { + /* + * Just free the page... VM_PAGE_FREE takes + * care of cleaning up all the state... + * including doing the vm_pageout_throttle_up + */ + VM_PAGE_FREE(m); + } else { + vm_page_lockspin_queues(); + + vm_pageout_throttle_up(m); + vm_page_activate(m); + + vm_page_unlock_queues(); + + /* + * And we are done with it. + */ + } + vm_object_activity_end(object); + vm_object_unlock(object); + + vm_page_lockspin_queues(); + continue; + } +#if 0 + /* + * we don't hold the page queue lock + * so this check isn't safe to make + */ + VM_PAGE_CHECK(m); +#endif + /* + * give back the activity_in_progress reference we + * took when we queued up this page and replace it + * it with a paging_in_progress reference that will + * also hold the paging offset from changing and + * prevent the object from terminating + */ + vm_object_activity_end(object); + vm_object_paging_begin(object); + vm_object_unlock(object); + + /* + * Send the data to the pager. + * any pageout clustering happens there + */ + memory_object_data_return(pager, + m->offset + object->paging_offset, + PAGE_SIZE, + NULL, + NULL, + FALSE, + FALSE, + 0); + + vm_object_lock(object); + vm_object_paging_end(object); + vm_object_unlock(object); + + vm_pageout_io_throttle(); + + vm_page_lockspin_queues(); + } + q->pgo_busy = FALSE; + q->pgo_idle = TRUE; + + assert_wait((event_t) &q->pgo_pending, THREAD_UNINT); + vm_page_unlock_queues(); + + thread_block_parameter((thread_continue_t)vm_pageout_iothread_external_continue, (void *) q); + /*NOTREACHED*/ +} - break; - - case FCS_DELAYED: - clock_get_system_nanotime(&sec, &nsec); - ts.tv_sec = (unsigned int) sec; - ts.tv_nsec = nsec; - if (CMP_MACH_TIMESPEC(&ts, &flow_control.ts) >= 0) { - /* - * the pageout thread for the default pager is potentially - * deadlocked since the - * default pager queue has been throttled for more than the - * allowable time... we need to move some clean pages or dirty - * pages belonging to the external pagers if they aren't throttled - * vm_page_free_wanted represents the number of threads currently - * blocked waiting for pages... we'll move one page for each of - * these plus a fixed amount to break the logjam... once we're done - * moving this number of pages, we'll re-enter the FSC_DELAYED state - * with a new timeout target since we have no way of knowing - * whether we've broken the deadlock except through observation - * of the queue associated with the default pager... we need to - * stop moving pages and allow the system to run to see what - * state it settles into. - */ - vm_pageout_deadlock_target = vm_pageout_deadlock_relief + vm_page_free_wanted + vm_page_free_wanted_privileged; - vm_pageout_scan_deadlock_detected++; - flow_control.state = FCS_DEADLOCK_DETECTED; +uint32_t vm_compressor_failed; - thread_wakeup((event_t) &vm_pageout_garbage_collect); - goto consider_inactive; - } - /* - * just resniff instead of trying - * to compute a new delay time... we're going to be - * awakened immediately upon a laundry completion, - * so we won't wait any longer than necessary - */ - msecs = vm_pageout_idle_wait; - break; +#define MAX_FREE_BATCH 32 +uint32_t vm_compressor_time_thread; /* Set via sysctl to record time accrued by + * this thread. + */ +uint64_t vm_compressor_thread_runtime; - case FCS_DEADLOCK_DETECTED: - if (vm_pageout_deadlock_target) - goto consider_inactive; - goto reset_deadlock_timer; +static void +vm_pageout_iothread_internal_continue(struct cq *cq) +{ + struct vm_pageout_queue *q; + vm_page_t m = NULL; + boolean_t pgo_draining; + vm_page_t local_q; + int local_cnt; + vm_page_t local_freeq = NULL; + int local_freed = 0; + int local_batch_size; - } - vm_pageout_scan_throttle++; - iq->pgo_throttled = TRUE; -vm_pageout_scan_delay: - if (object != NULL) { - vm_object_unlock(object); - object = NULL; - } - vm_pageout_scan_wants_object = VM_OBJECT_NULL; - if (local_freeq) { - vm_page_unlock_queues(); - vm_page_free_list(local_freeq, TRUE); - - local_freeq = NULL; - local_freed = 0; - vm_page_lock_queues(); + KERNEL_DEBUG(0xe040000c | DBG_FUNC_END, 0, 0, 0, 0, 0); - if (flow_control.state == FCS_DELAYED && - !VM_PAGE_Q_THROTTLED(iq)) { - flow_control.state = FCS_IDLE; - vm_pageout_scan_throttle_aborted++; - goto consider_inactive; - } - } -#if CONFIG_EMBEDDED - { - int percent_avail; + q = cq->q; + local_batch_size = q->pgo_maxlaundry / (vm_compressor_thread_count * 2); - /* - * Decide if we need to send a memory status notification. - */ - percent_avail = - (vm_page_active_count + vm_page_inactive_count + - vm_page_speculative_count + vm_page_free_count + - (IP_VALID(memory_manager_default)?0:vm_page_purgeable_count) ) * 100 / - atop_64(max_mem); - if (percent_avail >= (kern_memorystatus_level + 5) || - percent_avail <= (kern_memorystatus_level - 5)) { - kern_memorystatus_level = percent_avail; - thread_wakeup((event_t)&kern_memorystatus_wakeup); - } - } +#if RECORD_THE_COMPRESSED_DATA + if (q->pgo_laundry) + c_compressed_record_init(); #endif - assert_wait_timeout((event_t) &iq->pgo_laundry, THREAD_INTERRUPTIBLE, msecs, 1000*NSEC_PER_USEC); - counter(c_vm_pageout_scan_block++); + while (TRUE) { + int pages_left_on_q = 0; - vm_page_unlock_queues(); + local_cnt = 0; + local_q = NULL; - assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL); + KERNEL_DEBUG(0xe0400014 | DBG_FUNC_START, 0, 0, 0, 0, 0); + + vm_page_lock_queues(); - thread_block(THREAD_CONTINUE_NULL); + KERNEL_DEBUG(0xe0400014 | DBG_FUNC_END, 0, 0, 0, 0, 0); - vm_page_lock_queues(); - delayed_unlock = 1; + KERNEL_DEBUG(0xe0400018 | DBG_FUNC_START, q->pgo_laundry, 0, 0, 0, 0); - iq->pgo_throttled = FALSE; + while ( !vm_page_queue_empty(&q->pgo_pending) && local_cnt < local_batch_size) { - if (loop_count >= vm_page_inactive_count) - loop_count = 0; - inactive_burst_count = 0; + vm_page_queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq); + assert(m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q); + VM_PAGE_CHECK(m); + + m->vm_page_q_state = VM_PAGE_NOT_ON_Q; + VM_PAGE_ZERO_PAGEQ_ENTRY(m); + m->laundry = FALSE; - goto Restart; - /*NOTREACHED*/ + m->snext = local_q; + local_q = m; + local_cnt++; } + if (local_q == NULL) + break; + q->pgo_busy = TRUE; - flow_control.state = FCS_IDLE; -consider_inactive: - loop_count++; - inactive_burst_count++; - vm_pageout_inactive++; + if ((pgo_draining = q->pgo_draining) == FALSE) { + vm_pageout_throttle_up_batch(q, local_cnt); + pages_left_on_q = q->pgo_laundry; + } else + pages_left_on_q = q->pgo_laundry - local_cnt; - /* Choose a victim. */ - - while (1) { - m = NULL; - - if (IP_VALID(memory_manager_default)) { - assert(vm_page_throttled_count == 0); - assert(queue_empty(&vm_page_queue_throttled)); - } + vm_page_unlock_queues(); - /* - * The most eligible pages are ones we paged in speculatively, - * but which have not yet been touched. - */ - if ( !queue_empty(&sq->age_q) ) { - m = (vm_page_t) queue_first(&sq->age_q); - break; - } - /* - * Time for a zero-filled inactive page? - */ - if ( ((zf_run_count < zf_ratio) && vm_zf_queue_count >= zf_queue_min_count) || - queue_empty(&vm_page_queue_inactive)) { - if ( !queue_empty(&vm_page_queue_zf) ) { - m = (vm_page_t) queue_first(&vm_page_queue_zf); - zf_run_count++; - break; - } - } - /* - * It's either a normal inactive page or nothing. - */ - if ( !queue_empty(&vm_page_queue_inactive) ) { - m = (vm_page_t) queue_first(&vm_page_queue_inactive); - zf_run_count = 0; - break; - } +#if !RECORD_THE_COMPRESSED_DATA + if (pages_left_on_q >= local_batch_size && cq->id < (vm_compressor_thread_count - 1)) + thread_wakeup((event_t) ((uintptr_t)&q->pgo_pending + cq->id + 1)); +#endif + KERNEL_DEBUG(0xe0400018 | DBG_FUNC_END, q->pgo_laundry, 0, 0, 0, 0); - panic("vm_pageout: no victim"); - } + while (local_q) { - assert(!m->active && (m->inactive || m->speculative || m->throttled)); - assert(!m->laundry); - assert(m->object != kernel_object); - assert(m->phys_page != vm_page_guard_addr); + KERNEL_DEBUG(0xe0400024 | DBG_FUNC_START, local_cnt, 0, 0, 0, 0); - if (!m->speculative) { - vm_pageout_stats[vm_pageout_stat_now].considered++; - } + m = local_q; + local_q = m->snext; + m->snext = NULL; - DTRACE_VM2(scan, int, 1, (uint64_t *), NULL); + if (vm_pageout_compress_page(&cq->current_chead, cq->scratch_buf, m, FALSE) == KERN_SUCCESS) { - /* - * check to see if we currently are working - * with the same object... if so, we've - * already got the lock - */ - if (m->object != object) { - /* - * the object associated with candidate page is - * different from the one we were just working - * with... dump the lock if we still own it - */ - if (object != NULL) { - vm_object_unlock(object); - object = NULL; - vm_pageout_scan_wants_object = VM_OBJECT_NULL; - } - /* - * Try to lock object; since we've alread got the - * page queues lock, we can only 'try' for this one. - * if the 'try' fails, we need to do a mutex_pause - * to allow the owner of the object lock a chance to - * run... otherwise, we're likely to trip over this - * object in the same state as we work our way through - * the queue... clumps of pages associated with the same - * object are fairly typical on the inactive and active queues - */ - if (!vm_object_lock_try_scan(m->object)) { - vm_pageout_inactive_nolock++; + m->snext = local_freeq; + local_freeq = m; + local_freed++; - requeue_page: - /* - * Move page to end and continue. - * Don't re-issue ticket - */ - if (m->zero_fill) { - if (m->speculative) { - panic("vm_pageout_scan(): page %p speculative and zero-fill !?\n", m); - } - assert(!m->speculative); - queue_remove(&vm_page_queue_zf, m, - vm_page_t, pageq); - queue_enter(&vm_page_queue_zf, m, - vm_page_t, pageq); - } else if (m->speculative) { - remque(&m->pageq); - m->speculative = FALSE; - vm_page_speculative_count--; - - /* - * move to the head of the inactive queue - * to get it out of the way... the speculative - * queue is generally too small to depend - * on there being enough pages from other - * objects to make cycling it back on the - * same queue a winning proposition - */ - queue_enter_first(&vm_page_queue_inactive, m, - vm_page_t, pageq); - m->inactive = TRUE; - vm_page_inactive_count++; - token_new_pagecount++; - } else if (m->throttled) { - queue_remove(&vm_page_queue_throttled, m, - vm_page_t, pageq); - m->throttled = FALSE; - vm_page_throttled_count--; - - /* - * not throttled any more, so can stick - * it on the inactive queue. - */ - queue_enter(&vm_page_queue_inactive, m, - vm_page_t, pageq); - m->inactive = TRUE; - vm_page_inactive_count++; - token_new_pagecount++; - } else { - queue_remove(&vm_page_queue_inactive, m, - vm_page_t, pageq); -#if MACH_ASSERT - vm_page_inactive_count--; /* balance for purgeable queue asserts */ -#endif - vm_purgeable_q_advance_all(); + if (local_freed >= MAX_FREE_BATCH) { - queue_enter(&vm_page_queue_inactive, m, - vm_page_t, pageq); -#if MACH_ASSERT - vm_page_inactive_count++; /* balance for purgeable queue asserts */ -#endif - token_new_pagecount++; + vm_page_free_list(local_freeq, TRUE); + local_freeq = NULL; + local_freed = 0; } - pmap_clear_reference(m->phys_page); - m->reference = FALSE; + } +#if !CONFIG_JETSAM + while (vm_page_free_count < COMPRESSOR_FREE_RESERVED_LIMIT) { + kern_return_t wait_result; + int need_wakeup = 0; - if ( !queue_empty(&sq->age_q) ) - m = (vm_page_t) queue_first(&sq->age_q); - else if ( ((zf_run_count < zf_ratio) && vm_zf_queue_count >= zf_queue_min_count) || - queue_empty(&vm_page_queue_inactive)) { - if ( !queue_empty(&vm_page_queue_zf) ) - m = (vm_page_t) queue_first(&vm_page_queue_zf); - } else if ( !queue_empty(&vm_page_queue_inactive) ) { - m = (vm_page_t) queue_first(&vm_page_queue_inactive); - } - /* - * this is the next object we're going to be interested in - * try to make sure its available after the mutex_yield - * returns control - */ - vm_pageout_scan_wants_object = m->object; + if (local_freeq) { + vm_page_free_list(local_freeq, TRUE); - /* - * force us to dump any collected free pages - * and to pause before moving on - */ - try_failed = TRUE; + local_freeq = NULL; + local_freed = 0; - goto done_with_inactivepage; - } - object = m->object; - vm_pageout_scan_wants_object = VM_OBJECT_NULL; + continue; + } + lck_mtx_lock_spin(&vm_page_queue_free_lock); - try_failed = FALSE; - } + if (vm_page_free_count < COMPRESSOR_FREE_RESERVED_LIMIT) { - /* - * Paging out pages of external objects which - * are currently being created must be avoided. - * The pager may claim for memory, thus leading to a - * possible dead lock between it and the pageout thread, - * if such pages are finally chosen. The remaining assumption - * is that there will finally be enough available pages in the - * inactive pool to page out in order to satisfy all memory - * claimed by the thread which concurrently creates the pager. - */ - if (!object->pager_initialized && object->pager_created) { - /* - * Move page to end and continue, hoping that - * there will be enough other inactive pages to - * page out so that the thread which currently - * initializes the pager will succeed. - * Don't re-grant the ticket, the page should - * pulled from the queue and paged out whenever - * one of its logically adjacent fellows is - * targeted. - */ - vm_pageout_inactive_avoid++; - goto requeue_page; - } - /* - * Remove the page from its list. - */ - if (m->speculative) { - remque(&m->pageq); - page_prev_state = PAGE_STATE_SPECULATIVE; - m->speculative = FALSE; - vm_page_speculative_count--; - } else if (m->throttled) { - queue_remove(&vm_page_queue_throttled, m, vm_page_t, pageq); - page_prev_state = PAGE_STATE_THROTTLED; - m->throttled = FALSE; - vm_page_throttled_count--; - } else { - if (m->zero_fill) { - queue_remove(&vm_page_queue_zf, m, vm_page_t, pageq); - page_prev_state = PAGE_STATE_ZEROFILL; - vm_zf_queue_count--; - } else { - page_prev_state = PAGE_STATE_INACTIVE; - queue_remove(&vm_page_queue_inactive, m, vm_page_t, pageq); - } - m->inactive = FALSE; - if (!m->fictitious) - vm_page_inactive_count--; - vm_purgeable_q_advance_all(); - } + if (vm_page_free_wanted_privileged++ == 0) + need_wakeup = 1; + wait_result = assert_wait((event_t)&vm_page_free_wanted_privileged, THREAD_UNINT); - m->pageq.next = NULL; - m->pageq.prev = NULL; + lck_mtx_unlock(&vm_page_queue_free_lock); - if ( !m->fictitious && catch_up_count) - catch_up_count--; + if (need_wakeup) + thread_wakeup((event_t)&vm_page_free_wanted); - /* - * ENCRYPTED SWAP: - * if this page has already been picked up as part of a - * page-out cluster, it will be busy because it is being - * encrypted (see vm_object_upl_request()). But we still - * want to demote it from "clean-in-place" (aka "adjacent") - * to "clean-and-free" (aka "target"), so let's ignore its - * "busy" bit here and proceed to check for "cleaning" a - * little bit below... - */ - if ( !m->encrypted_cleaning && (m->busy || !object->alive)) { - /* - * Somebody is already playing with this page. - * Leave it off the pageout queues. - * - */ - vm_pageout_inactive_busy++; + if (wait_result == THREAD_WAITING) - goto done_with_inactivepage; + thread_block(THREAD_CONTINUE_NULL); + } else + lck_mtx_unlock(&vm_page_queue_free_lock); + } +#endif } + if (local_freeq) { + vm_page_free_list(local_freeq, TRUE); + + local_freeq = NULL; + local_freed = 0; + } + if (pgo_draining == TRUE) { + vm_page_lockspin_queues(); + vm_pageout_throttle_up_batch(q, local_cnt); + vm_page_unlock_queues(); + } + } + KERNEL_DEBUG(0xe040000c | DBG_FUNC_START, 0, 0, 0, 0, 0); - /* - * If it's absent or in error, we can reclaim the page. - */ + /* + * queue lock is held and our q is empty + */ + q->pgo_busy = FALSE; + q->pgo_idle = TRUE; - if (m->absent || m->error) { - vm_pageout_inactive_absent++; -reclaim_page: - if (vm_pageout_deadlock_target) { - vm_pageout_scan_inactive_throttle_success++; - vm_pageout_deadlock_target--; - } + assert_wait((event_t) ((uintptr_t)&q->pgo_pending + cq->id), THREAD_UNINT); + vm_page_unlock_queues(); - DTRACE_VM2(dfree, int, 1, (uint64_t *), NULL); + if (__improbable(vm_compressor_time_thread)) { + vm_compressor_thread_runtime = thread_get_runtime_self(); + } - if (object->internal) { - DTRACE_VM2(anonfree, int, 1, (uint64_t *), NULL); - } else { - DTRACE_VM2(fsfree, int, 1, (uint64_t *), NULL); - } - vm_page_free_prepare_queues(m); + KERNEL_DEBUG(0xe0400018 | DBG_FUNC_END, 0, 0, 0, 0, 0); - /* - * remove page from object here since we're already - * behind the object lock... defer the rest of the work - * we'd normally do in vm_page_free_prepare_object - * until 'vm_page_free_list' is called - */ - if (m->tabled) - vm_page_remove(m, TRUE); + thread_block_parameter((thread_continue_t)vm_pageout_iothread_internal_continue, (void *) cq); + /*NOTREACHED*/ +} - assert(m->pageq.next == NULL && - m->pageq.prev == NULL); - m->pageq.next = (queue_entry_t)local_freeq; - local_freeq = m; - local_freed++; - inactive_burst_count = 0; - if(page_prev_state != PAGE_STATE_SPECULATIVE) { - vm_pageout_stats[vm_pageout_stat_now].reclaimed++; - page_prev_state = 0; - } +static void +vm_pageout_immediate(vm_page_t m, boolean_t object_locked_by_caller) +{ + assert(vm_pageout_immediate_scratch_buf); - goto done_with_inactivepage; - } + if (vm_pageout_compress_page(&vm_pageout_immediate_chead, vm_pageout_immediate_scratch_buf, m, object_locked_by_caller) == KERN_SUCCESS) { - assert(!m->private); - assert(!m->fictitious); + vm_page_free_prepare_object(m, TRUE); + vm_page_release(m, TRUE); + } +} - /* - * If already cleaning this page in place, convert from - * "adjacent" to "target". We can leave the page mapped, - * and vm_pageout_object_terminate will determine whether - * to free or reactivate. - */ - if (m->cleaning) { - m->busy = TRUE; - m->pageout = TRUE; - m->dump_cleaning = TRUE; - vm_page_wire(m); +kern_return_t +vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m, boolean_t object_locked_by_caller) +{ + vm_object_t object; + memory_object_t pager; + int compressed_count_delta; + kern_return_t retval; + + object = VM_PAGE_OBJECT(m); - CLUSTER_STAT(vm_pageout_cluster_conversions++); + if (object->object_slid) { + panic("slid page %p not allowed on this path\n", m); + } + assert(!m->free_when_done); + assert(!m->laundry); - inactive_burst_count = 0; + pager = object->pager; - goto done_with_inactivepage; - } + if (object_locked_by_caller == FALSE && (!object->pager_initialized || pager == MEMORY_OBJECT_NULL)) { + + KERNEL_DEBUG(0xe0400010 | DBG_FUNC_START, object, pager, 0, 0, 0); - /* - * If the object is empty, the page must be reclaimed even - * if dirty or used. - * If the page belongs to a volatile object, we stick it back - * on. - */ - if (object->copy == VM_OBJECT_NULL) { - if (object->purgable == VM_PURGABLE_EMPTY) { - m->busy = TRUE; - if (m->pmapped == TRUE) { - /* unmap the page */ - refmod_state = pmap_disconnect(m->phys_page); - if (refmod_state & VM_MEM_MODIFIED) { - m->dirty = TRUE; - } - } - if (m->dirty || m->precious) { - /* we saved the cost of cleaning this page ! */ - vm_page_purged_count++; - } - goto reclaim_page; - } - if (object->purgable == VM_PURGABLE_VOLATILE) { - /* if it's wired, we can't put it on our queue */ - assert(!VM_PAGE_WIRED(m)); - /* just stick it back on! */ - goto reactivate_page; - } - } + vm_object_lock(object); /* - * If it's being used, reactivate. - * (Fictitious pages are either busy or absent.) - * First, update the reference and dirty bits - * to make sure the page is unreferenced. + * If there is no memory object for the page, create + * one and hand it to the compression pager. */ - refmod_state = -1; - if (m->reference == FALSE && m->pmapped == TRUE) { - refmod_state = pmap_get_refmod(m->phys_page); - - if (refmod_state & VM_MEM_REFERENCED) - m->reference = TRUE; - if (refmod_state & VM_MEM_MODIFIED) - m->dirty = TRUE; - } + if (!object->pager_initialized) + vm_object_collapse(object, (vm_object_offset_t) 0, TRUE); + if (!object->pager_initialized) + vm_object_compressor_pager_create(object); - if (m->reference || m->dirty) { - /* deal with a rogue "reusable" page */ - VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m); - } + pager = object->pager; - if (m->reference && !m->no_cache) { + if (!object->pager_initialized || pager == MEMORY_OBJECT_NULL) { /* - * The page we pulled off the inactive list has - * been referenced. It is possible for other - * processors to be touching pages faster than we - * can clear the referenced bit and traverse the - * inactive queue, so we limit the number of - * reactivations. + * Still no pager for the object, + * or the pager has been destroyed. + * Reactivate the page. + * + * Should only happen if there is no + * compression pager */ - if (++reactivated_this_call >= reactivate_limit) { - vm_pageout_reactivation_limit_exceeded++; - } else if (catch_up_count) { - vm_pageout_catch_ups++; - } else if (++inactive_reclaim_run >= VM_PAGEOUT_INACTIVE_FORCE_RECLAIM) { - vm_pageout_inactive_force_reclaim++; - } else { - uint32_t isinuse; -reactivate_page: - if ( !object->internal && object->pager != MEMORY_OBJECT_NULL && - vnode_pager_get_isinuse(object->pager, &isinuse) == KERN_SUCCESS && !isinuse) { - /* - * no explict mappings of this object exist - * and it's not open via the filesystem - */ - vm_page_deactivate(m); - vm_pageout_inactive_deactivated++; - } else { - /* - * The page was/is being used, so put back on active list. - */ - vm_page_activate(m); - VM_STAT_INCR(reactivations); - } - vm_pageout_inactive_used++; - inactive_burst_count = 0; + PAGE_WAKEUP_DONE(m); - goto done_with_inactivepage; - } - /* - * Make sure we call pmap_get_refmod() if it - * wasn't already called just above, to update - * the dirty bit. + vm_page_lockspin_queues(); + vm_page_activate(m); + vm_pageout_dirty_no_pager++; + vm_page_unlock_queues(); + + /* + * And we are done with it. */ - if ((refmod_state == -1) && !m->dirty && m->pmapped) { - refmod_state = pmap_get_refmod(m->phys_page); - if (refmod_state & VM_MEM_MODIFIED) - m->dirty = TRUE; - } - forced_reclaim = TRUE; - } else { - forced_reclaim = FALSE; + vm_object_activity_end(object); + vm_object_unlock(object); + + return KERN_FAILURE; } + vm_object_unlock(object); + + KERNEL_DEBUG(0xe0400010 | DBG_FUNC_END, object, pager, 0, 0, 0); + } + assert(object->pager_initialized && pager != MEMORY_OBJECT_NULL); - XPR(XPR_VM_PAGEOUT, - "vm_pageout_scan, replace object 0x%X offset 0x%X page 0x%X\n", - object, m->offset, m, 0,0); + if (object_locked_by_caller == FALSE) + assert(object->activity_in_progress > 0); - /* - * we've got a candidate page to steal... - * - * m->dirty is up to date courtesy of the - * preceding check for m->reference... if - * we get here, then m->reference had to be - * FALSE (or possibly "reactivate_limit" was - * exceeded), but in either case we called - * pmap_get_refmod() and updated both - * m->reference and m->dirty - * - * if it's dirty or precious we need to - * see if the target queue is throtttled - * it if is, we need to skip over it by moving it back - * to the end of the inactive queue - */ + retval = vm_compressor_pager_put( + pager, + m->offset + object->paging_offset, + VM_PAGE_GET_PHYS_PAGE(m), + current_chead, + scratch_buf, + &compressed_count_delta); - inactive_throttled = FALSE; + if (object_locked_by_caller == FALSE) { + vm_object_lock(object); - if (m->dirty || m->precious) { - if (object->internal) { - if (VM_PAGE_Q_THROTTLED(iq)) - inactive_throttled = TRUE; - } else if (VM_PAGE_Q_THROTTLED(eq)) { - inactive_throttled = TRUE; - } - } - if (inactive_throttled == TRUE) { -throttle_inactive: - if (!IP_VALID(memory_manager_default) && - object->internal && m->dirty && - (object->purgable == VM_PURGABLE_DENY || - object->purgable == VM_PURGABLE_NONVOLATILE || - object->purgable == VM_PURGABLE_VOLATILE)) { - queue_enter(&vm_page_queue_throttled, m, - vm_page_t, pageq); - m->throttled = TRUE; - vm_page_throttled_count++; - } else { - if (m->zero_fill) { - queue_enter(&vm_page_queue_zf, m, - vm_page_t, pageq); - vm_zf_queue_count++; - } else - queue_enter(&vm_page_queue_inactive, m, - vm_page_t, pageq); - m->inactive = TRUE; - if (!m->fictitious) { - vm_page_inactive_count++; - token_new_pagecount++; - } - } - vm_pageout_scan_inactive_throttled++; - goto done_with_inactivepage; - } + assert(object->activity_in_progress > 0); + assert(VM_PAGE_OBJECT(m) == object); + } - /* - * we've got a page that we can steal... - * eliminate all mappings and make sure - * we have the up-to-date modified state - * first take the page BUSY, so that no new - * mappings can be made - */ - m->busy = TRUE; - - /* - * if we need to do a pmap_disconnect then we - * need to re-evaluate m->dirty since the pmap_disconnect - * provides the true state atomically... the - * page was still mapped up to the pmap_disconnect - * and may have been dirtied at the last microsecond - * - * we also check for the page being referenced 'late' - * if it was, we first need to do a WAKEUP_DONE on it - * since we already set m->busy = TRUE, before - * going off to reactivate it - * - * Note that if 'pmapped' is FALSE then the page is not - * and has not been in any map, so there is no point calling - * pmap_disconnect(). m->dirty and/or m->reference could - * have been set in anticipation of likely usage of the page. - */ - if (m->pmapped == TRUE) { - refmod_state = pmap_disconnect(m->phys_page); + vm_compressor_pager_count(pager, + compressed_count_delta, + FALSE, /* shared_lock */ + object); - if (refmod_state & VM_MEM_MODIFIED) - m->dirty = TRUE; - if (refmod_state & VM_MEM_REFERENCED) { - - /* If m->reference is already set, this page must have - * already failed the reactivate_limit test, so don't - * bump the counts twice. - */ - if ( ! m->reference ) { - m->reference = TRUE; - if (forced_reclaim || - ++reactivated_this_call >= reactivate_limit) - vm_pageout_reactivation_limit_exceeded++; - else { - PAGE_WAKEUP_DONE(m); - goto reactivate_page; - } - } - } - } - /* - * reset our count of pages that have been reclaimed - * since the last page was 'stolen' - */ - inactive_reclaim_run = 0; + assert( !VM_PAGE_WIRED(m)); + if (retval == KERN_SUCCESS) { /* - * If it's clean and not precious, we can free the page. + * If the object is purgeable, its owner's + * purgeable ledgers will be updated in + * vm_page_remove() but the page still + * contributes to the owner's memory footprint, + * so account for it as such. */ - if (!m->dirty && !m->precious) { - if (m->zero_fill) - vm_pageout_inactive_zf++; - vm_pageout_inactive_clean++; - - goto reclaim_page; + if (object->purgable != VM_PURGABLE_DENY && + object->vo_purgeable_owner != NULL) { + /* one more compressed purgeable page */ + vm_purgeable_compressed_update(object, + +1); } + VM_STAT_INCR(compressions); + + if (m->tabled) + vm_page_remove(m, TRUE); - /* - * The page may have been dirtied since the last check - * for a throttled target queue (which may have been skipped - * if the page was clean then). With the dirty page - * disconnected here, we can make one final check. - */ - { - boolean_t disconnect_throttled = FALSE; - if (object->internal) { - if (VM_PAGE_Q_THROTTLED(iq)) - disconnect_throttled = TRUE; - } else if (VM_PAGE_Q_THROTTLED(eq)) { - disconnect_throttled = TRUE; - } + } else { + PAGE_WAKEUP_DONE(m); - if (disconnect_throttled == TRUE) { - PAGE_WAKEUP_DONE(m); - goto throttle_inactive; - } - } + vm_page_lockspin_queues(); + + vm_page_activate(m); + vm_compressor_failed++; + + vm_page_unlock_queues(); + } + if (object_locked_by_caller == FALSE) { + vm_object_activity_end(object); + vm_object_unlock(object); + } + return retval; +} - vm_pageout_stats[vm_pageout_stat_now].reclaimed++; - vm_pageout_cluster(m); +static void +vm_pageout_adjust_io_throttles(struct vm_pageout_queue *iq, struct vm_pageout_queue *eq, boolean_t req_lowpriority) +{ + uint32_t policy; + boolean_t set_iq = FALSE; + boolean_t set_eq = FALSE; + + if (hibernate_cleaning_in_progress == TRUE) + req_lowpriority = FALSE; - if (m->zero_fill) - vm_pageout_inactive_zf++; - vm_pageout_inactive_dirty++; + if (eq->pgo_inited == TRUE && eq->pgo_lowpriority != req_lowpriority) + set_eq = TRUE; + + if (set_iq == TRUE || set_eq == TRUE) { - inactive_burst_count = 0; + vm_page_unlock_queues(); -done_with_inactivepage: - if (delayed_unlock++ > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT || try_failed == TRUE) { + if (req_lowpriority == TRUE) { + policy = THROTTLE_LEVEL_PAGEOUT_THROTTLED; + DTRACE_VM(laundrythrottle); + } else { + policy = THROTTLE_LEVEL_PAGEOUT_UNTHROTTLED; + DTRACE_VM(laundryunthrottle); + } + if (set_iq == TRUE) { + proc_set_thread_policy_with_tid(kernel_task, iq->pgo_tid, + TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy); - if (object != NULL) { - vm_pageout_scan_wants_object = VM_OBJECT_NULL; - vm_object_unlock(object); - object = NULL; - } - if (local_freeq) { - vm_page_unlock_queues(); - vm_page_free_list(local_freeq, TRUE); - - local_freeq = NULL; - local_freed = 0; - vm_page_lock_queues(); - } else - lck_mtx_yield(&vm_page_queue_lock); + iq->pgo_lowpriority = req_lowpriority; + } + if (set_eq == TRUE) { + proc_set_thread_policy_with_tid(kernel_task, eq->pgo_tid, + TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy); - delayed_unlock = 1; + eq->pgo_lowpriority = req_lowpriority; } - /* - * back to top of pageout scan loop - */ + vm_page_lock_queues(); } } -int vm_page_free_count_init; - -void -vm_page_free_reserve( - int pages) +static void +vm_pageout_iothread_external(void) { - int free_after_reserve; + thread_t self = current_thread(); - vm_page_free_reserved += pages; + self->options |= TH_OPT_VMPRIV; - free_after_reserve = vm_page_free_count_init - vm_page_free_reserved; + DTRACE_VM2(laundrythrottle, int, 1, (uint64_t *), NULL); - vm_page_free_min = vm_page_free_reserved + - VM_PAGE_FREE_MIN(free_after_reserve); + proc_set_thread_policy(self, TASK_POLICY_EXTERNAL, + TASK_POLICY_IO, THROTTLE_LEVEL_PAGEOUT_THROTTLED); - if (vm_page_free_min > VM_PAGE_FREE_MIN_LIMIT) - vm_page_free_min = VM_PAGE_FREE_MIN_LIMIT; + vm_page_lock_queues(); - vm_page_free_target = vm_page_free_reserved + - VM_PAGE_FREE_TARGET(free_after_reserve); + vm_pageout_queue_external.pgo_tid = self->thread_id; + vm_pageout_queue_external.pgo_lowpriority = TRUE; + vm_pageout_queue_external.pgo_inited = TRUE; - if (vm_page_free_target > VM_PAGE_FREE_TARGET_LIMIT) - vm_page_free_target = VM_PAGE_FREE_TARGET_LIMIT; + vm_page_unlock_queues(); + + vm_pageout_iothread_external_continue(&vm_pageout_queue_external); + + /*NOTREACHED*/ +} + + +static void +vm_pageout_iothread_internal(struct cq *cq) +{ + thread_t self = current_thread(); + + self->options |= TH_OPT_VMPRIV; + + vm_page_lock_queues(); - if (vm_page_free_target < vm_page_free_min + 5) - vm_page_free_target = vm_page_free_min + 5; + vm_pageout_queue_internal.pgo_tid = self->thread_id; + vm_pageout_queue_internal.pgo_lowpriority = TRUE; + vm_pageout_queue_internal.pgo_inited = TRUE; - vm_page_throttle_limit = vm_page_free_target - (vm_page_free_target / 3); - vm_page_creation_throttle = vm_page_free_target / 2; -} + vm_page_unlock_queues(); -/* - * vm_pageout is the high level pageout daemon. - */ + if (vm_restricted_to_single_processor == TRUE) + thread_vm_bind_group_add(); -void -vm_pageout_continue(void) -{ - DTRACE_VM2(pgrrun, int, 1, (uint64_t *), NULL); - vm_pageout_scan_event_counter++; - vm_pageout_scan(); - /* we hold vm_page_queue_free_lock now */ - assert(vm_page_free_wanted == 0); - assert(vm_page_free_wanted_privileged == 0); - assert_wait((event_t) &vm_page_free_wanted, THREAD_UNINT); - lck_mtx_unlock(&vm_page_queue_free_lock); + vm_pageout_iothread_internal_continue(cq); - counter(c_vm_pageout_block++); - thread_block((thread_continue_t)vm_pageout_continue); /*NOTREACHED*/ } +kern_return_t +vm_set_buffer_cleanup_callout(boolean_t (*func)(int)) +{ + if (OSCompareAndSwapPtr(NULL, func, (void * volatile *) &consider_buffer_cache_collect)) { + return KERN_SUCCESS; + } else { + return KERN_FAILURE; /* Already set */ + } +} -#ifdef FAKE_DEADLOCK +extern boolean_t memorystatus_manual_testing_on; +extern unsigned int memorystatus_level; -#define FAKE_COUNT 5000 -int internal_count = 0; -int fake_deadlock = 0; +#if VM_PRESSURE_EVENTS -#endif +boolean_t vm_pressure_events_enabled = FALSE; -static void -vm_pageout_iothread_continue(struct vm_pageout_queue *q) +void +vm_pressure_response(void) { - vm_page_t m = NULL; - vm_object_t object; - memory_object_t pager; - thread_t self = current_thread(); - if ((vm_pageout_internal_iothread != THREAD_NULL) - && (self == vm_pageout_external_iothread ) - && (self->options & TH_OPT_VMPRIV)) - self->options &= ~TH_OPT_VMPRIV; + vm_pressure_level_t old_level = kVMPressureNormal; + int new_level = -1; + unsigned int total_pages; + uint64_t available_memory = 0; - vm_page_lockspin_queues(); + if (vm_pressure_events_enabled == FALSE) + return; - while ( !queue_empty(&q->pgo_pending) ) { - q->pgo_busy = TRUE; - queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq); - VM_PAGE_CHECK(m); - m->pageout_queue = FALSE; - m->pageq.next = NULL; - m->pageq.prev = NULL; - vm_page_unlock_queues(); + available_memory = (uint64_t) AVAILABLE_NON_COMPRESSED_MEMORY; -#ifdef FAKE_DEADLOCK - if (q == &vm_pageout_queue_internal) { - vm_offset_t addr; - int pg_count; - internal_count++; + total_pages = (unsigned int) atop_64(max_mem); +#if CONFIG_SECLUDED_MEMORY + total_pages -= vm_page_secluded_count; +#endif /* CONFIG_SECLUDED_MEMORY */ + memorystatus_level = (unsigned int) ((available_memory * 100) / total_pages); - if ((internal_count == FAKE_COUNT)) { + if (memorystatus_manual_testing_on) { + return; + } + + old_level = memorystatus_vm_pressure_level; - pg_count = vm_page_free_count + vm_page_free_reserved; + switch (memorystatus_vm_pressure_level) { - if (kmem_alloc(kernel_map, &addr, PAGE_SIZE * pg_count) == KERN_SUCCESS) { - kmem_free(kernel_map, addr, PAGE_SIZE * pg_count); - } - internal_count = 0; - fake_deadlock++; - } - } -#endif - object = m->object; + case kVMPressureNormal: + { + if (VM_PRESSURE_WARNING_TO_CRITICAL()) { + new_level = kVMPressureCritical; + } else if (VM_PRESSURE_NORMAL_TO_WARNING()) { + new_level = kVMPressureWarning; + } + break; + } - vm_object_lock(object); + case kVMPressureWarning: + case kVMPressureUrgent: + { + if (VM_PRESSURE_WARNING_TO_NORMAL()) { + new_level = kVMPressureNormal; + } else if (VM_PRESSURE_WARNING_TO_CRITICAL()) { + new_level = kVMPressureCritical; + } + break; + } - if (!object->pager_initialized) { + case kVMPressureCritical: + { + if (VM_PRESSURE_WARNING_TO_NORMAL()) { + new_level = kVMPressureNormal; + } else if (VM_PRESSURE_CRITICAL_TO_WARNING()) { + new_level = kVMPressureWarning; + } + break; + } - /* - * If there is no memory object for the page, create - * one and hand it to the default pager. - */ + default: + return; + } + + if (new_level != -1) { + memorystatus_vm_pressure_level = (vm_pressure_level_t) new_level; - if (!object->pager_initialized) - vm_object_collapse(object, - (vm_object_offset_t) 0, - TRUE); - if (!object->pager_initialized) - vm_object_pager_create(object); - if (!object->pager_initialized) { - /* - * Still no pager for the object. - * Reactivate the page. - * - * Should only happen if there is no - * default pager. - */ - vm_page_lockspin_queues(); + if ((memorystatus_vm_pressure_level != kVMPressureNormal) || (old_level != new_level)) { + if (vm_pressure_thread_running == FALSE) { + thread_wakeup(&vm_pressure_thread); + } - vm_pageout_queue_steal(m, TRUE); - vm_pageout_dirty_no_pager++; - vm_page_activate(m); + if (old_level != new_level) { + thread_wakeup(&vm_pressure_changed); + } + } + } - vm_page_unlock_queues(); +} +#endif /* VM_PRESSURE_EVENTS */ - /* - * And we are done with it. - */ - PAGE_WAKEUP_DONE(m); +kern_return_t +mach_vm_pressure_level_monitor(__unused boolean_t wait_for_pressure, __unused unsigned int *pressure_level) { - vm_object_paging_end(object); - vm_object_unlock(object); +#if !VM_PRESSURE_EVENTS + + return KERN_FAILURE; - vm_page_lockspin_queues(); - continue; - } - } - pager = object->pager; - if (pager == MEMORY_OBJECT_NULL) { - /* - * This pager has been destroyed by either - * memory_object_destroy or vm_object_destroy, and - * so there is nowhere for the page to go. - */ - if (m->pageout) { - /* - * Just free the page... VM_PAGE_FREE takes - * care of cleaning up all the state... - * including doing the vm_pageout_throttle_up - */ - VM_PAGE_FREE(m); - } else { - vm_page_lockspin_queues(); +#else /* VM_PRESSURE_EVENTS */ - vm_pageout_queue_steal(m, TRUE); - vm_page_activate(m); - - vm_page_unlock_queues(); + kern_return_t kr = KERN_SUCCESS; - /* - * And we are done with it. - */ - PAGE_WAKEUP_DONE(m); - } - vm_object_paging_end(object); - vm_object_unlock(object); + if (pressure_level != NULL) { - vm_page_lockspin_queues(); - continue; - } - VM_PAGE_CHECK(m); - vm_object_unlock(object); - /* - * we expect the paging_in_progress reference to have - * already been taken on the object before it was added - * to the appropriate pageout I/O queue... this will - * keep the object from being terminated and/or the - * paging_offset from changing until the I/O has - * completed... therefore no need to lock the object to - * pull the paging_offset from it. - * - * Send the data to the pager. - * any pageout clustering happens there - */ - memory_object_data_return(pager, - m->offset + object->paging_offset, - PAGE_SIZE, - NULL, - NULL, - FALSE, - FALSE, - 0); + vm_pressure_level_t old_level = memorystatus_vm_pressure_level; - vm_object_lock(object); - vm_object_paging_end(object); - vm_object_unlock(object); + if (wait_for_pressure == TRUE) { + wait_result_t wr = 0; - vm_page_lockspin_queues(); - } - assert_wait((event_t) q, THREAD_UNINT); + while (old_level == *pressure_level) { + wr = assert_wait((event_t) &vm_pressure_changed, + THREAD_INTERRUPTIBLE); + if (wr == THREAD_WAITING) { + wr = thread_block(THREAD_CONTINUE_NULL); + } + if (wr == THREAD_INTERRUPTED) { + return KERN_ABORTED; + } + if (wr == THREAD_AWAKENED) { + + old_level = memorystatus_vm_pressure_level; - if (q->pgo_throttled == TRUE && !VM_PAGE_Q_THROTTLED(q)) { - q->pgo_throttled = FALSE; - thread_wakeup((event_t) &q->pgo_laundry); - } - if (q->pgo_draining == TRUE && q->pgo_laundry == 0) { - q->pgo_draining = FALSE; - thread_wakeup((event_t) (&q->pgo_laundry+1)); + if (old_level != *pressure_level) { + break; + } + } + } + } + + *pressure_level = old_level; + kr = KERN_SUCCESS; + } else { + kr = KERN_INVALID_ARGUMENT; } - q->pgo_busy = FALSE; - q->pgo_idle = TRUE; - vm_page_unlock_queues(); - thread_block_parameter((thread_continue_t)vm_pageout_iothread_continue, (void *) &q->pgo_pending); - /*NOTREACHED*/ + return kr; +#endif /* VM_PRESSURE_EVENTS */ } +#if VM_PRESSURE_EVENTS +void +vm_pressure_thread(void) { + static boolean_t thread_initialized = FALSE; -static void -vm_pageout_iothread_external(void) -{ - thread_t self = current_thread(); - - self->options |= TH_OPT_VMPRIV; + if (thread_initialized == TRUE) { + vm_pressure_thread_running = TRUE; + consider_vm_pressure_events(); + vm_pressure_thread_running = FALSE; + } - vm_pageout_iothread_continue(&vm_pageout_queue_external); - /*NOTREACHED*/ + thread_initialized = TRUE; + assert_wait((event_t) &vm_pressure_thread, THREAD_UNINT); + thread_block((thread_continue_t)vm_pressure_thread); } +#endif /* VM_PRESSURE_EVENTS */ -static void -vm_pageout_iothread_internal(void) -{ - thread_t self = current_thread(); +uint32_t vm_pageout_considered_page_last = 0; - self->options |= TH_OPT_VMPRIV; +/* + * called once per-second via "compute_averages" + */ +void +compute_pageout_gc_throttle(__unused void *arg) +{ + if (vm_pageout_considered_page != vm_pageout_considered_page_last) { - vm_pageout_iothread_continue(&vm_pageout_queue_internal); - /*NOTREACHED*/ -} + vm_pageout_considered_page_last = vm_pageout_considered_page; -kern_return_t -vm_set_buffer_cleanup_callout(boolean_t (*func)(int)) -{ - if (OSCompareAndSwapPtr(NULL, func, (void * volatile *) &consider_buffer_cache_collect)) { - return KERN_SUCCESS; - } else { - return KERN_FAILURE; /* Already set */ + thread_wakeup((event_t) &vm_pageout_garbage_collect); } } + static void vm_pageout_garbage_collect(int collect) { + if (collect) { boolean_t buf_large_zfree = FALSE; + boolean_t first_try = TRUE; + stack_collect(); - /* - * consider_zone_gc should be last, because the other operations - * might return memory to zones. - */ consider_machine_collect(); - if (consider_buffer_cache_collect != NULL) { - buf_large_zfree = (*consider_buffer_cache_collect)(0); - } - consider_zone_gc(buf_large_zfree); + m_drain(); + + do { + if (consider_buffer_cache_collect != NULL) { + buf_large_zfree = (*consider_buffer_cache_collect)(0); + } + if (first_try == TRUE || buf_large_zfree == TRUE) { + /* + * consider_zone_gc should be last, because the other operations + * might return memory to zones. + */ + consider_zone_gc(); + } + first_try = FALSE; + + } while (buf_large_zfree == TRUE && vm_page_free_count < vm_page_free_target); consider_machine_adjust(); } - assert_wait((event_t) &vm_pageout_garbage_collect, THREAD_UNINT); thread_block_parameter((thread_continue_t) vm_pageout_garbage_collect, (void *)1); @@ -2568,6 +4536,62 @@ vm_pageout_garbage_collect(int collect) } +#if VM_PAGE_BUCKETS_CHECK +#if VM_PAGE_FAKE_BUCKETS +extern vm_map_offset_t vm_page_fake_buckets_start, vm_page_fake_buckets_end; +#endif /* VM_PAGE_FAKE_BUCKETS */ +#endif /* VM_PAGE_BUCKETS_CHECK */ + + +#define FBDP_TEST_COLLAPSE_COMPRESSOR 0 +#define FBDP_TEST_WIRE_AND_EXTRACT 0 +#define FBDP_TEST_PAGE_WIRE_OVERFLOW 0 + +#if FBDP_TEST_COLLAPSE_COMPRESSOR +extern boolean_t vm_object_collapse_compressor_allowed; +#include +#endif /* FBDP_TEST_COLLAPSE_COMPRESSOR */ + +#if FBDP_TEST_WIRE_AND_EXTRACT +extern ledger_template_t task_ledger_template; +#include +extern ppnum_t vm_map_get_phys_page(vm_map_t map, + vm_offset_t offset); +#endif /* FBDP_TEST_WIRE_AND_EXTRACT */ + + +void +vm_set_restrictions() +{ + host_basic_info_data_t hinfo; + mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT; + +#define BSD_HOST 1 + host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count); + + assert(hinfo.max_cpus > 0); + + if (hinfo.max_cpus <= 3) { + /* + * on systems with a limited number of CPUS, bind the + * 4 major threads that can free memory and that tend to use + * a fair bit of CPU under pressured conditions to a single processor. + * This insures that these threads don't hog all of the available CPUs + * (important for camera launch), while allowing them to run independently + * w/r to locks... the 4 threads are + * vm_pageout_scan, vm_pageout_iothread_internal (compressor), + * vm_compressor_swap_trigger_thread (minor and major compactions), + * memorystatus_thread (jetsams). + * + * the first time the thread is run, it is responsible for checking the + * state of vm_restricted_to_single_processor, and if TRUE it calls + * thread_bind_master... someday this should be replaced with a group + * scheduling mechanism and KPI. + */ + vm_restricted_to_single_processor = TRUE; + } +} + void vm_pageout(void) @@ -2581,20 +4605,27 @@ vm_pageout(void) * Set thread privileges. */ s = splsched(); + thread_lock(self); - self->priority = BASEPRI_PREEMPT - 1; - set_sched_pri(self, self->priority); + self->options |= TH_OPT_VMPRIV; + sched_set_thread_base_priority(self, BASEPRI_PREEMPT - 1); thread_unlock(self); if (!self->reserved_stack) self->reserved_stack = self->kernel_stack; + if (vm_restricted_to_single_processor == TRUE) + thread_vm_bind_group_add(); + splx(s); /* * Initialize some paging parameters. */ + if (vm_pageout_swap_wait == 0) + vm_pageout_swap_wait = VM_PAGEOUT_SWAP_WAIT; + if (vm_pageout_idle_wait == 0) vm_pageout_idle_wait = VM_PAGEOUT_IDLE_WAIT; @@ -2642,22 +4673,27 @@ vm_pageout(void) vm_page_free_reserve(0); - queue_init(&vm_pageout_queue_external.pgo_pending); + vm_page_queue_init(&vm_pageout_queue_external.pgo_pending); vm_pageout_queue_external.pgo_maxlaundry = VM_PAGE_LAUNDRY_MAX; vm_pageout_queue_external.pgo_laundry = 0; vm_pageout_queue_external.pgo_idle = FALSE; vm_pageout_queue_external.pgo_busy = FALSE; vm_pageout_queue_external.pgo_throttled = FALSE; vm_pageout_queue_external.pgo_draining = FALSE; + vm_pageout_queue_external.pgo_lowpriority = FALSE; + vm_pageout_queue_external.pgo_tid = -1; + vm_pageout_queue_external.pgo_inited = FALSE; - queue_init(&vm_pageout_queue_internal.pgo_pending); + vm_page_queue_init(&vm_pageout_queue_internal.pgo_pending); vm_pageout_queue_internal.pgo_maxlaundry = 0; vm_pageout_queue_internal.pgo_laundry = 0; vm_pageout_queue_internal.pgo_idle = FALSE; vm_pageout_queue_internal.pgo_busy = FALSE; vm_pageout_queue_internal.pgo_throttled = FALSE; vm_pageout_queue_internal.pgo_draining = FALSE; - + vm_pageout_queue_internal.pgo_lowpriority = FALSE; + vm_pageout_queue_internal.pgo_tid = -1; + vm_pageout_queue_internal.pgo_inited = FALSE; /* internal pageout thread started when default pager registered first time */ /* external pageout and garbage collection threads started here */ @@ -2671,16 +4707,366 @@ vm_pageout(void) thread_deallocate(vm_pageout_external_iothread); result = kernel_thread_start_priority((thread_continue_t)vm_pageout_garbage_collect, NULL, - MINPRI_KERNEL, + BASEPRI_DEFAULT, &thread); if (result != KERN_SUCCESS) panic("vm_pageout_garbage_collect: create failed"); thread_deallocate(thread); +#if VM_PRESSURE_EVENTS + result = kernel_thread_start_priority((thread_continue_t)vm_pressure_thread, NULL, + BASEPRI_DEFAULT, + &thread); + + if (result != KERN_SUCCESS) + panic("vm_pressure_thread: create failed"); + + thread_deallocate(thread); +#endif + vm_object_reaper_init(); + bzero(&vm_config, sizeof(vm_config)); + + switch(vm_compressor_mode) { + + case VM_PAGER_DEFAULT: + printf("mapping deprecated VM_PAGER_DEFAULT to VM_PAGER_COMPRESSOR_WITH_SWAP\n"); + + case VM_PAGER_COMPRESSOR_WITH_SWAP: + vm_config.compressor_is_present = TRUE; + vm_config.swap_is_present = TRUE; + vm_config.compressor_is_active = TRUE; + vm_config.swap_is_active = TRUE; + break; + + case VM_PAGER_COMPRESSOR_NO_SWAP: + vm_config.compressor_is_present = TRUE; + vm_config.swap_is_present = TRUE; + vm_config.compressor_is_active = TRUE; + break; + + case VM_PAGER_FREEZER_DEFAULT: + printf("mapping deprecated VM_PAGER_FREEZER_DEFAULT to VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP\n"); + + case VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP: + vm_config.compressor_is_present = TRUE; + vm_config.swap_is_present = TRUE; + break; + + case VM_PAGER_COMPRESSOR_NO_SWAP_PLUS_FREEZER_COMPRESSOR_WITH_SWAP: + vm_config.compressor_is_present = TRUE; + vm_config.swap_is_present = TRUE; + vm_config.compressor_is_active = TRUE; + vm_config.freezer_swap_is_active = TRUE; + break; + + case VM_PAGER_NOT_CONFIGURED: + break; + + default: + printf("unknown compressor mode - %x\n", vm_compressor_mode); + break; + } + if (VM_CONFIG_COMPRESSOR_IS_PRESENT) + vm_compressor_pager_init(); + +#if VM_PRESSURE_EVENTS + vm_pressure_events_enabled = TRUE; +#endif /* VM_PRESSURE_EVENTS */ + +#if CONFIG_PHANTOM_CACHE + vm_phantom_cache_init(); +#endif +#if VM_PAGE_BUCKETS_CHECK +#if VM_PAGE_FAKE_BUCKETS + printf("**** DEBUG: protecting fake buckets [0x%llx:0x%llx]\n", + (uint64_t) vm_page_fake_buckets_start, + (uint64_t) vm_page_fake_buckets_end); + pmap_protect(kernel_pmap, + vm_page_fake_buckets_start, + vm_page_fake_buckets_end, + VM_PROT_READ); +// *(char *) vm_page_fake_buckets_start = 'x'; /* panic! */ +#endif /* VM_PAGE_FAKE_BUCKETS */ +#endif /* VM_PAGE_BUCKETS_CHECK */ + +#if VM_OBJECT_TRACKING + vm_object_tracking_init(); +#endif /* VM_OBJECT_TRACKING */ + + +#if FBDP_TEST_COLLAPSE_COMPRESSOR + vm_object_size_t backing_size, top_size; + vm_object_t backing_object, top_object; + vm_map_offset_t backing_offset, top_offset; + unsigned char *backing_address, *top_address; + kern_return_t kr; + + printf("FBDP_TEST_COLLAPSE_COMPRESSOR:\n"); + + /* create backing object */ + backing_size = 15 * PAGE_SIZE; + backing_object = vm_object_allocate(backing_size); + assert(backing_object != VM_OBJECT_NULL); + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: created backing object %p\n", + backing_object); + /* map backing object */ + backing_offset = 0; + kr = vm_map_enter(kernel_map, &backing_offset, backing_size, 0, + VM_FLAGS_ANYWHERE, backing_object, 0, FALSE, + VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT); + assert(kr == KERN_SUCCESS); + backing_address = (unsigned char *) backing_offset; + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: " + "mapped backing object %p at 0x%llx\n", + backing_object, (uint64_t) backing_offset); + /* populate with pages to be compressed in backing object */ + backing_address[0x1*PAGE_SIZE] = 0xB1; + backing_address[0x4*PAGE_SIZE] = 0xB4; + backing_address[0x7*PAGE_SIZE] = 0xB7; + backing_address[0xa*PAGE_SIZE] = 0xBA; + backing_address[0xd*PAGE_SIZE] = 0xBD; + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: " + "populated pages to be compressed in " + "backing_object %p\n", backing_object); + /* compress backing object */ + vm_object_pageout(backing_object); + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: compressing backing_object %p\n", + backing_object); + /* wait for all the pages to be gone */ + while (*(volatile int *)&backing_object->resident_page_count != 0) + IODelay(10); + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: backing_object %p compressed\n", + backing_object); + /* populate with pages to be resident in backing object */ + backing_address[0x0*PAGE_SIZE] = 0xB0; + backing_address[0x3*PAGE_SIZE] = 0xB3; + backing_address[0x6*PAGE_SIZE] = 0xB6; + backing_address[0x9*PAGE_SIZE] = 0xB9; + backing_address[0xc*PAGE_SIZE] = 0xBC; + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: " + "populated pages to be resident in " + "backing_object %p\n", backing_object); + /* leave the other pages absent */ + /* mess with the paging_offset of the backing_object */ + assert(backing_object->paging_offset == 0); + backing_object->paging_offset = 0x3000; + + /* create top object */ + top_size = 9 * PAGE_SIZE; + top_object = vm_object_allocate(top_size); + assert(top_object != VM_OBJECT_NULL); + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: created top object %p\n", + top_object); + /* map top object */ + top_offset = 0; + kr = vm_map_enter(kernel_map, &top_offset, top_size, 0, + VM_FLAGS_ANYWHERE, top_object, 0, FALSE, + VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT); + assert(kr == KERN_SUCCESS); + top_address = (unsigned char *) top_offset; + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: " + "mapped top object %p at 0x%llx\n", + top_object, (uint64_t) top_offset); + /* populate with pages to be compressed in top object */ + top_address[0x3*PAGE_SIZE] = 0xA3; + top_address[0x4*PAGE_SIZE] = 0xA4; + top_address[0x5*PAGE_SIZE] = 0xA5; + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: " + "populated pages to be compressed in " + "top_object %p\n", top_object); + /* compress top object */ + vm_object_pageout(top_object); + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: compressing top_object %p\n", + top_object); + /* wait for all the pages to be gone */ + while (top_object->resident_page_count != 0); + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: top_object %p compressed\n", + top_object); + /* populate with pages to be resident in top object */ + top_address[0x0*PAGE_SIZE] = 0xA0; + top_address[0x1*PAGE_SIZE] = 0xA1; + top_address[0x2*PAGE_SIZE] = 0xA2; + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: " + "populated pages to be resident in " + "top_object %p\n", top_object); + /* leave the other pages absent */ + + /* link the 2 objects */ + vm_object_reference(backing_object); + top_object->shadow = backing_object; + top_object->vo_shadow_offset = 0x3000; + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: linked %p and %p\n", + top_object, backing_object); + + /* unmap backing object */ + vm_map_remove(kernel_map, + backing_offset, + backing_offset + backing_size, + 0); + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: " + "unmapped backing_object %p [0x%llx:0x%llx]\n", + backing_object, + (uint64_t) backing_offset, + (uint64_t) (backing_offset + backing_size)); + + /* collapse */ + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: collapsing %p\n", top_object); + vm_object_lock(top_object); + vm_object_collapse(top_object, 0, FALSE); + vm_object_unlock(top_object); + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: collapsed %p\n", top_object); + + /* did it work? */ + if (top_object->shadow != VM_OBJECT_NULL) { + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: not collapsed\n"); + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: FAIL\n"); + if (vm_object_collapse_compressor_allowed) { + panic("FBDP_TEST_COLLAPSE_COMPRESSOR: FAIL\n"); + } + } else { + /* check the contents of the mapping */ + unsigned char expect[9] = + { 0xA0, 0xA1, 0xA2, /* resident in top */ + 0xA3, 0xA4, 0xA5, /* compressed in top */ + 0xB9, /* resident in backing + shadow_offset */ + 0xBD, /* compressed in backing + shadow_offset + paging_offset */ + 0x00 }; /* absent in both */ + unsigned char actual[9]; + unsigned int i, errors; + + errors = 0; + for (i = 0; i < sizeof (actual); i++) { + actual[i] = (unsigned char) top_address[i*PAGE_SIZE]; + if (actual[i] != expect[i]) { + errors++; + } + } + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: " + "actual [%x %x %x %x %x %x %x %x %x] " + "expect [%x %x %x %x %x %x %x %x %x] " + "%d errors\n", + actual[0], actual[1], actual[2], actual[3], + actual[4], actual[5], actual[6], actual[7], + actual[8], + expect[0], expect[1], expect[2], expect[3], + expect[4], expect[5], expect[6], expect[7], + expect[8], + errors); + if (errors) { + panic("FBDP_TEST_COLLAPSE_COMPRESSOR: FAIL\n"); + } else { + printf("FBDP_TEST_COLLAPSE_COMPRESSOR: PASS\n"); + } + } +#endif /* FBDP_TEST_COLLAPSE_COMPRESSOR */ + +#if FBDP_TEST_WIRE_AND_EXTRACT + ledger_t ledger; + vm_map_t user_map, wire_map; + mach_vm_address_t user_addr, wire_addr; + mach_vm_size_t user_size, wire_size; + mach_vm_offset_t cur_offset; + vm_prot_t cur_prot, max_prot; + ppnum_t user_ppnum, wire_ppnum; + kern_return_t kr; + + ledger = ledger_instantiate(task_ledger_template, + LEDGER_CREATE_ACTIVE_ENTRIES); + user_map = vm_map_create(pmap_create(ledger, 0, PMAP_CREATE_64BIT), + 0x100000000ULL, + 0x200000000ULL, + TRUE); + wire_map = vm_map_create(NULL, + 0x100000000ULL, + 0x200000000ULL, + TRUE); + user_addr = 0; + user_size = 0x10000; + kr = mach_vm_allocate(user_map, + &user_addr, + user_size, + VM_FLAGS_ANYWHERE); + assert(kr == KERN_SUCCESS); + wire_addr = 0; + wire_size = user_size; + kr = mach_vm_remap(wire_map, + &wire_addr, + wire_size, + 0, + VM_FLAGS_ANYWHERE, + user_map, + user_addr, + FALSE, + &cur_prot, + &max_prot, + VM_INHERIT_NONE); + assert(kr == KERN_SUCCESS); + for (cur_offset = 0; + cur_offset < wire_size; + cur_offset += PAGE_SIZE) { + kr = vm_map_wire_and_extract(wire_map, + wire_addr + cur_offset, + VM_PROT_DEFAULT | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK), + TRUE, + &wire_ppnum); + assert(kr == KERN_SUCCESS); + user_ppnum = vm_map_get_phys_page(user_map, + user_addr + cur_offset); + printf("FBDP_TEST_WIRE_AND_EXTRACT: kr=0x%x " + "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n", + kr, + user_map, user_addr + cur_offset, user_ppnum, + wire_map, wire_addr + cur_offset, wire_ppnum); + if (kr != KERN_SUCCESS || + wire_ppnum == 0 || + wire_ppnum != user_ppnum) { + panic("FBDP_TEST_WIRE_AND_EXTRACT: FAIL\n"); + } + } + cur_offset -= PAGE_SIZE; + kr = vm_map_wire_and_extract(wire_map, + wire_addr + cur_offset, + VM_PROT_DEFAULT, + TRUE, + &wire_ppnum); + assert(kr == KERN_SUCCESS); + printf("FBDP_TEST_WIRE_AND_EXTRACT: re-wire kr=0x%x " + "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n", + kr, + user_map, user_addr + cur_offset, user_ppnum, + wire_map, wire_addr + cur_offset, wire_ppnum); + if (kr != KERN_SUCCESS || + wire_ppnum == 0 || + wire_ppnum != user_ppnum) { + panic("FBDP_TEST_WIRE_AND_EXTRACT: FAIL\n"); + } + + printf("FBDP_TEST_WIRE_AND_EXTRACT: PASS\n"); +#endif /* FBDP_TEST_WIRE_AND_EXTRACT */ + +#if FBDP_TEST_PAGE_WIRE_OVERFLOW + vm_object_t fbdp_object; + vm_page_t fbdp_page; + + printf("FBDP_TEST_PAGE_WIRE_OVERFLOW: starting...\n"); + + fbdp_object = vm_object_allocate(PAGE_SIZE); + vm_object_lock(fbdp_object); + fbdp_page = vm_page_alloc(fbdp_object, 0x0); + vm_page_lock_queues(); + do { + vm_page_wire(fbdp_page, 1, FALSE); + } while (fbdp_page->wire_count != 0); + vm_page_unlock_queues(); + vm_object_unlock(fbdp_object); + panic("FBDP(%p,%p): wire_count overflow not detected\n", + fbdp_object, fbdp_page); +#endif /* FBDP_TEST_PAGE_WIRE_OVERFLOW */ + vm_pageout_continue(); /* @@ -2707,71 +5093,109 @@ vm_pageout(void) /*NOTREACHED*/ } + + +int vm_compressor_thread_count = 2; + kern_return_t vm_pageout_internal_start(void) { - kern_return_t result; + kern_return_t result; + int i; + host_basic_info_data_t hinfo; + + assert (VM_CONFIG_COMPRESSOR_IS_PRESENT); - vm_pageout_queue_internal.pgo_maxlaundry = VM_PAGE_LAUNDRY_MAX; - result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal, NULL, BASEPRI_PREEMPT - 1, &vm_pageout_internal_iothread); - if (result == KERN_SUCCESS) - thread_deallocate(vm_pageout_internal_iothread); + mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT; +#define BSD_HOST 1 + host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count); + + assert(hinfo.max_cpus > 0); + + if (vm_compressor_thread_count >= hinfo.max_cpus) + vm_compressor_thread_count = hinfo.max_cpus - 1; + if (vm_compressor_thread_count <= 0) + vm_compressor_thread_count = 1; + else if (vm_compressor_thread_count > MAX_COMPRESSOR_THREAD_COUNT) + vm_compressor_thread_count = MAX_COMPRESSOR_THREAD_COUNT; + + if (vm_compressor_immediate_preferred == TRUE) { + vm_pageout_immediate_chead = NULL; + vm_pageout_immediate_scratch_buf = kalloc(vm_compressor_get_encode_scratch_size()); + + vm_compressor_thread_count = 1; + } + + vm_pageout_queue_internal.pgo_maxlaundry = (vm_compressor_thread_count * 4) * VM_PAGE_LAUNDRY_MAX; + + for (i = 0; i < vm_compressor_thread_count; i++) { + ciq[i].id = i; + ciq[i].q = &vm_pageout_queue_internal; + ciq[i].current_chead = NULL; + ciq[i].scratch_buf = kalloc(COMPRESSOR_SCRATCH_BUF_SIZE); + + result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal, (void *)&ciq[i], BASEPRI_PREEMPT - 1, &vm_pageout_internal_iothread); + + if (result == KERN_SUCCESS) + thread_deallocate(vm_pageout_internal_iothread); + else + break; + } return result; } - +#if CONFIG_IOSCHED /* - * when marshalling pages into a UPL and subsequently committing - * or aborting them, it is necessary to hold - * the vm_page_queue_lock (a hot global lock) for certain operations - * on the page... however, the majority of the work can be done - * while merely holding the object lock... in fact there are certain - * collections of pages that don't require any work brokered by the - * vm_page_queue_lock... to mitigate the time spent behind the global - * lock, go to a 2 pass algorithm... collect pages up to DELAYED_WORK_LIMIT - * while doing all of the work that doesn't require the vm_page_queue_lock... - * then call dw_do_work to acquire the vm_page_queue_lock and do the - * necessary work for each page... we will grab the busy bit on the page - * if it's not already held so that dw_do_work can drop the object lock - * if it can't immediately take the vm_page_queue_lock in order to compete - * for the locks in the same order that vm_pageout_scan takes them. - * the operation names are modeled after the names of the routines that - * need to be called in order to make the changes very obvious in the - * original loop + * To support I/O Expedite for compressed files we mark the upls with special flags. + * The way decmpfs works is that we create a big upl which marks all the pages needed to + * represent the compressed file as busy. We tag this upl with the flag UPL_DECMP_REQ. Decmpfs + * then issues smaller I/Os for compressed I/Os, deflates them and puts the data into the pages + * being held in the big original UPL. We mark each of these smaller UPLs with the flag + * UPL_DECMP_REAL_IO. Any outstanding real I/O UPL is tracked by the big req upl using the + * decmp_io_upl field (in the upl structure). This link is protected in the forward direction + * by the req upl lock (the reverse link doesnt need synch. since we never inspect this link + * unless the real I/O upl is being destroyed). */ -#define DELAYED_WORK_LIMIT 32 - -#define DW_vm_page_unwire 0x01 -#define DW_vm_page_wire 0x02 -#define DW_vm_page_free 0x04 -#define DW_vm_page_activate 0x08 -#define DW_vm_page_deactivate_internal 0x10 -#define DW_vm_page_speculate 0x20 -#define DW_vm_page_lru 0x40 -#define DW_vm_pageout_throttle_up 0x80 -#define DW_PAGE_WAKEUP 0x100 -#define DW_clear_busy 0x200 -#define DW_clear_reference 0x400 -#define DW_set_reference 0x800 - -struct dw { - vm_page_t dw_m; - int dw_mask; -}; - - -static void dw_do_work(vm_object_t object, struct dw *dwp, int dw_count); +static void +upl_set_decmp_info(upl_t upl, upl_t src_upl) +{ + assert((src_upl->flags & UPL_DECMP_REQ) != 0); + + upl_lock(src_upl); + if (src_upl->decmp_io_upl) { + /* + * If there is already an alive real I/O UPL, ignore this new UPL. + * This case should rarely happen and even if it does, it just means + * that we might issue a spurious expedite which the driver is expected + * to handle. + */ + upl_unlock(src_upl); + return; + } + src_upl->decmp_io_upl = (void *)upl; + src_upl->ref_count++; + + upl->flags |= UPL_DECMP_REAL_IO; + upl->decmp_io_upl = (void *)src_upl; + upl_unlock(src_upl); +} +#endif /* CONFIG_IOSCHED */ +#if UPL_DEBUG +int upl_debug_enabled = 1; +#else +int upl_debug_enabled = 0; +#endif static upl_t upl_create(int type, int flags, upl_size_t size) { upl_t upl; - int page_field_size = 0; + vm_size_t page_field_size = 0; int upl_flags = 0; - int upl_size = sizeof(struct upl); + vm_size_t upl_size = sizeof(struct upl); size = round_page_32(size); @@ -2782,7 +5206,7 @@ upl_create(int type, int flags, upl_size_t size) upl_flags |= UPL_LITE; } if (type & UPL_CREATE_INTERNAL) { - upl_size += (int) sizeof(struct upl_page_info) * atop(size); + upl_size += sizeof(struct upl_page_info) * atop(size); upl_flags |= UPL_INTERNAL; } @@ -2792,19 +5216,45 @@ upl_create(int type, int flags, upl_size_t size) bzero((char *)upl + upl_size, page_field_size); upl->flags = upl_flags | flags; - upl->src_object = NULL; upl->kaddr = (vm_offset_t)0; upl->size = 0; upl->map_object = NULL; upl->ref_count = 1; + upl->ext_ref_count = 0; upl->highest_page = 0; upl_lock_init(upl); upl->vector_upl = NULL; + upl->associated_upl = NULL; +#if CONFIG_IOSCHED + if (type & UPL_CREATE_IO_TRACKING) { + upl->upl_priority = proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO); + } + + upl->upl_reprio_info = 0; + upl->decmp_io_upl = 0; + if ((type & UPL_CREATE_INTERNAL) && (type & UPL_CREATE_EXPEDITE_SUP)) { + /* Only support expedite on internal UPLs */ + thread_t curthread = current_thread(); + upl->upl_reprio_info = (uint64_t *)kalloc(sizeof(uint64_t) * atop(size)); + bzero(upl->upl_reprio_info, (sizeof(uint64_t) * atop(size))); + upl->flags |= UPL_EXPEDITE_SUPPORTED; + if (curthread->decmp_upl != NULL) + upl_set_decmp_info(upl, curthread->decmp_upl); + } +#endif +#if CONFIG_IOSCHED || UPL_DEBUG + if ((type & UPL_CREATE_IO_TRACKING) || upl_debug_enabled) { + upl->upl_creator = current_thread(); + upl->uplq.next = 0; + upl->uplq.prev = 0; + upl->flags |= UPL_TRACKED_BY_OBJECT; + } +#endif + #if UPL_DEBUG upl->ubc_alias1 = 0; upl->ubc_alias2 = 0; - upl->upl_creator = current_thread(); upl->upl_state = 0; upl->upl_commit_index = 0; bzero(&upl->upl_commit_records[0], sizeof(upl->upl_commit_records)); @@ -2821,8 +5271,24 @@ upl_destroy(upl_t upl) int page_field_size; /* bit field in word size buf */ int size; -#if UPL_DEBUG - { + if (upl->ext_ref_count) { + panic("upl(%p) ext_ref_count", upl); + } + +#if CONFIG_IOSCHED + if ((upl->flags & UPL_DECMP_REAL_IO) && upl->decmp_io_upl) { + upl_t src_upl; + src_upl = upl->decmp_io_upl; + assert((src_upl->flags & UPL_DECMP_REQ) != 0); + upl_lock(src_upl); + src_upl->decmp_io_upl = NULL; + upl_unlock(src_upl); + upl_deallocate(src_upl); + } +#endif /* CONFIG_IOSCHED */ + +#if CONFIG_IOSCHED || UPL_DEBUG + if ((upl->flags & UPL_TRACKED_BY_OBJECT) && !(upl->flags & UPL_VECTOR)) { vm_object_t object; if (upl->flags & UPL_SHADOWED) { @@ -2830,11 +5296,14 @@ upl_destroy(upl_t upl) } else { object = upl->map_object; } + vm_object_lock(object); queue_remove(&object->uplq, upl, upl_t, uplq); + vm_object_activity_end(object); + vm_object_collapse(object, 0, TRUE); vm_object_unlock(object); } -#endif /* UPL_DEBUG */ +#endif /* * drop a reference on the map_object whether or * not a pageout object is inserted @@ -2854,6 +5323,12 @@ upl_destroy(upl_t upl) } upl_lock_destroy(upl); upl->vector_upl = (vector_upl_t) 0xfeedbeef; + +#if CONFIG_IOSCHED + if (upl->flags & UPL_EXPEDITE_SUPPORTED) + kfree(upl->upl_reprio_info, sizeof(uint64_t) * (size/PAGE_SIZE)); +#endif + if (upl->flags & UPL_INTERNAL) { kfree(upl, sizeof(struct upl) + @@ -2864,24 +5339,56 @@ upl_destroy(upl_t upl) } } -void uc_upl_dealloc(upl_t upl); -__private_extern__ void -uc_upl_dealloc(upl_t upl) -{ - if (--upl->ref_count == 0) - upl_destroy(upl); -} - void upl_deallocate(upl_t upl) { + upl_lock(upl); if (--upl->ref_count == 0) { if(vector_upl_is_valid(upl)) vector_upl_deallocate(upl); + upl_unlock(upl); upl_destroy(upl); } + else + upl_unlock(upl); +} + +#if CONFIG_IOSCHED +void +upl_mark_decmp(upl_t upl) +{ + if (upl->flags & UPL_TRACKED_BY_OBJECT) { + upl->flags |= UPL_DECMP_REQ; + upl->upl_creator->decmp_upl = (void *)upl; + } +} + +void +upl_unmark_decmp(upl_t upl) +{ + if(upl && (upl->flags & UPL_DECMP_REQ)) { + upl->upl_creator->decmp_upl = NULL; + } +} + +#endif /* CONFIG_IOSCHED */ + +#define VM_PAGE_Q_BACKING_UP(q) \ + ((q)->pgo_laundry >= (((q)->pgo_maxlaundry * 8) / 10)) + +boolean_t must_throttle_writes(void); + +boolean_t +must_throttle_writes() +{ + if (VM_PAGE_Q_BACKING_UP(&vm_pageout_queue_external) && + vm_page_pageable_external_count > (AVAILABLE_NON_COMPRESSED_MEMORY * 6) / 10) + return (TRUE); + + return (FALSE); } + #if DEVELOPMENT || DEBUG /*/* * Statistics about UPL enforcement of copy-on-write obligations. @@ -2947,11 +5454,12 @@ vm_object_upl_request( upl_t *upl_ptr, upl_page_info_array_t user_page_list, unsigned int *page_list_count, - int cntrl_flags) + upl_control_flags_t cntrl_flags) { vm_page_t dst_page = VM_PAGE_NULL; vm_object_offset_t dst_offset; upl_size_t xfer_size; + unsigned int size_in_pages; boolean_t dirty; boolean_t hw_dirty; upl_t upl = NULL; @@ -2963,9 +5471,13 @@ vm_object_upl_request( int refmod_state = 0; wpl_array_t lite_list = NULL; vm_object_t last_copy_object; - struct dw dw_array[DELAYED_WORK_LIMIT]; - struct dw *dwp; + struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT]; + struct vm_page_delayed_work *dwp; int dw_count; + int dw_limit; + int io_tracking_flag = 0; + int grab_options; + ppnum_t phys_page; if (cntrl_flags & ~UPL_VALID_FLAGS) { /* @@ -2980,16 +5492,25 @@ vm_object_upl_request( panic("vm_object_upl_request: contiguous object specified\n"); - if ((size / PAGE_SIZE) > MAX_UPL_SIZE) - size = MAX_UPL_SIZE * PAGE_SIZE; + if (size > MAX_UPL_SIZE_BYTES) + size = MAX_UPL_SIZE_BYTES; if ( (cntrl_flags & UPL_SET_INTERNAL) && page_list_count != NULL) - *page_list_count = MAX_UPL_SIZE; + *page_list_count = MAX_UPL_SIZE_BYTES >> PAGE_SHIFT; + +#if CONFIG_IOSCHED || UPL_DEBUG + if (object->io_tracking || upl_debug_enabled) + io_tracking_flag |= UPL_CREATE_IO_TRACKING; +#endif +#if CONFIG_IOSCHED + if (object->io_tracking) + io_tracking_flag |= UPL_CREATE_EXPEDITE_SUP; +#endif if (cntrl_flags & UPL_SET_INTERNAL) { if (cntrl_flags & UPL_SET_LITE) { - upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE, 0, size); + upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE | io_tracking_flag, 0, size); user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl)); lite_list = (wpl_array_t) @@ -3000,7 +5521,7 @@ vm_object_upl_request( lite_list = NULL; } } else { - upl = upl_create(UPL_CREATE_INTERNAL, 0, size); + upl = upl_create(UPL_CREATE_INTERNAL | io_tracking_flag, 0, size); user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl)); if (size == 0) { @@ -3010,14 +5531,14 @@ vm_object_upl_request( } else { if (cntrl_flags & UPL_SET_LITE) { - upl = upl_create(UPL_CREATE_EXTERNAL | UPL_CREATE_LITE, 0, size); + upl = upl_create(UPL_CREATE_EXTERNAL | UPL_CREATE_LITE | io_tracking_flag, 0, size); lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl)); if (size == 0) { lite_list = NULL; } } else { - upl = upl_create(UPL_CREATE_EXTERNAL, 0, size); + upl = upl_create(UPL_CREATE_EXTERNAL | io_tracking_flag, 0, size); } } *upl_ptr = upl; @@ -3037,7 +5558,7 @@ vm_object_upl_request( upl->map_object->pageout = TRUE; upl->map_object->can_persist = FALSE; upl->map_object->copy_strategy = MEMORY_OBJECT_COPY_NONE; - upl->map_object->shadow_offset = offset; + upl->map_object->vo_shadow_offset = offset; upl->map_object->wimg_bits = object->wimg_bits; VM_PAGE_GRAB_FICTITIOUS(alias_page); @@ -3060,16 +5581,25 @@ vm_object_upl_request( vm_object_lock(object); vm_object_activity_begin(object); + grab_options = 0; +#if CONFIG_SECLUDED_MEMORY + if (object->can_grab_secluded) { + grab_options |= VM_PAGE_GRAB_SECLUDED; + } +#endif /* CONFIG_SECLUDED_MEMORY */ + /* * we can lock in the paging_offset once paging_in_progress is set */ upl->size = size; upl->offset = offset + object->paging_offset; -#if UPL_DEBUG - queue_enter(&object->uplq, upl, upl_t, uplq); -#endif /* UPL_DEBUG */ - +#if CONFIG_IOSCHED || UPL_DEBUG + if (object->io_tracking || upl_debug_enabled) { + vm_object_activity_begin(object); + queue_enter(&object->uplq, upl, upl_t, uplq); + } +#endif if ((cntrl_flags & UPL_WILL_MODIFY) && object->copy != VM_OBJECT_NULL) { /* * Honor copy-on-write obligations @@ -3101,9 +5631,32 @@ vm_object_upl_request( xfer_size = size; dst_offset = offset; + size_in_pages = size / PAGE_SIZE; dwp = &dw_array[0]; dw_count = 0; + dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT); + + if (vm_page_free_count > (vm_page_free_target + size_in_pages) || + object->resident_page_count < ((MAX_UPL_SIZE_BYTES * 2) >> PAGE_SHIFT)) + object->scan_collisions = 0; + + if ((cntrl_flags & UPL_WILL_MODIFY) && must_throttle_writes() == TRUE) { + boolean_t isSSD = FALSE; + + vnode_pager_get_isSSD(object->pager, &isSSD); + vm_object_unlock(object); + + OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages); + + if (isSSD == TRUE) + delay(1000 * size_in_pages); + else + delay(5000 * size_in_pages); + OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages); + + vm_object_lock(object); + } while (xfer_size) { @@ -3121,13 +5674,16 @@ vm_object_upl_request( dst_page->fictitious || dst_page->absent || dst_page->error || - (VM_PAGE_WIRED(dst_page) && !dst_page->pageout && !dst_page->list_req_pending)) { - + dst_page->cleaning || + (VM_PAGE_WIRED(dst_page))) { + if (user_page_list) user_page_list[entry].phys_addr = 0; goto try_next_page; } + phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page); + /* * grab this up front... * a high percentange of the time we're going to @@ -3136,11 +5692,11 @@ vm_object_upl_request( * the pmap layer by grabbing it here and recording it */ if (dst_page->pmapped) - refmod_state = pmap_get_refmod(dst_page->phys_page); + refmod_state = pmap_get_refmod(phys_page); else refmod_state = 0; - if ( (refmod_state & VM_MEM_REFERENCED) && dst_page->inactive ) { + if ( (refmod_state & VM_MEM_REFERENCED) && VM_PAGE_INACTIVE(dst_page)) { /* * page is on inactive list and referenced... * reactivate it now... this gets it out of the @@ -3153,7 +5709,7 @@ vm_object_upl_request( /* * we're only asking for DIRTY pages to be returned */ - if (dst_page->list_req_pending || !(cntrl_flags & UPL_FOR_PAGEOUT)) { + if (dst_page->laundry || !(cntrl_flags & UPL_FOR_PAGEOUT)) { /* * if we were the page stolen by vm_pageout_scan to be * cleaned (as opposed to a buddy being clustered in @@ -3169,12 +5725,11 @@ vm_object_upl_request( * this is a request for a PAGEOUT cluster and this page * is merely along for the ride as a 'buddy'... not only * does it have to be dirty to be returned, but it also - * can't have been referenced recently... note that we've - * already filtered above based on whether this page is - * currently on the inactive queue or it meets the page - * ticket (generation count) check + * can't have been referenced recently... */ - if ( (cntrl_flags & UPL_CLEAN_IN_PLACE || !(refmod_state & VM_MEM_REFERENCED)) && + if ( (hibernate_cleaning_in_progress == TRUE || + (!((refmod_state & VM_MEM_REFERENCED) || dst_page->reference) || + (dst_page->vm_page_q_state == VM_PAGE_ON_THROTTLED_Q))) && ((refmod_state & VM_MEM_MODIFIED) || dst_page->dirty || dst_page->precious) ) { goto check_busy; } @@ -3183,16 +5738,31 @@ dont_return: * if we reach here, we're not to return * the page... go on to the next one */ + if (dst_page->laundry == TRUE) { + /* + * if we get here, the page is not 'cleaning' (filtered out above). + * since it has been referenced, remove it from the laundry + * so we don't pay the cost of an I/O to clean a page + * we're just going to take back + */ + vm_page_lockspin_queues(); + + vm_pageout_steal_laundry(dst_page, TRUE); + vm_page_activate(dst_page); + + vm_page_unlock_queues(); + } if (user_page_list) user_page_list[entry].phys_addr = 0; goto try_next_page; } check_busy: - if (dst_page->busy && (!(dst_page->list_req_pending && (dst_page->pageout || dst_page->cleaning)))) { - if (cntrl_flags & UPL_NOBLOCK) { - if (user_page_list) + if (dst_page->busy) { + if (cntrl_flags & UPL_NOBLOCK) { + if (user_page_list) user_page_list[entry].phys_addr = 0; + dwp->dw_mask = 0; goto try_next_page; } @@ -3204,15 +5774,6 @@ check_busy: continue; } - /* - * Someone else already cleaning the page? - */ - if ((dst_page->cleaning || dst_page->absent || VM_PAGE_WIRED(dst_page)) && !dst_page->list_req_pending) { - if (user_page_list) - user_page_list[entry].phys_addr = 0; - - goto try_next_page; - } /* * ENCRYPTED SWAP: * The caller is gathering this page and might @@ -3239,16 +5800,11 @@ check_busy: */ dst_page->busy = was_busy; } - if (dst_page->pageout_queue == TRUE) { + if (dst_page->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) { vm_page_lockspin_queues(); -#if CONFIG_EMBEDDED - if (dst_page->laundry) -#else - if (dst_page->pageout_queue == TRUE) -#endif - { + if (dst_page->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) { /* * we've buddied up a page for a clustered pageout * that has already been moved to the pageout @@ -3267,28 +5823,22 @@ check_busy: * were not counted in the initial * vm_pageout_scan work */ - if (dst_page->list_req_pending) + if (dst_page->pageout) encountered_lrp = TRUE; - if ((dst_page->dirty || (dst_page->object->internal && dst_page->precious)) && !dst_page->list_req_pending) { + if ((dst_page->dirty || (object->internal && dst_page->precious))) { if (encountered_lrp) CLUSTER_STAT(pages_at_higher_offsets++;) else CLUSTER_STAT(pages_at_lower_offsets++;) } #endif - /* - * Turn off busy indication on pending - * pageout. Note: we can only get here - * in the request pending case. - */ - dst_page->list_req_pending = FALSE; - dst_page->busy = FALSE; - hw_dirty = refmod_state & VM_MEM_MODIFIED; dirty = hw_dirty ? TRUE : dst_page->dirty; - if (dst_page->phys_page > upl->highest_page) - upl->highest_page = dst_page->phys_page; + if (phys_page > upl->highest_page) + upl->highest_page = phys_page; + + assert (!pmap_is_noencrypt(phys_page)); if (cntrl_flags & UPL_SET_LITE) { unsigned int pg_num; @@ -3298,7 +5848,7 @@ check_busy: lite_list[pg_num>>5] |= 1 << (pg_num & 31); if (hw_dirty) - pmap_clear_modify(dst_page->phys_page); + pmap_clear_modify(phys_page); /* * Mark original page as cleaning @@ -3319,21 +5869,15 @@ check_busy: alias_page->absent = FALSE; alias_page = NULL; } -#if MACH_PAGEMAP - /* - * Record that this page has been - * written out - */ - vm_external_state_set(object->existence_map, dst_page->offset); -#endif /*MACH_PAGEMAP*/ - dst_page->dirty = dirty; + if (dirty) { + SET_PAGE_DIRTY(dst_page, FALSE); + } else { + dst_page->dirty = FALSE; + } if (!dirty) dst_page->precious = TRUE; - if (dst_page->pageout) - dst_page->busy = TRUE; - if ( (cntrl_flags & UPL_ENCRYPT) ) { /* * ENCRYPTED SWAP: @@ -3353,16 +5897,8 @@ check_busy: dst_page->encrypted_cleaning = TRUE; } if ( !(cntrl_flags & UPL_CLEAN_IN_PLACE) ) { - /* - * deny access to the target page - * while it is being worked on - */ - if ((!dst_page->pageout) && ( !VM_PAGE_WIRED(dst_page))) { - dst_page->busy = TRUE; - dst_page->pageout = TRUE; - - dwp->dw_mask |= DW_vm_page_wire; - } + if ( !VM_PAGE_WIRED(dst_page)) + dst_page->free_when_done = TRUE; } } else { if ((cntrl_flags & UPL_WILL_MODIFY) && object->copy != last_copy_object) { @@ -3413,71 +5949,30 @@ check_busy: if (dst_page != VM_PAGE_NULL) { if ((cntrl_flags & UPL_RET_ONLY_ABSENT)) { + /* + * skip over pages already present in the cache + */ + if (user_page_list) + user_page_list[entry].phys_addr = 0; - if ( !(dst_page->absent && dst_page->list_req_pending) ) { - /* - * skip over pages already present in the cache - */ - if (user_page_list) - user_page_list[entry].phys_addr = 0; - - goto try_next_page; - } + goto try_next_page; + } + if (dst_page->fictitious) { + panic("need corner case for fictitious page"); } - if ( !(dst_page->list_req_pending) ) { - - if (dst_page->cleaning) { - /* - * someone else is writing to the page... wait... - */ - PAGE_SLEEP(object, dst_page, THREAD_UNINT); - - continue; - } - } else { - if (dst_page->fictitious && - dst_page->phys_page == vm_page_fictitious_addr) { - assert( !dst_page->speculative); - /* - * dump the fictitious page - */ - dst_page->list_req_pending = FALSE; - - VM_PAGE_FREE(dst_page); - - dst_page = NULL; - - } else if (dst_page->absent) { - /* - * the default_pager case - */ - dst_page->list_req_pending = FALSE; - dst_page->busy = FALSE; - } else if (dst_page->pageout || dst_page->cleaning) { - /* - * page was earmarked by vm_pageout_scan - * to be cleaned and stolen... we're going - * to take it back since we are not attempting - * to read that page and we don't want to stall - * waiting for it to be cleaned for 2 reasons... - * 1 - no use paging it out and back in - * 2 - if we stall, we may casue a deadlock in - * the FS trying to acquire the its locks - * on the VNOP_PAGEOUT path presuming that - * those locks are already held on the read - * path before trying to create this UPL - * - * so undo all of the state that vm_pageout_scan - * hung on this page - */ - dst_page->busy = FALSE; + if (dst_page->busy || dst_page->cleaning) { + /* + * someone else is playing with the + * page. We will have to wait. + */ + PAGE_SLEEP(object, dst_page, THREAD_UNINT); - vm_pageout_queue_steal(dst_page, FALSE); - } + continue; } - } - if (dst_page == VM_PAGE_NULL) { + if (dst_page->laundry) + vm_pageout_steal_laundry(dst_page, FALSE); + } else { if (object->private) { /* * This is a nasty wrinkle for users @@ -3493,11 +5988,32 @@ check_busy: goto try_next_page; } - /* - * need to allocate a page - */ - dst_page = vm_page_grab(); + if (object->scan_collisions) { + /* + * the pageout_scan thread is trying to steal + * pages from this object, but has run into our + * lock... grab 2 pages from the head of the object... + * the first is freed on behalf of pageout_scan, the + * 2nd is for our own use... we use vm_object_page_grab + * in both cases to avoid taking pages from the free + * list since we are under memory pressure and our + * lock on this object is getting in the way of + * relieving it + */ + dst_page = vm_object_page_grab(object); + + if (dst_page != VM_PAGE_NULL) + vm_page_release(dst_page, + FALSE); + dst_page = vm_object_page_grab(object); + } + if (dst_page == VM_PAGE_NULL) { + /* + * need to allocate a page + */ + dst_page = vm_page_grab_options(grab_options); + } if (dst_page == VM_PAGE_NULL) { if ( (cntrl_flags & (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) == (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) { /* @@ -3516,7 +6032,16 @@ check_busy: * offset... */ vm_object_unlock(object); + + OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages); + + VM_DEBUG_EVENT(vm_upl_page_wait, VM_UPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0); + VM_PAGE_WAIT(); + OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages); + + VM_DEBUG_EVENT(vm_upl_page_wait, VM_UPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0); + vm_object_lock(object); continue; @@ -3536,20 +6061,13 @@ check_busy: * speculative list */ dst_page->clustered = TRUE; + + if ( !(cntrl_flags & UPL_FILE_IO)) + VM_STAT_INCR(pageins); } } - if (dst_page->fictitious) { - panic("need corner case for fictitious page"); - } - if (dst_page->busy) { - /* - * someone else is playing with the - * page. We will have to wait. - */ - PAGE_SLEEP(object, dst_page, THREAD_UNINT); + phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page); - continue; - } /* * ENCRYPTED SWAP: */ @@ -3574,9 +6092,9 @@ check_busy: * eliminate all mappings from the * original object and its prodigy */ - refmod_state = pmap_disconnect(dst_page->phys_page); + refmod_state = pmap_disconnect(phys_page); else - refmod_state = pmap_get_refmod(dst_page->phys_page); + refmod_state = pmap_get_refmod(phys_page); } else refmod_state = 0; @@ -3591,7 +6109,7 @@ check_busy: lite_list[pg_num>>5] |= 1 << (pg_num & 31); if (hw_dirty) - pmap_clear_modify(dst_page->phys_page); + pmap_clear_modify(phys_page); /* * Mark original page as cleaning @@ -3613,7 +6131,12 @@ check_busy: alias_page = NULL; } - if (cntrl_flags & UPL_CLEAN_IN_PLACE) { + if (cntrl_flags & UPL_REQUEST_SET_DIRTY) { + upl->flags &= ~UPL_CLEAR_DIRTY; + upl->flags |= UPL_SET_DIRTY; + dirty = TRUE; + upl->flags |= UPL_SET_DIRTY; + } else if (cntrl_flags & UPL_CLEAN_IN_PLACE) { /* * clean in place for read implies * that a write will be done on all @@ -3649,26 +6172,39 @@ check_busy: */ dwp->dw_mask |= DW_set_reference; } - dst_page->precious = (cntrl_flags & UPL_PRECIOUS) ? TRUE : FALSE; + if (cntrl_flags & UPL_PRECIOUS) { + if (object->internal) { + SET_PAGE_DIRTY(dst_page, FALSE); + dst_page->precious = FALSE; + } else { + dst_page->precious = TRUE; + } + } else { + dst_page->precious = FALSE; + } } if (dst_page->busy) upl->flags |= UPL_HAS_BUSY; - if (dst_page->phys_page > upl->highest_page) - upl->highest_page = dst_page->phys_page; + if (phys_page > upl->highest_page) + upl->highest_page = phys_page; + assert (!pmap_is_noencrypt(phys_page)); if (user_page_list) { - user_page_list[entry].phys_addr = dst_page->phys_page; - user_page_list[entry].pageout = dst_page->pageout; + user_page_list[entry].phys_addr = phys_page; + user_page_list[entry].free_when_done = dst_page->free_when_done; user_page_list[entry].absent = dst_page->absent; user_page_list[entry].dirty = dst_page->dirty; user_page_list[entry].precious = dst_page->precious; user_page_list[entry].device = FALSE; + user_page_list[entry].needed = FALSE; if (dst_page->clustered == TRUE) - user_page_list[entry].speculative = dst_page->speculative; + user_page_list[entry].speculative = (dst_page->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE; else user_page_list[entry].speculative = FALSE; user_page_list[entry].cs_validated = dst_page->cs_validated; user_page_list[entry].cs_tainted = dst_page->cs_tainted; + user_page_list[entry].cs_nx = dst_page->cs_nx; + user_page_list[entry].mark = FALSE; } /* * if UPL_RET_ONLY_ABSENT is set, then @@ -3683,28 +6219,18 @@ check_busy: * update clustered and speculative state * */ - VM_PAGE_CONSUME_CLUSTERED(dst_page); + if (dst_page->clustered) + VM_PAGE_CONSUME_CLUSTERED(dst_page); } try_next_page: if (dwp->dw_mask) { if (dwp->dw_mask & DW_vm_page_activate) VM_STAT_INCR(reactivations); - if (dst_page->busy == FALSE) { - /* - * dw_do_work may need to drop the object lock - * if it does, we need the pages it's looking at to - * be held stable via the busy bit. - */ - dst_page->busy = TRUE; - dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); - } - dwp->dw_m = dst_page; - dwp++; - dw_count++; + VM_PAGE_ADD_DELAYED_WORK(dwp, dst_page, dw_count); - if (dw_count >= DELAYED_WORK_LIMIT) { - dw_do_work(object, &dw_array[0], dw_count); + if (dw_count >= dw_limit) { + vm_page_do_delayed_work(object, UPL_MEMORY_TAG(cntrl_flags), &dw_array[0], dw_count); dwp = &dw_array[0]; dw_count = 0; @@ -3715,7 +6241,7 @@ try_next_page: xfer_size -= PAGE_SIZE; } if (dw_count) - dw_do_work(object, &dw_array[0], dw_count); + vm_page_do_delayed_work(object, UPL_MEMORY_TAG(cntrl_flags), &dw_array[0], dw_count); if (alias_page != NULL) { VM_PAGE_FREE(alias_page); @@ -3735,60 +6261,6 @@ try_next_page: return KERN_SUCCESS; } -/* JMM - Backward compatability for now */ -kern_return_t -vm_fault_list_request( /* forward */ - memory_object_control_t control, - vm_object_offset_t offset, - upl_size_t size, - upl_t *upl_ptr, - upl_page_info_t **user_page_list_ptr, - unsigned int page_list_count, - int cntrl_flags); -kern_return_t -vm_fault_list_request( - memory_object_control_t control, - vm_object_offset_t offset, - upl_size_t size, - upl_t *upl_ptr, - upl_page_info_t **user_page_list_ptr, - unsigned int page_list_count, - int cntrl_flags) -{ - unsigned int local_list_count; - upl_page_info_t *user_page_list; - kern_return_t kr; - - if((cntrl_flags & UPL_VECTOR)==UPL_VECTOR) - return KERN_INVALID_ARGUMENT; - - if (user_page_list_ptr != NULL) { - local_list_count = page_list_count; - user_page_list = *user_page_list_ptr; - } else { - local_list_count = 0; - user_page_list = NULL; - } - kr = memory_object_upl_request(control, - offset, - size, - upl_ptr, - user_page_list, - &local_list_count, - cntrl_flags); - - if(kr != KERN_SUCCESS) - return kr; - - if ((user_page_list_ptr != NULL) && (cntrl_flags & UPL_INTERNAL)) { - *user_page_list_ptr = UPL_GET_INTERNAL_PAGE_LIST(*upl_ptr); - } - - return KERN_SUCCESS; -} - - - /* * Routine: vm_object_super_upl_request * Purpose: @@ -3809,7 +6281,7 @@ vm_object_super_upl_request( upl_t *upl, upl_page_info_t *user_page_list, unsigned int *page_list_count, - int cntrl_flags) + upl_control_flags_t cntrl_flags) { if (object->paging_offset > offset || ((cntrl_flags & UPL_VECTOR)==UPL_VECTOR)) return KERN_FAILURE; @@ -3825,7 +6297,7 @@ vm_object_super_upl_request( base_offset = (offset & ~((vm_object_offset_t) super_cluster - 1)); super_size = (offset + size) > (base_offset + super_cluster) ? super_cluster<<1 : super_cluster; - super_size_64 = ((base_offset + super_size) > object->size) ? (object->size - base_offset) : super_size; + super_size_64 = ((base_offset + super_size) > object->vo_size) ? (object->vo_size - base_offset) : super_size; super_size = (upl_size_t) super_size_64; assert(super_size == super_size_64); @@ -3861,16 +6333,18 @@ vm_map_create_upl( upl_t *upl, upl_page_info_array_t page_list, unsigned int *count, - int *flags) + upl_control_flags_t *flags) { - vm_map_entry_t entry; - int caller_flags; - int force_data_sync; - int sync_cow_data; - vm_object_t local_object; - vm_map_offset_t local_offset; - vm_map_offset_t local_start; - kern_return_t ret; + vm_map_entry_t entry; + upl_control_flags_t caller_flags; + int force_data_sync; + int sync_cow_data; + vm_object_t local_object; + vm_map_offset_t local_offset; + vm_map_offset_t local_start; + kern_return_t ret; + + assert(page_aligned(offset)); caller_flags = *flags; @@ -3890,176 +6364,327 @@ vm_map_create_upl( REDISCOVER_ENTRY: vm_map_lock_read(map); - if (vm_map_lookup_entry(map, offset, &entry)) { + if (!vm_map_lookup_entry(map, offset, &entry)) { + vm_map_unlock_read(map); + return KERN_FAILURE; + } - if ((entry->vme_end - offset) < *upl_size) { - *upl_size = (upl_size_t) (entry->vme_end - offset); - assert(*upl_size == entry->vme_end - offset); + if ((entry->vme_end - offset) < *upl_size) { + *upl_size = (upl_size_t) (entry->vme_end - offset); + assert(*upl_size == entry->vme_end - offset); + } + + if (caller_flags & UPL_QUERY_OBJECT_TYPE) { + *flags = 0; + + if (!entry->is_sub_map && + VME_OBJECT(entry) != VM_OBJECT_NULL) { + if (VME_OBJECT(entry)->private) + *flags = UPL_DEV_MEMORY; + + if (VME_OBJECT(entry)->phys_contiguous) + *flags |= UPL_PHYS_CONTIG; } + vm_map_unlock_read(map); + return KERN_SUCCESS; + } - if (caller_flags & UPL_QUERY_OBJECT_TYPE) { - *flags = 0; + if (VME_OBJECT(entry) == VM_OBJECT_NULL || + !VME_OBJECT(entry)->phys_contiguous) { + if (*upl_size > MAX_UPL_SIZE_BYTES) + *upl_size = MAX_UPL_SIZE_BYTES; + } - if ( !entry->is_sub_map && entry->object.vm_object != VM_OBJECT_NULL) { - if (entry->object.vm_object->private) - *flags = UPL_DEV_MEMORY; + /* + * Create an object if necessary. + */ + if (VME_OBJECT(entry) == VM_OBJECT_NULL) { - if (entry->object.vm_object->phys_contiguous) - *flags |= UPL_PHYS_CONTIG; - } - vm_map_unlock_read(map); + if (vm_map_lock_read_to_write(map)) + goto REDISCOVER_ENTRY; + + VME_OBJECT_SET(entry, + vm_object_allocate((vm_size_t) + (entry->vme_end - + entry->vme_start))); + VME_OFFSET_SET(entry, 0); + + vm_map_lock_write_to_read(map); + } + + if (!(caller_flags & UPL_COPYOUT_FROM) && + !(entry->protection & VM_PROT_WRITE)) { + vm_map_unlock_read(map); + return KERN_PROTECTION_FAILURE; + } + + + local_object = VME_OBJECT(entry); + assert(local_object != VM_OBJECT_NULL); + + if (!entry->is_sub_map && + !entry->needs_copy && + *upl_size != 0 && + local_object->vo_size > *upl_size && /* partial UPL */ + entry->wired_count == 0 && /* No COW for entries that are wired */ + (map->pmap != kernel_pmap) && /* alias checks */ + (vm_map_entry_should_cow_for_true_share(entry) /* case 1 */ + || + (/* case 2 */ + local_object->internal && + (local_object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) && + local_object->ref_count > 1))) { + vm_prot_t prot; - return KERN_SUCCESS; - } - if (entry->object.vm_object == VM_OBJECT_NULL || !entry->object.vm_object->phys_contiguous) { - if ((*upl_size/PAGE_SIZE) > MAX_UPL_SIZE) - *upl_size = MAX_UPL_SIZE * PAGE_SIZE; - } /* - * Create an object if necessary. + * Case 1: + * Set up the targeted range for copy-on-write to avoid + * applying true_share/copy_delay to the entire object. + * + * Case 2: + * This map entry covers only part of an internal + * object. There could be other map entries covering + * other areas of this object and some of these map + * entries could be marked as "needs_copy", which + * assumes that the object is COPY_SYMMETRIC. + * To avoid marking this object as COPY_DELAY and + * "true_share", let's shadow it and mark the new + * (smaller) object as "true_share" and COPY_DELAY. */ - if (entry->object.vm_object == VM_OBJECT_NULL) { - if (vm_map_lock_read_to_write(map)) - goto REDISCOVER_ENTRY; + if (vm_map_lock_read_to_write(map)) { + goto REDISCOVER_ENTRY; + } + vm_map_lock_assert_exclusive(map); + assert(VME_OBJECT(entry) == local_object); + + vm_map_clip_start(map, + entry, + vm_map_trunc_page(offset, + VM_MAP_PAGE_MASK(map))); + vm_map_clip_end(map, + entry, + vm_map_round_page(offset + *upl_size, + VM_MAP_PAGE_MASK(map))); + if ((entry->vme_end - offset) < *upl_size) { + *upl_size = (upl_size_t) (entry->vme_end - offset); + assert(*upl_size == entry->vme_end - offset); + } + + prot = entry->protection & ~VM_PROT_WRITE; + if (override_nx(map, VME_ALIAS(entry)) && prot) + prot |= VM_PROT_EXECUTE; + vm_object_pmap_protect(local_object, + VME_OFFSET(entry), + entry->vme_end - entry->vme_start, + ((entry->is_shared || + map->mapped_in_other_pmaps) + ? PMAP_NULL + : map->pmap), + entry->vme_start, + prot); - entry->object.vm_object = vm_object_allocate((vm_size_t)(entry->vme_end - entry->vme_start)); - entry->offset = 0; + assert(entry->wired_count == 0); - vm_map_lock_write_to_read(map); + /* + * Lock the VM object and re-check its status: if it's mapped + * in another address space, we could still be racing with + * another thread holding that other VM map exclusively. + */ + vm_object_lock(local_object); + if (local_object->true_share) { + /* object is already in proper state: no COW needed */ + assert(local_object->copy_strategy != + MEMORY_OBJECT_COPY_SYMMETRIC); + } else { + /* not true_share: ask for copy-on-write below */ + assert(local_object->copy_strategy == + MEMORY_OBJECT_COPY_SYMMETRIC); + entry->needs_copy = TRUE; } - if (!(caller_flags & UPL_COPYOUT_FROM)) { - if (!(entry->protection & VM_PROT_WRITE)) { - vm_map_unlock_read(map); - return KERN_PROTECTION_FAILURE; - } - if (entry->needs_copy) { - /* - * Honor copy-on-write for COPY_SYMMETRIC - * strategy. - */ - vm_map_t local_map; - vm_object_t object; - vm_object_offset_t new_offset; - vm_prot_t prot; - boolean_t wired; - vm_map_version_t version; - vm_map_t real_map; - - local_map = map; - - if (vm_map_lookup_locked(&local_map, - offset, VM_PROT_WRITE, - OBJECT_LOCK_EXCLUSIVE, - &version, &object, - &new_offset, &prot, &wired, - NULL, - &real_map) != KERN_SUCCESS) { - vm_map_unlock_read(local_map); - return KERN_FAILURE; - } - if (real_map != map) - vm_map_unlock(real_map); - vm_map_unlock_read(local_map); + vm_object_unlock(local_object); - vm_object_unlock(object); + vm_map_lock_write_to_read(map); + } - goto REDISCOVER_ENTRY; + if (entry->needs_copy) { + /* + * Honor copy-on-write for COPY_SYMMETRIC + * strategy. + */ + vm_map_t local_map; + vm_object_t object; + vm_object_offset_t new_offset; + vm_prot_t prot; + boolean_t wired; + vm_map_version_t version; + vm_map_t real_map; + vm_prot_t fault_type; + + local_map = map; + + if (caller_flags & UPL_COPYOUT_FROM) { + fault_type = VM_PROT_READ | VM_PROT_COPY; + vm_counters.create_upl_extra_cow++; + vm_counters.create_upl_extra_cow_pages += + (entry->vme_end - entry->vme_start) / PAGE_SIZE; + } else { + fault_type = VM_PROT_WRITE; + } + if (vm_map_lookup_locked(&local_map, + offset, fault_type, + OBJECT_LOCK_EXCLUSIVE, + &version, &object, + &new_offset, &prot, &wired, + NULL, + &real_map) != KERN_SUCCESS) { + if (fault_type == VM_PROT_WRITE) { + vm_counters.create_upl_lookup_failure_write++; + } else { + vm_counters.create_upl_lookup_failure_copy++; } + vm_map_unlock_read(local_map); + return KERN_FAILURE; } - if (entry->is_sub_map) { - vm_map_t submap; + if (real_map != map) + vm_map_unlock(real_map); + vm_map_unlock_read(local_map); - submap = entry->object.sub_map; - local_start = entry->vme_start; - local_offset = entry->offset; + vm_object_unlock(object); - vm_map_reference(submap); - vm_map_unlock_read(map); + goto REDISCOVER_ENTRY; + } - ret = vm_map_create_upl(submap, - local_offset + (offset - local_start), - upl_size, upl, page_list, count, flags); - vm_map_deallocate(submap); + if (entry->is_sub_map) { + vm_map_t submap; - return ret; - } - if (sync_cow_data) { - if (entry->object.vm_object->shadow || entry->object.vm_object->copy) { - local_object = entry->object.vm_object; - local_start = entry->vme_start; - local_offset = entry->offset; + submap = VME_SUBMAP(entry); + local_start = entry->vme_start; + local_offset = VME_OFFSET(entry); + + vm_map_reference(submap); + vm_map_unlock_read(map); - vm_object_reference(local_object); - vm_map_unlock_read(map); + ret = vm_map_create_upl(submap, + local_offset + (offset - local_start), + upl_size, upl, page_list, count, flags); + vm_map_deallocate(submap); - if (local_object->shadow && local_object->copy) { - vm_object_lock_request( - local_object->shadow, - (vm_object_offset_t) - ((offset - local_start) + - local_offset) + - local_object->shadow_offset, - *upl_size, FALSE, - MEMORY_OBJECT_DATA_SYNC, - VM_PROT_NO_CHANGE); - } - sync_cow_data = FALSE; - vm_object_deallocate(local_object); + return ret; + } - goto REDISCOVER_ENTRY; - } - } - if (force_data_sync) { - local_object = entry->object.vm_object; - local_start = entry->vme_start; - local_offset = entry->offset; + if (sync_cow_data && + (VME_OBJECT(entry)->shadow || + VME_OBJECT(entry)->copy)) { + local_object = VME_OBJECT(entry); + local_start = entry->vme_start; + local_offset = VME_OFFSET(entry); - vm_object_reference(local_object); - vm_map_unlock_read(map); + vm_object_reference(local_object); + vm_map_unlock_read(map); - vm_object_lock_request( - local_object, - (vm_object_offset_t) - ((offset - local_start) + local_offset), - (vm_object_size_t)*upl_size, FALSE, + if (local_object->shadow && local_object->copy) { + vm_object_lock_request(local_object->shadow, + ((vm_object_offset_t) + ((offset - local_start) + + local_offset) + + local_object->vo_shadow_offset), + *upl_size, FALSE, MEMORY_OBJECT_DATA_SYNC, VM_PROT_NO_CHANGE); - - force_data_sync = FALSE; - vm_object_deallocate(local_object); - - goto REDISCOVER_ENTRY; } - if (entry->object.vm_object->private) - *flags = UPL_DEV_MEMORY; - else - *flags = 0; - - if (entry->object.vm_object->phys_contiguous) - *flags |= UPL_PHYS_CONTIG; + sync_cow_data = FALSE; + vm_object_deallocate(local_object); - local_object = entry->object.vm_object; - local_offset = entry->offset; + goto REDISCOVER_ENTRY; + } + if (force_data_sync) { + local_object = VME_OBJECT(entry); local_start = entry->vme_start; + local_offset = VME_OFFSET(entry); vm_object_reference(local_object); vm_map_unlock_read(map); - ret = vm_object_iopl_request(local_object, - (vm_object_offset_t) ((offset - local_start) + local_offset), - *upl_size, - upl, - page_list, - count, - caller_flags); + vm_object_lock_request(local_object, + ((vm_object_offset_t) + ((offset - local_start) + + local_offset)), + (vm_object_size_t)*upl_size, + FALSE, + MEMORY_OBJECT_DATA_SYNC, + VM_PROT_NO_CHANGE); + + force_data_sync = FALSE; vm_object_deallocate(local_object); - return(ret); - } + goto REDISCOVER_ENTRY; + } + if (VME_OBJECT(entry)->private) + *flags = UPL_DEV_MEMORY; + else + *flags = 0; + + if (VME_OBJECT(entry)->phys_contiguous) + *flags |= UPL_PHYS_CONTIG; + + local_object = VME_OBJECT(entry); + local_offset = VME_OFFSET(entry); + local_start = entry->vme_start; + + + vm_object_lock(local_object); + + /* + * Ensure that this object is "true_share" and "copy_delay" now, + * while we're still holding the VM map lock. After we unlock the map, + * anything could happen to that mapping, including some copy-on-write + * activity. We need to make sure that the IOPL will point at the + * same memory as the mapping. + */ + if (local_object->true_share) { + assert(local_object->copy_strategy != + MEMORY_OBJECT_COPY_SYMMETRIC); + } else if (local_object != kernel_object && + local_object != compressor_object && + !local_object->phys_contiguous) { +#if VM_OBJECT_TRACKING_OP_TRUESHARE + if (!local_object->true_share && + vm_object_tracking_inited) { + void *bt[VM_OBJECT_TRACKING_BTDEPTH]; + int num = 0; + num = OSBacktrace(bt, + VM_OBJECT_TRACKING_BTDEPTH); + btlog_add_entry(vm_object_tracking_btlog, + local_object, + VM_OBJECT_TRACKING_OP_TRUESHARE, + bt, + num); + } +#endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */ + local_object->true_share = TRUE; + if (local_object->copy_strategy == + MEMORY_OBJECT_COPY_SYMMETRIC) { + local_object->copy_strategy = MEMORY_OBJECT_COPY_DELAY; + } + } + + vm_object_reference_locked(local_object); + vm_object_unlock(local_object); + vm_map_unlock_read(map); - return(KERN_FAILURE); + ret = vm_object_iopl_request(local_object, + ((vm_object_offset_t) + ((offset - local_start) + local_offset)), + *upl_size, + upl, + page_list, + count, + caller_flags); + vm_object_deallocate(local_object); + + return ret; } /* @@ -4132,6 +6757,7 @@ process_upl_to_enter: upl = vector_upl_subupl_byindex(vector_upl, curr_upl++ ); if(upl == NULL) goto process_upl_to_enter; + vector_upl_get_iostate(vector_upl, upl, &subupl_offset, &subupl_size); *dst_addr = (vm_map_offset_t)(vector_upl_dst_addr + (vm_map_offset_t)subupl_offset); } else { @@ -4169,9 +6795,9 @@ process_upl_to_enter: upl->map_object->pageout = TRUE; upl->map_object->can_persist = FALSE; upl->map_object->copy_strategy = MEMORY_OBJECT_COPY_NONE; - upl->map_object->shadow_offset = upl->offset - object->paging_offset; + upl->map_object->vo_shadow_offset = upl->offset - object->paging_offset; upl->map_object->wimg_bits = object->wimg_bits; - offset = upl->map_object->shadow_offset; + offset = upl->map_object->vo_shadow_offset; new_offset = 0; size = upl->size; @@ -4199,25 +6825,25 @@ process_upl_to_enter: assert(alias_page->fictitious); alias_page->fictitious = FALSE; alias_page->private = TRUE; - alias_page->pageout = TRUE; + alias_page->free_when_done = TRUE; /* * since m is a page in the upl it must * already be wired or BUSY, so it's * safe to assign the underlying physical * page to the alias */ - alias_page->phys_page = m->phys_page; + VM_PAGE_SET_PHYS_PAGE(alias_page, VM_PAGE_GET_PHYS_PAGE(m)); vm_object_unlock(object); vm_page_lockspin_queues(); - vm_page_wire(alias_page); + vm_page_wire(alias_page, VM_KERN_MEMORY_NONE, TRUE); vm_page_unlock_queues(); /* * ENCRYPTED SWAP: * The virtual page ("m") has to be wired in some way - * here or its physical page ("m->phys_page") could + * here or its backing physical page could * be recycled at any time. * Assuming this is enforced by the caller, we can't * get an encrypted page here. Since the encryption @@ -4230,7 +6856,7 @@ process_upl_to_enter: */ ASSERT_PAGE_DECRYPTED(m); - vm_page_insert(alias_page, upl->map_object, new_offset); + vm_page_insert_wired(alias_page, upl->map_object, new_offset, VM_KERN_MEMORY_NONE); assert(!alias_page->wanted); alias_page->busy = FALSE; @@ -4246,6 +6872,7 @@ process_upl_to_enter: offset = 0; else offset = upl->offset - upl->map_object->paging_offset; + size = upl->size; vm_object_reference(upl->map_object); @@ -4256,17 +6883,20 @@ process_upl_to_enter: * NEED A UPL_MAP ALIAS */ kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0, - VM_FLAGS_ANYWHERE, upl->map_object, offset, FALSE, + VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_OSFMK), + upl->map_object, offset, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); if (kr != KERN_SUCCESS) { + vm_object_deallocate(upl->map_object); upl_unlock(upl); return(kr); } } else { kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0, - VM_FLAGS_FIXED, upl->map_object, offset, FALSE, + VM_FLAGS_FIXED | VM_MAKE_TAG(VM_KERN_MEMORY_OSFMK), + upl->map_object, offset, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); if(kr) panic("vm_map_enter failed for a Vector UPL\n"); @@ -4277,18 +6907,15 @@ process_upl_to_enter: m = vm_page_lookup(upl->map_object, offset); if (m) { - unsigned int cache_attr; - cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK; - m->pmapped = TRUE; /* CODE SIGNING ENFORCEMENT: page has been wpmapped, * but only in kernel space. If this was on a user map, * we'd have to set the wpmapped bit. */ /* m->wpmapped = TRUE; */ - assert(map==kernel_map); + assert(map->pmap == kernel_pmap); - PMAP_ENTER(map->pmap, addr, m, VM_PROT_ALL, cache_attr, TRUE); + PMAP_ENTER(map->pmap, addr, m, VM_PROT_DEFAULT, VM_PROT_NONE, 0, TRUE); } offset += PAGE_SIZE_64; } @@ -4390,9 +7017,12 @@ process_upl_to_remove: if(!isVectorUPL) { upl_unlock(upl); - vm_map_remove(map, - vm_map_trunc_page(addr), - vm_map_round_page(addr + size), + vm_map_remove( + map, + vm_map_trunc_page(addr, + VM_MAP_PAGE_MASK(map)), + vm_map_round_page(addr + size, + VM_MAP_PAGE_MASK(map)), VM_MAP_NO_FLAGS); return KERN_SUCCESS; @@ -4411,88 +7041,6 @@ process_upl_to_remove: return KERN_FAILURE; } -static void -dw_do_work( - vm_object_t object, - struct dw *dwp, - int dw_count) -{ - int j; - boolean_t held_as_spin = TRUE; - - /* - * pageout_scan takes the vm_page_lock_queues first - * then tries for the object lock... to avoid what - * is effectively a lock inversion, we'll go to the - * trouble of taking them in that same order... otherwise - * if this object contains the majority of the pages resident - * in the UBC (or a small set of large objects actively being - * worked on contain the majority of the pages), we could - * cause the pageout_scan thread to 'starve' in its attempt - * to find pages to move to the free queue, since it has to - * successfully acquire the object lock of any candidate page - * before it can steal/clean it. - */ - if (!vm_page_trylockspin_queues()) { - vm_object_unlock(object); - - vm_page_lockspin_queues(); - - for (j = 0; ; j++) { - if (!vm_object_lock_avoid(object) && - _vm_object_lock_try(object)) - break; - vm_page_unlock_queues(); - mutex_pause(j); - vm_page_lockspin_queues(); - } - } - for (j = 0; j < dw_count; j++, dwp++) { - - if (dwp->dw_mask & DW_vm_pageout_throttle_up) - vm_pageout_throttle_up(dwp->dw_m); - - if (dwp->dw_mask & DW_vm_page_wire) - vm_page_wire(dwp->dw_m); - else if (dwp->dw_mask & DW_vm_page_unwire) { - boolean_t queueit; - - queueit = (dwp->dw_mask & DW_vm_page_free) ? FALSE : TRUE; - - vm_page_unwire(dwp->dw_m, queueit); - } - if (dwp->dw_mask & DW_vm_page_free) { - if (held_as_spin == TRUE) { - vm_page_lockconvert_queues(); - held_as_spin = FALSE; - } - vm_page_free(dwp->dw_m); - } else { - if (dwp->dw_mask & DW_vm_page_deactivate_internal) - vm_page_deactivate_internal(dwp->dw_m, FALSE); - else if (dwp->dw_mask & DW_vm_page_activate) - vm_page_activate(dwp->dw_m); - else if (dwp->dw_mask & DW_vm_page_speculate) - vm_page_speculate(dwp->dw_m, TRUE); - else if (dwp->dw_mask & DW_vm_page_lru) - vm_page_lru(dwp->dw_m); - - if (dwp->dw_mask & DW_set_reference) - dwp->dw_m->reference = TRUE; - else if (dwp->dw_mask & DW_clear_reference) - dwp->dw_m->reference = FALSE; - - if (dwp->dw_mask & DW_clear_busy) - dwp->dw_m->busy = FALSE; - - if (dwp->dw_mask & DW_PAGE_WAKEUP) - PAGE_WAKEUP(dwp->dw_m); - } - } - vm_page_unlock_queues(); -} - - kern_return_t upl_commit_range( @@ -4507,6 +7055,7 @@ upl_commit_range( upl_size_t xfer_size, subupl_size = size; vm_object_t shadow_object; vm_object_t object; + vm_object_t m_object; vm_object_offset_t target_offset; upl_offset_t subupl_offset = offset; int entry; @@ -4514,10 +7063,21 @@ upl_commit_range( int occupied; int clear_refmod = 0; int pgpgout_count = 0; - struct dw dw_array[DELAYED_WORK_LIMIT]; - struct dw *dwp; - int dw_count, isVectorUPL = 0; + struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT]; + struct vm_page_delayed_work *dwp; + int dw_count; + int dw_limit; + int isVectorUPL = 0; upl_t vector_upl = NULL; + boolean_t should_be_throttled = FALSE; + + vm_page_t nxt_page = VM_PAGE_NULL; + int fast_path_possible = 0; + int fast_path_full_commit = 0; + int throttle_page = 0; + int unwired_count = 0; + int local_queue_count = 0; + vm_page_t first_local, last_local; *empty = FALSE; @@ -4575,6 +7135,8 @@ process_upl_to_commit: } return KERN_FAILURE; } + if (upl->flags & UPL_SET_DIRTY) + flags |= UPL_COMMIT_SET_DIRTY; if (upl->flags & UPL_CLEAR_DIRTY) flags |= UPL_COMMIT_CLEAR_DIRTY; @@ -4595,6 +7157,9 @@ process_upl_to_commit: entry = offset/PAGE_SIZE; target_offset = (vm_object_offset_t)offset; + assert(!(target_offset & PAGE_MASK)); + assert(!(xfer_size & PAGE_MASK)); + if (upl->flags & UPL_KERNEL_OBJECT) vm_object_lock_shared(shadow_object); else @@ -4622,9 +7187,37 @@ process_upl_to_commit: */ flags &= ~UPL_COMMIT_CS_VALIDATED; } + if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal) + should_be_throttled = TRUE; dwp = &dw_array[0]; dw_count = 0; + dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT); + + if ((upl->flags & UPL_IO_WIRE) && + !(flags & UPL_COMMIT_FREE_ABSENT) && + !isVectorUPL && + shadow_object->purgable != VM_PURGABLE_VOLATILE && + shadow_object->purgable != VM_PURGABLE_EMPTY) { + + if (!vm_page_queue_empty(&shadow_object->memq)) { + + if (size == shadow_object->vo_size) { + nxt_page = (vm_page_t)vm_page_queue_first(&shadow_object->memq); + fast_path_full_commit = 1; + } + fast_path_possible = 1; + + if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal && + (shadow_object->purgable == VM_PURGABLE_DENY || + shadow_object->purgable == VM_PURGABLE_NONVOLATILE || + shadow_object->purgable == VM_PURGABLE_VOLATILE)) { + throttle_page = 1; + } + } + } + first_local = VM_PAGE_NULL; + last_local = VM_PAGE_NULL; while (xfer_size) { vm_page_t t, m; @@ -4637,29 +7230,44 @@ process_upl_to_commit: if (upl->flags & UPL_LITE) { unsigned int pg_num; + if (nxt_page != VM_PAGE_NULL) { + m = nxt_page; + nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->listq); + target_offset = m->offset; + } pg_num = (unsigned int) (target_offset/PAGE_SIZE); assert(pg_num == target_offset/PAGE_SIZE); if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) { lite_list[pg_num>>5] &= ~(1 << (pg_num & 31)); - if (!(upl->flags & UPL_KERNEL_OBJECT)) + if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL) m = vm_page_lookup(shadow_object, target_offset + (upl->offset - shadow_object->paging_offset)); - } + } else + m = NULL; } if (upl->flags & UPL_SHADOWED) { if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) { - t->pageout = FALSE; + t->free_when_done = FALSE; VM_PAGE_FREE(t); - if (m == VM_PAGE_NULL) - m = vm_page_lookup(shadow_object, target_offset + object->shadow_offset); + if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL) + m = vm_page_lookup(shadow_object, target_offset + object->vo_shadow_offset); } } - if ((upl->flags & UPL_KERNEL_OBJECT) || m == VM_PAGE_NULL) + if (m == VM_PAGE_NULL) + goto commit_next_page; + + m_object = VM_PAGE_OBJECT(m); + + if (m->vm_page_q_state == VM_PAGE_USED_BY_COMPRESSOR) { + assert(m->busy); + + dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); goto commit_next_page; + } if (flags & UPL_COMMIT_CS_VALIDATED) { /* @@ -4669,15 +7277,19 @@ process_upl_to_commit: */ m->cs_validated = page_list[entry].cs_validated; m->cs_tainted = page_list[entry].cs_tainted; + m->cs_nx = page_list[entry].cs_nx; } + if (flags & UPL_COMMIT_WRITTEN_BY_KERNEL) + m->written_by_kernel = TRUE; + if (upl->flags & UPL_IO_WIRE) { if (page_list) page_list[entry].phys_addr = 0; - if (flags & UPL_COMMIT_SET_DIRTY) - m->dirty = TRUE; - else if (flags & UPL_COMMIT_CLEAR_DIRTY) { + if (flags & UPL_COMMIT_SET_DIRTY) { + SET_PAGE_DIRTY(m, FALSE); + } else if (flags & UPL_COMMIT_CLEAR_DIRTY) { m->dirty = FALSE; if (! (flags & UPL_COMMIT_CS_VALIDATED) && @@ -4689,18 +7301,19 @@ process_upl_to_commit: * so it will need to be * re-validated. */ + if (m->slid) { + panic("upl_commit_range(%p): page %p was slid\n", + upl, m); + } + assert(!m->slid); m->cs_validated = FALSE; #if DEVELOPMENT || DEBUG vm_cs_validated_resets++; #endif - pmap_disconnect(m->phys_page); + pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)); } clear_refmod |= VM_MEM_MODIFIED; } - if (flags & UPL_COMMIT_INACTIVATE) { - dwp->dw_mask |= DW_vm_page_deactivate_internal; - clear_refmod |= VM_MEM_REFERENCED; - } if (upl->flags & UPL_ACCESS_BLOCKED) { /* * We blocked access to the pages in this UPL. @@ -4709,18 +7322,88 @@ process_upl_to_commit: */ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); } - if (m->absent) { - if (flags & UPL_COMMIT_FREE_ABSENT) - dwp->dw_mask |= DW_vm_page_free; - else { + if (fast_path_possible) { + assert(m_object->purgable != VM_PURGABLE_EMPTY); + assert(m_object->purgable != VM_PURGABLE_VOLATILE); + if (m->absent) { + assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q); + assert(m->wire_count == 0); + assert(m->busy); + m->absent = FALSE; dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); + } else { + if (m->wire_count == 0) + panic("wire_count == 0, m = %p, obj = %p\n", m, shadow_object); + assert(m->vm_page_q_state == VM_PAGE_IS_WIRED); + + /* + * XXX FBDP need to update some other + * counters here (purgeable_wired_count) + * (ledgers), ... + */ + assert(m->wire_count > 0); + m->wire_count--; + + if (m->wire_count == 0) { + m->vm_page_q_state = VM_PAGE_NOT_ON_Q; + unwired_count++; + } } - } else - dwp->dw_mask |= DW_vm_page_unwire; + if (m->wire_count == 0) { + assert(m->pageq.next == 0 && m->pageq.prev == 0); + + if (last_local == VM_PAGE_NULL) { + assert(first_local == VM_PAGE_NULL); + + last_local = m; + first_local = m; + } else { + assert(first_local != VM_PAGE_NULL); + + m->pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local); + first_local->pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(m); + first_local = m; + } + local_queue_count++; + if (throttle_page) { + m->vm_page_q_state = VM_PAGE_ON_THROTTLED_Q; + } else { + if (flags & UPL_COMMIT_INACTIVATE) { + if (shadow_object->internal) + m->vm_page_q_state = VM_PAGE_ON_INACTIVE_INTERNAL_Q; + else + m->vm_page_q_state = VM_PAGE_ON_INACTIVE_EXTERNAL_Q; + } else + m->vm_page_q_state = VM_PAGE_ON_ACTIVE_Q; + } + } + } else { + if (flags & UPL_COMMIT_INACTIVATE) { + dwp->dw_mask |= DW_vm_page_deactivate_internal; + clear_refmod |= VM_MEM_REFERENCED; + } + if (m->absent) { + if (flags & UPL_COMMIT_FREE_ABSENT) + dwp->dw_mask |= DW_vm_page_free; + else { + m->absent = FALSE; + dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); + + if ( !(dwp->dw_mask & DW_vm_page_deactivate_internal)) + dwp->dw_mask |= DW_vm_page_activate; + } + } else + dwp->dw_mask |= DW_vm_page_unwire; + } goto commit_next_page; } + assert(m->vm_page_q_state != VM_PAGE_USED_BY_COMPRESSOR); + + if (page_list) + page_list[entry].phys_addr = 0; + /* * make sure to clear the hardware * modify or reference bits before @@ -4731,87 +7414,89 @@ process_upl_to_commit: if (flags & UPL_COMMIT_CLEAR_DIRTY) { m->dirty = FALSE; - if (! (flags & UPL_COMMIT_CS_VALIDATED) && - m->cs_validated && !m->cs_tainted) { - /* - * CODE SIGNING: - * This page is no longer dirty - * but could have been modified, - * so it will need to be - * re-validated. - */ - m->cs_validated = FALSE; -#if DEVELOPMENT || DEBUG - vm_cs_validated_resets++; -#endif - pmap_disconnect(m->phys_page); - } clear_refmod |= VM_MEM_MODIFIED; } - if (page_list) { - upl_page_info_t *p; - - p = &(page_list[entry]); - - if (p->phys_addr && p->pageout && !m->pageout) { - m->busy = TRUE; - m->pageout = TRUE; - - dwp->dw_mask |= DW_vm_page_wire; + if (m->laundry) + dwp->dw_mask |= DW_vm_pageout_throttle_up; - } else if (p->phys_addr && - !p->pageout && m->pageout && - !m->dump_cleaning) { - m->pageout = FALSE; + if (VM_PAGE_WIRED(m)) + m->free_when_done = FALSE; + + if (! (flags & UPL_COMMIT_CS_VALIDATED) && + m->cs_validated && !m->cs_tainted) { + /* + * CODE SIGNING: + * This page is no longer dirty + * but could have been modified, + * so it will need to be + * re-validated. + */ + if (m->slid) { + panic("upl_commit_range(%p): page %p was slid\n", + upl, m); + } + assert(!m->slid); + m->cs_validated = FALSE; +#if DEVELOPMENT || DEBUG + vm_cs_validated_resets++; +#endif + pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)); + } + if (m->overwriting) { + /* + * the (COPY_OUT_FROM == FALSE) request_page_list case + */ + if (m->busy) { +#if CONFIG_PHANTOM_CACHE + if (m->absent && !m_object->internal) + dwp->dw_mask |= DW_vm_phantom_cache_update; +#endif m->absent = FALSE; - m->overwriting = FALSE; - dwp->dw_mask |= (DW_vm_page_unwire | DW_clear_busy | DW_PAGE_WAKEUP); + dwp->dw_mask |= DW_clear_busy; + } else { + /* + * alternate (COPY_OUT_FROM == FALSE) page_list case + * Occurs when the original page was wired + * at the time of the list request + */ + assert(VM_PAGE_WIRED(m)); + + dwp->dw_mask |= DW_vm_page_unwire; /* reactivates */ } - page_list[entry].phys_addr = 0; + m->overwriting = FALSE; } - m->dump_cleaning = FALSE; + if (m->encrypted_cleaning == TRUE) { + m->encrypted_cleaning = FALSE; - if (m->laundry) - dwp->dw_mask |= DW_vm_pageout_throttle_up; + dwp->dw_mask |= DW_clear_busy | DW_PAGE_WAKEUP; + } + m->cleaning = FALSE; - if (m->pageout) { - m->cleaning = FALSE; - m->encrypted_cleaning = FALSE; - m->pageout = FALSE; + if (m->free_when_done) { + /* + * With the clean queue enabled, UPL_PAGEOUT should + * no longer set the pageout bit. It's pages now go + * to the clean queue. + */ + assert(!(flags & UPL_PAGEOUT)); + assert(!m_object->internal); + + m->free_when_done = FALSE; #if MACH_CLUSTER_STATS if (m->wanted) vm_pageout_target_collisions++; #endif - m->dirty = FALSE; - - if (! (flags & UPL_COMMIT_CS_VALIDATED) && - m->cs_validated && !m->cs_tainted) { - /* - * CODE SIGNING: - * This page is no longer dirty - * but could have been modified, - * so it will need to be - * re-validated. - */ - m->cs_validated = FALSE; -#if DEVELOPMENT || DEBUG - vm_cs_validated_resets++; -#endif - pmap_disconnect(m->phys_page); - } - if ((flags & UPL_COMMIT_SET_DIRTY) || - (m->pmapped && (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED))) - m->dirty = TRUE; - - if (m->dirty) { + (m->pmapped && (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED))) { /* * page was re-dirtied after we started * the pageout... reactivate it since * we don't know whether the on-disk * copy matches what is now in memory */ - dwp->dw_mask |= (DW_vm_page_unwire | DW_clear_busy | DW_PAGE_WAKEUP); + SET_PAGE_DIRTY(m, FALSE); + + dwp->dw_mask |= DW_vm_page_activate | DW_PAGE_WAKEUP; if (upl->flags & UPL_PAGEOUT) { CLUSTER_STAT(vm_pageout_target_page_dirtied++;) @@ -4823,76 +7508,26 @@ process_upl_to_commit: * page has been successfully cleaned * go ahead and free it for other use */ - - if (m->object->internal) { + if (m_object->internal) { DTRACE_VM2(anonpgout, int, 1, (uint64_t *), NULL); } else { DTRACE_VM2(fspgout, int, 1, (uint64_t *), NULL); } - dwp->dw_mask |= DW_vm_page_free; - - if (upl->flags & UPL_PAGEOUT) { - CLUSTER_STAT(vm_pageout_target_page_freed++;) + m->dirty = FALSE; + m->busy = TRUE; - if (page_list[entry].dirty) { - VM_STAT_INCR(pageouts); - DTRACE_VM2(pgout, int, 1, (uint64_t *), NULL); - pgpgout_count++; - } - } + dwp->dw_mask |= DW_vm_page_free; } goto commit_next_page; } #if MACH_CLUSTER_STATS if (m->wpmapped) - m->dirty = pmap_is_modified(m->phys_page); + m->dirty = pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m)); if (m->dirty) vm_pageout_cluster_dirtied++; else vm_pageout_cluster_cleaned++; if (m->wanted) vm_pageout_cluster_collisions++; #endif - m->dirty = FALSE; - - if (! (flags & UPL_COMMIT_CS_VALIDATED) && - m->cs_validated && !m->cs_tainted) { - /* - * CODE SIGNING: - * This page is no longer dirty - * but could have been modified, - * so it will need to be - * re-validated. - */ - m->cs_validated = FALSE; -#if DEVELOPMENT || DEBUG - vm_cs_validated_resets++; -#endif - pmap_disconnect(m->phys_page); - } - - if ((m->busy) && (m->cleaning)) { - /* - * the request_page_list case - */ - m->absent = FALSE; - m->overwriting = FALSE; - - dwp->dw_mask |= DW_clear_busy; - - } else if (m->overwriting) { - /* - * alternate request page list, write to - * page_list case. Occurs when the original - * page was wired at the time of the list - * request - */ - assert(VM_PAGE_WIRED(m)); - m->overwriting = FALSE; - - dwp->dw_mask |= DW_vm_page_unwire; /* reactivates */ - } - m->cleaning = FALSE; - m->encrypted_cleaning = FALSE; - /* * It is a part of the semantic of COPYOUT_FROM * UPLs that a commit implies cache sync @@ -4903,22 +7538,45 @@ process_upl_to_commit: if ((upl->flags & UPL_PAGE_SYNC_DONE) || (flags & UPL_COMMIT_CLEAR_PRECIOUS)) m->precious = FALSE; - if (flags & UPL_COMMIT_SET_DIRTY) - m->dirty = TRUE; + if (flags & UPL_COMMIT_SET_DIRTY) { + SET_PAGE_DIRTY(m, FALSE); + } else { + m->dirty = FALSE; + } - if ((flags & UPL_COMMIT_INACTIVATE) && !m->clustered && !m->speculative) { - dwp->dw_mask |= DW_vm_page_deactivate_internal; - clear_refmod |= VM_MEM_REFERENCED; + /* with the clean queue on, move *all* cleaned pages to the clean queue */ + if (hibernate_cleaning_in_progress == FALSE && !m->dirty && (upl->flags & UPL_PAGEOUT)) { + pgpgout_count++; - } else if (!m->active && !m->inactive && !m->speculative) { + VM_STAT_INCR(pageouts); + DTRACE_VM2(pgout, int, 1, (uint64_t *), NULL); - if (m->clustered || (flags & UPL_COMMIT_SPECULATE)) - dwp->dw_mask |= DW_vm_page_speculate; - else if (m->reference) - dwp->dw_mask |= DW_vm_page_activate; - else { + dwp->dw_mask |= DW_enqueue_cleaned; + vm_pageout_enqueued_cleaned_from_inactive_dirty++; + } else if (should_be_throttled == TRUE && (m->vm_page_q_state == VM_PAGE_NOT_ON_Q)) { + /* + * page coming back in from being 'frozen'... + * it was dirty before it was frozen, so keep it so + * the vm_page_activate will notice that it really belongs + * on the throttle queue and put it there + */ + SET_PAGE_DIRTY(m, FALSE); + dwp->dw_mask |= DW_vm_page_activate; + + } else { + if ((flags & UPL_COMMIT_INACTIVATE) && !m->clustered && (m->vm_page_q_state != VM_PAGE_ON_SPECULATIVE_Q)) { dwp->dw_mask |= DW_vm_page_deactivate_internal; clear_refmod |= VM_MEM_REFERENCED; + } else if ( !VM_PAGE_PAGEABLE(m)) { + + if (m->clustered || (flags & UPL_COMMIT_SPECULATE)) + dwp->dw_mask |= DW_vm_page_speculate; + else if (m->reference) + dwp->dw_mask |= DW_vm_page_activate; + else { + dwp->dw_mask |= DW_vm_page_deactivate_internal; + clear_refmod |= VM_MEM_REFERENCED; + } } } if (upl->flags & UPL_ACCESS_BLOCKED) { @@ -4936,7 +7594,7 @@ process_upl_to_commit: commit_next_page: if (clear_refmod) - pmap_clear_refmod(m->phys_page, clear_refmod); + pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(m), clear_refmod); target_offset += PAGE_SIZE_64; xfer_size -= PAGE_SIZE; @@ -4944,37 +7602,100 @@ commit_next_page: if (dwp->dw_mask) { if (dwp->dw_mask & ~(DW_clear_busy | DW_PAGE_WAKEUP)) { - if (m->busy == FALSE) { - /* - * dw_do_work may need to drop the object lock - * if it does, we need the pages it's looking at to - * be held stable via the busy bit. - */ - m->busy = TRUE; - dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); + VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count); + + if (dw_count >= dw_limit) { + vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count); + + dwp = &dw_array[0]; + dw_count = 0; + } + } else { + if (dwp->dw_mask & DW_clear_busy) + m->busy = FALSE; + + if (dwp->dw_mask & DW_PAGE_WAKEUP) + PAGE_WAKEUP(m); + } + } + } + if (dw_count) + vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count); + + if (fast_path_possible) { + + assert(shadow_object->purgable != VM_PURGABLE_VOLATILE); + assert(shadow_object->purgable != VM_PURGABLE_EMPTY); + + if (local_queue_count || unwired_count) { + + if (local_queue_count) { + vm_page_t first_target; + vm_page_queue_head_t *target_queue; + + if (throttle_page) + target_queue = &vm_page_queue_throttled; + else { + if (flags & UPL_COMMIT_INACTIVATE) { + if (shadow_object->internal) + target_queue = &vm_page_queue_anonymous; + else + target_queue = &vm_page_queue_inactive; + } else + target_queue = &vm_page_queue_active; } - dwp->dw_m = m; - dwp++; - dw_count++; + /* + * Transfer the entire local queue to a regular LRU page queues. + */ + vm_page_lockspin_queues(); - if (dw_count >= DELAYED_WORK_LIMIT) { - dw_do_work(shadow_object, &dw_array[0], dw_count); - - dwp = &dw_array[0]; - dw_count = 0; + first_target = (vm_page_t) vm_page_queue_first(target_queue); + + if (vm_page_queue_empty(target_queue)) + target_queue->prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local); + else + first_target->pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local); + + target_queue->next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local); + first_local->pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(target_queue); + last_local->pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_target); + + /* + * Adjust the global page counts. + */ + if (throttle_page) { + vm_page_throttled_count += local_queue_count; + } else { + if (flags & UPL_COMMIT_INACTIVATE) { + if (shadow_object->internal) + vm_page_anonymous_count += local_queue_count; + vm_page_inactive_count += local_queue_count; + + token_new_pagecount += local_queue_count; + } else + vm_page_active_count += local_queue_count; + + if (shadow_object->internal) + vm_page_pageable_internal_count += local_queue_count; + else + vm_page_pageable_external_count += local_queue_count; } } else { - if (dwp->dw_mask & DW_clear_busy) - m->busy = FALSE; + vm_page_lockspin_queues(); + } + if (unwired_count) { + vm_page_wire_count -= unwired_count; + VM_CHECK_MEMORYSTATUS; + } + vm_page_unlock_queues(); - if (dwp->dw_mask & DW_PAGE_WAKEUP) - PAGE_WAKEUP(m); + shadow_object->wired_page_count -= unwired_count; + + if (!shadow_object->wired_page_count) { + VM_OBJECT_UNWIRED(shadow_object); } } } - if (dw_count) - dw_do_work(shadow_object, &dw_array[0], dw_count); - occupied = 1; if (upl->flags & UPL_DEVICE_MEMORY) { @@ -4983,18 +7704,21 @@ commit_next_page: int pg_num; int i; - pg_num = upl->size/PAGE_SIZE; - pg_num = (pg_num + 31) >> 5; occupied = 0; - for (i = 0; i < pg_num; i++) { - if (lite_list[i] != 0) { - occupied = 1; - break; + if (!fast_path_full_commit) { + pg_num = upl->size/PAGE_SIZE; + pg_num = (pg_num + 31) >> 5; + + for (i = 0; i < pg_num; i++) { + if (lite_list[i] != 0) { + occupied = 1; + break; + } } } } else { - if (queue_empty(&upl->map_object->memq)) + if (vm_page_queue_empty(&upl->map_object->memq)) occupied = 0; } if (occupied == 0) { @@ -5017,6 +7741,7 @@ commit_next_page: * against this object */ vm_object_activity_end(shadow_object); + vm_object_collapse(shadow_object, 0, TRUE); } else { /* * we dontated the paging reference to @@ -5047,7 +7772,6 @@ commit_next_page: } goto process_upl_to_commit; } - if (pgpgout_count) { DTRACE_VM2(pgpgout, int, pgpgout_count, (uint64_t *), NULL); } @@ -5063,6 +7787,7 @@ upl_abort_range( int error, boolean_t *empty) { + upl_page_info_t *user_page_list = NULL; upl_size_t xfer_size, subupl_size = size; vm_object_t shadow_object; vm_object_t object; @@ -5071,9 +7796,11 @@ upl_abort_range( int entry; wpl_array_t lite_list; int occupied; - struct dw dw_array[DELAYED_WORK_LIMIT]; - struct dw *dwp; - int dw_count, isVectorUPL = 0; + struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT]; + struct vm_page_delayed_work *dwp; + int dw_count; + int dw_limit; + int isVectorUPL = 0; upl_t vector_upl = NULL; *empty = FALSE; @@ -5138,6 +7865,8 @@ process_upl_to_abort: lite_list = (wpl_array_t) ((((uintptr_t)upl) + sizeof(struct upl)) + ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t))); + + user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl)); } else { lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl)); @@ -5153,6 +7882,9 @@ process_upl_to_abort: entry = offset/PAGE_SIZE; target_offset = (vm_object_offset_t)offset; + assert(!(target_offset & PAGE_MASK)); + assert(!(xfer_size & PAGE_MASK)); + if (upl->flags & UPL_KERNEL_OBJECT) vm_object_lock_shared(shadow_object); else @@ -5166,23 +7898,28 @@ process_upl_to_abort: dwp = &dw_array[0]; dw_count = 0; + dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT); if ((error & UPL_ABORT_DUMP_PAGES) && (upl->flags & UPL_KERNEL_OBJECT)) panic("upl_abort_range: kernel_object being DUMPED"); while (xfer_size) { vm_page_t t, m; + unsigned int pg_num; + boolean_t needed; - dwp->dw_mask = 0; + pg_num = (unsigned int) (target_offset/PAGE_SIZE); + assert(pg_num == target_offset/PAGE_SIZE); + + needed = FALSE; + if (user_page_list) + needed = user_page_list[pg_num].needed; + + dwp->dw_mask = 0; m = VM_PAGE_NULL; if (upl->flags & UPL_LITE) { - unsigned int pg_num; - - pg_num = (unsigned int) (target_offset/PAGE_SIZE); - assert(pg_num == target_offset/PAGE_SIZE); - if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) { lite_list[pg_num>>5] &= ~(1 << (pg_num & 31)); @@ -5194,12 +7931,12 @@ process_upl_to_abort: } if (upl->flags & UPL_SHADOWED) { if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) { - t->pageout = FALSE; + t->free_when_done = FALSE; VM_PAGE_FREE(t); if (m == VM_PAGE_NULL) - m = vm_page_lookup(shadow_object, target_offset + object->shadow_offset); + m = vm_page_lookup(shadow_object, target_offset + object->vo_shadow_offset); } } if ((upl->flags & UPL_KERNEL_OBJECT)) @@ -5207,10 +7944,11 @@ process_upl_to_abort: if (m != VM_PAGE_NULL) { + assert(m->vm_page_q_state != VM_PAGE_USED_BY_COMPRESSOR); + if (m->absent) { boolean_t must_free = TRUE; - m->clustered = FALSE; /* * COPYOUT = FALSE case * check for error conditions which must @@ -5232,6 +7970,18 @@ process_upl_to_abort: m->unusual = TRUE; must_free = FALSE; } + if (m->clustered && needed == FALSE) { + /* + * This page was a part of a speculative + * read-ahead initiated by the kernel + * itself. No one is expecting this + * page and no one will clean up its + * error state if it ever becomes valid + * in the future. + * We have to free it here. + */ + must_free = TRUE; + } /* * ENCRYPTED SWAP: @@ -5244,6 +7994,21 @@ process_upl_to_abort: m->cleaning = FALSE; m->encrypted_cleaning = FALSE; + + if (m->overwriting && !m->busy) { + /* + * this shouldn't happen since + * this is an 'absent' page, but + * it doesn't hurt to check for + * the 'alternate' method of + * stabilizing the page... + * we will mark 'busy' to be cleared + * in the following code which will + * take care of the primary stabilzation + * method (i.e. setting 'busy' to TRUE) + */ + dwp->dw_mask |= DW_vm_page_unwire; + } m->overwriting = FALSE; dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); @@ -5259,35 +8024,58 @@ process_upl_to_abort: if (m->laundry) dwp->dw_mask |= DW_vm_pageout_throttle_up; - if (m->pageout) { - assert(m->busy); - assert(m->wire_count == 1); - m->pageout = FALSE; + if (upl->flags & UPL_ACCESS_BLOCKED) { + /* + * We blocked access to the pages in this UPL. + * Clear the "busy" bit and wake up any waiter + * for this page. + */ + dwp->dw_mask |= DW_clear_busy; + } + if (m->overwriting) { + if (m->busy) + dwp->dw_mask |= DW_clear_busy; + else { + /* + * deal with the 'alternate' method + * of stabilizing the page... + * we will either free the page + * or mark 'busy' to be cleared + * in the following code which will + * take care of the primary stabilzation + * method (i.e. setting 'busy' to TRUE) + */ + dwp->dw_mask |= DW_vm_page_unwire; + } + m->overwriting = FALSE; + } + if (m->encrypted_cleaning == TRUE) { + m->encrypted_cleaning = FALSE; - dwp->dw_mask |= DW_vm_page_unwire; + dwp->dw_mask |= DW_clear_busy; } - m->dump_cleaning = FALSE; + m->free_when_done = FALSE; m->cleaning = FALSE; - m->encrypted_cleaning = FALSE; - m->overwriting = FALSE; -#if MACH_PAGEMAP - vm_external_state_clr(m->object->existence_map, m->offset); -#endif /* MACH_PAGEMAP */ + if (error & UPL_ABORT_DUMP_PAGES) { - pmap_disconnect(m->phys_page); + pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)); dwp->dw_mask |= DW_vm_page_free; } else { - if (error & UPL_ABORT_REFERENCE) { - /* - * we've been told to explictly - * reference this page... for - * file I/O, this is done by - * implementing an LRU on the inactive q - */ - dwp->dw_mask |= DW_vm_page_lru; + if (!(dwp->dw_mask & DW_vm_page_unwire)) { + if (error & UPL_ABORT_REFERENCE) { + /* + * we've been told to explictly + * reference this page... for + * file I/O, this is done by + * implementing an LRU on the inactive q + */ + dwp->dw_mask |= DW_vm_page_lru; + + } else if ( !VM_PAGE_PAGEABLE(m)) + dwp->dw_mask |= DW_vm_page_deactivate_internal; } - dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); + dwp->dw_mask |= DW_PAGE_WAKEUP; } } } @@ -5298,21 +8086,10 @@ abort_next_page: if (dwp->dw_mask) { if (dwp->dw_mask & ~(DW_clear_busy | DW_PAGE_WAKEUP)) { - if (m->busy == FALSE) { - /* - * dw_do_work may need to drop the object lock - * if it does, we need the pages it's looking at to - * be held stable via the busy bit. - */ - m->busy = TRUE; - dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); - } - dwp->dw_m = m; - dwp++; - dw_count++; + VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count); - if (dw_count >= DELAYED_WORK_LIMIT) { - dw_do_work(shadow_object, &dw_array[0], dw_count); + if (dw_count >= dw_limit) { + vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count); dwp = &dw_array[0]; dw_count = 0; @@ -5327,7 +8104,7 @@ abort_next_page: } } if (dw_count) - dw_do_work(shadow_object, &dw_array[0], dw_count); + vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count); occupied = 1; @@ -5348,7 +8125,7 @@ abort_next_page: } } } else { - if (queue_empty(&upl->map_object->memq)) + if (vm_page_queue_empty(&upl->map_object->memq)) occupied = 0; } if (occupied == 0) { @@ -5371,6 +8148,7 @@ abort_next_page: * against this object */ vm_object_activity_end(shadow_object); + vm_object_collapse(shadow_object, 0, TRUE); } else { /* * we dontated the paging reference to @@ -5413,6 +8191,9 @@ upl_abort( { boolean_t empty; + if (upl == UPL_NULL) + return KERN_INVALID_ARGUMENT; + return upl_abort_range(upl, 0, upl->size, error, &empty); } @@ -5426,12 +8207,393 @@ upl_commit( { boolean_t empty; + if (upl == UPL_NULL) + return KERN_INVALID_ARGUMENT; + return upl_commit_range(upl, 0, upl->size, 0, page_list, count, &empty); } +void +iopl_valid_data( + upl_t upl) +{ + vm_object_t object; + vm_offset_t offset; + vm_page_t m, nxt_page = VM_PAGE_NULL; + upl_size_t size; + int wired_count = 0; + + if (upl == NULL) + panic("iopl_valid_data: NULL upl"); + if (vector_upl_is_valid(upl)) + panic("iopl_valid_data: vector upl"); + if ((upl->flags & (UPL_DEVICE_MEMORY|UPL_SHADOWED|UPL_ACCESS_BLOCKED|UPL_IO_WIRE|UPL_INTERNAL)) != UPL_IO_WIRE) + panic("iopl_valid_data: unsupported upl, flags = %x", upl->flags); + + object = upl->map_object; + + if (object == kernel_object || object == compressor_object) + panic("iopl_valid_data: object == kernel or compressor"); + + if (object->purgable == VM_PURGABLE_VOLATILE || + object->purgable == VM_PURGABLE_EMPTY) + panic("iopl_valid_data: object %p purgable %d", + object, object->purgable); + + size = upl->size; + + vm_object_lock(object); + + if (object->vo_size == size && object->resident_page_count == (size / PAGE_SIZE)) + nxt_page = (vm_page_t)vm_page_queue_first(&object->memq); + else + offset = 0 + upl->offset - object->paging_offset; + + while (size) { + + if (nxt_page != VM_PAGE_NULL) { + m = nxt_page; + nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->listq); + } else { + m = vm_page_lookup(object, offset); + offset += PAGE_SIZE; + + if (m == VM_PAGE_NULL) + panic("iopl_valid_data: missing expected page at offset %lx", (long)offset); + } + if (m->busy) { + if (!m->absent) + panic("iopl_valid_data: busy page w/o absent"); + + if (m->pageq.next || m->pageq.prev) + panic("iopl_valid_data: busy+absent page on page queue"); + if (m->reusable) { + panic("iopl_valid_data: %p is reusable", m); + } + + m->absent = FALSE; + m->dirty = TRUE; + assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q); + assert(m->wire_count == 0); + m->wire_count++; + assert(m->wire_count); + if (m->wire_count == 1) { + m->vm_page_q_state = VM_PAGE_IS_WIRED; + wired_count++; + } else { + panic("iopl_valid_data: %p already wired\n", m); + } + + PAGE_WAKEUP_DONE(m); + } + size -= PAGE_SIZE; + } + if (wired_count) { + + if (!object->wired_page_count) { + VM_OBJECT_WIRED(object); + } + object->wired_page_count += wired_count; + assert(object->resident_page_count >= object->wired_page_count); + + /* no need to adjust purgeable accounting for this object: */ + assert(object->purgable != VM_PURGABLE_VOLATILE); + assert(object->purgable != VM_PURGABLE_EMPTY); + + vm_page_lockspin_queues(); + vm_page_wire_count += wired_count; + vm_page_unlock_queues(); + } + vm_object_unlock(object); +} + +vm_tag_t +iopl_set_tag( + upl_t upl, + vm_tag_t tag) +{ + vm_object_t object; + vm_tag_t prior_tag; + + if (upl == NULL) + panic("%s: NULL upl", __FUNCTION__); + if (vector_upl_is_valid(upl)) + panic("%s: vector upl", __FUNCTION__); + if (kernel_object == upl->map_object) + return (tag); + if ((upl->flags & (UPL_DEVICE_MEMORY|UPL_SHADOWED|UPL_ACCESS_BLOCKED|UPL_IO_WIRE|UPL_INTERNAL)) != UPL_IO_WIRE) + return (tag); + + object = upl->map_object; + vm_object_lock(object); + + prior_tag = object->wire_tag; + object->wire_tag = tag; + if (VM_KERN_MEMORY_NONE == prior_tag) prior_tag = tag; + vm_object_unlock(object); + + return (prior_tag); +} + + +void +vm_object_set_pmap_cache_attr( + vm_object_t object, + upl_page_info_array_t user_page_list, + unsigned int num_pages, + boolean_t batch_pmap_op) +{ + unsigned int cache_attr = 0; + + cache_attr = object->wimg_bits & VM_WIMG_MASK; + assert(user_page_list); + if (cache_attr != VM_WIMG_USE_DEFAULT) { + PMAP_BATCH_SET_CACHE_ATTR(object, user_page_list, cache_attr, num_pages, batch_pmap_op); + } +} + + +boolean_t vm_object_iopl_wire_full(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t); +kern_return_t vm_object_iopl_wire_empty(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t, vm_object_offset_t *, int); + + + +boolean_t +vm_object_iopl_wire_full(vm_object_t object, upl_t upl, upl_page_info_array_t user_page_list, + wpl_array_t lite_list, upl_control_flags_t cntrl_flags) +{ + vm_page_t dst_page; + vm_tag_t tag; + unsigned int entry; + int page_count; + int delayed_unlock = 0; + boolean_t retval = TRUE; + ppnum_t phys_page; + + vm_object_lock_assert_exclusive(object); + assert(object->purgable != VM_PURGABLE_VOLATILE); + assert(object->purgable != VM_PURGABLE_EMPTY); + assert(object->pager == NULL); + assert(object->copy == NULL); + assert(object->shadow == NULL); + + tag = UPL_MEMORY_TAG(cntrl_flags); + page_count = object->resident_page_count; + dst_page = (vm_page_t)vm_page_queue_first(&object->memq); + + vm_page_lock_queues(); + + while (page_count--) { + + if (dst_page->busy || + dst_page->fictitious || + dst_page->absent || + dst_page->error || + dst_page->cleaning || + dst_page->restart || + dst_page->encrypted || + dst_page->laundry) { + retval = FALSE; + goto done; + } + if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->written_by_kernel == TRUE) { + retval = FALSE; + goto done; + } + dst_page->reference = TRUE; + + vm_page_wire(dst_page, tag, FALSE); + + if (!(cntrl_flags & UPL_COPYOUT_FROM)) { + SET_PAGE_DIRTY(dst_page, FALSE); + } + entry = (unsigned int)(dst_page->offset / PAGE_SIZE); + assert(entry >= 0 && entry < object->resident_page_count); + lite_list[entry>>5] |= 1 << (entry & 31); + + phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page); + + if (phys_page > upl->highest_page) + upl->highest_page = phys_page; + + if (user_page_list) { + user_page_list[entry].phys_addr = phys_page; + user_page_list[entry].absent = dst_page->absent; + user_page_list[entry].dirty = dst_page->dirty; + user_page_list[entry].free_when_done = dst_page->free_when_done; + user_page_list[entry].precious = dst_page->precious; + user_page_list[entry].device = FALSE; + user_page_list[entry].speculative = FALSE; + user_page_list[entry].cs_validated = FALSE; + user_page_list[entry].cs_tainted = FALSE; + user_page_list[entry].cs_nx = FALSE; + user_page_list[entry].needed = FALSE; + user_page_list[entry].mark = FALSE; + } + if (delayed_unlock++ > 256) { + delayed_unlock = 0; + lck_mtx_yield(&vm_page_queue_lock); + + VM_CHECK_MEMORYSTATUS; + } + dst_page = (vm_page_t)vm_page_queue_next(&dst_page->listq); + } +done: + vm_page_unlock_queues(); + + VM_CHECK_MEMORYSTATUS; + + return (retval); +} + + +kern_return_t +vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t user_page_list, + wpl_array_t lite_list, upl_control_flags_t cntrl_flags, vm_object_offset_t *dst_offset, int page_count) +{ + vm_page_t dst_page; + vm_tag_t tag; + boolean_t no_zero_fill = FALSE; + int interruptible; + int pages_wired = 0; + int pages_inserted = 0; + int entry = 0; + uint64_t delayed_ledger_update = 0; + kern_return_t ret = KERN_SUCCESS; + int grab_options; + ppnum_t phys_page; + + vm_object_lock_assert_exclusive(object); + assert(object->purgable != VM_PURGABLE_VOLATILE); + assert(object->purgable != VM_PURGABLE_EMPTY); + assert(object->pager == NULL); + assert(object->copy == NULL); + assert(object->shadow == NULL); + + if (cntrl_flags & UPL_SET_INTERRUPTIBLE) + interruptible = THREAD_ABORTSAFE; + else + interruptible = THREAD_UNINT; + + if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO)) + no_zero_fill = TRUE; + + tag = UPL_MEMORY_TAG(cntrl_flags); + + grab_options = 0; +#if CONFIG_SECLUDED_MEMORY + if (object->can_grab_secluded) { + grab_options |= VM_PAGE_GRAB_SECLUDED; + } +#endif /* CONFIG_SECLUDED_MEMORY */ + + while (page_count--) { + + while ((dst_page = vm_page_grab_options(grab_options)) + == VM_PAGE_NULL) { + + OSAddAtomic(page_count, &vm_upl_wait_for_pages); + + VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0); + + if (vm_page_wait(interruptible) == FALSE) { + /* + * interrupted case + */ + OSAddAtomic(-page_count, &vm_upl_wait_for_pages); + + VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1); + + ret = MACH_SEND_INTERRUPTED; + goto done; + } + OSAddAtomic(-page_count, &vm_upl_wait_for_pages); + + VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0); + } + if (no_zero_fill == FALSE) + vm_page_zero_fill(dst_page); + else + dst_page->absent = TRUE; + + dst_page->reference = TRUE; + + if (!(cntrl_flags & UPL_COPYOUT_FROM)) { + SET_PAGE_DIRTY(dst_page, FALSE); + } + if (dst_page->absent == FALSE) { + assert(dst_page->vm_page_q_state == VM_PAGE_NOT_ON_Q); + assert(dst_page->wire_count == 0); + dst_page->wire_count++; + dst_page->vm_page_q_state = VM_PAGE_IS_WIRED; + assert(dst_page->wire_count); + pages_wired++; + PAGE_WAKEUP_DONE(dst_page); + } + pages_inserted++; + + vm_page_insert_internal(dst_page, object, *dst_offset, tag, FALSE, TRUE, TRUE, TRUE, &delayed_ledger_update); + + lite_list[entry>>5] |= 1 << (entry & 31); + + phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page); + + if (phys_page > upl->highest_page) + upl->highest_page = phys_page; + + if (user_page_list) { + user_page_list[entry].phys_addr = phys_page; + user_page_list[entry].absent = dst_page->absent; + user_page_list[entry].dirty = dst_page->dirty; + user_page_list[entry].free_when_done = FALSE; + user_page_list[entry].precious = FALSE; + user_page_list[entry].device = FALSE; + user_page_list[entry].speculative = FALSE; + user_page_list[entry].cs_validated = FALSE; + user_page_list[entry].cs_tainted = FALSE; + user_page_list[entry].cs_nx = FALSE; + user_page_list[entry].needed = FALSE; + user_page_list[entry].mark = FALSE; + } + entry++; + *dst_offset += PAGE_SIZE_64; + } +done: + if (pages_wired) { + vm_page_lockspin_queues(); + vm_page_wire_count += pages_wired; + vm_page_unlock_queues(); + } + if (pages_inserted) { + if (object->internal) { + OSAddAtomic(pages_inserted, &vm_page_internal_count); + } else { + OSAddAtomic(pages_inserted, &vm_page_external_count); + } + } + if (delayed_ledger_update) { + task_t owner; + + owner = object->vo_purgeable_owner; + assert(owner); + + /* more non-volatile bytes */ + ledger_credit(owner->ledger, + task_ledgers.purgeable_nonvolatile, + delayed_ledger_update); + /* more footprint */ + ledger_credit(owner->ledger, + task_ledgers.phys_footprint, + delayed_ledger_update); + } + return (ret); +} + + unsigned int vm_object_iopl_request_sleep_for_cleaning = 0; + kern_return_t vm_object_iopl_request( vm_object_t object, @@ -5440,7 +8602,7 @@ vm_object_iopl_request( upl_t *upl_ptr, upl_page_info_array_t user_page_list, unsigned int *page_list_count, - int cntrl_flags) + upl_control_flags_t cntrl_flags) { vm_page_t dst_page; vm_object_offset_t dst_offset; @@ -5449,14 +8611,25 @@ vm_object_iopl_request( unsigned int entry; wpl_array_t lite_list = NULL; int no_zero_fill = FALSE; + unsigned int size_in_pages; u_int32_t psize; kern_return_t ret; vm_prot_t prot; struct vm_object_fault_info fault_info; - struct dw dw_array[DELAYED_WORK_LIMIT]; - struct dw *dwp; + struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT]; + struct vm_page_delayed_work *dwp; int dw_count; + int dw_limit; int dw_index; + boolean_t caller_lookup; + int io_tracking_flag = 0; + int interruptible; + ppnum_t phys_page; + + boolean_t set_cache_attr_needed = FALSE; + boolean_t free_wired_pages = FALSE; + boolean_t fast_path_empty_req = FALSE; + boolean_t fast_path_full_req = FALSE; if (cntrl_flags & ~UPL_VALID_FLAGS) { /* @@ -5473,10 +8646,10 @@ vm_object_iopl_request( return KERN_INVALID_VALUE; if (object->phys_contiguous) { - if ((offset + object->shadow_offset) >= (vm_object_offset_t)max_valid_dma_address) + if ((offset + object->vo_shadow_offset) >= (vm_object_offset_t)max_valid_dma_address) return KERN_INVALID_ADDRESS; - if (((offset + object->shadow_offset) + size) >= (vm_object_offset_t)max_valid_dma_address) + if (((offset + object->vo_shadow_offset) + size) >= (vm_object_offset_t)max_valid_dma_address) return KERN_INVALID_ADDRESS; } } @@ -5490,7 +8663,7 @@ vm_object_iopl_request( */ assert(! (cntrl_flags & UPL_ENCRYPT)); } - if (cntrl_flags & UPL_NOZEROFILL) + if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO)) no_zero_fill = TRUE; if (cntrl_flags & UPL_COPYOUT_FROM) @@ -5498,20 +8671,21 @@ vm_object_iopl_request( else prot = VM_PROT_READ | VM_PROT_WRITE; - if (((size/PAGE_SIZE) > MAX_UPL_SIZE) && !object->phys_contiguous) - size = MAX_UPL_SIZE * PAGE_SIZE; - - if (cntrl_flags & UPL_SET_INTERNAL) { - if (page_list_count != NULL) - *page_list_count = MAX_UPL_SIZE; - } - if (((cntrl_flags & UPL_SET_INTERNAL) && !(object->phys_contiguous)) && - ((page_list_count != NULL) && (*page_list_count != 0) && *page_list_count < (size/page_size))) - return KERN_INVALID_ARGUMENT; - if ((!object->internal) && (object->paging_offset != 0)) panic("vm_object_iopl_request: external object with non-zero paging offset\n"); +#if CONFIG_IOSCHED || UPL_DEBUG + if ((object->io_tracking && object != kernel_object) || upl_debug_enabled) + io_tracking_flag |= UPL_CREATE_IO_TRACKING; +#endif + +#if CONFIG_IOSCHED + if (object->io_tracking) { + /* Check if we're dealing with the kernel object. We do not support expedite on kernel object UPLs */ + if (object != kernel_object) + io_tracking_flag |= UPL_CREATE_EXPEDITE_SUP; + } +#endif if (object->phys_contiguous) psize = PAGE_SIZE; @@ -5519,7 +8693,7 @@ vm_object_iopl_request( psize = size; if (cntrl_flags & UPL_SET_INTERNAL) { - upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE, UPL_IO_WIRE, psize); + upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE | io_tracking_flag, UPL_IO_WIRE, psize); user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl)); lite_list = (wpl_array_t) (((uintptr_t)user_page_list) + @@ -5529,7 +8703,7 @@ vm_object_iopl_request( lite_list = NULL; } } else { - upl = upl_create(UPL_CREATE_LITE, UPL_IO_WIRE, psize); + upl = upl_create(UPL_CREATE_LITE | io_tracking_flag, UPL_IO_WIRE, psize); lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl)); if (size == 0) { @@ -5543,6 +8717,8 @@ vm_object_iopl_request( upl->map_object = object; upl->size = size; + size_in_pages = size / PAGE_SIZE; + if (object == kernel_object && !(cntrl_flags & (UPL_NEED_32BIT_ADDR | UPL_BLOCK_ACCESS))) { upl->flags |= UPL_KERNEL_OBJECT; @@ -5562,16 +8738,20 @@ vm_object_iopl_request( if (cntrl_flags & UPL_BLOCK_ACCESS) { /* - * The user requested that access to the pages in this URL + * The user requested that access to the pages in this UPL * be blocked until the UPL is commited or aborted. */ upl->flags |= UPL_ACCESS_BLOCKED; } - if (object->phys_contiguous) { -#if UPL_DEBUG +#if CONFIG_IOSCHED || UPL_DEBUG + if (upl->flags & UPL_TRACKED_BY_OBJECT) { + vm_object_activity_begin(object); queue_enter(&object->uplq, upl, upl_t, uplq); -#endif /* UPL_DEBUG */ + } +#endif + + if (object->phys_contiguous) { if (upl->flags & UPL_ACCESS_BLOCKED) { assert(!object->blocked_access); @@ -5586,10 +8766,10 @@ vm_object_iopl_request( */ upl->flags |= UPL_DEVICE_MEMORY; - upl->highest_page = (ppnum_t) ((offset + object->shadow_offset + size - 1)>>PAGE_SHIFT); + upl->highest_page = (ppnum_t) ((offset + object->vo_shadow_offset + size - 1)>>PAGE_SHIFT); if (user_page_list) { - user_page_list[0].phys_addr = (ppnum_t) ((offset + object->shadow_offset)>>PAGE_SHIFT); + user_page_list[0].phys_addr = (ppnum_t) ((offset + object->vo_shadow_offset)>>PAGE_SHIFT); user_page_list[0].device = TRUE; } if (page_list_count != NULL) { @@ -5600,20 +8780,33 @@ vm_object_iopl_request( } return KERN_SUCCESS; } - if (object != kernel_object) { + if (object != kernel_object && object != compressor_object) { /* * Protect user space from future COW operations */ +#if VM_OBJECT_TRACKING_OP_TRUESHARE + if (!object->true_share && + vm_object_tracking_inited) { + void *bt[VM_OBJECT_TRACKING_BTDEPTH]; + int num = 0; + + num = OSBacktrace(bt, + VM_OBJECT_TRACKING_BTDEPTH); + btlog_add_entry(vm_object_tracking_btlog, + object, + VM_OBJECT_TRACKING_OP_TRUESHARE, + bt, + num); + } +#endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */ + + vm_object_lock_assert_exclusive(object); object->true_share = TRUE; if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) object->copy_strategy = MEMORY_OBJECT_COPY_DELAY; } -#if UPL_DEBUG - queue_enter(&object->uplq, upl, upl_t, uplq); -#endif /* UPL_DEBUG */ - if (!(cntrl_flags & UPL_COPYOUT_FROM) && object->copy != VM_OBJECT_NULL) { /* @@ -5646,12 +8839,64 @@ vm_object_iopl_request( iopl_cow_pages += size >> PAGE_SHIFT; #endif } + if (!(cntrl_flags & (UPL_NEED_32BIT_ADDR | UPL_BLOCK_ACCESS)) && + object->purgable != VM_PURGABLE_VOLATILE && + object->purgable != VM_PURGABLE_EMPTY && + object->copy == NULL && + size == object->vo_size && + offset == 0 && + object->shadow == NULL && + object->pager == NULL) + { + if (object->resident_page_count == size_in_pages) + { + assert(object != compressor_object); + assert(object != kernel_object); + fast_path_full_req = TRUE; + } + else if (object->resident_page_count == 0) + { + assert(object != compressor_object); + assert(object != kernel_object); + fast_path_empty_req = TRUE; + set_cache_attr_needed = TRUE; + } + } + if (cntrl_flags & UPL_SET_INTERRUPTIBLE) + interruptible = THREAD_ABORTSAFE; + else + interruptible = THREAD_UNINT; entry = 0; xfer_size = size; dst_offset = offset; + dw_count = 0; + + if (fast_path_full_req) { + + if (vm_object_iopl_wire_full(object, upl, user_page_list, lite_list, cntrl_flags) == TRUE) + goto finish; + /* + * we couldn't complete the processing of this request on the fast path + * so fall through to the slow path and finish up + */ + + } else if (fast_path_empty_req) { + + if (cntrl_flags & UPL_REQUEST_NO_FAULT) { + ret = KERN_MEMORY_ERROR; + goto return_err; + } + ret = vm_object_iopl_wire_empty(object, upl, user_page_list, lite_list, cntrl_flags, &dst_offset, size_in_pages); + + if (ret) { + free_wired_pages = TRUE; + goto return_err; + } + goto finish; + } fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL; fault_info.user_tag = 0; @@ -5659,17 +8904,32 @@ vm_object_iopl_request( fault_info.hi_offset = offset + xfer_size; fault_info.no_cache = FALSE; fault_info.stealth = FALSE; + fault_info.io_sync = FALSE; + fault_info.cs_bypass = FALSE; fault_info.mark_zf_absent = TRUE; + fault_info.interruptible = interruptible; + fault_info.batch_pmap_op = TRUE; dwp = &dw_array[0]; - dw_count = 0; + dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT); while (xfer_size) { vm_fault_return_t result; - unsigned int pg_num; dwp->dw_mask = 0; + if (fast_path_full_req) { + /* + * if we get here, it means that we ran into a page + * state we couldn't handle in the fast path and + * bailed out to the slow path... since the order + * we look at pages is different between the 2 paths, + * the following check is needed to determine whether + * this page was already processed in the fast path + */ + if (lite_list[entry>>5] & (1 << (entry & 31))) + goto skip_page; + } dst_page = vm_page_lookup(object, dst_offset); /* @@ -5687,29 +8947,42 @@ vm_object_iopl_request( if (object == kernel_object) panic("vm_object_iopl_request: missing/bad page in kernel object\n"); + if (object == compressor_object) + panic("vm_object_iopl_request: missing/bad page in compressor object\n"); + + if (cntrl_flags & UPL_REQUEST_NO_FAULT) { + ret = KERN_MEMORY_ERROR; + goto return_err; + } + set_cache_attr_needed = TRUE; + + /* + * We just looked up the page and the result remains valid + * until the object lock is release, so send it to + * vm_fault_page() (as "dst_page"), to avoid having to + * look it up again there. + */ + caller_lookup = TRUE; do { vm_page_t top_page; kern_return_t error_code; - int interruptible; - - if (cntrl_flags & UPL_SET_INTERRUPTIBLE) - interruptible = THREAD_ABORTSAFE; - else - interruptible = THREAD_UNINT; - fault_info.interruptible = interruptible; fault_info.cluster_size = xfer_size; vm_object_paging_begin(object); result = vm_fault_page(object, dst_offset, - prot | VM_PROT_WRITE, FALSE, + prot | VM_PROT_WRITE, FALSE, + caller_lookup, &prot, &dst_page, &top_page, (int *)0, &error_code, no_zero_fill, FALSE, &fault_info); + /* our lookup is no longer valid at this point */ + caller_lookup = FALSE; + switch (result) { case VM_FAULT_SUCCESS: @@ -5737,9 +9010,12 @@ vm_object_iopl_request( if (top_page != VM_PAGE_NULL) { vm_object_t local_object; - local_object = top_page->object; - - if (top_page->object != dst_page->object) { + local_object = VM_PAGE_OBJECT(top_page); + + /* + * comparing 2 packed pointers + */ + if (top_page->vm_page_object != dst_page->vm_page_object) { vm_object_lock(local_object); VM_PAGE_FREE(top_page); vm_object_paging_end(local_object); @@ -5756,17 +9032,23 @@ vm_object_iopl_request( vm_object_lock(object); break; - case VM_FAULT_FICTITIOUS_SHORTAGE: - vm_page_more_fictitious(); + case VM_FAULT_MEMORY_SHORTAGE: + OSAddAtomic((size_in_pages - entry), &vm_upl_wait_for_pages); - vm_object_lock(object); - break; + VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0); - case VM_FAULT_MEMORY_SHORTAGE: if (vm_page_wait(interruptible)) { + OSAddAtomic(-(size_in_pages - entry), &vm_upl_wait_for_pages); + + VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0); vm_object_lock(object); + break; } + OSAddAtomic(-(size_in_pages - entry), &vm_upl_wait_for_pages); + + VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1); + /* fall thru */ case VM_FAULT_INTERRUPTED: @@ -5791,13 +9073,19 @@ vm_object_iopl_request( } while (result != VM_FAULT_SUCCESS); } + phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page); if (upl->flags & UPL_KERNEL_OBJECT) goto record_phys_addr; + if (dst_page->vm_page_q_state == VM_PAGE_USED_BY_COMPRESSOR) { + dst_page->busy = TRUE; + goto record_phys_addr; + } + if (dst_page->cleaning) { /* - * Someone else is cleaning this page in place.as + * Someone else is cleaning this page in place. * In theory, we should be able to proceed and use this * page but they'll probably end up clearing the "busy" * bit on it in upl_commit_range() but they didn't set @@ -5810,8 +9098,11 @@ vm_object_iopl_request( PAGE_SLEEP(object, dst_page, THREAD_UNINT); continue; } + if (dst_page->laundry) + vm_pageout_steal_laundry(dst_page, FALSE); + if ( (cntrl_flags & UPL_NEED_32BIT_ADDR) && - dst_page->phys_page >= (max_valid_dma_address >> PAGE_SHIFT) ) { + phys_page >= (max_valid_dma_address >> PAGE_SHIFT) ) { vm_page_t low_page; int refmod; @@ -5841,11 +9132,11 @@ vm_object_iopl_request( * to find the new page being substituted. */ if (dst_page->pmapped) - refmod = pmap_disconnect(dst_page->phys_page); + refmod = pmap_disconnect(phys_page); else refmod = 0; - if ( !dst_page->absent) + if (!dst_page->absent) vm_page_copy(dst_page, low_page); low_page->reference = dst_page->reference; @@ -5854,8 +9145,9 @@ vm_object_iopl_request( if (refmod & VM_MEM_REFERENCED) low_page->reference = TRUE; - if (refmod & VM_MEM_MODIFIED) - low_page->dirty = TRUE; + if (refmod & VM_MEM_MODIFIED) { + SET_PAGE_DIRTY(low_page, FALSE); + } vm_page_replace(low_page, object, dst_offset); @@ -5867,6 +9159,8 @@ vm_object_iopl_request( */ if ( !dst_page->absent) dst_page->busy = FALSE; + + phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page); } if ( !dst_page->busy) dwp->dw_mask |= DW_vm_page_wire; @@ -5874,7 +9168,8 @@ vm_object_iopl_request( if (cntrl_flags & UPL_BLOCK_ACCESS) { /* * Mark the page "busy" to block any future page fault - * on this page. We'll also remove the mapping + * on this page in addition to wiring it. + * We'll also remove the mapping * of all these pages before leaving this routine. */ assert(!dst_page->fictitious); @@ -5886,75 +9181,78 @@ vm_object_iopl_request( */ dwp->dw_mask |= DW_set_reference; - if (!(cntrl_flags & UPL_COPYOUT_FROM)) - dst_page->dirty = TRUE; + if (!(cntrl_flags & UPL_COPYOUT_FROM)) { + SET_PAGE_DIRTY(dst_page, TRUE); + } + if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->written_by_kernel == TRUE) { + pmap_sync_page_attributes_phys(phys_page); + dst_page->written_by_kernel = FALSE; + } + record_phys_addr: if (dst_page->busy) upl->flags |= UPL_HAS_BUSY; - pg_num = (unsigned int) ((dst_offset-offset)/PAGE_SIZE); - assert(pg_num == (dst_offset-offset)/PAGE_SIZE); - lite_list[pg_num>>5] |= 1 << (pg_num & 31); + lite_list[entry>>5] |= 1 << (entry & 31); - if (dst_page->phys_page > upl->highest_page) - upl->highest_page = dst_page->phys_page; + if (phys_page > upl->highest_page) + upl->highest_page = phys_page; if (user_page_list) { - user_page_list[entry].phys_addr = dst_page->phys_page; - user_page_list[entry].pageout = dst_page->pageout; + user_page_list[entry].phys_addr = phys_page; + user_page_list[entry].free_when_done = dst_page->free_when_done; user_page_list[entry].absent = dst_page->absent; user_page_list[entry].dirty = dst_page->dirty; user_page_list[entry].precious = dst_page->precious; user_page_list[entry].device = FALSE; + user_page_list[entry].needed = FALSE; if (dst_page->clustered == TRUE) - user_page_list[entry].speculative = dst_page->speculative; + user_page_list[entry].speculative = (dst_page->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE; else user_page_list[entry].speculative = FALSE; user_page_list[entry].cs_validated = dst_page->cs_validated; user_page_list[entry].cs_tainted = dst_page->cs_tainted; + user_page_list[entry].cs_nx = dst_page->cs_nx; + user_page_list[entry].mark = FALSE; } - if (object != kernel_object) { + if (object != kernel_object && object != compressor_object) { /* * someone is explicitly grabbing this page... * update clustered and speculative state * */ - VM_PAGE_CONSUME_CLUSTERED(dst_page); + if (dst_page->clustered) + VM_PAGE_CONSUME_CLUSTERED(dst_page); } +skip_page: entry++; dst_offset += PAGE_SIZE_64; xfer_size -= PAGE_SIZE; if (dwp->dw_mask) { - if (dst_page->busy == FALSE) { - /* - * dw_do_work may need to drop the object lock - * if it does, we need the pages it's looking at to - * be held stable via the busy bit. - */ - dst_page->busy = TRUE; - dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); - } - dwp->dw_m = dst_page; - dwp++; - dw_count++; + VM_PAGE_ADD_DELAYED_WORK(dwp, dst_page, dw_count); - if (dw_count >= DELAYED_WORK_LIMIT) { - dw_do_work(object, &dw_array[0], dw_count); + if (dw_count >= dw_limit) { + vm_page_do_delayed_work(object, UPL_MEMORY_TAG(cntrl_flags), &dw_array[0], dw_count); dwp = &dw_array[0]; dw_count = 0; } } } + assert(entry == size_in_pages); + if (dw_count) - dw_do_work(object, &dw_array[0], dw_count); + vm_page_do_delayed_work(object, UPL_MEMORY_TAG(cntrl_flags), &dw_array[0], dw_count); +finish: + if (user_page_list && set_cache_attr_needed == TRUE) + vm_object_set_pmap_cache_attr(object, user_page_list, size_in_pages, TRUE); if (page_list_count != NULL) { if (upl->flags & UPL_INTERNAL) *page_list_count = 0; - else if (*page_list_count > entry) - *page_list_count = entry; + else if (*page_list_count > size_in_pages) + *page_list_count = size_in_pages; } vm_object_unlock(object); @@ -5970,6 +9268,7 @@ record_phys_addr: assert(!object->blocked_access); object->blocked_access = TRUE; } + return KERN_SUCCESS; return_err: @@ -6010,7 +9309,7 @@ return_err: } vm_page_lock_queues(); - if (dst_page->absent) { + if (dst_page->absent || free_wired_pages == TRUE) { vm_page_free(dst_page); need_unwire = FALSE; @@ -6019,7 +9318,7 @@ return_err: vm_page_unwire(dst_page, TRUE); PAGE_WAKEUP_DONE(dst_page); - } + } vm_page_unlock_queues(); if (need_unwire == TRUE) @@ -6030,6 +9329,7 @@ return_err: #endif if (! (upl->flags & UPL_KERNEL_OBJECT)) { vm_object_activity_end(object); + vm_object_collapse(object, 0, TRUE); } vm_object_unlock(object); upl_destroy(upl); @@ -6091,15 +9391,28 @@ upl_transpose( * Make each UPL point to the correct VM object, i.e. the * object holding the pages that the UPL refers to... */ -#if UPL_DEBUG - queue_remove(&object1->uplq, upl1, upl_t, uplq); - queue_remove(&object2->uplq, upl2, upl_t, uplq); +#if CONFIG_IOSCHED || UPL_DEBUG + if ((upl1->flags & UPL_TRACKED_BY_OBJECT) || (upl2->flags & UPL_TRACKED_BY_OBJECT)) { + vm_object_lock(object1); + vm_object_lock(object2); + } + if (upl1->flags & UPL_TRACKED_BY_OBJECT) + queue_remove(&object1->uplq, upl1, upl_t, uplq); + if (upl2->flags & UPL_TRACKED_BY_OBJECT) + queue_remove(&object2->uplq, upl2, upl_t, uplq); #endif upl1->map_object = object2; upl2->map_object = object1; -#if UPL_DEBUG - queue_enter(&object1->uplq, upl2, upl_t, uplq); - queue_enter(&object2->uplq, upl1, upl_t, uplq); + +#if CONFIG_IOSCHED || UPL_DEBUG + if (upl1->flags & UPL_TRACKED_BY_OBJECT) + queue_enter(&object2->uplq, upl1, upl_t, uplq); + if (upl2->flags & UPL_TRACKED_BY_OBJECT) + queue_enter(&object1->uplq, upl2, upl_t, uplq); + if ((upl1->flags & UPL_TRACKED_BY_OBJECT) || (upl2->flags & UPL_TRACKED_BY_OBJECT)) { + vm_object_unlock(object2); + vm_object_unlock(object1); + } #endif } @@ -6116,6 +9429,27 @@ done: return retval; } +void +upl_range_needed( + upl_t upl, + int index, + int count) +{ + upl_page_info_t *user_page_list; + int size_in_pages; + + if ( !(upl->flags & UPL_INTERNAL) || count <= 0) + return; + + size_in_pages = upl->size / PAGE_SIZE; + + user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl)); + + while (count-- && index < size_in_pages) + user_page_list[index++].needed = TRUE; +} + + /* * ENCRYPTED SWAP: * @@ -6195,8 +9529,11 @@ vm_paging_map_init(void) if (kr != KERN_SUCCESS) { panic("vm_paging_map_init: kernel_map full\n"); } - map_entry->object.vm_object = kernel_object; - map_entry->offset = page_map_offset; + VME_OBJECT_SET(map_entry, kernel_object); + VME_OFFSET_SET(map_entry, page_map_offset); + map_entry->protection = VM_PROT_NONE; + map_entry->max_protection = VM_PROT_NONE; + map_entry->permanent = TRUE; vm_object_reference(kernel_object); vm_map_unlock(kernel_map); @@ -6221,13 +9558,14 @@ vm_paging_map_init(void) */ kern_return_t vm_paging_map_object( - vm_map_offset_t *address, vm_page_t page, vm_object_t object, vm_object_offset_t offset, - vm_map_size_t *size, vm_prot_t protection, - boolean_t can_unlock_object) + boolean_t can_unlock_object, + vm_map_size_t *size, /* IN/OUT */ + vm_map_offset_t *address, /* OUT */ + boolean_t *need_unmap) /* OUT */ { kern_return_t kr; vm_map_offset_t page_map_offset; @@ -6235,8 +9573,18 @@ vm_paging_map_object( vm_object_offset_t object_offset; int i; - if (page != VM_PAGE_NULL && *size == PAGE_SIZE) { + /* use permanent 1-to-1 kernel mapping of physical memory ? */ +#if __x86_64__ + *address = (vm_map_offset_t) + PHYSMAP_PTOV((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(page) << + PAGE_SHIFT); + *need_unmap = FALSE; + return KERN_SUCCESS; +#else +#warn "vm_paging_map_object: no 1-to-1 kernel mapping of physical memory..." +#endif + assert(page->busy); /* * Use one of the pre-allocated kernel virtual addresses @@ -6277,9 +9625,12 @@ vm_paging_map_object( */ vm_paging_page_waiter_total++; vm_paging_page_waiter++; - thread_sleep_fast_usimple_lock(&vm_paging_page_waiter, - &vm_paging_lock, - THREAD_UNINT); + kr = assert_wait((event_t)&vm_paging_page_waiter, THREAD_UNINT); + if (kr == THREAD_WAITING) { + simple_unlock(&vm_paging_lock); + kr = thread_block(THREAD_CONTINUE_NULL); + simple_lock(&vm_paging_lock); + } vm_paging_page_waiter--; /* ... and try again */ } @@ -6295,9 +9646,6 @@ vm_paging_map_object( vm_paging_page_inuse[i] = TRUE; simple_unlock(&vm_paging_lock); - if (page->pmapped == FALSE) { - pmap_sync_page_data_phys(page->phys_page); - } page->pmapped = TRUE; /* @@ -6310,12 +9658,13 @@ vm_paging_map_object( page_map_offset, page, protection, - ((int) page->object->wimg_bits & - VM_WIMG_MASK), + VM_PROT_NONE, + 0, TRUE); vm_paging_objects_mapped++; vm_paging_pages_mapped++; *address = page_map_offset; + *need_unmap = TRUE; /* all done and mapped, ready to use ! */ return KERN_SUCCESS; @@ -6331,11 +9680,15 @@ vm_paging_map_object( } if (! can_unlock_object) { + *address = 0; + *size = 0; + *need_unmap = FALSE; return KERN_NOT_SUPPORTED; } object_offset = vm_object_trunc_page(offset); - map_size = vm_map_round_page(*size); + map_size = vm_map_round_page(*size, + VM_MAP_PAGE_MASK(kernel_map)); /* * Try and map the required range of the object @@ -6359,6 +9712,7 @@ vm_paging_map_object( if (kr != KERN_SUCCESS) { *address = 0; *size = 0; + *need_unmap = FALSE; vm_object_deallocate(object); /* for the map entry */ vm_object_lock(object); return kr; @@ -6380,7 +9734,6 @@ vm_paging_map_object( for (page_map_offset = 0; map_size != 0; map_size -= PAGE_SIZE_64, page_map_offset += PAGE_SIZE_64) { - unsigned int cache_attr; page = vm_page_lookup(object, offset + page_map_offset); if (page == VM_PAGE_NULL) { @@ -6391,27 +9744,27 @@ vm_paging_map_object( assert(kr == KERN_SUCCESS); *address = 0; *size = 0; + *need_unmap = FALSE; vm_object_lock(object); return KERN_MEMORY_ERROR; } - if (page->pmapped == FALSE) { - pmap_sync_page_data_phys(page->phys_page); - } page->pmapped = TRUE; - cache_attr = ((unsigned int) object->wimg_bits) & VM_WIMG_MASK; - //assert(pmap_verify_free(page->phys_page)); + //assert(pmap_verify_free(VM_PAGE_GET_PHYS_PAGE(page))); PMAP_ENTER(kernel_pmap, *address + page_map_offset, page, protection, - cache_attr, + VM_PROT_NONE, + 0, TRUE); } vm_paging_objects_mapped_slow++; vm_paging_pages_mapped_slow += (unsigned long) (map_size / PAGE_SIZE_64); + *need_unmap = TRUE; + return KERN_SUCCESS; } @@ -6472,7 +9825,7 @@ vm_paging_unmap_object( } } -#if CRYPTO +#if ENCRYPTED_SWAP /* * Encryption data. * "iv" is the "initial vector". Ideally, we want to @@ -6481,7 +9834,7 @@ vm_paging_unmap_object( */ #define SWAP_CRYPT_AES_KEY_SIZE 128 /* XXX 192 and 256 don't work ! */ boolean_t swap_crypt_ctx_initialized = FALSE; -aes_32t swap_crypt_key[8]; /* big enough for a 256 key */ +uint32_t swap_crypt_key[8]; /* big enough for a 256 key */ aes_ctx swap_crypt_ctx; const unsigned char swap_crypt_null_iv[AES_BLOCK_SIZE] = {0xa, }; @@ -6596,7 +9949,9 @@ vm_page_encrypt( { kern_return_t kr; vm_map_size_t kernel_mapping_size; + boolean_t kernel_mapping_needs_unmap; vm_offset_t kernel_vaddr; + vm_object_t page_object; union { unsigned char aes_iv[AES_BLOCK_SIZE]; struct { @@ -6610,7 +9965,6 @@ vm_page_encrypt( } assert(page->busy); - assert(page->dirty || page->precious); if (page->encrypted) { /* @@ -6619,14 +9973,18 @@ vm_page_encrypt( vm_page_encrypt_already_encrypted_counter++; return; } + assert(page->dirty || page->precious); + ASSERT_PAGE_DECRYPTED(page); + page_object = VM_PAGE_OBJECT(page); + /* * Take a paging-in-progress reference to keep the object * alive even if we have to unlock it (in vm_paging_map_object() * for example)... */ - vm_object_paging_begin(page->object); + vm_object_paging_begin(page_object); if (kernel_mapping_offset == 0) { /* @@ -6635,13 +9993,15 @@ vm_page_encrypt( * its contents and encrypt them. */ kernel_mapping_size = PAGE_SIZE; - kr = vm_paging_map_object(&kernel_mapping_offset, - page, - page->object, + kernel_mapping_needs_unmap = FALSE; + kr = vm_paging_map_object(page, + page_object, page->offset, - &kernel_mapping_size, VM_PROT_READ | VM_PROT_WRITE, - FALSE); + FALSE, + &kernel_mapping_size, + &kernel_mapping_offset, + &kernel_mapping_needs_unmap); if (kr != KERN_SUCCESS) { panic("vm_page_encrypt: " "could not map page in kernel: 0x%x\n", @@ -6649,6 +10009,7 @@ vm_page_encrypt( } } else { kernel_mapping_size = 0; + kernel_mapping_needs_unmap = FALSE; } kernel_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping_offset); @@ -6665,9 +10026,9 @@ vm_page_encrypt( * use to break the key. */ bzero(&encrypt_iv.aes_iv[0], sizeof (encrypt_iv.aes_iv)); - encrypt_iv.vm.pager_object = page->object->pager; + encrypt_iv.vm.pager_object = page_object->pager; encrypt_iv.vm.paging_offset = - page->object->paging_offset + page->offset; + page_object->paging_offset + page->offset; /* encrypt the "initial vector" */ aes_encrypt_cbc((const unsigned char *) &encrypt_iv.aes_iv[0], @@ -6692,8 +10053,8 @@ vm_page_encrypt( * if we had to map it ourselves. Otherwise, let * the caller undo the mapping if needed. */ - if (kernel_mapping_size != 0) { - vm_paging_unmap_object(page->object, + if (kernel_mapping_needs_unmap) { + vm_paging_unmap_object(page_object, kernel_mapping_offset, kernel_mapping_offset + kernel_mapping_size); } @@ -6708,11 +10069,11 @@ vm_page_encrypt( * The software bits will be reset later after the I/O * has completed (in upl_commit_range()). */ - pmap_clear_refmod(page->phys_page, VM_MEM_REFERENCED | VM_MEM_MODIFIED); + pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(page), VM_MEM_REFERENCED | VM_MEM_MODIFIED); page->encrypted = TRUE; - vm_object_paging_end(page->object); + vm_object_paging_end(page_object); } /* @@ -6735,6 +10096,8 @@ vm_page_decrypt( kern_return_t kr; vm_map_size_t kernel_mapping_size; vm_offset_t kernel_vaddr; + boolean_t kernel_mapping_needs_unmap; + vm_object_t page_object; union { unsigned char aes_iv[AES_BLOCK_SIZE]; struct { @@ -6742,16 +10105,20 @@ vm_page_decrypt( vm_object_offset_t paging_offset; } vm; } decrypt_iv; + boolean_t was_dirty; assert(page->busy); assert(page->encrypted); + page_object = VM_PAGE_OBJECT(page); + was_dirty = page->dirty; + /* * Take a paging-in-progress reference to keep the object * alive even if we have to unlock it (in vm_paging_map_object() * for example)... */ - vm_object_paging_begin(page->object); + vm_object_paging_begin(page_object); if (kernel_mapping_offset == 0) { /* @@ -6760,13 +10127,15 @@ vm_page_decrypt( * its contents and decrypt them. */ kernel_mapping_size = PAGE_SIZE; - kr = vm_paging_map_object(&kernel_mapping_offset, - page, - page->object, + kernel_mapping_needs_unmap = FALSE; + kr = vm_paging_map_object(page, + page_object, page->offset, - &kernel_mapping_size, VM_PROT_READ | VM_PROT_WRITE, - FALSE); + FALSE, + &kernel_mapping_size, + &kernel_mapping_offset, + &kernel_mapping_needs_unmap); if (kr != KERN_SUCCESS) { panic("vm_page_decrypt: " "could not map page in kernel: 0x%x\n", @@ -6774,6 +10143,7 @@ vm_page_decrypt( } } else { kernel_mapping_size = 0; + kernel_mapping_needs_unmap = FALSE; } kernel_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping_offset); @@ -6785,9 +10155,9 @@ vm_page_decrypt( * used to encrypt that page. */ bzero(&decrypt_iv.aes_iv[0], sizeof (decrypt_iv.aes_iv)); - decrypt_iv.vm.pager_object = page->object->pager; + decrypt_iv.vm.pager_object = page_object->pager; decrypt_iv.vm.paging_offset = - page->object->paging_offset + page->offset; + page_object->paging_offset + page->offset; /* encrypt the "initial vector" */ aes_encrypt_cbc((const unsigned char *) &decrypt_iv.aes_iv[0], @@ -6811,22 +10181,30 @@ vm_page_decrypt( * if we had to map it ourselves. Otherwise, let * the caller undo the mapping if needed. */ - if (kernel_mapping_size != 0) { - vm_paging_unmap_object(page->object, + if (kernel_mapping_needs_unmap) { + vm_paging_unmap_object(page_object, kernel_vaddr, kernel_vaddr + PAGE_SIZE); } - /* - * After decryption, the page is actually clean. - * It was encrypted as part of paging, which "cleans" - * the "dirty" pages. - * Noone could access it after it was encrypted - * and the decryption doesn't count. - */ - page->dirty = FALSE; - assert (page->cs_validated == FALSE); - pmap_clear_refmod(page->phys_page, VM_MEM_MODIFIED | VM_MEM_REFERENCED); + if (was_dirty) { + /* + * The pager did not specify that the page would be + * clean when it got paged in, so let's not clean it here + * either. + */ + } else { + /* + * After decryption, the page is actually still clean. + * It was encrypted as part of paging, which "cleans" + * the "dirty" pages. + * Noone could access it after it was encrypted + * and the decryption doesn't count. + */ + page->dirty = FALSE; + assert (page->cs_validated == FALSE); + pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(page), VM_MEM_MODIFIED | VM_MEM_REFERENCED); + } page->encrypted = FALSE; /* @@ -6836,18 +10214,18 @@ vm_page_decrypt( * be part of a DMA transfer from a driver that expects the memory to * be coherent at this point, we have to flush the data cache. */ - pmap_sync_page_attributes_phys(page->phys_page); + pmap_sync_page_attributes_phys(VM_PAGE_GET_PHYS_PAGE(page)); /* * Since the page is not mapped yet, some code might assume that it * doesn't need to invalidate the instruction cache when writing to * that page. That code relies on "pmapped" being FALSE, so that the * caches get synchronized when the page is first mapped. */ - assert(pmap_verify_free(page->phys_page)); + assert(pmap_verify_free(VM_PAGE_GET_PHYS_PAGE(page))); page->pmapped = FALSE; page->wpmapped = FALSE; - vm_object_paging_end(page->object); + vm_object_paging_end(page_object); } #if DEVELOPMENT || DEBUG @@ -6943,7 +10321,7 @@ process_upl_to_encrypt: base_offset + offset_in_upl); if (page == VM_PAGE_NULL) { panic("upl_encrypt: " - "no page for (obj=%p,off=%lld+%d)!\n", + "no page for (obj=%p,off=0x%llx+0x%x)!\n", shadow_object, base_offset, offset_in_upl); @@ -6956,7 +10334,7 @@ process_upl_to_encrypt: * encryption completes, any access will cause a * page fault and the page gets decrypted at that time. */ - pmap_disconnect(page->phys_page); + pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(page)); vm_page_encrypt(page, 0); if (vm_object_lock_avoid(shadow_object)) { @@ -6979,7 +10357,7 @@ process_upl_to_encrypt: goto process_upl_to_encrypt; } -#else /* CRYPTO */ +#else /* ENCRYPTED_SWAP */ void upl_encrypt( __unused upl_t upl, @@ -7002,23 +10380,19 @@ vm_page_decrypt( { } -#endif /* CRYPTO */ +#endif /* ENCRYPTED_SWAP */ +/* + * page->object must be locked + */ void -vm_pageout_queue_steal(vm_page_t page, boolean_t queues_locked) +vm_pageout_steal_laundry(vm_page_t page, boolean_t queues_locked) { - boolean_t pageout; - - pageout = page->pageout; - - page->list_req_pending = FALSE; - page->cleaning = FALSE; - page->pageout = FALSE; - if (!queues_locked) { vm_page_lockspin_queues(); } + page->free_when_done = FALSE; /* * need to drop the laundry count... * we may also need to remove it @@ -7029,14 +10403,6 @@ vm_pageout_queue_steal(vm_page_t page, boolean_t queues_locked) */ vm_pageout_throttle_up(page); - if (pageout == TRUE) { - /* - * toss the wire count we picked up - * when we intially set this page up - * to be cleaned... - */ - vm_page_unwire(page, TRUE); - } vm_page_steal_pageout_page++; if (!queues_locked) { @@ -7084,7 +10450,7 @@ vector_upl_deallocate(upl_t upl) vector_upl->size = 0; vector_upl->offset = 0; kfree(vector_upl, sizeof(struct _vector_upl)); - vector_upl = (vector_upl_t)0xdeadbeef; + vector_upl = (vector_upl_t)0xfeedfeed; } else panic("vector_upl_deallocate was passed a non-vectored upl\n"); @@ -7098,7 +10464,7 @@ vector_upl_is_valid(upl_t upl) { if(upl && ((upl->flags & UPL_VECTOR)==UPL_VECTOR)) { vector_upl_t vector_upl = upl->vector_upl; - if(vector_upl == NULL || vector_upl == (vector_upl_t)0xdeadbeef || vector_upl == (vector_upl_t)0xfeedbeef) + if(vector_upl == NULL || vector_upl == (vector_upl_t)0xfeedfeed || vector_upl == (vector_upl_t)0xfeedbeef) return FALSE; else return TRUE; @@ -7373,6 +10739,202 @@ upl_clear_dirty( } } +void +upl_set_referenced( + upl_t upl, + boolean_t value) +{ + upl_lock(upl); + if (value) { + upl->ext_ref_count++; + } else { + if (!upl->ext_ref_count) { + panic("upl_set_referenced not %p\n", upl); + } + upl->ext_ref_count--; + } + upl_unlock(upl); +} + +#if CONFIG_IOSCHED +void +upl_set_blkno( + upl_t upl, + vm_offset_t upl_offset, + int io_size, + int64_t blkno) +{ + int i,j; + if ((upl->flags & UPL_EXPEDITE_SUPPORTED) == 0) + return; + + assert(upl->upl_reprio_info != 0); + for(i = (int)(upl_offset / PAGE_SIZE), j = 0; j < io_size; i++, j += PAGE_SIZE) { + UPL_SET_REPRIO_INFO(upl, i, blkno, io_size); + } +} +#endif + +boolean_t +vm_page_is_slideable(vm_page_t m) +{ + boolean_t result = FALSE; + vm_shared_region_slide_info_t si; + vm_object_t m_object; + + m_object = VM_PAGE_OBJECT(m); + + vm_object_lock_assert_held(m_object); + + /* make sure our page belongs to the one object allowed to do this */ + if (!m_object->object_slid) { + goto done; + } + + si = m_object->vo_slide_info; + if (si == NULL) { + goto done; + } + + if(!m->slid && (si->start <= m->offset && si->end > m->offset)) { + result = TRUE; + } + +done: + return result; +} + +int vm_page_slide_counter = 0; +int vm_page_slide_errors = 0; +kern_return_t +vm_page_slide( + vm_page_t page, + vm_map_offset_t kernel_mapping_offset) +{ + kern_return_t kr; + vm_map_size_t kernel_mapping_size; + boolean_t kernel_mapping_needs_unmap; + vm_offset_t kernel_vaddr; + uint32_t pageIndex; + uint32_t slide_chunk; + vm_object_t page_object; + + page_object = VM_PAGE_OBJECT(page); + + assert(!page->slid); + assert(page_object->object_slid); + vm_object_lock_assert_exclusive(page_object); + + if (page->error) + return KERN_FAILURE; + + /* + * Take a paging-in-progress reference to keep the object + * alive even if we have to unlock it (in vm_paging_map_object() + * for example)... + */ + vm_object_paging_begin(page_object); + + if (kernel_mapping_offset == 0) { + /* + * The page hasn't already been mapped in kernel space + * by the caller. Map it now, so that we can access + * its contents and decrypt them. + */ + kernel_mapping_size = PAGE_SIZE; + kernel_mapping_needs_unmap = FALSE; + kr = vm_paging_map_object(page, + page_object, + page->offset, + VM_PROT_READ | VM_PROT_WRITE, + FALSE, + &kernel_mapping_size, + &kernel_mapping_offset, + &kernel_mapping_needs_unmap); + if (kr != KERN_SUCCESS) { + panic("vm_page_slide: " + "could not map page in kernel: 0x%x\n", + kr); + } + } else { + kernel_mapping_size = 0; + kernel_mapping_needs_unmap = FALSE; + } + kernel_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping_offset); + + /* + * Slide the pointers on the page. + */ + + /*assert that slide_file_info.start/end are page-aligned?*/ + + assert(!page->slid); + assert(page_object->object_slid); + + pageIndex = (uint32_t)((page->offset - + page_object->vo_slide_info->start) / + PAGE_SIZE_FOR_SR_SLIDE); + for (slide_chunk = 0; + slide_chunk < PAGE_SIZE / PAGE_SIZE_FOR_SR_SLIDE; + slide_chunk++) { + kr = vm_shared_region_slide_page(page_object->vo_slide_info, + (kernel_vaddr + + (slide_chunk * + PAGE_SIZE_FOR_SR_SLIDE)), + (pageIndex + slide_chunk)); + if (kr != KERN_SUCCESS) { + break; + } + } + + vm_page_slide_counter++; + + /* + * Unmap the page from the kernel's address space, + */ + if (kernel_mapping_needs_unmap) { + vm_paging_unmap_object(page_object, + kernel_vaddr, + kernel_vaddr + PAGE_SIZE); + } + + page->dirty = FALSE; + pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(page), VM_MEM_MODIFIED | VM_MEM_REFERENCED); + + if (kr != KERN_SUCCESS || cs_debug > 1) { + printf("vm_page_slide(%p): " + "obj %p off 0x%llx mobj %p moff 0x%llx\n", + page, + page_object, page->offset, + page_object->pager, + page->offset + page_object->paging_offset); + } + + if (kr == KERN_SUCCESS) { + page->slid = TRUE; + } else { + page->error = TRUE; + vm_page_slide_errors++; + } + + vm_object_paging_end(page_object); + + return kr; +} + +void inline memoryshot(unsigned int event, unsigned int control) +{ + if (vm_debug_events) { + KERNEL_DEBUG_CONSTANT1((MACHDBG_CODE(DBG_MACH_VM_PRESSURE, event)) | control, + vm_page_active_count, vm_page_inactive_count, + vm_page_free_count, vm_page_speculative_count, + vm_page_throttled_count); + } else { + (void) event; + (void) control; + } + +} #ifdef MACH_BSD @@ -7401,6 +10963,15 @@ ppnum_t upl_phys_page(upl_page_info_t *upl, int index) return(UPL_PHYS_PAGE(upl, index)); } +void upl_page_set_mark(upl_page_info_t *upl, int index, boolean_t v) +{ + upl[index].mark = v; +} + +boolean_t upl_page_get_mark(upl_page_info_t *upl, int index) +{ + return upl[index].mark; +} void vm_countdirtypages(void) @@ -7416,50 +10987,50 @@ vm_countdirtypages(void) precpages=0; vm_page_lock_queues(); - m = (vm_page_t) queue_first(&vm_page_queue_inactive); + m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive); do { if (m ==(vm_page_t )0) break; if(m->dirty) dpages++; - if(m->pageout) pgopages++; + if(m->free_when_done) pgopages++; if(m->precious) precpages++; - assert(m->object != kernel_object); - m = (vm_page_t) queue_next(&m->pageq); + assert(VM_PAGE_OBJECT(m) != kernel_object); + m = (vm_page_t) vm_page_queue_next(&m->pageq); if (m ==(vm_page_t )0) break; - } while (!queue_end(&vm_page_queue_inactive,(queue_entry_t) m)); + } while (!vm_page_queue_end(&vm_page_queue_inactive, (vm_page_queue_entry_t) m)); vm_page_unlock_queues(); vm_page_lock_queues(); - m = (vm_page_t) queue_first(&vm_page_queue_throttled); + m = (vm_page_t) vm_page_queue_first(&vm_page_queue_throttled); do { if (m ==(vm_page_t )0) break; dpages++; assert(m->dirty); - assert(!m->pageout); - assert(m->object != kernel_object); - m = (vm_page_t) queue_next(&m->pageq); + assert(!m->free_when_done); + assert(VM_PAGE_OBJECT(m) != kernel_object); + m = (vm_page_t) vm_page_queue_next(&m->pageq); if (m ==(vm_page_t )0) break; - } while (!queue_end(&vm_page_queue_throttled,(queue_entry_t) m)); + } while (!vm_page_queue_end(&vm_page_queue_throttled, (vm_page_queue_entry_t) m)); vm_page_unlock_queues(); vm_page_lock_queues(); - m = (vm_page_t) queue_first(&vm_page_queue_zf); + m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous); do { if (m ==(vm_page_t )0) break; if(m->dirty) dpages++; - if(m->pageout) pgopages++; + if(m->free_when_done) pgopages++; if(m->precious) precpages++; - assert(m->object != kernel_object); - m = (vm_page_t) queue_next(&m->pageq); + assert(VM_PAGE_OBJECT(m) != kernel_object); + m = (vm_page_t) vm_page_queue_next(&m->pageq); if (m ==(vm_page_t )0) break; - } while (!queue_end(&vm_page_queue_zf,(queue_entry_t) m)); + } while (!vm_page_queue_end(&vm_page_queue_anonymous, (vm_page_queue_entry_t) m)); vm_page_unlock_queues(); printf("IN Q: %d : %d : %d\n", dpages, pgopages, precpages); @@ -7469,19 +11040,19 @@ vm_countdirtypages(void) precpages=0; vm_page_lock_queues(); - m = (vm_page_t) queue_first(&vm_page_queue_active); + m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active); do { if(m == (vm_page_t )0) break; if(m->dirty) dpages++; - if(m->pageout) pgopages++; + if(m->free_when_done) pgopages++; if(m->precious) precpages++; - assert(m->object != kernel_object); - m = (vm_page_t) queue_next(&m->pageq); + assert(VM_PAGE_OBJECT(m) != kernel_object); + m = (vm_page_t) vm_page_queue_next(&m->pageq); if(m == (vm_page_t )0) break; - } while (!queue_end(&vm_page_queue_active,(queue_entry_t) m)); + } while (!vm_page_queue_end(&vm_page_queue_active, (vm_page_queue_entry_t) m)); vm_page_unlock_queues(); printf("AC Q: %d : %d : %d\n", dpages, pgopages, precpages); @@ -7501,6 +11072,24 @@ upl_size_t upl_get_size( return upl->size; } +upl_t upl_associated_upl(upl_t upl) +{ + return upl->associated_upl; +} + +void upl_set_associated_upl(upl_t upl, upl_t associated_upl) +{ + upl->associated_upl = associated_upl; +} + +struct vnode * upl_lookup_vnode(upl_t upl) +{ + if (!upl->map_object->internal) + return vnode_pager_lookup_vnode(upl->map_object->pager); + else + return NULL; +} + #if UPL_DEBUG kern_return_t upl_ubc_alias_set(upl_t upl, uintptr_t alias1, uintptr_t alias2) { @@ -7518,79 +11107,82 @@ int upl_ubc_alias_get(upl_t upl, uintptr_t * al, uintptr_t * al2) } #endif /* UPL_DEBUG */ +#if VM_PRESSURE_EVENTS +/* + * Upward trajectory. + */ +extern boolean_t vm_compressor_low_on_space(void); +boolean_t +VM_PRESSURE_NORMAL_TO_WARNING(void) { -#if MACH_KDB -#include -#include -#include + if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) { -#define printf kdbprintf -void db_pageout(void); + /* Available pages below our threshold */ + if (memorystatus_available_pages < memorystatus_available_pages_pressure) { + /* No frozen processes to kill */ + if (memorystatus_frozen_count == 0) { + /* Not enough suspended processes available. */ + if (memorystatus_suspended_count < MEMORYSTATUS_SUSPENDED_THRESHOLD) { + return TRUE; + } + } + } + return FALSE; -void -db_vm(void) -{ + } else { + return ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0); + } +} - iprintf("VM Statistics:\n"); - db_indent += 2; - iprintf("pages:\n"); - db_indent += 2; - iprintf("activ %5d inact %5d free %5d", - vm_page_active_count, vm_page_inactive_count, - vm_page_free_count); - printf(" wire %5d gobbl %5d\n", - vm_page_wire_count, vm_page_gobble_count); - db_indent -= 2; - iprintf("target:\n"); - db_indent += 2; - iprintf("min %5d inact %5d free %5d", - vm_page_free_min, vm_page_inactive_target, - vm_page_free_target); - printf(" resrv %5d\n", vm_page_free_reserved); - db_indent -= 2; - iprintf("pause:\n"); - db_pageout(); - db_indent -= 2; +boolean_t +VM_PRESSURE_WARNING_TO_CRITICAL(void) { + + if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) { + + /* Available pages below our threshold */ + if (memorystatus_available_pages < memorystatus_available_pages_critical) { + return TRUE; + } + return FALSE; + } else { + return (vm_compressor_low_on_space() || (AVAILABLE_NON_COMPRESSED_MEMORY < ((12 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0); + } } -#if MACH_COUNTERS -extern int c_laundry_pages_freed; -#endif /* MACH_COUNTERS */ +/* + * Downward trajectory. + */ +boolean_t +VM_PRESSURE_WARNING_TO_NORMAL(void) { -void -db_pageout(void) -{ - iprintf("Pageout Statistics:\n"); - db_indent += 2; - iprintf("active %5d inactv %5d\n", - vm_pageout_active, vm_pageout_inactive); - iprintf("nolock %5d avoid %5d busy %5d absent %5d\n", - vm_pageout_inactive_nolock, vm_pageout_inactive_avoid, - vm_pageout_inactive_busy, vm_pageout_inactive_absent); - iprintf("used %5d clean %5d dirty %5d\n", - vm_pageout_inactive_used, vm_pageout_inactive_clean, - vm_pageout_inactive_dirty); -#if MACH_COUNTERS - iprintf("laundry_pages_freed %d\n", c_laundry_pages_freed); -#endif /* MACH_COUNTERS */ -#if MACH_CLUSTER_STATS - iprintf("Cluster Statistics:\n"); - db_indent += 2; - iprintf("dirtied %5d cleaned %5d collisions %5d\n", - vm_pageout_cluster_dirtied, vm_pageout_cluster_cleaned, - vm_pageout_cluster_collisions); - iprintf("clusters %5d conversions %5d\n", - vm_pageout_cluster_clusters, vm_pageout_cluster_conversions); - db_indent -= 2; - iprintf("Target Statistics:\n"); - db_indent += 2; - iprintf("collisions %5d page_dirtied %5d page_freed %5d\n", - vm_pageout_target_collisions, vm_pageout_target_page_dirtied, - vm_pageout_target_page_freed); - db_indent -= 2; -#endif /* MACH_CLUSTER_STATS */ - db_indent -= 2; + if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) { + + /* Available pages above our threshold */ + unsigned int target_threshold = memorystatus_available_pages_pressure + ((15 * memorystatus_available_pages_pressure) / 100); + if (memorystatus_available_pages > target_threshold) { + return TRUE; + } + return FALSE; + } else { + return ((AVAILABLE_NON_COMPRESSED_MEMORY > ((12 * VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) / 10)) ? 1 : 0); + } +} + +boolean_t +VM_PRESSURE_CRITICAL_TO_WARNING(void) { + + if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) { + + /* Available pages above our threshold */ + unsigned int target_threshold = memorystatus_available_pages_critical + ((15 * memorystatus_available_pages_critical) / 100); + if (memorystatus_available_pages > target_threshold) { + return TRUE; + } + return FALSE; + } else { + return ((AVAILABLE_NON_COMPRESSED_MEMORY > ((14 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0); + } } +#endif /* VM_PRESSURE_EVENTS */ -#endif /* MACH_KDB */