X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/a39ff7e25e19b3a8c3020042a3872ca9ec9659f1..0a7de7458d150b5d4dffc935ba399be265ef0a1a:/osfmk/vm/vm_pageout.c diff --git a/osfmk/vm/vm_pageout.c b/osfmk/vm/vm_pageout.c index 20eac579d..6b6e3d04d 100644 --- a/osfmk/vm/vm_pageout.c +++ b/osfmk/vm/vm_pageout.c @@ -2,7 +2,7 @@ * Copyright (c) 2000-2014 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,34 +22,34 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ -/* +/* * Mach Operating System * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University * All Rights Reserved. - * + * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. - * + * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. - * + * * Carnegie Mellon requests users of this software to return to - * + * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 - * + * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ @@ -114,13 +114,13 @@ #include #endif -extern int cs_debug; - #if UPL_DEBUG #include #endif -extern void m_drain(void); +extern int cs_debug; + +extern void mbuf_drain(boolean_t); #if VM_PRESSURE_EVENTS #if CONFIG_JETSAM @@ -135,27 +135,17 @@ extern uint64_t memorystatus_available_pages_critical; extern unsigned int memorystatus_frozen_count; extern unsigned int memorystatus_suspended_count; - extern vm_pressure_level_t memorystatus_vm_pressure_level; -int memorystatus_purge_on_warning = 2; -int memorystatus_purge_on_urgent = 5; -int memorystatus_purge_on_critical = 8; void vm_pressure_response(void); -boolean_t vm_pressure_thread_running = FALSE; extern void consider_vm_pressure_events(void); #define MEMORYSTATUS_SUSPENDED_THRESHOLD 4 #endif /* VM_PRESSURE_EVENTS */ -boolean_t vm_pressure_changed = FALSE; - -#ifndef VM_PAGEOUT_BURST_ACTIVE_THROTTLE /* maximum iterations of the active queue to move pages to inactive */ -#define VM_PAGEOUT_BURST_ACTIVE_THROTTLE 100 -#endif #ifndef VM_PAGEOUT_BURST_INACTIVE_THROTTLE /* maximum iterations of the inactive queue w/o stealing/cleaning a page */ -#ifdef CONFIG_EMBEDDED +#ifdef CONFIG_EMBEDDED #define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 1024 #else #define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 4096 @@ -163,50 +153,36 @@ boolean_t vm_pressure_changed = FALSE; #endif #ifndef VM_PAGEOUT_DEADLOCK_RELIEF -#define VM_PAGEOUT_DEADLOCK_RELIEF 100 /* number of pages to move to break deadlock */ -#endif - -#ifndef VM_PAGEOUT_INACTIVE_RELIEF -#define VM_PAGEOUT_INACTIVE_RELIEF 50 /* minimum number of pages to move to the inactive q */ +#define VM_PAGEOUT_DEADLOCK_RELIEF 100 /* number of pages to move to break deadlock */ #endif -#ifndef VM_PAGE_LAUNDRY_MAX -#define VM_PAGE_LAUNDRY_MAX 128UL /* maximum pageouts on a given pageout queue */ -#endif /* VM_PAGEOUT_LAUNDRY_MAX */ - -#ifndef VM_PAGEOUT_BURST_WAIT -#define VM_PAGEOUT_BURST_WAIT 10 /* milliseconds */ -#endif /* VM_PAGEOUT_BURST_WAIT */ +#ifndef VM_PAGE_LAUNDRY_MAX +#define VM_PAGE_LAUNDRY_MAX 128UL /* maximum pageouts on a given pageout queue */ +#endif /* VM_PAGEOUT_LAUNDRY_MAX */ -#ifndef VM_PAGEOUT_EMPTY_WAIT -#define VM_PAGEOUT_EMPTY_WAIT 200 /* milliseconds */ -#endif /* VM_PAGEOUT_EMPTY_WAIT */ +#ifndef VM_PAGEOUT_BURST_WAIT +#define VM_PAGEOUT_BURST_WAIT 1 /* milliseconds */ +#endif /* VM_PAGEOUT_BURST_WAIT */ -#ifndef VM_PAGEOUT_DEADLOCK_WAIT -#define VM_PAGEOUT_DEADLOCK_WAIT 300 /* milliseconds */ -#endif /* VM_PAGEOUT_DEADLOCK_WAIT */ +#ifndef VM_PAGEOUT_EMPTY_WAIT +#define VM_PAGEOUT_EMPTY_WAIT 50 /* milliseconds */ +#endif /* VM_PAGEOUT_EMPTY_WAIT */ -#ifndef VM_PAGEOUT_IDLE_WAIT -#define VM_PAGEOUT_IDLE_WAIT 10 /* milliseconds */ -#endif /* VM_PAGEOUT_IDLE_WAIT */ +#ifndef VM_PAGEOUT_DEADLOCK_WAIT +#define VM_PAGEOUT_DEADLOCK_WAIT 100 /* milliseconds */ +#endif /* VM_PAGEOUT_DEADLOCK_WAIT */ -#ifndef VM_PAGEOUT_SWAP_WAIT -#define VM_PAGEOUT_SWAP_WAIT 50 /* milliseconds */ -#endif /* VM_PAGEOUT_SWAP_WAIT */ +#ifndef VM_PAGEOUT_IDLE_WAIT +#define VM_PAGEOUT_IDLE_WAIT 10 /* milliseconds */ +#endif /* VM_PAGEOUT_IDLE_WAIT */ -#ifndef VM_PAGEOUT_PRESSURE_PAGES_CONSIDERED -#define VM_PAGEOUT_PRESSURE_PAGES_CONSIDERED 1000 /* maximum pages considered before we issue a pressure event */ -#endif /* VM_PAGEOUT_PRESSURE_PAGES_CONSIDERED */ +#ifndef VM_PAGEOUT_SWAP_WAIT +#define VM_PAGEOUT_SWAP_WAIT 10 /* milliseconds */ +#endif /* VM_PAGEOUT_SWAP_WAIT */ -#ifndef VM_PAGEOUT_PRESSURE_EVENT_MONITOR_SECS -#define VM_PAGEOUT_PRESSURE_EVENT_MONITOR_SECS 5 /* seconds */ -#endif /* VM_PAGEOUT_PRESSURE_EVENT_MONITOR_SECS */ - -unsigned int vm_page_speculative_q_age_ms = VM_PAGE_SPECULATIVE_Q_AGE_MS; -unsigned int vm_page_speculative_percentage = 5; #ifndef VM_PAGE_SPECULATIVE_TARGET -#define VM_PAGE_SPECULATIVE_TARGET(total) ((total) * 1 / (100 / vm_page_speculative_percentage)) +#define VM_PAGE_SPECULATIVE_TARGET(total) ((total) * 1 / (100 / vm_pageout_state.vm_page_speculative_percentage)) #endif /* VM_PAGE_SPECULATIVE_TARGET */ @@ -222,26 +198,22 @@ unsigned int vm_page_speculative_percentage = 5; * then the pageout daemon starts running. */ -#ifndef VM_PAGE_INACTIVE_TARGET -#ifdef CONFIG_EMBEDDED -#define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 1 / 3) -#else -#define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 1 / 2) -#endif -#endif /* VM_PAGE_INACTIVE_TARGET */ +#ifndef VM_PAGE_INACTIVE_TARGET +#define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 1 / 2) +#endif /* VM_PAGE_INACTIVE_TARGET */ /* * Once the pageout daemon starts running, it keeps going * until vm_page_free_count meets or exceeds vm_page_free_target. */ -#ifndef VM_PAGE_FREE_TARGET -#ifdef CONFIG_EMBEDDED -#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 100) +#ifndef VM_PAGE_FREE_TARGET +#ifdef CONFIG_EMBEDDED +#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 100) #else -#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80) +#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80) #endif -#endif /* VM_PAGE_FREE_TARGET */ +#endif /* VM_PAGE_FREE_TARGET */ /* @@ -249,22 +221,22 @@ unsigned int vm_page_speculative_percentage = 5; * falls below vm_page_free_min. */ -#ifndef VM_PAGE_FREE_MIN -#ifdef CONFIG_EMBEDDED -#define VM_PAGE_FREE_MIN(free) (10 + (free) / 200) +#ifndef VM_PAGE_FREE_MIN +#ifdef CONFIG_EMBEDDED +#define VM_PAGE_FREE_MIN(free) (10 + (free) / 200) #else -#define VM_PAGE_FREE_MIN(free) (10 + (free) / 100) +#define VM_PAGE_FREE_MIN(free) (10 + (free) / 100) #endif -#endif /* VM_PAGE_FREE_MIN */ +#endif /* VM_PAGE_FREE_MIN */ -#ifdef CONFIG_EMBEDDED -#define VM_PAGE_FREE_RESERVED_LIMIT 100 -#define VM_PAGE_FREE_MIN_LIMIT 1500 -#define VM_PAGE_FREE_TARGET_LIMIT 2000 +#ifdef CONFIG_EMBEDDED +#define VM_PAGE_FREE_RESERVED_LIMIT 100 +#define VM_PAGE_FREE_MIN_LIMIT 1500 +#define VM_PAGE_FREE_TARGET_LIMIT 2000 #else -#define VM_PAGE_FREE_RESERVED_LIMIT 1700 -#define VM_PAGE_FREE_MIN_LIMIT 3500 -#define VM_PAGE_FREE_TARGET_LIMIT 4000 +#define VM_PAGE_FREE_RESERVED_LIMIT 1700 +#define VM_PAGE_FREE_MIN_LIMIT 3500 +#define VM_PAGE_FREE_TARGET_LIMIT 4000 #endif /* @@ -275,10 +247,10 @@ unsigned int vm_page_speculative_percentage = 5; * operation by dipping into the reserved pool of pages. */ -#ifndef VM_PAGE_FREE_RESERVED -#define VM_PAGE_FREE_RESERVED(n) \ +#ifndef VM_PAGE_FREE_RESERVED +#define VM_PAGE_FREE_RESERVED(n) \ ((unsigned) (6 * VM_PAGE_LAUNDRY_MAX) + (n)) -#endif /* VM_PAGE_FREE_RESERVED */ +#endif /* VM_PAGE_FREE_RESERVED */ /* * When we dequeue pages from the inactive list, they are @@ -289,35 +261,26 @@ unsigned int vm_page_speculative_percentage = 5; * we will make per call of vm_pageout_scan(). */ #define VM_PAGE_REACTIVATE_LIMIT_MAX 20000 -#ifndef VM_PAGE_REACTIVATE_LIMIT -#ifdef CONFIG_EMBEDDED -#define VM_PAGE_REACTIVATE_LIMIT(avail) (VM_PAGE_INACTIVE_TARGET(avail) / 2) + +#ifndef VM_PAGE_REACTIVATE_LIMIT +#ifdef CONFIG_EMBEDDED +#define VM_PAGE_REACTIVATE_LIMIT(avail) (VM_PAGE_INACTIVE_TARGET(avail) / 2) #else -#define VM_PAGE_REACTIVATE_LIMIT(avail) (MAX((avail) * 1 / 20,VM_PAGE_REACTIVATE_LIMIT_MAX)) +#define VM_PAGE_REACTIVATE_LIMIT(avail) (MAX((avail) * 1 / 20,VM_PAGE_REACTIVATE_LIMIT_MAX)) #endif -#endif /* VM_PAGE_REACTIVATE_LIMIT */ -#define VM_PAGEOUT_INACTIVE_FORCE_RECLAIM 1000 - +#endif /* VM_PAGE_REACTIVATE_LIMIT */ +#define VM_PAGEOUT_INACTIVE_FORCE_RECLAIM 1000 extern boolean_t hibernate_cleaning_in_progress; -/* - * Exported variable used to broadcast the activation of the pageout scan - * Working Set uses this to throttle its use of pmap removes. In this - * way, code which runs within memory in an uncontested context does - * not keep encountering soft faults. - */ - -unsigned int vm_pageout_scan_event_counter = 0; - /* * Forward declarations for internal routines. */ struct cq { struct vm_pageout_queue *q; - void *current_chead; - char *scratch_buf; - int id; + void *current_chead; + char *scratch_buf; + int id; }; struct cq ciq[MAX_COMPRESSOR_THREAD_COUNT]; @@ -332,6 +295,7 @@ boolean_t VM_PRESSURE_WARNING_TO_CRITICAL(void); boolean_t VM_PRESSURE_WARNING_TO_NORMAL(void); boolean_t VM_PRESSURE_CRITICAL_TO_WARNING(void); #endif + void vm_pageout_garbage_collect(int); static void vm_pageout_iothread_external(void); static void vm_pageout_iothread_internal(struct cq *cq); @@ -339,184 +303,58 @@ static void vm_pageout_adjust_eq_iothrottle(struct vm_pageout_queue *, boolean_t extern void vm_pageout_continue(void); extern void vm_pageout_scan(void); + void vm_tests(void); /* forward */ -boolean_t vm_restricted_to_single_processor = FALSE; #if !CONFIG_EMBEDDED static boolean_t vm_pageout_waiter = FALSE; static boolean_t vm_pageout_running = FALSE; #endif /* !CONFIG_EMBEDDED */ -static thread_t vm_pageout_external_iothread = THREAD_NULL; -static thread_t vm_pageout_internal_iothread = THREAD_NULL; - -unsigned int vm_pageout_reserved_internal = 0; -unsigned int vm_pageout_reserved_really = 0; - -unsigned int vm_pageout_swap_wait = 0; -unsigned int vm_pageout_idle_wait = 0; /* milliseconds */ -unsigned int vm_pageout_empty_wait = 0; /* milliseconds */ -unsigned int vm_pageout_burst_wait = 0; /* milliseconds */ -unsigned int vm_pageout_deadlock_wait = 0; /* milliseconds */ -unsigned int vm_pageout_deadlock_relief = 0; -unsigned int vm_pageout_inactive_relief = 0; -unsigned int vm_pageout_burst_active_throttle = 0; -unsigned int vm_pageout_burst_inactive_throttle = 0; - -int vm_upl_wait_for_pages = 0; - - -/* - * These variables record the pageout daemon's actions: - * how many pages it looks at and what happens to those pages. - * No locking needed because only one thread modifies the variables. - */ - -unsigned int vm_pageout_active = 0; /* debugging */ -unsigned int vm_pageout_inactive = 0; /* debugging */ -unsigned int vm_pageout_inactive_throttled = 0; /* debugging */ -unsigned int vm_pageout_inactive_forced = 0; /* debugging */ -unsigned int vm_pageout_inactive_nolock = 0; /* debugging */ -unsigned int vm_pageout_inactive_avoid = 0; /* debugging */ -unsigned int vm_pageout_inactive_busy = 0; /* debugging */ -unsigned int vm_pageout_inactive_error = 0; /* debugging */ -unsigned int vm_pageout_inactive_absent = 0; /* debugging */ -unsigned int vm_pageout_inactive_notalive = 0; /* debugging */ -unsigned int vm_pageout_inactive_used = 0; /* debugging */ -unsigned int vm_pageout_cache_evicted = 0; /* debugging */ -unsigned int vm_pageout_inactive_clean = 0; /* debugging */ -unsigned int vm_pageout_speculative_clean = 0; /* debugging */ -unsigned int vm_pageout_speculative_dirty = 0; /* debugging */ - -unsigned int vm_pageout_freed_from_cleaned = 0; -unsigned int vm_pageout_freed_from_speculative = 0; -unsigned int vm_pageout_freed_from_inactive_clean = 0; -unsigned int vm_pageout_freed_after_compression = 0; - -extern uint32_t vm_compressor_pages_grabbed; -extern uint32_t c_segment_pages_compressed; - -unsigned int vm_pageout_enqueued_cleaned_from_inactive_dirty = 0; - -unsigned int vm_pageout_cleaned_reclaimed = 0; /* debugging; how many cleaned pages are reclaimed by the pageout scan */ -unsigned int vm_pageout_cleaned_reactivated = 0; /* debugging; how many cleaned pages are found to be referenced on pageout (and are therefore reactivated) */ -unsigned int vm_pageout_cleaned_reference_reactivated = 0; -unsigned int vm_pageout_cleaned_volatile_reactivated = 0; -unsigned int vm_pageout_cleaned_fault_reactivated = 0; -unsigned int vm_pageout_cleaned_commit_reactivated = 0; /* debugging; how many cleaned pages are found to be referenced on commit (and are therefore reactivated) */ -unsigned int vm_pageout_cleaned_busy = 0; -unsigned int vm_pageout_cleaned_nolock = 0; - -unsigned int vm_pageout_inactive_dirty_internal = 0; /* debugging */ -unsigned int vm_pageout_inactive_dirty_external = 0; /* debugging */ -unsigned int vm_pageout_inactive_deactivated = 0; /* debugging */ -unsigned int vm_pageout_inactive_anonymous = 0; /* debugging */ -unsigned int vm_pageout_dirty_no_pager = 0; /* debugging */ -unsigned int vm_pageout_purged_objects = 0; /* used for sysctl vm stats */ -unsigned int vm_stat_discard = 0; /* debugging */ -unsigned int vm_stat_discard_sent = 0; /* debugging */ -unsigned int vm_stat_discard_failure = 0; /* debugging */ -unsigned int vm_stat_discard_throttle = 0; /* debugging */ -unsigned int vm_pageout_reactivation_limit_exceeded = 0; /* debugging */ -unsigned int vm_pageout_inactive_force_reclaim = 0; /* debugging */ -unsigned int vm_pageout_skipped_external = 0; /* debugging */ - -unsigned int vm_pageout_scan_reclaimed_throttled = 0; -unsigned int vm_pageout_scan_active_throttled = 0; -unsigned int vm_pageout_scan_inactive_throttled_internal = 0; -unsigned int vm_pageout_scan_inactive_throttled_external = 0; -unsigned int vm_pageout_scan_throttle = 0; /* debugging */ -unsigned int vm_pageout_scan_burst_throttle = 0; /* debugging */ -unsigned int vm_pageout_scan_empty_throttle = 0; /* debugging */ -unsigned int vm_pageout_scan_swap_throttle = 0; /* debugging */ -unsigned int vm_pageout_scan_deadlock_detected = 0; /* debugging */ -unsigned int vm_pageout_scan_active_throttle_success = 0; /* debugging */ -unsigned int vm_pageout_scan_inactive_throttle_success = 0; /* debugging */ -unsigned int vm_pageout_inactive_external_forced_jetsam_count = 0; /* debugging */ -unsigned int vm_pageout_scan_throttle_deferred = 0; /* debugging */ -unsigned int vm_pageout_scan_yield_unthrottled = 0; /* debugging */ -unsigned int vm_page_speculative_count_drifts = 0; -unsigned int vm_page_speculative_count_drift_max = 0; - -uint32_t vm_compressor_failed; - -/* - * Backing store throttle when BS is exhausted - */ -unsigned int vm_backing_store_low = 0; - -unsigned int vm_pageout_out_of_line = 0; -unsigned int vm_pageout_in_place = 0; - -unsigned int vm_page_steal_pageout_page = 0; - -struct vm_config vm_config; - -struct vm_pageout_queue vm_pageout_queue_internal __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT))); -struct vm_pageout_queue vm_pageout_queue_external __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT))); +#if DEVELOPMENT || DEBUG +struct vm_pageout_debug vm_pageout_debug; +#endif +struct vm_pageout_vminfo vm_pageout_vminfo; +struct vm_pageout_state vm_pageout_state; +struct vm_config vm_config; -unsigned int vm_page_speculative_target = 0; +struct vm_pageout_queue vm_pageout_queue_internal __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT))); +struct vm_pageout_queue vm_pageout_queue_external __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT))); -vm_object_t vm_pageout_scan_wants_object = VM_OBJECT_NULL; +int vm_upl_wait_for_pages = 0; +vm_object_t vm_pageout_scan_wants_object = VM_OBJECT_NULL; -boolean_t (* volatile consider_buffer_cache_collect)(int) = NULL; +boolean_t(*volatile consider_buffer_cache_collect)(int) = NULL; -#if DEVELOPMENT || DEBUG -unsigned long vm_cs_validated_resets = 0; -#endif +int vm_debug_events = 0; -int vm_debug_events = 0; +lck_grp_t vm_pageout_lck_grp; #if CONFIG_MEMORYSTATUS -#if !CONFIG_JETSAM -extern boolean_t memorystatus_idle_exit_from_VM(void); -#endif extern boolean_t memorystatus_kill_on_VM_page_shortage(boolean_t async); -extern void memorystatus_on_pageout_scan_end(void); uint32_t vm_pageout_memorystatus_fb_factor_nr = 5; uint32_t vm_pageout_memorystatus_fb_factor_dr = 2; -#if DEVELOPMENT || DEBUG -uint32_t vm_grab_anon_overrides = 0; -uint32_t vm_grab_anon_nops = 0; -#endif #endif -#if MACH_CLUSTER_STATS -unsigned long vm_pageout_cluster_dirtied = 0; -unsigned long vm_pageout_cluster_cleaned = 0; -unsigned long vm_pageout_cluster_collisions = 0; -unsigned long vm_pageout_cluster_clusters = 0; -unsigned long vm_pageout_cluster_conversions = 0; -unsigned long vm_pageout_target_collisions = 0; -unsigned long vm_pageout_target_page_dirtied = 0; -unsigned long vm_pageout_target_page_freed = 0; -#define CLUSTER_STAT(clause) clause -#else /* MACH_CLUSTER_STATS */ -#define CLUSTER_STAT(clause) -#endif /* MACH_CLUSTER_STATS */ -#if DEVELOPMENT || DEBUG -vmct_stats_t vmct_stats; -#endif - -/* +/* * Routine: vm_pageout_object_terminate * Purpose: * Destroy the pageout_object, and perform all of the * required cleanup actions. - * + * * In/Out conditions: * The object must be locked, and will be returned locked. */ void vm_pageout_object_terminate( - vm_object_t object) + vm_object_t object) { - vm_object_t shadow_object; + vm_object_t shadow_object; /* * Deal with the deallocation (last reference) of a pageout object @@ -529,37 +367,39 @@ vm_pageout_object_terminate( vm_object_lock(shadow_object); while (!vm_page_queue_empty(&object->memq)) { - vm_page_t p, m; - vm_object_offset_t offset; + vm_page_t p, m; + vm_object_offset_t offset; p = (vm_page_t) vm_page_queue_first(&object->memq); - assert(p->private); - assert(p->free_when_done); - p->free_when_done = FALSE; - assert(!p->cleaning); - assert(!p->laundry); + assert(p->vmp_private); + assert(p->vmp_free_when_done); + p->vmp_free_when_done = FALSE; + assert(!p->vmp_cleaning); + assert(!p->vmp_laundry); - offset = p->offset; + offset = p->vmp_offset; VM_PAGE_FREE(p); p = VM_PAGE_NULL; m = vm_page_lookup(shadow_object, - offset + object->vo_shadow_offset); + offset + object->vo_shadow_offset); - if(m == VM_PAGE_NULL) + if (m == VM_PAGE_NULL) { continue; + } - assert((m->dirty) || (m->precious) || - (m->busy && m->cleaning)); + assert((m->vmp_dirty) || (m->vmp_precious) || + (m->vmp_busy && m->vmp_cleaning)); /* * Handle the trusted pager throttle. * Also decrement the burst throttle (if external). */ vm_page_lock_queues(); - if (m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) + if (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) { vm_pageout_throttle_up(m); + } /* * Handle the "target" page(s). These pages are to be freed if @@ -569,15 +409,12 @@ vm_pageout_object_terminate( * pages may have been modified between the selection as an * adjacent page and conversion to a target. */ - if (m->free_when_done) { - assert(m->busy); - assert(m->vm_page_q_state == VM_PAGE_IS_WIRED); - assert(m->wire_count == 1); - m->cleaning = FALSE; - m->free_when_done = FALSE; -#if MACH_CLUSTER_STATS - if (m->wanted) vm_pageout_target_collisions++; -#endif + if (m->vmp_free_when_done) { + assert(m->vmp_busy); + assert(m->vmp_q_state == VM_PAGE_IS_WIRED); + assert(m->vmp_wire_count == 1); + m->vmp_cleaning = FALSE; + m->vmp_free_when_done = FALSE; /* * Revoke all access to the page. Since the object is * locked, and the page is busy, this prevents the page @@ -591,17 +428,15 @@ vm_pageout_object_terminate( if (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED) { SET_PAGE_DIRTY(m, FALSE); } else { - m->dirty = FALSE; + m->vmp_dirty = FALSE; } - if (m->dirty) { - CLUSTER_STAT(vm_pageout_target_page_dirtied++;) - vm_page_unwire(m, TRUE); /* reactivates */ + if (m->vmp_dirty) { + vm_page_unwire(m, TRUE); /* reactivates */ VM_STAT_INCR(reactivations); PAGE_WAKEUP_DONE(m); } else { - CLUSTER_STAT(vm_pageout_target_page_freed++;) - vm_page_free(m);/* clears busy, etc. */ + vm_page_free(m); /* clears busy, etc. */ } vm_page_unlock_queues(); continue; @@ -612,19 +447,20 @@ vm_pageout_object_terminate( * If prep_pin_count is nonzero, then someone is using the * page, so make it active. */ - if ((m->vm_page_q_state == VM_PAGE_NOT_ON_Q) && !m->private) { - if (m->reference) + if ((m->vmp_q_state == VM_PAGE_NOT_ON_Q) && !m->vmp_private) { + if (m->vmp_reference) { vm_page_activate(m); - else + } else { vm_page_deactivate(m); + } } - if (m->overwriting) { + if (m->vmp_overwriting) { /* * the (COPY_OUT_FROM == FALSE) request_page_list case */ - if (m->busy) { + if (m->vmp_busy) { /* - * We do not re-set m->dirty ! + * We do not re-set m->vmp_dirty ! * The page was busy so no extraneous activity * could have occurred. COPY_INTO is a read into the * new pages. CLEAN_IN_PLACE does actually write @@ -634,39 +470,22 @@ vm_pageout_object_terminate( */ pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m)); - m->busy = FALSE; - m->absent = FALSE; + m->vmp_busy = FALSE; + m->vmp_absent = FALSE; } else { /* * alternate (COPY_OUT_FROM == FALSE) request_page_list case * Occurs when the original page was wired * at the time of the list request */ - assert(VM_PAGE_WIRED(m)); - vm_page_unwire(m, TRUE); /* reactivates */ + assert(VM_PAGE_WIRED(m)); + vm_page_unwire(m, TRUE); /* reactivates */ } - m->overwriting = FALSE; + m->vmp_overwriting = FALSE; } else { - /* - * Set the dirty state according to whether or not the page was - * modified during the pageout. Note that we purposefully do - * NOT call pmap_clear_modify since the page is still mapped. - * If the page were to be dirtied between the 2 calls, this - * this fact would be lost. This code is only necessary to - * maintain statistics, since the pmap module is always - * consulted if m->dirty is false. - */ -#if MACH_CLUSTER_STATS - m->dirty = pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m)); - - if (m->dirty) vm_pageout_cluster_dirtied++; - else vm_pageout_cluster_cleaned++; - if (m->wanted) vm_pageout_cluster_collisions++; -#else - m->dirty = FALSE; -#endif + m->vmp_dirty = FALSE; } - m->cleaning = FALSE; + m->vmp_cleaning = FALSE; /* * Wakeup any thread waiting for the page to be un-cleaning. @@ -700,39 +519,39 @@ vm_pageout_object_terminate( */ static void vm_pageclean_setup( - vm_page_t m, - vm_page_t new_m, - vm_object_t new_object, - vm_object_offset_t new_offset) + vm_page_t m, + vm_page_t new_m, + vm_object_t new_object, + vm_object_offset_t new_offset) { - assert(!m->busy); + assert(!m->vmp_busy); #if 0 - assert(!m->cleaning); + assert(!m->vmp_cleaning); #endif XPR(XPR_VM_PAGEOUT, "vm_pageclean_setup, obj 0x%X off 0x%X page 0x%X new 0x%X new_off 0x%X\n", - VM_PAGE_OBJECT(m), m->offset, m, - new_m, new_offset); + VM_PAGE_OBJECT(m), m->vmp_offset, m, + new_m, new_offset); pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m)); /* * Mark original page as cleaning in place. */ - m->cleaning = TRUE; + m->vmp_cleaning = TRUE; SET_PAGE_DIRTY(m, FALSE); - m->precious = FALSE; + m->vmp_precious = FALSE; /* * Convert the fictitious page to a private shadow of * the real page. */ - assert(new_m->fictitious); + assert(new_m->vmp_fictitious); assert(VM_PAGE_GET_PHYS_PAGE(new_m) == vm_page_fictitious_addr); - new_m->fictitious = FALSE; - new_m->private = TRUE; - new_m->free_when_done = TRUE; + new_m->vmp_fictitious = FALSE; + new_m->vmp_private = TRUE; + new_m->vmp_free_when_done = TRUE; VM_PAGE_SET_PHYS_PAGE(new_m, VM_PAGE_GET_PHYS_PAGE(m)); vm_page_lockspin_queues(); @@ -740,8 +559,8 @@ vm_pageclean_setup( vm_page_unlock_queues(); vm_page_insert_wired(new_m, new_object, new_offset, VM_KERN_MEMORY_NONE); - assert(!new_m->wanted); - new_m->busy = FALSE; + assert(!new_m->vmp_wanted); + new_m->vmp_busy = FALSE; } /* @@ -762,38 +581,38 @@ vm_pageclean_setup( * Implementation: * Move this page to a completely new object. */ -void +void vm_pageout_initialize_page( - vm_page_t m) + vm_page_t m) { - vm_object_t object; - vm_object_offset_t paging_offset; - memory_object_t pager; + vm_object_t object; + vm_object_offset_t paging_offset; + memory_object_t pager; XPR(XPR_VM_PAGEOUT, - "vm_pageout_initialize_page, page 0x%X\n", - m, 0, 0, 0, 0); + "vm_pageout_initialize_page, page 0x%X\n", + m, 0, 0, 0, 0); assert(VM_CONFIG_COMPRESSOR_IS_PRESENT); object = VM_PAGE_OBJECT(m); - assert(m->busy); + assert(m->vmp_busy); assert(object->internal); /* * Verify that we really want to clean this page */ - assert(!m->absent); - assert(!m->error); - assert(m->dirty); + assert(!m->vmp_absent); + assert(!m->vmp_error); + assert(m->vmp_dirty); /* * Create a paging reference to let us play with the object. */ - paging_offset = m->offset + object->paging_offset; + paging_offset = m->vmp_offset + object->paging_offset; - if (m->absent || m->error || m->restart || (!m->dirty && !m->precious)) { + if (m->vmp_absent || m->vmp_error || m->vmp_restart || (!m->vmp_dirty && !m->vmp_precious)) { panic("reservation without pageout?"); /* alan */ VM_PAGE_FREE(m); @@ -803,7 +622,7 @@ vm_pageout_initialize_page( } /* - * If there's no pager, then we can't clean the page. This should + * If there's no pager, then we can't clean the page. This should * never happen since this should be a copy object and therefore not * an external object, so the pager should always be there. */ @@ -843,15 +662,6 @@ vm_pageout_initialize_page( vm_object_paging_end(object); } -#if MACH_CLUSTER_STATS -#define MAXCLUSTERPAGES 16 -struct { - unsigned long pages_in_cluster; - unsigned long pages_at_higher_offsets; - unsigned long pages_at_lower_offsets; -} cluster_stats[MAXCLUSTERPAGES]; -#endif /* MACH_CLUSTER_STATS */ - /* * vm_pageout_cluster: @@ -867,24 +677,32 @@ struct { * * The page must not be on any pageout queue. */ +#if DEVELOPMENT || DEBUG +vmct_stats_t vmct_stats; + int32_t vmct_active = 0; +uint64_t vm_compressor_epoch_start = 0; +uint64_t vm_compressor_epoch_stop = 0; + typedef enum vmct_state_t { VMCT_IDLE, VMCT_AWAKENED, VMCT_ACTIVE, } vmct_state_t; vmct_state_t vmct_state[MAX_COMPRESSOR_THREAD_COUNT]; +#endif + void vm_pageout_cluster(vm_page_t m) { - vm_object_t object = VM_PAGE_OBJECT(m); - struct vm_pageout_queue *q; + vm_object_t object = VM_PAGE_OBJECT(m); + struct vm_pageout_queue *q; XPR(XPR_VM_PAGEOUT, - "vm_pageout_cluster, object 0x%X offset 0x%X page 0x%X\n", - object, m->offset, m, 0, 0); + "vm_pageout_cluster, object 0x%X offset 0x%X page 0x%X\n", + object, m->vmp_offset, m, 0, 0); VM_PAGE_CHECK(m); LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); @@ -893,9 +711,9 @@ vm_pageout_cluster(vm_page_t m) /* * Only a certain kind of page is appreciated here. */ - assert((m->dirty || m->precious) && (!VM_PAGE_WIRED(m))); - assert(!m->cleaning && !m->laundry); - assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q); + assert((m->vmp_dirty || m->vmp_precious) && (!VM_PAGE_WIRED(m))); + assert(!m->vmp_cleaning && !m->vmp_laundry); + assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q); /* * protect the object from collapse or termination @@ -905,20 +723,21 @@ vm_pageout_cluster(vm_page_t m) if (object->internal == TRUE) { assert(VM_CONFIG_COMPRESSOR_IS_PRESENT); - m->busy = TRUE; + m->vmp_busy = TRUE; - q = &vm_pageout_queue_internal; - } else - q = &vm_pageout_queue_external; + q = &vm_pageout_queue_internal; + } else { + q = &vm_pageout_queue_external; + } - /* + /* * pgo_laundry count is tied to the laundry bit */ - m->laundry = TRUE; + m->vmp_laundry = TRUE; q->pgo_laundry++; - m->vm_page_q_state = VM_PAGE_ON_PAGEOUT_Q; - vm_page_queue_enter(&q->pgo_pending, m, vm_page_t, pageq); + m->vmp_q_state = VM_PAGE_ON_PAGEOUT_Q; + vm_page_queue_enter(&q->pgo_pending, m, vmp_pageq); if (q->pgo_idle == TRUE) { q->pgo_idle = FALSE; @@ -928,10 +747,8 @@ vm_pageout_cluster(vm_page_t m) } -unsigned long vm_pageout_throttle_up_count = 0; - /* - * A page is back from laundry or we are stealing it back from + * A page is back from laundry or we are stealing it back from * the laundering state. See if there are some pages waiting to * go to laundry and if we can let some of them go now. * @@ -939,48 +756,48 @@ unsigned long vm_pageout_throttle_up_count = 0; */ void vm_pageout_throttle_up( - vm_page_t m) + vm_page_t m) { - struct vm_pageout_queue *q; - vm_object_t m_object; - - m_object = VM_PAGE_OBJECT(m); - - assert(m_object != VM_OBJECT_NULL); - assert(m_object != kernel_object); + struct vm_pageout_queue *q; + vm_object_t m_object; - LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); - vm_object_lock_assert_exclusive(m_object); + m_object = VM_PAGE_OBJECT(m); - vm_pageout_throttle_up_count++; + assert(m_object != VM_OBJECT_NULL); + assert(m_object != kernel_object); - if (m_object->internal == TRUE) - q = &vm_pageout_queue_internal; - else - q = &vm_pageout_queue_external; + LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); + vm_object_lock_assert_exclusive(m_object); - if (m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) { + if (m_object->internal == TRUE) { + q = &vm_pageout_queue_internal; + } else { + q = &vm_pageout_queue_external; + } - vm_page_queue_remove(&q->pgo_pending, m, vm_page_t, pageq); - m->vm_page_q_state = VM_PAGE_NOT_ON_Q; + if (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) { + vm_page_queue_remove(&q->pgo_pending, m, vmp_pageq); + m->vmp_q_state = VM_PAGE_NOT_ON_Q; - VM_PAGE_ZERO_PAGEQ_ENTRY(m); + VM_PAGE_ZERO_PAGEQ_ENTRY(m); - vm_object_activity_end(m_object); - } - if (m->laundry == TRUE) { + vm_object_activity_end(m_object); - m->laundry = FALSE; - q->pgo_laundry--; + VM_PAGEOUT_DEBUG(vm_page_steal_pageout_page, 1); + } + if (m->vmp_laundry == TRUE) { + m->vmp_laundry = FALSE; + q->pgo_laundry--; - if (q->pgo_throttled == TRUE) { - q->pgo_throttled = FALSE; - thread_wakeup((event_t) &q->pgo_laundry); - } - if (q->pgo_draining == TRUE && q->pgo_laundry == 0) { - q->pgo_draining = FALSE; - thread_wakeup((event_t) (&q->pgo_laundry+1)); - } + if (q->pgo_throttled == TRUE) { + q->pgo_throttled = FALSE; + thread_wakeup((event_t) &q->pgo_laundry); + } + if (q->pgo_draining == TRUE && q->pgo_laundry == 0) { + q->pgo_draining = FALSE; + thread_wakeup((event_t) (&q->pgo_laundry + 1)); + } + VM_PAGEOUT_DEBUG(vm_pageout_throttle_up_count, 1); } } @@ -988,22 +805,22 @@ vm_pageout_throttle_up( static void vm_pageout_throttle_up_batch( struct vm_pageout_queue *q, - int batch_cnt) + int batch_cnt) { - LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); - vm_pageout_throttle_up_count += batch_cnt; + VM_PAGEOUT_DEBUG(vm_pageout_throttle_up_count, batch_cnt); - q->pgo_laundry -= batch_cnt; + q->pgo_laundry -= batch_cnt; - if (q->pgo_throttled == TRUE) { - q->pgo_throttled = FALSE; - thread_wakeup((event_t) &q->pgo_laundry); - } - if (q->pgo_draining == TRUE && q->pgo_laundry == 0) { - q->pgo_draining = FALSE; - thread_wakeup((event_t) (&q->pgo_laundry+1)); - } + if (q->pgo_throttled == TRUE) { + q->pgo_throttled = FALSE; + thread_wakeup((event_t) &q->pgo_laundry); + } + if (q->pgo_draining == TRUE && q->pgo_laundry == 0) { + q->pgo_draining = FALSE; + thread_wakeup((event_t) (&q->pgo_laundry + 1)); + } } @@ -1025,20 +842,64 @@ vm_pageout_throttle_up_batch( * also returns the number of pages the system still needs to reclaim at this * moment in time. */ -#define VM_PAGEOUT_STAT_SIZE 31 +#if DEVELOPMENT || DEBUG +#define VM_PAGEOUT_STAT_SIZE (30 * 8) + 1 +#else +#define VM_PAGEOUT_STAT_SIZE (1 * 8) + 1 +#endif struct vm_pageout_stat { - unsigned int considered; - unsigned int reclaimed_clean; + unsigned long vm_page_active_count; + unsigned long vm_page_speculative_count; + unsigned long vm_page_inactive_count; + unsigned long vm_page_anonymous_count; + + unsigned long vm_page_free_count; + unsigned long vm_page_wire_count; + unsigned long vm_page_compressor_count; + + unsigned long vm_page_pages_compressed; + unsigned long vm_page_pageable_internal_count; + unsigned long vm_page_pageable_external_count; + unsigned long vm_page_xpmapped_external_count; + + unsigned int pages_grabbed; + unsigned int pages_freed; + unsigned int pages_compressed; unsigned int pages_grabbed_by_compressor; + unsigned int failed_compressions; + + unsigned int pages_evicted; + unsigned int pages_purged; + + unsigned int considered; + unsigned int considered_bq_internal; + unsigned int considered_bq_external; + + unsigned int skipped_external; + unsigned int filecache_min_reactivations; + + unsigned int freed_speculative; + unsigned int freed_cleaned; + unsigned int freed_internal; + unsigned int freed_external; + unsigned int cleaned_dirty_external; + unsigned int cleaned_dirty_internal; + + unsigned int inactive_referenced; + unsigned int inactive_nolock; + unsigned int reactivation_limit_exceeded; + unsigned int forced_inactive_reclaim; + unsigned int throttled_internal_q; unsigned int throttled_external_q; - unsigned int failed_compressions; -} vm_pageout_stats[VM_PAGEOUT_STAT_SIZE] = {{0,0,0,0,0,0,0,0}, }; + + unsigned int phantom_ghosts_found; + unsigned int phantom_ghosts_added; +} vm_pageout_stats[VM_PAGEOUT_STAT_SIZE] = {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, }; unsigned int vm_pageout_stat_now = 0; -unsigned int vm_memory_pressure = 0; #define VM_PAGEOUT_STAT_BEFORE(i) \ (((i) == 0) ? VM_PAGEOUT_STAT_SIZE - 1 : (i) - 1) @@ -1046,15 +907,14 @@ unsigned int vm_memory_pressure = 0; (((i) == VM_PAGEOUT_STAT_SIZE - 1) ? 0 : (i) + 1) #if VM_PAGE_BUCKETS_CHECK -int vm_page_buckets_check_interval = 10; /* in seconds */ +int vm_page_buckets_check_interval = 80; /* in eighths of a second */ #endif /* VM_PAGE_BUCKETS_CHECK */ -/* - * Called from compute_averages(). - */ + void -compute_memory_pressure( - __unused void *arg) +record_memory_pressure(void); +void +record_memory_pressure(void) { unsigned int vm_pageout_next; @@ -1066,21 +926,18 @@ compute_memory_pressure( } #endif /* VM_PAGE_BUCKETS_CHECK */ - vm_memory_pressure = - vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].reclaimed_clean; + vm_pageout_state.vm_memory_pressure = + vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].freed_speculative + + vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].freed_cleaned + + vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].freed_internal + + vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].freed_external; - commpage_set_memory_pressure( vm_memory_pressure ); + commpage_set_memory_pressure((unsigned int)vm_pageout_state.vm_memory_pressure ); /* move "now" forward */ vm_pageout_next = VM_PAGEOUT_STAT_AFTER(vm_pageout_stat_now); - vm_pageout_stats[vm_pageout_next].considered = 0; - vm_pageout_stats[vm_pageout_next].reclaimed_clean = 0; - vm_pageout_stats[vm_pageout_next].throttled_internal_q = 0; - vm_pageout_stats[vm_pageout_next].throttled_external_q = 0; - vm_pageout_stats[vm_pageout_next].cleaned_dirty_external = 0; - vm_pageout_stats[vm_pageout_next].pages_compressed = 0; - vm_pageout_stats[vm_pageout_next].pages_grabbed_by_compressor = 0; - vm_pageout_stats[vm_pageout_next].failed_compressions = 0; + + bzero(&vm_pageout_stats[vm_pageout_next], sizeof(struct vm_pageout_stat)); vm_pageout_stat_now = vm_pageout_next; } @@ -1089,8 +946,8 @@ compute_memory_pressure( /* * IMPORTANT * mach_vm_ctl_page_free_wanted() is called indirectly, via - * mach_vm_pressure_monitor(), when taking a stackshot. Therefore, - * it must be safe in the restricted stackshot context. Locks and/or + * mach_vm_pressure_monitor(), when taking a stackshot. Therefore, + * it must be safe in the restricted stackshot context. Locks and/or * blocking are not allowable. */ unsigned int @@ -1112,7 +969,7 @@ mach_vm_ctl_page_free_wanted(void) /* * IMPORTANT: - * mach_vm_pressure_monitor() is called when taking a stackshot, with + * mach_vm_pressure_monitor() is called when taking a stackshot, with * wait_for_pressure FALSE, so that code path must remain safe in the * restricted stackshot context. No blocking or locks are allowable. * on that code path. @@ -1120,15 +977,17 @@ mach_vm_ctl_page_free_wanted(void) kern_return_t mach_vm_pressure_monitor( - boolean_t wait_for_pressure, - unsigned int nsecs_monitored, - unsigned int *pages_reclaimed_p, - unsigned int *pages_wanted_p) + boolean_t wait_for_pressure, + unsigned int nsecs_monitored, + unsigned int *pages_reclaimed_p, + unsigned int *pages_wanted_p) { - wait_result_t wr; - unsigned int vm_pageout_then, vm_pageout_now; - unsigned int pages_reclaimed; + wait_result_t wr; + unsigned int vm_pageout_then, vm_pageout_now; + unsigned int pages_reclaimed; + unsigned int units_of_monitor; + units_of_monitor = 8 * nsecs_monitored; /* * We don't take the vm_page_queue_lock here because we don't want * vm_pressure_monitor() to get in the way of the vm_pageout_scan() @@ -1140,7 +999,7 @@ mach_vm_pressure_monitor( /* wait until there's memory pressure */ while (vm_page_free_count >= vm_page_free_target) { wr = assert_wait((event_t) &vm_page_free_wanted, - THREAD_INTERRUPTIBLE); + THREAD_INTERRUPTIBLE); if (wr == THREAD_WAITING) { wr = thread_block(THREAD_CONTINUE_NULL); } @@ -1172,12 +1031,15 @@ mach_vm_pressure_monitor( vm_pageout_now = vm_pageout_stat_now; pages_reclaimed = 0; for (vm_pageout_then = - VM_PAGEOUT_STAT_BEFORE(vm_pageout_now); - vm_pageout_then != vm_pageout_now && - nsecs_monitored-- != 0; - vm_pageout_then = - VM_PAGEOUT_STAT_BEFORE(vm_pageout_then)) { - pages_reclaimed += vm_pageout_stats[vm_pageout_then].reclaimed_clean; + VM_PAGEOUT_STAT_BEFORE(vm_pageout_now); + vm_pageout_then != vm_pageout_now && + units_of_monitor-- != 0; + vm_pageout_then = + VM_PAGEOUT_STAT_BEFORE(vm_pageout_then)) { + pages_reclaimed += vm_pageout_stats[vm_pageout_then].freed_speculative; + pages_reclaimed += vm_pageout_stats[vm_pageout_then].freed_cleaned; + pages_reclaimed += vm_pageout_stats[vm_pageout_then].freed_internal; + pages_reclaimed += vm_pageout_stats[vm_pageout_then].freed_external; } *pages_reclaimed_p = pages_reclaimed; @@ -1195,7 +1057,7 @@ vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *, int); * condition variable used to make sure there is * only a single sweep going on at a time */ -boolean_t vm_pageout_disconnect_all_pages_active = FALSE; +boolean_t vm_pageout_disconnect_all_pages_active = FALSE; void @@ -1221,23 +1083,22 @@ vm_pageout_disconnect_all_pages() void vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *q, int qcount) { - vm_page_t m; - vm_object_t t_object = NULL; - vm_object_t l_object = NULL; - vm_object_t m_object = NULL; - int delayed_unlock = 0; - int try_failed_count = 0; - int disconnected_count = 0; - int paused_count = 0; - int object_locked_count = 0; + vm_page_t m; + vm_object_t t_object = NULL; + vm_object_t l_object = NULL; + vm_object_t m_object = NULL; + int delayed_unlock = 0; + int try_failed_count = 0; + int disconnected_count = 0; + int paused_count = 0; + int object_locked_count = 0; KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_DISCONNECT_ALL_PAGE_MAPPINGS)) | DBG_FUNC_START, - q, qcount, 0, 0, 0); + q, qcount, 0, 0, 0); vm_page_lock_queues(); while (qcount && !vm_page_queue_empty(q)) { - LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); m = (vm_page_t) vm_page_queue_first(q); @@ -1249,27 +1110,27 @@ vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *q, int qcount) * already got the lock */ if (m_object != l_object) { - /* - * the object associated with candidate page is + /* + * the object associated with candidate page is * different from the one we were just working * with... dump the lock if we still own it */ - if (l_object != NULL) { - vm_object_unlock(l_object); + if (l_object != NULL) { + vm_object_unlock(l_object); l_object = NULL; } - if (m_object != t_object) + if (m_object != t_object) { try_failed_count = 0; + } /* * Try to lock object; since we've alread got the * page queues lock, we can only 'try' for this one. * if the 'try' fails, we need to do a mutex_pause * to allow the owner of the object lock a chance to - * run... + * run... */ - if ( !vm_object_lock_try_scan(m_object)) { - + if (!vm_object_lock_try_scan(m_object)) { if (try_failed_count > 20) { goto reenter_pg_on_q; } @@ -1287,27 +1148,25 @@ vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *q, int qcount) l_object = m_object; } - if ( !m_object->alive || m->cleaning || m->laundry || m->busy || m->absent || m->error || m->free_when_done) { + if (!m_object->alive || m->vmp_cleaning || m->vmp_laundry || m->vmp_busy || m->vmp_absent || m->vmp_error || m->vmp_free_when_done) { /* * put it back on the head of its queue */ goto reenter_pg_on_q; } - if (m->pmapped == TRUE) { - + if (m->vmp_pmapped == TRUE) { pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)); disconnected_count++; } reenter_pg_on_q: - vm_page_queue_remove(q, m, vm_page_t, pageq); - vm_page_queue_enter(q, m, vm_page_t, pageq); + vm_page_queue_remove(q, m, vmp_pageq); + vm_page_queue_enter(q, m, vmp_pageq); qcount--; try_failed_count = 0; if (delayed_unlock++ > 128) { - if (l_object != NULL) { vm_object_unlock(l_object); l_object = NULL; @@ -1323,7 +1182,7 @@ reenter_pg_on_q: vm_page_unlock_queues(); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_DISCONNECT_ALL_PAGE_MAPPINGS)) | DBG_FUNC_END, - q, disconnected_count, object_locked_count, paused_count, 0); + q, disconnected_count, object_locked_count, paused_count, 0); } #endif @@ -1336,14 +1195,13 @@ vm_pageout_page_queue(vm_page_queue_head_t *, int); * condition variable used to make sure there is * only a single sweep going on at a time */ -boolean_t vm_pageout_anonymous_pages_active = FALSE; +boolean_t vm_pageout_anonymous_pages_active = FALSE; void vm_pageout_anonymous_pages() { if (VM_CONFIG_COMPRESSOR_IS_PRESENT) { - vm_page_lock_queues(); if (vm_pageout_anonymous_pages_active == TRUE) { @@ -1357,8 +1215,9 @@ vm_pageout_anonymous_pages() vm_pageout_page_queue(&vm_page_queue_anonymous, vm_page_anonymous_count); vm_pageout_page_queue(&vm_page_queue_active, vm_page_active_count); - if (VM_CONFIG_SWAP_IS_PRESENT) + if (VM_CONFIG_SWAP_IS_PRESENT) { vm_consider_swapping(); + } vm_page_lock_queues(); vm_pageout_anonymous_pages_active = FALSE; @@ -1370,39 +1229,37 @@ vm_pageout_anonymous_pages() void vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount) { - vm_page_t m; - vm_object_t t_object = NULL; - vm_object_t l_object = NULL; - vm_object_t m_object = NULL; - int delayed_unlock = 0; - int try_failed_count = 0; - int refmod_state; - int pmap_options; - struct vm_pageout_queue *iq; - ppnum_t phys_page; + vm_page_t m; + vm_object_t t_object = NULL; + vm_object_t l_object = NULL; + vm_object_t m_object = NULL; + int delayed_unlock = 0; + int try_failed_count = 0; + int refmod_state; + int pmap_options; + struct vm_pageout_queue *iq; + ppnum_t phys_page; iq = &vm_pageout_queue_internal; - + vm_page_lock_queues(); while (qcount && !vm_page_queue_empty(q)) { - LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); if (VM_PAGE_Q_THROTTLED(iq)) { - - if (l_object != NULL) { - vm_object_unlock(l_object); + if (l_object != NULL) { + vm_object_unlock(l_object); l_object = NULL; } iq->pgo_draining = TRUE; - + assert_wait((event_t) (&iq->pgo_laundry + 1), THREAD_INTERRUPTIBLE); vm_page_unlock_queues(); - + thread_block(THREAD_CONTINUE_NULL); - + vm_page_lock_queues(); delayed_unlock = 0; continue; @@ -1416,30 +1273,31 @@ vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount) * already got the lock */ if (m_object != l_object) { - if ( !m_object->internal) + if (!m_object->internal) { goto reenter_pg_on_q; + } - /* - * the object associated with candidate page is + /* + * the object associated with candidate page is * different from the one we were just working * with... dump the lock if we still own it */ - if (l_object != NULL) { - vm_object_unlock(l_object); + if (l_object != NULL) { + vm_object_unlock(l_object); l_object = NULL; } - if (m_object != t_object) + if (m_object != t_object) { try_failed_count = 0; + } /* * Try to lock object; since we've alread got the * page queues lock, we can only 'try' for this one. * if the 'try' fails, we need to do a mutex_pause * to allow the owner of the object lock a chance to - * run... + * run... */ - if ( !vm_object_lock_try_scan(m_object)) { - + if (!vm_object_lock_try_scan(m_object)) { if (try_failed_count > 20) { goto reenter_pg_on_q; } @@ -1453,7 +1311,7 @@ vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount) } l_object = m_object; } - if ( !m_object->alive || m->cleaning || m->laundry || m->busy || m->absent || m->error || m->free_when_done) { + if (!m_object->alive || m->vmp_cleaning || m->vmp_laundry || m->vmp_busy || m->vmp_absent || m->vmp_error || m->vmp_free_when_done) { /* * page is not to be cleaned * put it back on the head of its queue @@ -1462,22 +1320,23 @@ vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount) } phys_page = VM_PAGE_GET_PHYS_PAGE(m); - if (m->reference == FALSE && m->pmapped == TRUE) { + if (m->vmp_reference == FALSE && m->vmp_pmapped == TRUE) { refmod_state = pmap_get_refmod(phys_page); - - if (refmod_state & VM_MEM_REFERENCED) - m->reference = TRUE; + + if (refmod_state & VM_MEM_REFERENCED) { + m->vmp_reference = TRUE; + } if (refmod_state & VM_MEM_MODIFIED) { - SET_PAGE_DIRTY(m, FALSE); + SET_PAGE_DIRTY(m, FALSE); } } - if (m->reference == TRUE) { - m->reference = FALSE; + if (m->vmp_reference == TRUE) { + m->vmp_reference = FALSE; pmap_clear_refmod_options(phys_page, VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL); goto reenter_pg_on_q; } - if (m->pmapped == TRUE) { - if (m->dirty || m->precious) { + if (m->vmp_pmapped == TRUE) { + if (m->vmp_dirty || m->vmp_precious) { pmap_options = PMAP_OPTIONS_COMPRESSOR; } else { pmap_options = PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED; @@ -1487,7 +1346,8 @@ vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount) SET_PAGE_DIRTY(m, FALSE); } } - if ( !m->dirty && !m->precious) { + + if (!m->vmp_dirty && !m->vmp_precious) { vm_page_unlock_queues(); VM_PAGE_FREE(m); vm_page_lock_queues(); @@ -1495,22 +1355,22 @@ vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount) goto next_pg; } - if (!m_object->pager_initialized || m_object->pager == MEMORY_OBJECT_NULL) { - + if (!m_object->pager_initialized || m_object->pager == MEMORY_OBJECT_NULL) { if (!m_object->pager_initialized) { - vm_page_unlock_queues(); vm_object_collapse(m_object, (vm_object_offset_t) 0, TRUE); - if (!m_object->pager_initialized) + if (!m_object->pager_initialized) { vm_object_compressor_pager_create(m_object); + } vm_page_lock_queues(); delayed_unlock = 0; } - if (!m_object->pager_initialized || m_object->pager == MEMORY_OBJECT_NULL) + if (!m_object->pager_initialized || m_object->pager == MEMORY_OBJECT_NULL) { goto reenter_pg_on_q; + } /* * vm_object_compressor_pager_create will drop the object lock * which means 'm' may no longer be valid to use @@ -1531,14 +1391,13 @@ vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount) goto next_pg; reenter_pg_on_q: - vm_page_queue_remove(q, m, vm_page_t, pageq); - vm_page_queue_enter(q, m, vm_page_t, pageq); + vm_page_queue_remove(q, m, vmp_pageq); + vm_page_queue_enter(q, m, vmp_pageq); next_pg: qcount--; try_failed_count = 0; if (delayed_unlock++ > 128) { - if (l_object != NULL) { vm_object_unlock(l_object); l_object = NULL; @@ -1561,81 +1420,74 @@ next_pg: */ extern void vm_pageout_io_throttle(void); -#define VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, obj) \ - MACRO_BEGIN \ - /* \ - * If a "reusable" page somehow made it back into \ - * the active queue, it's been re-used and is not \ - * quite re-usable. \ - * If the VM object was "all_reusable", consider it \ - * as "all re-used" instead of converting it to \ - * "partially re-used", which could be expensive. \ - */ \ - assert(VM_PAGE_OBJECT((m)) == (obj)); \ - if ((m)->reusable || \ - (obj)->all_reusable) { \ - vm_object_reuse_pages((obj), \ - (m)->offset, \ - (m)->offset + PAGE_SIZE_64, \ - FALSE); \ - } \ - MACRO_END - - -#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT 64 -#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX 1024 - -#define FCS_IDLE 0 -#define FCS_DELAYED 1 -#define FCS_DEADLOCK_DETECTED 2 +#define VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, obj) \ + MACRO_BEGIN \ + /* \ + * If a "reusable" page somehow made it back into \ + * the active queue, it's been re-used and is not \ + * quite re-usable. \ + * If the VM object was "all_reusable", consider it \ + * as "all re-used" instead of converting it to \ + * "partially re-used", which could be expensive. \ + */ \ + assert(VM_PAGE_OBJECT((m)) == (obj)); \ + if ((m)->vmp_reusable || \ + (obj)->all_reusable) { \ + vm_object_reuse_pages((obj), \ + (m)->vmp_offset, \ + (m)->vmp_offset + PAGE_SIZE_64, \ + FALSE); \ + } \ + MACRO_END + + +#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT 64 +#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX 1024 + +#define FCS_IDLE 0 +#define FCS_DELAYED 1 +#define FCS_DEADLOCK_DETECTED 2 struct flow_control { - int state; - mach_timespec_t ts; + int state; + mach_timespec_t ts; }; + #if CONFIG_BACKGROUND_QUEUE -uint64_t vm_pageout_skipped_bq_internal = 0; -uint64_t vm_pageout_considered_bq_internal = 0; -uint64_t vm_pageout_considered_bq_external = 0; uint64_t vm_pageout_rejected_bq_internal = 0; uint64_t vm_pageout_rejected_bq_external = 0; +uint64_t vm_pageout_skipped_bq_internal = 0; #endif -uint32_t vm_pageout_no_victim = 0; -uint32_t vm_pageout_considered_page = 0; -uint32_t vm_page_filecache_min = 0; - -#define ANONS_GRABBED_LIMIT 2 - -#if CONFIG_SECLUDED_MEMORY -extern vm_page_t vm_page_grab_secluded(void); -uint64_t vm_pageout_secluded_burst_count = 0; -#endif /* CONFIG_SECLUDED_MEMORY */ +#define ANONS_GRABBED_LIMIT 2 +#if 0 static void vm_pageout_delayed_unlock(int *, int *, vm_page_t *); +#endif static void vm_pageout_prepare_to_block(vm_object_t *, int *, vm_page_t *, int *, int); -#define VM_PAGEOUT_PB_NO_ACTION 0 -#define VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER 1 -#define VM_PAGEOUT_PB_THREAD_YIELD 2 +#define VM_PAGEOUT_PB_NO_ACTION 0 +#define VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER 1 +#define VM_PAGEOUT_PB_THREAD_YIELD 2 +#if 0 static void vm_pageout_delayed_unlock(int *delayed_unlock, int *local_freed, vm_page_t *local_freeq) { if (*local_freeq) { vm_page_unlock_queues(); - VM_DEBUG_EVENT( + VM_DEBUG_CONSTANT_EVENT( vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START, - vm_page_free_count, *local_freed, 0, 1); + vm_page_free_count, 0, 0, 1); vm_page_free_list(*local_freeq, TRUE); - VM_DEBUG_EVENT(vm_pageout_freelist,VM_PAGEOUT_FREELIST, DBG_FUNC_END, - vm_page_free_count, 0, 0, 1); + VM_DEBUG_CONSTANT_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END, + vm_page_free_count, *local_freed, 0, 1); *local_freeq = NULL; *local_freed = 0; @@ -1646,11 +1498,12 @@ vm_pageout_delayed_unlock(int *delayed_unlock, int *local_freed, vm_page_t *loca } *delayed_unlock = 1; } +#endif static void vm_pageout_prepare_to_block(vm_object_t *object, int *delayed_unlock, - vm_page_t *local_freeq, int *local_freed, int action) + vm_page_t *local_freeq, int *local_freed, int action) { vm_page_unlock_queues(); @@ -1658,17 +1511,8 @@ vm_pageout_prepare_to_block(vm_object_t *object, int *delayed_unlock, vm_object_unlock(*object); *object = NULL; } - vm_pageout_scan_wants_object = VM_OBJECT_NULL; - if (*local_freeq) { - - VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START, - vm_page_free_count, *local_freed, 0, 2); - vm_page_free_list(*local_freeq, TRUE); - - VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END, - vm_page_free_count, 0, 0, 2); *local_freeq = NULL; *local_freed = 0; @@ -1676,7 +1520,6 @@ vm_pageout_prepare_to_block(vm_object_t *object, int *delayed_unlock, *delayed_unlock = 1; switch (action) { - case VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER: vm_consider_waking_compactor_swapper(); break; @@ -1691,130 +1534,269 @@ vm_pageout_prepare_to_block(vm_object_t *object, int *delayed_unlock, } -int last_vm_pageout_freed_from_inactive_clean = 0; -int last_vm_pageout_freed_from_cleaned = 0; -int last_vm_pageout_freed_from_speculative = 0; -int last_vm_pageout_freed_after_compression = 0; -int last_vm_pageout_enqueued_cleaned_from_inactive_dirty = 0; -int last_vm_pageout_inactive_force_reclaim = 0; -int last_vm_pageout_scan_inactive_throttled_external = 0; -int last_vm_pageout_scan_inactive_throttled_internal = 0; -int last_vm_pageout_reactivation_limit_exceeded = 0; -int last_vm_pageout_considered_page = 0; -int last_vm_compressor_pages_grabbed = 0; -int last_vm_compressor_failed = 0; -int last_vm_pageout_skipped_external = 0; +static struct vm_pageout_vminfo last; + +uint64_t last_vm_page_pages_grabbed = 0; +extern uint32_t c_segment_pages_compressed; + +extern uint64_t shared_region_pager_reclaimed; +extern struct memory_object_pager_ops shared_region_pager_ops; -void update_vm_info(void) +void +update_vm_info(void) { - int tmp1, tmp2, tmp3, tmp4; + uint64_t tmp; + + vm_pageout_stats[vm_pageout_stat_now].vm_page_active_count = vm_page_active_count; + vm_pageout_stats[vm_pageout_stat_now].vm_page_speculative_count = vm_page_speculative_count; + vm_pageout_stats[vm_pageout_stat_now].vm_page_inactive_count = vm_page_inactive_count; + vm_pageout_stats[vm_pageout_stat_now].vm_page_anonymous_count = vm_page_anonymous_count; + + vm_pageout_stats[vm_pageout_stat_now].vm_page_free_count = vm_page_free_count; + vm_pageout_stats[vm_pageout_stat_now].vm_page_wire_count = vm_page_wire_count; + vm_pageout_stats[vm_pageout_stat_now].vm_page_compressor_count = VM_PAGE_COMPRESSOR_COUNT; + + vm_pageout_stats[vm_pageout_stat_now].vm_page_pages_compressed = c_segment_pages_compressed; + vm_pageout_stats[vm_pageout_stat_now].vm_page_pageable_internal_count = vm_page_pageable_internal_count; + vm_pageout_stats[vm_pageout_stat_now].vm_page_pageable_external_count = vm_page_pageable_external_count; + vm_pageout_stats[vm_pageout_stat_now].vm_page_xpmapped_external_count = vm_page_xpmapped_external_count; + + + tmp = vm_pageout_vminfo.vm_pageout_considered_page; + vm_pageout_stats[vm_pageout_stat_now].considered = (unsigned int)(tmp - last.vm_pageout_considered_page); + last.vm_pageout_considered_page = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_compressions; + vm_pageout_stats[vm_pageout_stat_now].pages_compressed = (unsigned int)(tmp - last.vm_pageout_compressions); + last.vm_pageout_compressions = tmp; + + tmp = vm_pageout_vminfo.vm_compressor_failed; + vm_pageout_stats[vm_pageout_stat_now].failed_compressions = (unsigned int)(tmp - last.vm_compressor_failed); + last.vm_compressor_failed = tmp; + + tmp = vm_pageout_vminfo.vm_compressor_pages_grabbed; + vm_pageout_stats[vm_pageout_stat_now].pages_grabbed_by_compressor = (unsigned int)(tmp - last.vm_compressor_pages_grabbed); + last.vm_compressor_pages_grabbed = tmp; + + tmp = vm_pageout_vminfo.vm_phantom_cache_found_ghost; + vm_pageout_stats[vm_pageout_stat_now].phantom_ghosts_found = (unsigned int)(tmp - last.vm_phantom_cache_found_ghost); + last.vm_phantom_cache_found_ghost = tmp; + + tmp = vm_pageout_vminfo.vm_phantom_cache_added_ghost; + vm_pageout_stats[vm_pageout_stat_now].phantom_ghosts_added = (unsigned int)(tmp - last.vm_phantom_cache_added_ghost); + last.vm_phantom_cache_added_ghost = tmp; + + tmp = get_pages_grabbed_count(); + vm_pageout_stats[vm_pageout_stat_now].pages_grabbed = (unsigned int)(tmp - last_vm_page_pages_grabbed); + last_vm_page_pages_grabbed = tmp; + + tmp = vm_pageout_vminfo.vm_page_pages_freed; + vm_pageout_stats[vm_pageout_stat_now].pages_freed = (unsigned int)(tmp - last.vm_page_pages_freed); + last.vm_page_pages_freed = tmp; + + + if (vm_pageout_stats[vm_pageout_stat_now].considered) { + tmp = vm_pageout_vminfo.vm_pageout_pages_evicted; + vm_pageout_stats[vm_pageout_stat_now].pages_evicted = (unsigned int)(tmp - last.vm_pageout_pages_evicted); + last.vm_pageout_pages_evicted = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_pages_purged; + vm_pageout_stats[vm_pageout_stat_now].pages_purged = (unsigned int)(tmp - last.vm_pageout_pages_purged); + last.vm_pageout_pages_purged = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_freed_speculative; + vm_pageout_stats[vm_pageout_stat_now].freed_speculative = (unsigned int)(tmp - last.vm_pageout_freed_speculative); + last.vm_pageout_freed_speculative = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_freed_external; + vm_pageout_stats[vm_pageout_stat_now].freed_external = (unsigned int)(tmp - last.vm_pageout_freed_external); + last.vm_pageout_freed_external = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_inactive_referenced; + vm_pageout_stats[vm_pageout_stat_now].inactive_referenced = (unsigned int)(tmp - last.vm_pageout_inactive_referenced); + last.vm_pageout_inactive_referenced = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_scan_inactive_throttled_external; + vm_pageout_stats[vm_pageout_stat_now].throttled_external_q = (unsigned int)(tmp - last.vm_pageout_scan_inactive_throttled_external); + last.vm_pageout_scan_inactive_throttled_external = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_inactive_dirty_external; + vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_external = (unsigned int)(tmp - last.vm_pageout_inactive_dirty_external); + last.vm_pageout_inactive_dirty_external = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_freed_cleaned; + vm_pageout_stats[vm_pageout_stat_now].freed_cleaned = (unsigned int)(tmp - last.vm_pageout_freed_cleaned); + last.vm_pageout_freed_cleaned = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_inactive_nolock; + vm_pageout_stats[vm_pageout_stat_now].inactive_nolock = (unsigned int)(tmp - last.vm_pageout_inactive_nolock); + last.vm_pageout_inactive_nolock = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_scan_inactive_throttled_internal; + vm_pageout_stats[vm_pageout_stat_now].throttled_internal_q = (unsigned int)(tmp - last.vm_pageout_scan_inactive_throttled_internal); + last.vm_pageout_scan_inactive_throttled_internal = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_skipped_external; + vm_pageout_stats[vm_pageout_stat_now].skipped_external = (unsigned int)(tmp - last.vm_pageout_skipped_external); + last.vm_pageout_skipped_external = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_reactivation_limit_exceeded; + vm_pageout_stats[vm_pageout_stat_now].reactivation_limit_exceeded = (unsigned int)(tmp - last.vm_pageout_reactivation_limit_exceeded); + last.vm_pageout_reactivation_limit_exceeded = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_inactive_force_reclaim; + vm_pageout_stats[vm_pageout_stat_now].forced_inactive_reclaim = (unsigned int)(tmp - last.vm_pageout_inactive_force_reclaim); + last.vm_pageout_inactive_force_reclaim = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_freed_internal; + vm_pageout_stats[vm_pageout_stat_now].freed_internal = (unsigned int)(tmp - last.vm_pageout_freed_internal); + last.vm_pageout_freed_internal = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_considered_bq_internal; + vm_pageout_stats[vm_pageout_stat_now].considered_bq_internal = (unsigned int)(tmp - last.vm_pageout_considered_bq_internal); + last.vm_pageout_considered_bq_internal = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_considered_bq_external; + vm_pageout_stats[vm_pageout_stat_now].considered_bq_external = (unsigned int)(tmp - last.vm_pageout_considered_bq_external); + last.vm_pageout_considered_bq_external = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_filecache_min_reactivated; + vm_pageout_stats[vm_pageout_stat_now].filecache_min_reactivations = (unsigned int)(tmp - last.vm_pageout_filecache_min_reactivated); + last.vm_pageout_filecache_min_reactivated = tmp; + + tmp = vm_pageout_vminfo.vm_pageout_inactive_dirty_internal; + vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_internal = (unsigned int)(tmp - last.vm_pageout_inactive_dirty_internal); + last.vm_pageout_inactive_dirty_internal = tmp; + } - if (!kdebug_enable) - return; - KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO1)) | DBG_FUNC_NONE, - vm_page_active_count, - vm_page_speculative_count, - vm_page_inactive_count, - vm_page_anonymous_count, - 0); + vm_pageout_stats[vm_pageout_stat_now].vm_page_active_count, + vm_pageout_stats[vm_pageout_stat_now].vm_page_speculative_count, + vm_pageout_stats[vm_pageout_stat_now].vm_page_inactive_count, + vm_pageout_stats[vm_pageout_stat_now].vm_page_anonymous_count, + 0); KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO2)) | DBG_FUNC_NONE, - vm_page_free_count, - vm_page_wire_count, - VM_PAGE_COMPRESSOR_COUNT, - 0, 0); + vm_pageout_stats[vm_pageout_stat_now].vm_page_free_count, + vm_pageout_stats[vm_pageout_stat_now].vm_page_wire_count, + vm_pageout_stats[vm_pageout_stat_now].vm_page_compressor_count, + 0, + 0); KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO3)) | DBG_FUNC_NONE, - c_segment_pages_compressed, - vm_page_internal_count, - vm_page_external_count, - vm_page_xpmapped_external_count, - 0); - - - if ((vm_pageout_considered_page - last_vm_pageout_considered_page) == 0 && - (vm_pageout_enqueued_cleaned_from_inactive_dirty - last_vm_pageout_enqueued_cleaned_from_inactive_dirty == 0) && - (vm_pageout_freed_after_compression - last_vm_pageout_freed_after_compression == 0)) - return; - - - tmp1 = vm_pageout_considered_page; - tmp2 = vm_pageout_freed_from_speculative; - tmp3 = vm_pageout_freed_from_inactive_clean; - - KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO4)) | DBG_FUNC_NONE, - tmp1 - last_vm_pageout_considered_page, - tmp2 - last_vm_pageout_freed_from_speculative, - tmp3 - last_vm_pageout_freed_from_inactive_clean, - 0, 0); - - last_vm_pageout_considered_page = tmp1; - last_vm_pageout_freed_from_speculative = tmp2; - last_vm_pageout_freed_from_inactive_clean = tmp3; - - - tmp1 = vm_pageout_scan_inactive_throttled_external; - tmp2 = vm_pageout_enqueued_cleaned_from_inactive_dirty; - tmp3 = vm_pageout_freed_from_cleaned; - - KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO5)) | DBG_FUNC_NONE, - tmp1 - last_vm_pageout_scan_inactive_throttled_external, - tmp2 - last_vm_pageout_enqueued_cleaned_from_inactive_dirty, - tmp3 - last_vm_pageout_freed_from_cleaned, - 0, 0); - - vm_pageout_stats[vm_pageout_stat_now].throttled_external_q += (tmp1 - last_vm_pageout_scan_inactive_throttled_external); - vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_external += (tmp2 - last_vm_pageout_enqueued_cleaned_from_inactive_dirty); - - last_vm_pageout_scan_inactive_throttled_external = tmp1; - last_vm_pageout_enqueued_cleaned_from_inactive_dirty = tmp2; - last_vm_pageout_freed_from_cleaned = tmp3; - - - tmp1 = vm_pageout_scan_inactive_throttled_internal; - tmp2 = vm_pageout_freed_after_compression; - tmp3 = vm_compressor_pages_grabbed; - tmp4 = vm_pageout_skipped_external; - - KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO6)) | DBG_FUNC_NONE, - tmp1 - last_vm_pageout_scan_inactive_throttled_internal, - tmp2 - last_vm_pageout_freed_after_compression, - tmp3 - last_vm_compressor_pages_grabbed, - tmp4 - last_vm_pageout_skipped_external, - 0); - - vm_pageout_stats[vm_pageout_stat_now].throttled_internal_q += (tmp1 - last_vm_pageout_scan_inactive_throttled_internal); - vm_pageout_stats[vm_pageout_stat_now].pages_compressed += (tmp2 - last_vm_pageout_freed_after_compression); - vm_pageout_stats[vm_pageout_stat_now].pages_grabbed_by_compressor += (tmp3 - last_vm_compressor_pages_grabbed); - - last_vm_pageout_scan_inactive_throttled_internal = tmp1; - last_vm_pageout_freed_after_compression = tmp2; - last_vm_compressor_pages_grabbed = tmp3; - last_vm_pageout_skipped_external = tmp4; - - - if ((vm_pageout_reactivation_limit_exceeded - last_vm_pageout_reactivation_limit_exceeded) == 0 && - (vm_pageout_inactive_force_reclaim - last_vm_pageout_inactive_force_reclaim) == 0 && - (vm_compressor_failed - last_vm_compressor_failed) == 0) - return; - - tmp1 = vm_pageout_reactivation_limit_exceeded; - tmp2 = vm_pageout_inactive_force_reclaim; - tmp3 = vm_compressor_failed; - - KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO7)) | DBG_FUNC_NONE, - tmp1 - last_vm_pageout_reactivation_limit_exceeded, - tmp2 - last_vm_pageout_inactive_force_reclaim, - tmp3 - last_vm_compressor_failed, - 0, 0); - - vm_pageout_stats[vm_pageout_stat_now].failed_compressions += (tmp3 - last_vm_compressor_failed); - - last_vm_pageout_reactivation_limit_exceeded = tmp1; - last_vm_pageout_inactive_force_reclaim = tmp2; - last_vm_compressor_failed = tmp3; + vm_pageout_stats[vm_pageout_stat_now].vm_page_pages_compressed, + vm_pageout_stats[vm_pageout_stat_now].vm_page_pageable_internal_count, + vm_pageout_stats[vm_pageout_stat_now].vm_page_pageable_external_count, + vm_pageout_stats[vm_pageout_stat_now].vm_page_xpmapped_external_count, + 0); + + if (vm_pageout_stats[vm_pageout_stat_now].considered || + vm_pageout_stats[vm_pageout_stat_now].pages_compressed || + vm_pageout_stats[vm_pageout_stat_now].failed_compressions) { + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO4)) | DBG_FUNC_NONE, + vm_pageout_stats[vm_pageout_stat_now].considered, + vm_pageout_stats[vm_pageout_stat_now].freed_speculative, + vm_pageout_stats[vm_pageout_stat_now].freed_external, + vm_pageout_stats[vm_pageout_stat_now].inactive_referenced, + 0); + + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO5)) | DBG_FUNC_NONE, + vm_pageout_stats[vm_pageout_stat_now].throttled_external_q, + vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_external, + vm_pageout_stats[vm_pageout_stat_now].freed_cleaned, + vm_pageout_stats[vm_pageout_stat_now].inactive_nolock, + 0); + + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO6)) | DBG_FUNC_NONE, + vm_pageout_stats[vm_pageout_stat_now].throttled_internal_q, + vm_pageout_stats[vm_pageout_stat_now].pages_compressed, + vm_pageout_stats[vm_pageout_stat_now].pages_grabbed_by_compressor, + vm_pageout_stats[vm_pageout_stat_now].skipped_external, + 0); + + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO7)) | DBG_FUNC_NONE, + vm_pageout_stats[vm_pageout_stat_now].reactivation_limit_exceeded, + vm_pageout_stats[vm_pageout_stat_now].forced_inactive_reclaim, + vm_pageout_stats[vm_pageout_stat_now].failed_compressions, + vm_pageout_stats[vm_pageout_stat_now].freed_internal, + 0); + + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO8)) | DBG_FUNC_NONE, + vm_pageout_stats[vm_pageout_stat_now].considered_bq_internal, + vm_pageout_stats[vm_pageout_stat_now].considered_bq_external, + vm_pageout_stats[vm_pageout_stat_now].filecache_min_reactivations, + vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_internal, + 0); + } + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO9)) | DBG_FUNC_NONE, + vm_pageout_stats[vm_pageout_stat_now].pages_grabbed, + vm_pageout_stats[vm_pageout_stat_now].pages_freed, + vm_pageout_stats[vm_pageout_stat_now].phantom_ghosts_found, + vm_pageout_stats[vm_pageout_stat_now].phantom_ghosts_added, + 0); + + record_memory_pressure(); +} + +extern boolean_t hibernation_vmqueues_inspection; + +void +vm_page_balance_inactive(int max_to_move) +{ + vm_page_t m; + + LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); + + if (hibernation_vmqueues_inspection == TRUE) { + /* + * It is likely that the hibernation code path is + * dealing with these very queues as we are about + * to move pages around in/from them and completely + * change the linkage of the pages. + * + * And so we skip the rebalancing of these queues. + */ + return; + } + vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count + + vm_page_inactive_count + + vm_page_speculative_count); + + while (max_to_move-- && (vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) { + VM_PAGEOUT_DEBUG(vm_pageout_balanced, 1); + + m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active); + + assert(m->vmp_q_state == VM_PAGE_ON_ACTIVE_Q); + assert(!m->vmp_laundry); + assert(VM_PAGE_OBJECT(m) != kernel_object); + assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr); + + DTRACE_VM2(scan, int, 1, (uint64_t *), NULL); + + /* + * by not passing in a pmap_flush_context we will forgo any TLB flushing, local or otherwise... + * + * a TLB flush isn't really needed here since at worst we'll miss the reference bit being + * updated in the PTE if a remote processor still has this mapping cached in its TLB when the + * new reference happens. If no futher references happen on the page after that remote TLB flushes + * we'll see a clean, non-referenced page when it eventually gets pulled out of the inactive queue + * by pageout_scan, which is just fine since the last reference would have happened quite far + * in the past (TLB caches don't hang around for very long), and of course could just as easily + * have happened before we moved the page + */ + if (m->vmp_pmapped == TRUE) { + pmap_clear_refmod_options(VM_PAGE_GET_PHYS_PAGE(m), VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL); + } + + /* + * The page might be absent or busy, + * but vm_page_deactivate can handle that. + * FALSE indicates that we don't want a H/W clear reference + */ + vm_page_deactivate_internal(m, FALSE); + } } @@ -1828,48 +1810,50 @@ vm_pageout_scan(void) { unsigned int loop_count = 0; unsigned int inactive_burst_count = 0; - unsigned int active_burst_count = 0; unsigned int reactivated_this_call; unsigned int reactivate_limit; vm_page_t local_freeq = NULL; int local_freed = 0; int delayed_unlock; - int delayed_unlock_limit = 0; - int refmod_state = 0; - int vm_pageout_deadlock_target = 0; - struct vm_pageout_queue *iq; - struct vm_pageout_queue *eq; - struct vm_speculative_age_q *sq; - struct flow_control flow_control = { 0, { 0, 0 } }; - boolean_t inactive_throttled = FALSE; - boolean_t try_failed; - mach_timespec_t ts; - unsigned int msecs = 0; - vm_object_t object = NULL; - uint32_t inactive_reclaim_run; - boolean_t exceeded_burst_throttle; - boolean_t grab_anonymous = FALSE; - boolean_t force_anonymous = FALSE; - boolean_t force_speculative_aging = FALSE; - int anons_grabbed = 0; - int page_prev_q_state = 0; + int delayed_unlock_limit = 0; + int refmod_state = 0; + int vm_pageout_deadlock_target = 0; + struct vm_pageout_queue *iq; + struct vm_pageout_queue *eq; + struct vm_speculative_age_q *sq; + struct flow_control flow_control = { 0, { 0, 0 } }; + boolean_t inactive_throttled = FALSE; + mach_timespec_t ts; + unsigned int msecs = 0; + vm_object_t object = NULL; + uint32_t inactive_reclaim_run; + boolean_t exceeded_burst_throttle; + boolean_t grab_anonymous = FALSE; + boolean_t force_anonymous = FALSE; + boolean_t force_speculative_aging = FALSE; + int anons_grabbed = 0; + int page_prev_q_state = 0; #if CONFIG_BACKGROUND_QUEUE - boolean_t page_from_bg_q = FALSE; + boolean_t page_from_bg_q = FALSE; #endif - int cache_evict_throttle = 0; - uint32_t vm_pageout_inactive_external_forced_reactivate_limit = 0; - int force_purge = 0; -#define DELAY_SPECULATIVE_AGE 1000 - int delay_speculative_age = 0; - vm_object_t m_object = VM_OBJECT_NULL; + int cache_evict_throttle = 0; + uint32_t vm_pageout_inactive_external_forced_reactivate_limit = 0; + uint32_t inactive_external_count; + int force_purge = 0; + int divisor; +#define DELAY_SPECULATIVE_AGE 1000 + int delay_speculative_age = 0; + vm_object_t m_object = VM_OBJECT_NULL; #if VM_PRESSURE_EVENTS vm_pressure_level_t pressure_level; #endif /* VM_PRESSURE_EVENTS */ VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_START, - vm_pageout_speculative_clean, vm_pageout_inactive_clean, - vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external); + vm_pageout_vminfo.vm_pageout_freed_speculative, + vm_pageout_state.vm_pageout_inactive_clean, + vm_pageout_vminfo.vm_pageout_inactive_dirty_internal, + vm_pageout_vminfo.vm_pageout_inactive_dirty_external); flow_control.state = FCS_IDLE; iq = &vm_pageout_queue_internal; @@ -1877,12 +1861,15 @@ vm_pageout_scan(void) sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q]; - XPR(XPR_VM_PAGEOUT, "vm_pageout_scan\n", 0, 0, 0, 0, 0); + XPR(XPR_VM_PAGEOUT, "vm_pageout_scan\n", 0, 0, 0, 0, 0); /* Ask the pmap layer to return any pages it no longer needs. */ - pmap_release_pages_fast(); + uint64_t pmap_wired_pages_freed = pmap_release_pages_fast(); vm_page_lock_queues(); + + vm_page_wire_count -= pmap_wired_pages_freed; + delayed_unlock = 1; /* @@ -1891,125 +1878,85 @@ vm_pageout_scan(void) */ reactivated_this_call = 0; reactivate_limit = VM_PAGE_REACTIVATE_LIMIT(vm_page_active_count + - vm_page_inactive_count); + vm_page_inactive_count); inactive_reclaim_run = 0; vm_pageout_inactive_external_forced_reactivate_limit = vm_page_active_count + vm_page_inactive_count; /* - * We want to gradually dribble pages from the active queue - * to the inactive queue. If we let the inactive queue get - * very small, and then suddenly dump many pages into it, - * those pages won't get a sufficient chance to be referenced - * before we start taking them from the inactive queue. - * * We must limit the rate at which we send pages to the pagers * so that we don't tie up too many pages in the I/O queues. * We implement a throttling mechanism using the laundry count - * to limit the number of pages outstanding to the default + * to limit the number of pages outstanding to the default * and external pagers. We can bypass the throttles and look * for clean pages if the pageout queues don't drain in a timely * fashion since this may indicate that the pageout paths are * stalled waiting for memory, which only we can provide. */ - Restart: assert(object == NULL); assert(delayed_unlock != 0); - - /* - * Recalculate vm_page_inactivate_target. - */ - vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count + - vm_page_inactive_count + - vm_page_speculative_count); vm_page_anonymous_min = vm_page_inactive_target / 20; + if (vm_pageout_state.vm_page_speculative_percentage > 50) { + vm_pageout_state.vm_page_speculative_percentage = 50; + } else if (vm_pageout_state.vm_page_speculative_percentage <= 0) { + vm_pageout_state.vm_page_speculative_percentage = 1; + } - /* - * don't want to wake the pageout_scan thread up everytime we fall below - * the targets... set a low water mark at 0.25% below the target - */ - vm_page_inactive_min = vm_page_inactive_target - (vm_page_inactive_target / 400); - - if (vm_page_speculative_percentage > 50) - vm_page_speculative_percentage = 50; - else if (vm_page_speculative_percentage <= 0) - vm_page_speculative_percentage = 1; - - vm_page_speculative_target = VM_PAGE_SPECULATIVE_TARGET(vm_page_active_count + - vm_page_inactive_count); - - try_failed = FALSE; + vm_pageout_state.vm_page_speculative_target = VM_PAGE_SPECULATIVE_TARGET(vm_page_active_count + + vm_page_inactive_count); for (;;) { vm_page_t m; DTRACE_VM2(rev, int, 1, (uint64_t *), NULL); - if (vm_upl_wait_for_pages < 0) + if (vm_upl_wait_for_pages < 0) { vm_upl_wait_for_pages = 0; + } delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT + vm_upl_wait_for_pages; - if (delayed_unlock_limit > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX) + if (delayed_unlock_limit > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX) { delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX; + } #if CONFIG_SECLUDED_MEMORY /* * Deal with secluded_q overflow. */ if (vm_page_secluded_count > vm_page_secluded_target) { - unsigned int secluded_overflow; vm_page_t secluded_page; - if (object != NULL) { - vm_object_unlock(object); - object = NULL; - vm_pageout_scan_wants_object = VM_OBJECT_NULL; - } /* * SECLUDED_AGING_BEFORE_ACTIVE: * Excess secluded pages go to the active queue and * will later go to the inactive queue. */ - active_burst_count = MIN(vm_pageout_burst_active_throttle, - vm_page_secluded_count_inuse); - secluded_overflow = (vm_page_secluded_count - - vm_page_secluded_target); - while (secluded_overflow-- > 0 && - vm_page_secluded_count > vm_page_secluded_target) { - assert((vm_page_secluded_count_free + - vm_page_secluded_count_inuse) == - vm_page_secluded_count); - secluded_page = (vm_page_t)vm_page_queue_first(&vm_page_queue_secluded); - assert(secluded_page->vm_page_q_state == - VM_PAGE_ON_SECLUDED_Q); - vm_page_queues_remove(secluded_page, FALSE); - assert(!secluded_page->fictitious); - assert(!VM_PAGE_WIRED(secluded_page)); - if (secluded_page->vm_page_object == 0) { - /* transfer to free queue */ - assert(secluded_page->busy); - secluded_page->snext = local_freeq; - local_freeq = secluded_page; - local_freed++; - } else { - /* transfer to head of active queue */ - vm_page_enqueue_active(secluded_page, FALSE); - if (active_burst_count-- == 0) { - vm_pageout_secluded_burst_count++; - break; - } - } + assert((vm_page_secluded_count_free + + vm_page_secluded_count_inuse) == + vm_page_secluded_count); + secluded_page = (vm_page_t)vm_page_queue_first(&vm_page_queue_secluded); + assert(secluded_page->vmp_q_state == VM_PAGE_ON_SECLUDED_Q); + + vm_page_queues_remove(secluded_page, FALSE); + assert(!secluded_page->vmp_fictitious); + assert(!VM_PAGE_WIRED(secluded_page)); + + if (secluded_page->vmp_object == 0) { + /* transfer to free queue */ + assert(secluded_page->vmp_busy); + secluded_page->vmp_snext = local_freeq; + local_freeq = secluded_page; + local_freed++; + } else { + /* transfer to head of active queue */ + vm_page_enqueue_active(secluded_page, FALSE); secluded_page = VM_PAGE_NULL; - - if (delayed_unlock++ > delayed_unlock_limit) { - vm_pageout_delayed_unlock(&delayed_unlock, &local_freed, &local_freeq); - } } } #endif /* CONFIG_SECLUDED_MEMORY */ @@ -2017,88 +1964,25 @@ Restart: assert(delayed_unlock); /* - * Move pages from active to inactive if we're below the target + * maintain our balance */ - if ((vm_page_inactive_count + vm_page_speculative_count) >= vm_page_inactive_target) - goto done_moving_active_pages; + vm_page_balance_inactive(1); - if (object != NULL) { - vm_object_unlock(object); - object = NULL; - vm_pageout_scan_wants_object = VM_OBJECT_NULL; - } - /* - * Don't sweep through active queue more than the throttle - * which should be kept relatively low - */ - active_burst_count = MIN(vm_pageout_burst_active_throttle, vm_page_active_count); - - VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_START, - vm_pageout_inactive, vm_pageout_inactive_used, vm_page_free_count, local_freed); - - VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_NONE, - vm_pageout_speculative_clean, vm_pageout_inactive_clean, - vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external); - memoryshot(VM_PAGEOUT_BALANCE, DBG_FUNC_START); - - - while (!vm_page_queue_empty(&vm_page_queue_active) && active_burst_count--) { - - vm_pageout_active++; - - m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active); - - assert(m->vm_page_q_state == VM_PAGE_ON_ACTIVE_Q); - assert(!m->laundry); - assert(VM_PAGE_OBJECT(m) != kernel_object); - assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr); - - DTRACE_VM2(scan, int, 1, (uint64_t *), NULL); - - /* - * by not passing in a pmap_flush_context we will forgo any TLB flushing, local or otherwise... - * - * a TLB flush isn't really needed here since at worst we'll miss the reference bit being - * updated in the PTE if a remote processor still has this mapping cached in its TLB when the - * new reference happens. If no futher references happen on the page after that remote TLB flushes - * we'll see a clean, non-referenced page when it eventually gets pulled out of the inactive queue - * by pageout_scan, which is just fine since the last reference would have happened quite far - * in the past (TLB caches don't hang around for very long), and of course could just as easily - * have happened before we moved the page - */ - pmap_clear_refmod_options(VM_PAGE_GET_PHYS_PAGE(m), VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL); - - /* - * The page might be absent or busy, - * but vm_page_deactivate can handle that. - * FALSE indicates that we don't want a H/W clear reference - */ - vm_page_deactivate_internal(m, FALSE); - - if (delayed_unlock++ > delayed_unlock_limit) { - vm_pageout_delayed_unlock(&delayed_unlock, &local_freed, &local_freeq); - } - } - - VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_END, - vm_page_active_count, vm_page_inactive_count, vm_page_speculative_count, vm_page_inactive_target); - memoryshot(VM_PAGEOUT_BALANCE, DBG_FUNC_END); /********************************************************************** - * above this point we're playing with the active and secluded queues - * below this point we're playing with the throttling mechanisms - * and the inactive queue - **********************************************************************/ + * above this point we're playing with the active and secluded queues + * below this point we're playing with the throttling mechanisms + * and the inactive queue + **********************************************************************/ -done_moving_active_pages: + if (vm_page_free_count + local_freed >= vm_page_free_target) { + vm_pageout_scan_wants_object = VM_OBJECT_NULL; - if (vm_page_free_count + local_freed >= vm_page_free_target) - { vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed, - VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER); + VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER); /* * make sure the pageout I/O threads are running - * throttled in case there are still requests + * throttled in case there are still requests * in the laundry... since we have met our targets * we don't need the laundry to be cleaned in a timely * fashion... so let's avoid interfering with foreground @@ -2106,23 +1990,7 @@ done_moving_active_pages: */ vm_pageout_adjust_eq_iothrottle(eq, TRUE); - /* - * recalculate vm_page_inactivate_target - */ - vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count + - vm_page_inactive_count + - vm_page_speculative_count); -#ifndef CONFIG_EMBEDDED - if (((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) && - !vm_page_queue_empty(&vm_page_queue_active)) { - /* - * inactive target still not met... keep going - * until we get the queues balanced... - */ - continue; - } -#endif - lck_mtx_lock(&vm_page_queue_free_lock); + lck_mtx_lock(&vm_page_queue_free_lock); if ((vm_page_free_count >= vm_page_free_target) && (vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) { @@ -2134,46 +2002,47 @@ return_from_scan: assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL); VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_NONE, - vm_pageout_inactive, vm_pageout_inactive_used, 0, 0); + vm_pageout_state.vm_pageout_inactive, + vm_pageout_state.vm_pageout_inactive_used, 0, 0); VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_END, - vm_pageout_speculative_clean, vm_pageout_inactive_clean, - vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external); + vm_pageout_vminfo.vm_pageout_freed_speculative, + vm_pageout_state.vm_pageout_inactive_clean, + vm_pageout_vminfo.vm_pageout_inactive_dirty_internal, + vm_pageout_vminfo.vm_pageout_inactive_dirty_external); return; } lck_mtx_unlock(&vm_page_queue_free_lock); } - + /* - * Before anything, we check if we have any ripe volatile + * Before anything, we check if we have any ripe volatile * objects around. If so, try to purge the first object. * If the purge fails, fall through to reclaim a page instead. * If the purge succeeds, go back to the top and reevalute * the new memory situation. */ - - assert (available_for_purge>=0); + + assert(available_for_purge >= 0); force_purge = 0; /* no force-purging */ #if VM_PRESSURE_EVENTS pressure_level = memorystatus_vm_pressure_level; if (pressure_level > kVMPressureNormal) { - if (pressure_level >= kVMPressureCritical) { - force_purge = memorystatus_purge_on_critical; + force_purge = vm_pageout_state.memorystatus_purge_on_critical; } else if (pressure_level >= kVMPressureUrgent) { - force_purge = memorystatus_purge_on_urgent; + force_purge = vm_pageout_state.memorystatus_purge_on_urgent; } else if (pressure_level >= kVMPressureWarning) { - force_purge = memorystatus_purge_on_warning; + force_purge = vm_pageout_state.memorystatus_purge_on_warning; } } #endif /* VM_PRESSURE_EVENTS */ if (available_for_purge || force_purge) { - - if (object != NULL) { - vm_object_unlock(object); + if (object != NULL) { + vm_object_unlock(object); object = NULL; } @@ -2181,7 +2050,7 @@ return_from_scan: VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_START, vm_page_free_count, 0, 0, 0); if (vm_purgeable_object_purge_one(force_purge, C_DONT_BLOCK)) { - vm_pageout_purged_objects++; + VM_PAGEOUT_DEBUG(vm_pageout_purged_objects, 1); VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_END, vm_page_free_count, 0, 0, 0); memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_END); continue; @@ -2191,26 +2060,26 @@ return_from_scan: } if (vm_page_queue_empty(&sq->age_q) && vm_page_speculative_count) { - /* + /* * try to pull pages from the aging bins... * see vm_page.h for an explanation of how * this mechanism works */ - struct vm_speculative_age_q *aq; - boolean_t can_steal = FALSE; + struct vm_speculative_age_q *aq; + boolean_t can_steal = FALSE; int num_scanned_queues; - + aq = &vm_page_queue_speculative[speculative_steal_index]; num_scanned_queues = 0; while (vm_page_queue_empty(&aq->age_q) && - num_scanned_queues++ != VM_PAGE_MAX_SPECULATIVE_AGE_Q) { + num_scanned_queues++ != VM_PAGE_MAX_SPECULATIVE_AGE_Q) { + speculative_steal_index++; - speculative_steal_index++; + if (speculative_steal_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q) { + speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q; + } - if (speculative_steal_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q) - speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q; - aq = &vm_page_queue_speculative[speculative_steal_index]; } @@ -2220,34 +2089,28 @@ return_from_scan: * queues but still haven't found one * that is not empty, even though * vm_page_speculative_count is not 0. - * - * report the anomaly... */ - printf("vm_pageout_scan: " - "all speculative queues empty " - "but count=%d. Re-adjusting.\n", - vm_page_speculative_count); - if (vm_page_speculative_count > vm_page_speculative_count_drift_max) - vm_page_speculative_count_drift_max = vm_page_speculative_count; - vm_page_speculative_count_drifts++; + if (!vm_page_queue_empty(&sq->age_q)) { + continue; + } #if DEVELOPMENT || DEBUG panic("vm_pageout_scan: vm_page_speculative_count=%d but queues are empty", vm_page_speculative_count); -#endif /* DEVELOPMENT || DEBUG */ +#endif /* readjust... */ vm_page_speculative_count = 0; /* ... and continue */ continue; } - if (vm_page_speculative_count > vm_page_speculative_target || force_speculative_aging == TRUE) - can_steal = TRUE; - else { + if (vm_page_speculative_count > vm_pageout_state.vm_page_speculative_target || force_speculative_aging == TRUE) { + can_steal = TRUE; + } else { if (!delay_speculative_age) { - mach_timespec_t ts_fully_aged; + mach_timespec_t ts_fully_aged; - ts_fully_aged.tv_sec = (VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_page_speculative_q_age_ms) / 1000; - ts_fully_aged.tv_nsec = ((VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_page_speculative_q_age_ms) % 1000) - * 1000 * NSEC_PER_USEC; + ts_fully_aged.tv_sec = (VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_pageout_state.vm_page_speculative_q_age_ms) / 1000; + ts_fully_aged.tv_nsec = ((VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_pageout_state.vm_page_speculative_q_age_ms) % 1000) + * 1000 * NSEC_PER_USEC; ADD_MACH_TIMESPEC(&ts_fully_aged, &aq->age_ts); @@ -2257,42 +2120,42 @@ return_from_scan: ts.tv_sec = (unsigned int) sec; ts.tv_nsec = nsec; - if (CMP_MACH_TIMESPEC(&ts, &ts_fully_aged) >= 0) + if (CMP_MACH_TIMESPEC(&ts, &ts_fully_aged) >= 0) { can_steal = TRUE; - else + } else { delay_speculative_age++; + } } else { delay_speculative_age++; - if (delay_speculative_age == DELAY_SPECULATIVE_AGE) + if (delay_speculative_age == DELAY_SPECULATIVE_AGE) { delay_speculative_age = 0; + } } } - if (can_steal == TRUE) + if (can_steal == TRUE) { vm_page_speculate_ageit(aq); + } } force_speculative_aging = FALSE; -#if CONFIG_BACKGROUND_QUEUE - if (vm_page_queue_empty(&sq->age_q) && cache_evict_throttle == 0 && - ((vm_page_background_mode == VM_PAGE_BG_DISABLED) || (vm_page_background_count <= vm_page_background_target))) -#else - if (vm_page_queue_empty(&sq->age_q) && cache_evict_throttle == 0) -#endif - { - int pages_evicted; + if (vm_page_queue_empty(&sq->age_q) && cache_evict_throttle == 0) { + int pages_evicted; - if (object != NULL) { - vm_object_unlock(object); + if (object != NULL) { + vm_object_unlock(object); object = NULL; } + KERNEL_DEBUG_CONSTANT(0x13001ec | DBG_FUNC_START, 0, 0, 0, 0, 0); + pages_evicted = vm_object_cache_evict(100, 10); - if (pages_evicted) { + KERNEL_DEBUG_CONSTANT(0x13001ec | DBG_FUNC_END, pages_evicted, 0, 0, 0, 0); - vm_pageout_cache_evicted += pages_evicted; + if (pages_evicted) { + vm_pageout_vminfo.vm_pageout_pages_evicted += pages_evicted; VM_DEBUG_EVENT(vm_pageout_cache_evict, VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE, - vm_page_free_count, pages_evicted, vm_pageout_cache_evicted, 0); + vm_page_free_count, pages_evicted, vm_pageout_vminfo.vm_pageout_pages_evicted, 0); memoryshot(VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE); /* @@ -2301,11 +2164,15 @@ return_from_scan: * and re-evaulate the memory situation */ continue; - } else + } else { cache_evict_throttle = 1000; + } } - if (cache_evict_throttle) + if (cache_evict_throttle) { cache_evict_throttle--; + } + + divisor = vm_pageout_state.vm_page_filecache_min_divisor; #if CONFIG_JETSAM /* @@ -2319,22 +2186,26 @@ return_from_scan: * throttled queue (which isn't counted as available) which * effectively disables this filter */ - if (vm_compressor_low_on_space()) - vm_page_filecache_min = 0; - else - vm_page_filecache_min = (AVAILABLE_NON_COMPRESSED_MEMORY / 7); + if (vm_compressor_low_on_space() || divisor == 0) { + vm_pageout_state.vm_page_filecache_min = 0; + } else { + vm_pageout_state.vm_page_filecache_min = + ((AVAILABLE_NON_COMPRESSED_MEMORY) * 10) / divisor; + } #else - if (vm_compressor_out_of_space()) - vm_page_filecache_min = 0; - else { + if (vm_compressor_out_of_space() || divisor == 0) { + vm_pageout_state.vm_page_filecache_min = 0; + } else { /* - * don't let the filecache_min fall below 33% of available memory... + * don't let the filecache_min fall below the specified critical level */ - vm_page_filecache_min = (AVAILABLE_NON_COMPRESSED_MEMORY / 3); + vm_pageout_state.vm_page_filecache_min = + ((AVAILABLE_NON_COMPRESSED_MEMORY) * 10) / divisor; } #endif - if (vm_page_free_count < (vm_page_free_reserved / 4)) - vm_page_filecache_min = 0; + if (vm_page_free_count < (vm_page_free_reserved / 4)) { + vm_pageout_state.vm_page_filecache_min = 0; + } exceeded_burst_throttle = FALSE; /* @@ -2346,74 +2217,66 @@ return_from_scan: */ if (vm_page_queue_empty(&vm_page_queue_inactive) && vm_page_queue_empty(&vm_page_queue_anonymous) && + vm_page_queue_empty(&vm_page_queue_cleaned) && vm_page_queue_empty(&sq->age_q)) { - vm_pageout_scan_empty_throttle++; - msecs = vm_pageout_empty_wait; + VM_PAGEOUT_DEBUG(vm_pageout_scan_empty_throttle, 1); + msecs = vm_pageout_state.vm_pageout_empty_wait; goto vm_pageout_scan_delay; - - } else if (inactive_burst_count >= - MIN(vm_pageout_burst_inactive_throttle, - (vm_page_inactive_count + - vm_page_speculative_count))) { - vm_pageout_scan_burst_throttle++; - msecs = vm_pageout_burst_wait; + } else if (inactive_burst_count >= + MIN(vm_pageout_state.vm_pageout_burst_inactive_throttle, + (vm_page_inactive_count + + vm_page_speculative_count))) { + VM_PAGEOUT_DEBUG(vm_pageout_scan_burst_throttle, 1); + msecs = vm_pageout_state.vm_pageout_burst_wait; exceeded_burst_throttle = TRUE; goto vm_pageout_scan_delay; - - } else if (vm_page_free_count > (vm_page_free_reserved / 4) && - VM_PAGEOUT_SCAN_NEEDS_TO_THROTTLE()) { - vm_pageout_scan_swap_throttle++; - msecs = vm_pageout_swap_wait; - goto vm_pageout_scan_delay; - - } else if (VM_PAGE_Q_THROTTLED(iq) && - VM_DYNAMIC_PAGING_ENABLED()) { + } else if (VM_PAGE_Q_THROTTLED(iq) && + VM_DYNAMIC_PAGING_ENABLED()) { clock_sec_t sec; clock_nsec_t nsec; - switch (flow_control.state) { - + switch (flow_control.state) { case FCS_IDLE: - if ((vm_page_free_count + local_freed) < vm_page_free_target) { + if ((vm_page_free_count + local_freed) < vm_page_free_target && + vm_pageout_state.vm_restricted_to_single_processor == FALSE) { + /* + * since the compressor is running independently of vm_pageout_scan + * let's not wait for it just yet... as long as we have a healthy supply + * of filecache pages to work with, let's keep stealing those. + */ + inactive_external_count = vm_page_inactive_count - vm_page_anonymous_count; - vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed, - VM_PAGEOUT_PB_THREAD_YIELD); - if (!VM_PAGE_Q_THROTTLED(iq)) { - vm_pageout_scan_yield_unthrottled++; - continue; - } - if (vm_page_pageable_external_count > vm_page_filecache_min && - !vm_page_queue_empty(&vm_page_queue_inactive)) { + if (vm_page_pageable_external_count > vm_pageout_state.vm_page_filecache_min && + (inactive_external_count >= VM_PAGE_INACTIVE_TARGET(vm_page_pageable_external_count))) { anons_grabbed = ANONS_GRABBED_LIMIT; - vm_pageout_scan_throttle_deferred++; + VM_PAGEOUT_DEBUG(vm_pageout_scan_throttle_deferred, 1); goto consider_inactive; } - if (((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) && vm_page_active_count) - continue; } reset_deadlock_timer: - ts.tv_sec = vm_pageout_deadlock_wait / 1000; - ts.tv_nsec = (vm_pageout_deadlock_wait % 1000) * 1000 * NSEC_PER_USEC; - clock_get_system_nanotime(&sec, &nsec); + ts.tv_sec = vm_pageout_state.vm_pageout_deadlock_wait / 1000; + ts.tv_nsec = (vm_pageout_state.vm_pageout_deadlock_wait % 1000) * 1000 * NSEC_PER_USEC; + clock_get_system_nanotime(&sec, &nsec); flow_control.ts.tv_sec = (unsigned int) sec; flow_control.ts.tv_nsec = nsec; ADD_MACH_TIMESPEC(&flow_control.ts, &ts); - + flow_control.state = FCS_DELAYED; - msecs = vm_pageout_deadlock_wait; + msecs = vm_pageout_state.vm_pageout_deadlock_wait; + vm_pageout_vminfo.vm_pageout_scan_inactive_throttled_internal++; break; - + case FCS_DELAYED: - clock_get_system_nanotime(&sec, &nsec); + clock_get_system_nanotime(&sec, &nsec); ts.tv_sec = (unsigned int) sec; ts.tv_nsec = nsec; if (CMP_MACH_TIMESPEC(&ts, &flow_control.ts) >= 0) { - /* + /* * the pageout thread for the default pager is potentially - * deadlocked since the + * deadlocked since the * default pager queue has been throttled for more than the * allowable time... we need to move some clean pages or dirty * pages belonging to the external pagers if they aren't throttled @@ -2421,14 +2284,15 @@ reset_deadlock_timer: * blocked waiting for pages... we'll move one page for each of * these plus a fixed amount to break the logjam... once we're done * moving this number of pages, we'll re-enter the FSC_DELAYED state - * with a new timeout target since we have no way of knowing + * with a new timeout target since we have no way of knowing * whether we've broken the deadlock except through observation * of the queue associated with the default pager... we need to * stop moving pages and allow the system to run to see what * state it settles into. */ - vm_pageout_deadlock_target = vm_pageout_deadlock_relief + vm_page_free_wanted + vm_page_free_wanted_privileged; - vm_pageout_scan_deadlock_detected++; + vm_pageout_deadlock_target = vm_pageout_state.vm_pageout_deadlock_relief + + vm_page_free_wanted + vm_page_free_wanted_privileged; + VM_PAGEOUT_DEBUG(vm_pageout_scan_deadlock_detected, 1); flow_control.state = FCS_DEADLOCK_DETECTED; thread_wakeup((event_t) &vm_pageout_garbage_collect); goto consider_inactive; @@ -2439,38 +2303,34 @@ reset_deadlock_timer: * awakened immediately upon a laundry completion, * so we won't wait any longer than necessary */ - msecs = vm_pageout_idle_wait; + msecs = vm_pageout_state.vm_pageout_idle_wait; break; case FCS_DEADLOCK_DETECTED: - if (vm_pageout_deadlock_target) - goto consider_inactive; + if (vm_pageout_deadlock_target) { + goto consider_inactive; + } goto reset_deadlock_timer; - } vm_pageout_scan_delay: - vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed, - VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER); + vm_pageout_scan_wants_object = VM_OBJECT_NULL; + + vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed, + VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER); - if (flow_control.state == FCS_DELAYED && - !VM_PAGE_Q_THROTTLED(iq)) { - flow_control.state = FCS_IDLE; - goto consider_inactive; - } - if (vm_page_free_count >= vm_page_free_target) { /* * we're here because * 1) someone else freed up some pages while we had * the queues unlocked above - * and we've hit one of the 3 conditions that + * and we've hit one of the 3 conditions that * cause us to pause the pageout scan thread * * since we already have enough free pages, * let's avoid stalling and return normally * * before we return, make sure the pageout I/O threads - * are running throttled in case there are still requests + * are running throttled in case there are still requests * in the laundry... since we have enough free pages * we don't need the laundry to be cleaned in a timely * fashion... so let's avoid interfering with foreground @@ -2493,7 +2353,7 @@ vm_pageout_scan_delay: goto return_from_scan; } lck_mtx_unlock(&vm_page_queue_free_lock); - + if ((vm_page_free_count + vm_page_cleaned_count) < vm_page_free_target) { /* * we're most likely about to block due to one of @@ -2523,35 +2383,43 @@ vm_pageout_scan_delay: flow_control.state = FCS_IDLE; goto consider_inactive; } + if (flow_control.state == FCS_DELAYED && !VM_PAGE_Q_THROTTLED(iq)) { + flow_control.state = FCS_IDLE; + goto consider_inactive; + } + VM_CHECK_MEMORYSTATUS; - if (flow_control.state != FCS_IDLE) - vm_pageout_scan_throttle++; + if (flow_control.state != FCS_IDLE) { + VM_PAGEOUT_DEBUG(vm_pageout_scan_throttle, 1); + } + iq->pgo_throttled = TRUE; + assert_wait_timeout((event_t) &iq->pgo_laundry, THREAD_INTERRUPTIBLE, msecs, 1000 * NSEC_PER_USEC); - assert_wait_timeout((event_t) &iq->pgo_laundry, THREAD_INTERRUPTIBLE, msecs, 1000*NSEC_PER_USEC); counter(c_vm_pageout_scan_block++); vm_page_unlock_queues(); assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL); - VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START, - iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0); + VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START, + iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0); memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START); thread_block(THREAD_CONTINUE_NULL); VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END, - iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0); + iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0); memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END); vm_page_lock_queues(); iq->pgo_throttled = FALSE; - if (loop_count >= vm_page_inactive_count) + if (loop_count >= vm_page_inactive_count) { loop_count = 0; + } inactive_burst_count = 0; goto Restart; @@ -2561,26 +2429,23 @@ vm_pageout_scan_delay: flow_control.state = FCS_IDLE; consider_inactive: - vm_pageout_inactive_external_forced_reactivate_limit = MIN((vm_page_active_count + vm_page_inactive_count), - vm_pageout_inactive_external_forced_reactivate_limit); + vm_pageout_inactive_external_forced_reactivate_limit = MIN((vm_page_active_count + vm_page_inactive_count), + vm_pageout_inactive_external_forced_reactivate_limit); loop_count++; inactive_burst_count++; - vm_pageout_inactive++; - + vm_pageout_state.vm_pageout_inactive++; /* * Choose a victim. */ while (1) { - uint32_t inactive_external_count; - #if CONFIG_BACKGROUND_QUEUE page_from_bg_q = FALSE; #endif /* CONFIG_BACKGROUND_QUEUE */ m = NULL; m_object = VM_OBJECT_NULL; - + if (VM_DYNAMIC_PAGING_ENABLED()) { assert(vm_page_throttled_count == 0); assert(vm_page_queue_empty(&vm_page_queue_throttled)); @@ -2593,9 +2458,9 @@ consider_inactive: */ if (!vm_page_queue_empty(&vm_page_queue_cleaned)) { m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned); - - assert(m->vm_page_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q); - + + assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q); + break; } @@ -2606,17 +2471,18 @@ consider_inactive: if (!vm_page_queue_empty(&sq->age_q)) { m = (vm_page_t) vm_page_queue_first(&sq->age_q); - assert(m->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q); + assert(m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q); - if (!m->dirty || force_anonymous == FALSE) - break; - else - m = NULL; + if (!m->vmp_dirty || force_anonymous == FALSE) { + break; + } else { + m = NULL; + } } #if CONFIG_BACKGROUND_QUEUE if (vm_page_background_mode != VM_PAGE_BG_DISABLED && (vm_page_background_count > vm_page_background_target)) { - vm_object_t bg_m_object = NULL; + vm_object_t bg_m_object = NULL; m = (vm_page_t) vm_page_queue_first(&vm_page_queue_background); @@ -2634,37 +2500,36 @@ consider_inactive: * page. */ } else if (force_anonymous == FALSE || bg_m_object->internal) { - if (bg_m_object->internal && - ((vm_compressor_out_of_space() == TRUE) || - (vm_page_free_count < (vm_page_free_reserved / 4)))) { - + (VM_PAGE_Q_THROTTLED(iq) || + vm_compressor_out_of_space() == TRUE || + vm_page_free_count < (vm_page_free_reserved / 4))) { vm_pageout_skipped_bq_internal++; } else { page_from_bg_q = TRUE; - - if (bg_m_object->internal) - vm_pageout_considered_bq_internal++; - else - vm_pageout_considered_bq_external++; - + + if (bg_m_object->internal) { + vm_pageout_vminfo.vm_pageout_considered_bq_internal++; + } else { + vm_pageout_vminfo.vm_pageout_considered_bq_external++; + } break; } } } #endif - - grab_anonymous = (vm_page_anonymous_count > vm_page_anonymous_min); inactive_external_count = vm_page_inactive_count - vm_page_anonymous_count; - if ((vm_page_pageable_external_count < vm_page_filecache_min || force_anonymous == TRUE) || - ((inactive_external_count < vm_page_anonymous_count) && (inactive_external_count < (vm_page_pageable_external_count / 3)))) { + if ((vm_page_pageable_external_count < vm_pageout_state.vm_page_filecache_min || force_anonymous == TRUE) || + (inactive_external_count < VM_PAGE_INACTIVE_TARGET(vm_page_pageable_external_count))) { grab_anonymous = TRUE; anons_grabbed = 0; - vm_pageout_skipped_external++; + vm_pageout_vminfo.vm_pageout_skipped_external++; goto want_anonymous; } + grab_anonymous = (vm_page_anonymous_count > vm_page_anonymous_min); + #if CONFIG_JETSAM /* If the file-backed pool has accumulated * significantly more pages than the jetsam @@ -2677,50 +2542,50 @@ consider_inactive: */ if (grab_anonymous == TRUE && !VM_PAGE_Q_THROTTLED(eq)) { if (vm_page_pageable_external_count > - vm_page_filecache_min) { + vm_pageout_state.vm_page_filecache_min) { if ((vm_page_pageable_external_count * - vm_pageout_memorystatus_fb_factor_dr) > + vm_pageout_memorystatus_fb_factor_dr) > (memorystatus_available_pages_critical * vm_pageout_memorystatus_fb_factor_nr)) { grab_anonymous = FALSE; -#if DEVELOPMENT || DEBUG - vm_grab_anon_overrides++; -#endif + + VM_PAGEOUT_DEBUG(vm_grab_anon_overrides, 1); } } -#if DEVELOPMENT || DEBUG if (grab_anonymous) { - vm_grab_anon_nops++; + VM_PAGEOUT_DEBUG(vm_grab_anon_nops, 1); } -#endif } #endif /* CONFIG_JETSAM */ want_anonymous: if (grab_anonymous == FALSE || anons_grabbed >= ANONS_GRABBED_LIMIT || vm_page_queue_empty(&vm_page_queue_anonymous)) { - - if ( !vm_page_queue_empty(&vm_page_queue_inactive) ) { + if (!vm_page_queue_empty(&vm_page_queue_inactive)) { m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive); - - assert(m->vm_page_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q); + + assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q); anons_grabbed = 0; - if (vm_page_pageable_external_count < vm_page_filecache_min) { - if ((++reactivated_this_call % 100)) - goto must_activate_page; - /* - * steal 1% of the file backed pages even if - * we are under the limit that has been set - * for a healthy filecache - */ + if (vm_page_pageable_external_count < vm_pageout_state.vm_page_filecache_min) { + if (!vm_page_queue_empty(&vm_page_queue_anonymous)) { + if ((++reactivated_this_call % 100)) { + vm_pageout_vminfo.vm_pageout_filecache_min_reactivated++; + goto must_activate_page; + } + /* + * steal 1% of the file backed pages even if + * we are under the limit that has been set + * for a healthy filecache + */ + } } break; } } - if ( !vm_page_queue_empty(&vm_page_queue_anonymous) ) { + if (!vm_page_queue_empty(&vm_page_queue_anonymous)) { m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous); - assert(m->vm_page_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q); + assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q); anons_grabbed++; break; @@ -2729,33 +2594,31 @@ want_anonymous: /* * if we've gotten here, we have no victim page. * check to see if we've not finished balancing the queues - * or we have a page on the aged speculative queue that we + * or we have a page on the aged speculative queue that we * skipped due to force_anonymous == TRUE.. or we have * speculative pages that we can prematurely age... if * one of these cases we'll keep going, else panic */ force_anonymous = FALSE; - vm_pageout_no_victim++; + VM_PAGEOUT_DEBUG(vm_pageout_no_victim, 1); - if ((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) - goto done_with_inactivepage; - - if (!vm_page_queue_empty(&sq->age_q)) + if (!vm_page_queue_empty(&sq->age_q)) { goto done_with_inactivepage; + } if (vm_page_speculative_count) { force_speculative_aging = TRUE; goto done_with_inactivepage; } panic("vm_pageout: no victim"); - + /* NOTREACHED */ } assert(VM_PAGE_PAGEABLE(m)); m_object = VM_PAGE_OBJECT(m); force_anonymous = FALSE; - - page_prev_q_state = m->vm_page_q_state; + + page_prev_q_state = m->vmp_q_state; /* * we just found this page on one of our queues... * it can't also be on the pageout queue, so safe @@ -2763,14 +2626,13 @@ want_anonymous: */ vm_page_queues_remove(m, TRUE); - assert(!m->laundry); - assert(!m->private); - assert(!m->fictitious); + assert(!m->vmp_laundry); + assert(!m->vmp_private); + assert(!m->vmp_fictitious); assert(m_object != kernel_object); assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr); - vm_pageout_stats[vm_pageout_stat_now].considered++; - vm_pageout_considered_page++; + vm_pageout_vminfo.vm_pageout_considered_page++; DTRACE_VM2(scan, int, 1, (uint64_t *), NULL); @@ -2780,15 +2642,14 @@ want_anonymous: * already got the lock */ if (m_object != object) { - /* - * the object associated with candidate page is + /* + * the object associated with candidate page is * different from the one we were just working * with... dump the lock if we still own it */ - if (object != NULL) { - vm_object_unlock(object); + if (object != NULL) { + vm_object_unlock(object); object = NULL; - vm_pageout_scan_wants_object = VM_OBJECT_NULL; } /* * Try to lock object; since we've alread got the @@ -2803,93 +2664,112 @@ want_anonymous: if (!vm_object_lock_try_scan(m_object)) { vm_page_t m_want = NULL; - vm_pageout_inactive_nolock++; + vm_pageout_vminfo.vm_pageout_inactive_nolock++; - if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) - vm_pageout_cleaned_nolock++; + if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) { + VM_PAGEOUT_DEBUG(vm_pageout_cleaned_nolock, 1); + } pmap_clear_reference(VM_PAGE_GET_PHYS_PAGE(m)); - m->reference = FALSE; - -#if !CONFIG_EMBEDDED - /* - * m->object must be stable since we hold the page queues lock... - * we can update the scan_collisions field sans the object lock - * since it is a separate field and this is the only spot that does - * a read-modify-write operation and it is never executed concurrently... - * we can asynchronously set this field to 0 when creating a UPL, so it - * is possible for the value to be a bit non-determistic, but that's ok - * since it's only used as a hint - */ - /* - * This is not used on EMBEDDED because having this variable set *could* lead - * us to self-cannibalize pages from m_object to fill a UPL for a pagein. - * And, there's a high probability that the object that vm_pageout_scan - * wants and collides on is a very popular object e.g. the shared cache on EMBEDDED. - * The older pages that we cannibalize from the shared cache could be really - * important text pages e.g. the system call stubs. - */ - m_object->scan_collisions = 1; -#endif /* !CONFIG_EMBEDDED */ + m->vmp_reference = FALSE; - if ( !vm_page_queue_empty(&sq->age_q) ) - m_want = (vm_page_t) vm_page_queue_first(&sq->age_q); - else if ( !vm_page_queue_empty(&vm_page_queue_cleaned)) + if (!m_object->object_is_shared_cache) { + /* + * don't apply this optimization if this is the shared cache + * object, it's too easy to get rid of very hot and important + * pages... + * m->vmp_object must be stable since we hold the page queues lock... + * we can update the scan_collisions field sans the object lock + * since it is a separate field and this is the only spot that does + * a read-modify-write operation and it is never executed concurrently... + * we can asynchronously set this field to 0 when creating a UPL, so it + * is possible for the value to be a bit non-determistic, but that's ok + * since it's only used as a hint + */ + m_object->scan_collisions = 1; + } + if (!vm_page_queue_empty(&vm_page_queue_cleaned)) { m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned); - else if ( !vm_page_queue_empty(&vm_page_queue_inactive) && - (anons_grabbed >= ANONS_GRABBED_LIMIT || vm_page_queue_empty(&vm_page_queue_anonymous))) + } else if (!vm_page_queue_empty(&sq->age_q)) { + m_want = (vm_page_t) vm_page_queue_first(&sq->age_q); + } else if ((grab_anonymous == FALSE || anons_grabbed >= ANONS_GRABBED_LIMIT || + vm_page_queue_empty(&vm_page_queue_anonymous)) && + !vm_page_queue_empty(&vm_page_queue_inactive)) { m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive); - else if ( !vm_page_queue_empty(&vm_page_queue_anonymous)) + } else if (!vm_page_queue_empty(&vm_page_queue_anonymous)) { m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous); + } /* * this is the next object we're going to be interested in - * try to make sure its available after the mutex_yield + * try to make sure its available after the mutex_pause * returns control */ - if (m_want) + if (m_want) { vm_pageout_scan_wants_object = VM_PAGE_OBJECT(m_want); - - /* - * force us to dump any collected free pages - * and to pause before moving on - */ - try_failed = TRUE; + } goto requeue_page; } object = m_object; vm_pageout_scan_wants_object = VM_OBJECT_NULL; - - try_failed = FALSE; } assert(m_object == object); assert(VM_PAGE_OBJECT(m) == m_object); - if (m->busy) { + if (m->vmp_busy) { /* * Somebody is already playing with this page. * Put it back on the appropriate queue * */ - vm_pageout_inactive_busy++; + VM_PAGEOUT_DEBUG(vm_pageout_inactive_busy, 1); - if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) - vm_pageout_cleaned_busy++; + if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) { + VM_PAGEOUT_DEBUG(vm_pageout_cleaned_busy, 1); + } requeue_page: - if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) + if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) { vm_page_enqueue_inactive(m, FALSE); - else + } else { vm_page_activate(m); + } #if CONFIG_BACKGROUND_QUEUE +#if DEVELOPMENT || DEBUG if (page_from_bg_q == TRUE) { - if (m_object->internal) + if (m_object->internal) { vm_pageout_rejected_bq_internal++; - else + } else { vm_pageout_rejected_bq_external++; + } } #endif +#endif + goto done_with_inactivepage; + } + + /* + * if (m->vmp_cleaning && !m->vmp_free_when_done) + * If already cleaning this page in place + * just leave if off the paging queues. + * We can leave the page mapped, and upl_commit_range + * will put it on the clean queue. + * + * if (m->vmp_free_when_done && !m->vmp_cleaning) + * an msync INVALIDATE is in progress... + * this page has been marked for destruction + * after it has been cleaned, + * but not yet gathered into a UPL + * where 'cleaning' will be set... + * just leave it off the paging queues + * + * if (m->vmp_free_when_done && m->vmp_clenaing) + * an msync INVALIDATE is in progress + * and the UPL has already gathered this page... + * just leave it off the paging queues + */ + if (m->vmp_free_when_done || m->vmp_cleaning) { goto done_with_inactivepage; } @@ -2901,18 +2781,18 @@ requeue_page: * from reclaiming it - busy or cleaning - that we've already * dealt with */ - if (m->absent || m->error || !object->alive) { - - if (m->absent) - vm_pageout_inactive_absent++; - else if (!object->alive) - vm_pageout_inactive_notalive++; - else - vm_pageout_inactive_error++; -reclaim_page: + if (m->vmp_absent || m->vmp_error || !object->alive) { + if (m->vmp_absent) { + VM_PAGEOUT_DEBUG(vm_pageout_inactive_absent, 1); + } else if (!object->alive) { + VM_PAGEOUT_DEBUG(vm_pageout_inactive_notalive, 1); + } else { + VM_PAGEOUT_DEBUG(vm_pageout_inactive_error, 1); + } +reclaim_page: if (vm_pageout_deadlock_target) { - vm_pageout_scan_inactive_throttle_success++; - vm_pageout_deadlock_target--; + VM_PAGEOUT_DEBUG(vm_pageout_scan_inactive_throttle_success, 1); + vm_pageout_deadlock_target--; } DTRACE_VM2(dfree, int, 1, (uint64_t *), NULL); @@ -2922,10 +2802,16 @@ reclaim_page: } else { DTRACE_VM2(fsfree, int, 1, (uint64_t *), NULL); } - assert(!m->cleaning); - assert(!m->laundry); + assert(!m->vmp_cleaning); + assert(!m->vmp_laundry); + + if (!object->internal && + object->pager != NULL && + object->pager->mo_pager_ops == &shared_region_pager_ops) { + shared_region_pager_reclaimed++; + } - m->busy = TRUE; + m->vmp_busy = TRUE; /* * remove page from object here since we're already @@ -2933,42 +2819,45 @@ reclaim_page: * we'd normally do in vm_page_free_prepare_object * until 'vm_page_free_list' is called */ - if (m->tabled) + if (m->vmp_tabled) { vm_page_remove(m, TRUE); + } - assert(m->pageq.next == 0 && m->pageq.prev == 0); - m->snext = local_freeq; + assert(m->vmp_pageq.next == 0 && m->vmp_pageq.prev == 0); + m->vmp_snext = local_freeq; local_freeq = m; local_freed++; - - if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) - vm_pageout_freed_from_speculative++; - else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) - vm_pageout_freed_from_cleaned++; - else - vm_pageout_freed_from_inactive_clean++; - vm_pageout_stats[vm_pageout_stat_now].reclaimed_clean++; + if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) { + vm_pageout_vminfo.vm_pageout_freed_speculative++; + } else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) { + vm_pageout_vminfo.vm_pageout_freed_cleaned++; + } else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q) { + vm_pageout_vminfo.vm_pageout_freed_internal++; + } else { + vm_pageout_vminfo.vm_pageout_freed_external++; + } inactive_burst_count = 0; goto done_with_inactivepage; } - /* - * If the object is empty, the page must be reclaimed even - * if dirty or used. - * If the page belongs to a volatile object, we stick it back - * on. - */ if (object->copy == VM_OBJECT_NULL) { + /* + * No one else can have any interest in this page. + * If this is an empty purgable object, the page can be + * reclaimed even if dirty. + * If the page belongs to a volatile purgable object, we + * reactivate it if the compressor isn't active. + */ if (object->purgable == VM_PURGABLE_EMPTY) { - if (m->pmapped == TRUE) { + if (m->vmp_pmapped == TRUE) { /* unmap the page */ refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)); if (refmod_state & VM_MEM_MODIFIED) { SET_PAGE_DIRTY(m, FALSE); } } - if (m->dirty || m->precious) { + if (m->vmp_dirty || m->vmp_precious) { /* we saved the cost of cleaning this page ! */ vm_page_purged_count++; } @@ -2996,8 +2885,9 @@ reclaim_page: /* just stick it back on! */ reactivated_this_call++; - if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) - vm_pageout_cleaned_volatile_reactivated++; + if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) { + VM_PAGEOUT_DEBUG(vm_pageout_cleaned_volatile_reactivated, 1); + } goto reactivate_page; } @@ -3010,66 +2900,35 @@ reclaim_page: */ refmod_state = -1; - if (m->reference == FALSE && m->pmapped == TRUE) { - refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m)); - - if (refmod_state & VM_MEM_REFERENCED) - m->reference = TRUE; - if (refmod_state & VM_MEM_MODIFIED) { + if (m->vmp_reference == FALSE && m->vmp_pmapped == TRUE) { + refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m)); + + if (refmod_state & VM_MEM_REFERENCED) { + m->vmp_reference = TRUE; + } + if (refmod_state & VM_MEM_MODIFIED) { SET_PAGE_DIRTY(m, FALSE); } } - - /* - * if (m->cleaning && !m->free_when_done) - * If already cleaning this page in place and it hasn't - * been recently referenced, just pull off the queue. - * We can leave the page mapped, and upl_commit_range - * will put it on the clean queue. - * - * if (m->free_when_done && !m->cleaning) - * an msync INVALIDATE is in progress... - * this page has been marked for destruction - * after it has been cleaned, - * but not yet gathered into a UPL - * where 'cleaning' will be set... - * just leave it off the paging queues - * - * if (m->free_when_done && m->clenaing) - * an msync INVALIDATE is in progress - * and the UPL has already gathered this page... - * just leave it off the paging queues - */ - - /* - * page with m->free_when_done and still on the queues means that an - * MS_INVALIDATE is in progress on this page... leave it alone - */ - if (m->free_when_done) { - goto done_with_inactivepage; - } - - /* if cleaning, reactivate if referenced. otherwise, just pull off queue */ - if (m->cleaning) { - if (m->reference == TRUE) { - reactivated_this_call++; - goto reactivate_page; - } else { - goto done_with_inactivepage; - } + + if (m->vmp_reference || m->vmp_dirty) { + /* deal with a rogue "reusable" page */ + VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, m_object); } + divisor = vm_pageout_state.vm_page_xpmapped_min_divisor; - if (m->reference || m->dirty) { - /* deal with a rogue "reusable" page */ - VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, m_object); - } + if (divisor == 0) { + vm_pageout_state.vm_page_xpmapped_min = 0; + } else { + vm_pageout_state.vm_page_xpmapped_min = (vm_page_external_count * 10) / divisor; + } - if (!m->no_cache && + if (!m->vmp_no_cache && #if CONFIG_BACKGROUND_QUEUE page_from_bg_q == FALSE && #endif - (m->reference || - (m->xpmapped && !object->internal && (vm_page_xpmapped_external_count < (vm_page_external_count / 4))))) { + (m->vmp_reference || (m->vmp_xpmapped && !object->internal && + (vm_page_xpmapped_external_count < vm_pageout_state.vm_page_xpmapped_min)))) { /* * The page we pulled off the inactive list has * been referenced. It is possible for other @@ -3079,23 +2938,26 @@ reclaim_page: * reactivations. */ if (++reactivated_this_call >= reactivate_limit) { - vm_pageout_reactivation_limit_exceeded++; + vm_pageout_vminfo.vm_pageout_reactivation_limit_exceeded++; } else if (++inactive_reclaim_run >= VM_PAGEOUT_INACTIVE_FORCE_RECLAIM) { - vm_pageout_inactive_force_reclaim++; + vm_pageout_vminfo.vm_pageout_inactive_force_reclaim++; } else { uint32_t isinuse; - if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) - vm_pageout_cleaned_reference_reactivated++; + if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) { + VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reference_reactivated, 1); + } + + vm_pageout_vminfo.vm_pageout_inactive_referenced++; reactivate_page: - if ( !object->internal && object->pager != MEMORY_OBJECT_NULL && - vnode_pager_get_isinuse(object->pager, &isinuse) == KERN_SUCCESS && !isinuse) { + if (!object->internal && object->pager != MEMORY_OBJECT_NULL && + vnode_pager_get_isinuse(object->pager, &isinuse) == KERN_SUCCESS && !isinuse) { /* * no explict mappings of this object exist * and it's not open via the filesystem */ vm_page_deactivate(m); - vm_pageout_inactive_deactivated++; + VM_PAGEOUT_DEBUG(vm_pageout_inactive_deactivated, 1); } else { must_activate_page: /* @@ -3106,25 +2968,29 @@ must_activate_page: inactive_burst_count = 0; } #if CONFIG_BACKGROUND_QUEUE +#if DEVELOPMENT || DEBUG if (page_from_bg_q == TRUE) { - if (m_object->internal) + if (m_object->internal) { vm_pageout_rejected_bq_internal++; - else + } else { vm_pageout_rejected_bq_external++; + } } #endif - if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) - vm_pageout_cleaned_reactivated++; - vm_pageout_inactive_used++; +#endif + if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) { + VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reactivated, 1); + } + vm_pageout_state.vm_pageout_inactive_used++; - goto done_with_inactivepage; + goto done_with_inactivepage; } - /* + /* * Make sure we call pmap_get_refmod() if it * wasn't already called just above, to update * the dirty bit. */ - if ((refmod_state == -1) && !m->dirty && m->pmapped) { + if ((refmod_state == -1) && !m->vmp_dirty && m->vmp_pmapped) { refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m)); if (refmod_state & VM_MEM_MODIFIED) { SET_PAGE_DIRTY(m, FALSE); @@ -3132,20 +2998,20 @@ must_activate_page: } } - XPR(XPR_VM_PAGEOUT, - "vm_pageout_scan, replace object 0x%X offset 0x%X page 0x%X\n", - object, m->offset, m, 0,0); + XPR(XPR_VM_PAGEOUT, + "vm_pageout_scan, replace object 0x%X offset 0x%X page 0x%X\n", + object, m->vmp_offset, m, 0, 0); /* * we've got a candidate page to steal... * - * m->dirty is up to date courtesy of the - * preceding check for m->reference... if - * we get here, then m->reference had to be + * m->vmp_dirty is up to date courtesy of the + * preceding check for m->vmp_reference... if + * we get here, then m->vmp_reference had to be * FALSE (or possibly "reactivate_limit" was - * exceeded), but in either case we called - * pmap_get_refmod() and updated both - * m->reference and m->dirty + * exceeded), but in either case we called + * pmap_get_refmod() and updated both + * m->vmp_reference and m->vmp_dirty * * if it's dirty or precious we need to * see if the target queue is throtttled @@ -3155,36 +3021,35 @@ must_activate_page: inactive_throttled = FALSE; - if (m->dirty || m->precious) { - if (object->internal) { - if (VM_PAGE_Q_THROTTLED(iq)) - inactive_throttled = TRUE; + if (m->vmp_dirty || m->vmp_precious) { + if (object->internal) { + if (VM_PAGE_Q_THROTTLED(iq)) { + inactive_throttled = TRUE; + } } else if (VM_PAGE_Q_THROTTLED(eq)) { inactive_throttled = TRUE; } } throttle_inactive: if (!VM_DYNAMIC_PAGING_ENABLED() && - object->internal && m->dirty && + object->internal && m->vmp_dirty && (object->purgable == VM_PURGABLE_DENY || - object->purgable == VM_PURGABLE_NONVOLATILE || - object->purgable == VM_PURGABLE_VOLATILE)) { + object->purgable == VM_PURGABLE_NONVOLATILE || + object->purgable == VM_PURGABLE_VOLATILE)) { vm_page_check_pageable_safe(m); - assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q); - vm_page_queue_enter(&vm_page_queue_throttled, m, - vm_page_t, pageq); - m->vm_page_q_state = VM_PAGE_ON_THROTTLED_Q; + assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q); + vm_page_queue_enter(&vm_page_queue_throttled, m, vmp_pageq); + m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q; vm_page_throttled_count++; - vm_pageout_scan_reclaimed_throttled++; + VM_PAGEOUT_DEBUG(vm_pageout_scan_reclaimed_throttled, 1); inactive_burst_count = 0; goto done_with_inactivepage; } if (inactive_throttled == TRUE) { - if (object->internal == FALSE) { - /* + /* * we need to break up the following potential deadlock case... * a) The external pageout thread is stuck on the truncate lock for a file that is being extended i.e. written. * b) The thread doing the writing is waiting for pages while holding the truncate lock @@ -3209,12 +3074,12 @@ throttle_inactive: * that we can try to find clean pages in the active/inactive queues before * deciding to jetsam a process */ - vm_pageout_scan_inactive_throttled_external++; + vm_pageout_vminfo.vm_pageout_scan_inactive_throttled_external++; vm_page_check_pageable_safe(m); - assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q); - vm_page_queue_enter(&vm_page_queue_active, m, vm_page_t, pageq); - m->vm_page_q_state = VM_PAGE_ON_ACTIVE_Q; + assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q); + vm_page_queue_enter(&vm_page_queue_active, m, vmp_pageq); + m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q; vm_page_active_count++; vm_page_pageable_external_count++; @@ -3234,17 +3099,17 @@ throttle_inactive: vm_page_unlock_queues(); VM_DEBUG_CONSTANT_EVENT(vm_pageout_jetsam, VM_PAGEOUT_JETSAM, DBG_FUNC_START, - vm_page_active_count, vm_page_inactive_count, vm_page_free_count, vm_page_free_count); + vm_page_active_count, vm_page_inactive_count, vm_page_free_count, vm_page_free_count); - /* Kill first suitable process. If this call returned FALSE, we might have simply purged a process instead. */ + /* Kill first suitable process. If this call returned FALSE, we might have simply purged a process instead. */ if (memorystatus_kill_on_VM_page_shortage(FALSE) == TRUE) { - vm_pageout_inactive_external_forced_jetsam_count++; + VM_PAGEOUT_DEBUG(vm_pageout_inactive_external_forced_jetsam_count, 1); } - + VM_DEBUG_CONSTANT_EVENT(vm_pageout_jetsam, VM_PAGEOUT_JETSAM, DBG_FUNC_END, - vm_page_active_count, vm_page_inactive_count, vm_page_free_count, vm_page_free_count); + vm_page_active_count, vm_page_inactive_count, vm_page_free_count, vm_page_free_count); - vm_page_lock_queues(); + vm_page_lock_queues(); delayed_unlock = 1; } #else /* CONFIG_MEMORYSTATUS && CONFIG_JETSAM */ @@ -3253,7 +3118,6 @@ throttle_inactive: inactive_burst_count = 0; goto done_with_inactivepage; } else { - vm_pageout_scan_inactive_throttled_internal++; goto must_activate_page; } } @@ -3264,17 +3128,17 @@ throttle_inactive: * we have the up-to-date modified state * * if we need to do a pmap_disconnect then we - * need to re-evaluate m->dirty since the pmap_disconnect - * provides the true state atomically... the + * need to re-evaluate m->vmp_dirty since the pmap_disconnect + * provides the true state atomically... the * page was still mapped up to the pmap_disconnect * and may have been dirtied at the last microsecond * * Note that if 'pmapped' is FALSE then the page is not * and has not been in any map, so there is no point calling - * pmap_disconnect(). m->dirty could have been set in anticipation + * pmap_disconnect(). m->vmp_dirty could have been set in anticipation * of likely usage of the page. */ - if (m->pmapped == TRUE) { + if (m->vmp_pmapped == TRUE) { int pmap_options; /* @@ -3288,10 +3152,10 @@ throttle_inactive: * 3) This page belongs to a file and hence will not be * sent into the compressor */ - if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE || + if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE || object->internal == FALSE) { pmap_options = 0; - } else if (m->dirty || m->precious) { + } else if (m->vmp_dirty || m->vmp_precious) { /* * VM knows that this page is dirty (or * precious) and needs to be compressed @@ -3310,17 +3174,18 @@ throttle_inactive: * modified. */ pmap_options = - PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED; + PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED; } refmod_state = pmap_disconnect_options(VM_PAGE_GET_PHYS_PAGE(m), - pmap_options, - NULL); + pmap_options, + NULL); if (refmod_state & VM_MEM_MODIFIED) { SET_PAGE_DIRTY(m, FALSE); } } + /* - * reset our count of pages that have been reclaimed + * reset our count of pages that have been reclaimed * since the last page was 'stolen' */ inactive_reclaim_run = 0; @@ -3328,24 +3193,16 @@ throttle_inactive: /* * If it's clean and not precious, we can free the page. */ - if (!m->dirty && !m->precious) { - - if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) - vm_pageout_speculative_clean++; - else { - if (page_prev_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q) - vm_pageout_inactive_anonymous++; - else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) - vm_pageout_cleaned_reclaimed++; + if (!m->vmp_dirty && !m->vmp_precious) { + vm_pageout_state.vm_pageout_inactive_clean++; - vm_pageout_inactive_clean++; - } /* * OK, at this point we have found a page we are going to free. */ #if CONFIG_PHANTOM_CACHE - if (!object->internal) + if (!object->internal) { vm_phantom_cache_add_ghost(m); + } #endif goto reclaim_page; } @@ -3357,15 +3214,17 @@ throttle_inactive: * disconnected here, we can make one final check. */ if (object->internal) { - if (VM_PAGE_Q_THROTTLED(iq)) + if (VM_PAGE_Q_THROTTLED(iq)) { inactive_throttled = TRUE; + } } else if (VM_PAGE_Q_THROTTLED(eq)) { inactive_throttled = TRUE; } - if (inactive_throttled == TRUE) + if (inactive_throttled == TRUE) { goto throttle_inactive; - + } + #if VM_PRESSURE_EVENTS #if CONFIG_JETSAM @@ -3377,40 +3236,46 @@ throttle_inactive: */ #else /* CONFIG_JETSAM */ - + vm_pressure_response(); #endif /* CONFIG_JETSAM */ #endif /* VM_PRESSURE_EVENTS */ - - if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) - vm_pageout_speculative_dirty++; - else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q) - vm_pageout_inactive_anonymous++; - if (object->internal) - vm_pageout_inactive_dirty_internal++; - else - vm_pageout_inactive_dirty_external++; + if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) { + VM_PAGEOUT_DEBUG(vm_pageout_speculative_dirty, 1); + } + + if (object->internal) { + vm_pageout_vminfo.vm_pageout_inactive_dirty_internal++; + } else { + vm_pageout_vminfo.vm_pageout_inactive_dirty_external++; + } /* - * do NOT set the pageout bit! - * sure, we might need free pages, but this page is going to take time to become free - * anyway, so we may as well put it on the clean queue first and take it from there later - * if necessary. that way, we'll ensure we don't free up too much. -mj + * internal pages will go to the compressor... + * external pages will go to the appropriate pager to be cleaned + * and upon completion will end up on 'vm_page_queue_cleaned' which + * is a preferred queue to steal from */ vm_pageout_cluster(m); + inactive_burst_count = 0; done_with_inactivepage: - if (delayed_unlock++ > delayed_unlock_limit || try_failed == TRUE) { + if (delayed_unlock++ > delayed_unlock_limit) { + int freed = local_freed; - vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed, - VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER); - if (try_failed == TRUE) + vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed, + VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER); + if (freed == 0) { lck_mtx_yield(&vm_page_queue_lock); + } + } else if (vm_pageout_scan_wants_object) { + vm_page_unlock_queues(); + mutex_pause(0); + vm_page_lock_queues(); } - /* * back to top of pageout scan loop */ @@ -3418,43 +3283,44 @@ done_with_inactivepage: } -int vm_page_free_count_init; - void vm_page_free_reserve( int pages) { - int free_after_reserve; + int free_after_reserve; if (VM_CONFIG_COMPRESSOR_IS_PRESENT) { - - if ((vm_page_free_reserved + pages + COMPRESSOR_FREE_RESERVED_LIMIT) >= (VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT)) + if ((vm_page_free_reserved + pages + COMPRESSOR_FREE_RESERVED_LIMIT) >= (VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT)) { vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT; - else + } else { vm_page_free_reserved += (pages + COMPRESSOR_FREE_RESERVED_LIMIT); - + } } else { - if ((vm_page_free_reserved + pages) >= VM_PAGE_FREE_RESERVED_LIMIT) + if ((vm_page_free_reserved + pages) >= VM_PAGE_FREE_RESERVED_LIMIT) { vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT; - else + } else { vm_page_free_reserved += pages; + } } - free_after_reserve = vm_page_free_count_init - vm_page_free_reserved; + free_after_reserve = vm_pageout_state.vm_page_free_count_init - vm_page_free_reserved; vm_page_free_min = vm_page_free_reserved + - VM_PAGE_FREE_MIN(free_after_reserve); + VM_PAGE_FREE_MIN(free_after_reserve); - if (vm_page_free_min > VM_PAGE_FREE_MIN_LIMIT) - vm_page_free_min = VM_PAGE_FREE_MIN_LIMIT; + if (vm_page_free_min > VM_PAGE_FREE_MIN_LIMIT) { + vm_page_free_min = VM_PAGE_FREE_MIN_LIMIT; + } vm_page_free_target = vm_page_free_reserved + - VM_PAGE_FREE_TARGET(free_after_reserve); + VM_PAGE_FREE_TARGET(free_after_reserve); - if (vm_page_free_target > VM_PAGE_FREE_TARGET_LIMIT) - vm_page_free_target = VM_PAGE_FREE_TARGET_LIMIT; + if (vm_page_free_target > VM_PAGE_FREE_TARGET_LIMIT) { + vm_page_free_target = VM_PAGE_FREE_TARGET_LIMIT; + } - if (vm_page_free_target < vm_page_free_min + 5) + if (vm_page_free_target < vm_page_free_min + 5) { vm_page_free_target = vm_page_free_min + 5; + } vm_page_throttle_limit = vm_page_free_target - (vm_page_free_target / 2); } @@ -3467,7 +3333,7 @@ void vm_pageout_continue(void) { DTRACE_VM2(pgrrun, int, 1, (uint64_t *), NULL); - vm_pageout_scan_event_counter++; + VM_PAGEOUT_DEBUG(vm_pageout_scan_event_counter, 1); #if !CONFIG_EMBEDDED lck_mtx_lock(&vm_page_queue_free_lock); @@ -3507,17 +3373,17 @@ vm_pageout_wait(uint64_t deadline) kern_return_t kr; lck_mtx_lock(&vm_page_queue_free_lock); - for (kr = KERN_SUCCESS; vm_pageout_running && (KERN_SUCCESS == kr); ) { + for (kr = KERN_SUCCESS; vm_pageout_running && (KERN_SUCCESS == kr);) { vm_pageout_waiter = TRUE; if (THREAD_AWAKENED != lck_mtx_sleep_deadline( - &vm_page_queue_free_lock, LCK_SLEEP_DEFAULT, - (event_t) &vm_pageout_waiter, THREAD_UNINT, deadline)) { + &vm_page_queue_free_lock, LCK_SLEEP_DEFAULT, + (event_t) &vm_pageout_waiter, THREAD_UNINT, deadline)) { kr = KERN_OPERATION_TIMED_OUT; } } lck_mtx_unlock(&vm_page_queue_free_lock); - return (kr); + return kr; } #endif /* !CONFIG_EMBEDDED */ @@ -3525,139 +3391,136 @@ vm_pageout_wait(uint64_t deadline) static void vm_pageout_iothread_external_continue(struct vm_pageout_queue *q) { - vm_page_t m = NULL; - vm_object_t object; + vm_page_t m = NULL; + vm_object_t object; vm_object_offset_t offset; - memory_object_t pager; + memory_object_t pager; - /* On systems without a compressor, the external IO thread clears its + /* On systems with a compressor, the external IO thread clears its * VM privileged bit to accommodate large allocations (e.g. bulk UPL * creation) */ - if (vm_pageout_internal_iothread != THREAD_NULL) + if (vm_pageout_state.vm_pageout_internal_iothread != THREAD_NULL) { current_thread()->options &= ~TH_OPT_VMPRIV; + } vm_page_lockspin_queues(); - while ( !vm_page_queue_empty(&q->pgo_pending) ) { - - q->pgo_busy = TRUE; - vm_page_queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq); - - assert(m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q); - VM_PAGE_CHECK(m); - /* - * grab a snapshot of the object and offset this - * page is tabled in so that we can relookup this - * page after we've taken the object lock - these - * fields are stable while we hold the page queues lock - * but as soon as we drop it, there is nothing to keep - * this page in this object... we hold an activity_in_progress - * on this object which will keep it from terminating - */ - object = VM_PAGE_OBJECT(m); - offset = m->offset; - - if (object->object_slid) { - panic("slid page %p not allowed on this path\n", m); - } - m->vm_page_q_state = VM_PAGE_NOT_ON_Q; - VM_PAGE_ZERO_PAGEQ_ENTRY(m); - - vm_page_unlock_queues(); - - vm_object_lock(object); - - m = vm_page_lookup(object, offset); - - if (m == NULL || - m->busy || m->cleaning || !m->laundry || (m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q)) { - /* - * it's either the same page that someone else has - * started cleaning (or it's finished cleaning or - * been put back on the pageout queue), or - * the page has been freed or we have found a - * new page at this offset... in all of these cases - * we merely need to release the activity_in_progress - * we took when we put the page on the pageout queue - */ - vm_object_activity_end(object); - vm_object_unlock(object); - - vm_page_lockspin_queues(); - continue; - } - pager = object->pager; - - if (pager == MEMORY_OBJECT_NULL) { - /* - * This pager has been destroyed by either - * memory_object_destroy or vm_object_destroy, and - * so there is nowhere for the page to go. - */ - if (m->free_when_done) { - /* - * Just free the page... VM_PAGE_FREE takes - * care of cleaning up all the state... - * including doing the vm_pageout_throttle_up - */ - VM_PAGE_FREE(m); - } else { - vm_page_lockspin_queues(); - - vm_pageout_throttle_up(m); - vm_page_activate(m); - - vm_page_unlock_queues(); - - /* - * And we are done with it. - */ - } - vm_object_activity_end(object); - vm_object_unlock(object); - - vm_page_lockspin_queues(); - continue; - } + while (!vm_page_queue_empty(&q->pgo_pending)) { + q->pgo_busy = TRUE; + vm_page_queue_remove_first(&q->pgo_pending, m, vmp_pageq); + + assert(m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q); + VM_PAGE_CHECK(m); + /* + * grab a snapshot of the object and offset this + * page is tabled in so that we can relookup this + * page after we've taken the object lock - these + * fields are stable while we hold the page queues lock + * but as soon as we drop it, there is nothing to keep + * this page in this object... we hold an activity_in_progress + * on this object which will keep it from terminating + */ + object = VM_PAGE_OBJECT(m); + offset = m->vmp_offset; + + m->vmp_q_state = VM_PAGE_NOT_ON_Q; + VM_PAGE_ZERO_PAGEQ_ENTRY(m); + + vm_page_unlock_queues(); + + vm_object_lock(object); + + m = vm_page_lookup(object, offset); + + if (m == NULL || m->vmp_busy || m->vmp_cleaning || + !m->vmp_laundry || (m->vmp_q_state != VM_PAGE_NOT_ON_Q)) { + /* + * it's either the same page that someone else has + * started cleaning (or it's finished cleaning or + * been put back on the pageout queue), or + * the page has been freed or we have found a + * new page at this offset... in all of these cases + * we merely need to release the activity_in_progress + * we took when we put the page on the pageout queue + */ + vm_object_activity_end(object); + vm_object_unlock(object); + + vm_page_lockspin_queues(); + continue; + } + pager = object->pager; + + if (pager == MEMORY_OBJECT_NULL) { + /* + * This pager has been destroyed by either + * memory_object_destroy or vm_object_destroy, and + * so there is nowhere for the page to go. + */ + if (m->vmp_free_when_done) { + /* + * Just free the page... VM_PAGE_FREE takes + * care of cleaning up all the state... + * including doing the vm_pageout_throttle_up + */ + VM_PAGE_FREE(m); + } else { + vm_page_lockspin_queues(); + + vm_pageout_throttle_up(m); + vm_page_activate(m); + + vm_page_unlock_queues(); + + /* + * And we are done with it. + */ + } + vm_object_activity_end(object); + vm_object_unlock(object); + + vm_page_lockspin_queues(); + continue; + } #if 0 - /* - * we don't hold the page queue lock - * so this check isn't safe to make - */ - VM_PAGE_CHECK(m); + /* + * we don't hold the page queue lock + * so this check isn't safe to make + */ + VM_PAGE_CHECK(m); #endif - /* - * give back the activity_in_progress reference we - * took when we queued up this page and replace it - * it with a paging_in_progress reference that will - * also hold the paging offset from changing and - * prevent the object from terminating - */ - vm_object_activity_end(object); - vm_object_paging_begin(object); - vm_object_unlock(object); - - /* - * Send the data to the pager. - * any pageout clustering happens there - */ - memory_object_data_return(pager, - m->offset + object->paging_offset, - PAGE_SIZE, - NULL, - NULL, - FALSE, - FALSE, - 0); - - vm_object_lock(object); - vm_object_paging_end(object); - vm_object_unlock(object); - - vm_pageout_io_throttle(); - - vm_page_lockspin_queues(); + /* + * give back the activity_in_progress reference we + * took when we queued up this page and replace it + * it with a paging_in_progress reference that will + * also hold the paging offset from changing and + * prevent the object from terminating + */ + vm_object_activity_end(object); + vm_object_paging_begin(object); + vm_object_unlock(object); + + /* + * Send the data to the pager. + * any pageout clustering happens there + */ + memory_object_data_return(pager, + m->vmp_offset + object->paging_offset, + PAGE_SIZE, + NULL, + NULL, + FALSE, + FALSE, + 0); + + vm_object_lock(object); + vm_object_paging_end(object); + vm_object_unlock(object); + + vm_pageout_io_throttle(); + + vm_page_lockspin_queues(); } q->pgo_busy = FALSE; q->pgo_idle = TRUE; @@ -3670,15 +3533,11 @@ vm_pageout_iothread_external_continue(struct vm_pageout_queue *q) } -#define MAX_FREE_BATCH 32 +#define MAX_FREE_BATCH 32 uint32_t vm_compressor_time_thread; /* Set via sysctl to record time accrued by - * this thread. - */ - + * this thread. + */ -#if DEVELOPMENT || DEBUG -uint64_t compressor_epoch_start, compressor_epoch_stop, compressor_threads_runtime; -#endif void vm_pageout_iothread_internal_continue(struct cq *); @@ -3686,28 +3545,29 @@ void vm_pageout_iothread_internal_continue(struct cq *cq) { struct vm_pageout_queue *q; - vm_page_t m = NULL; - boolean_t pgo_draining; + vm_page_t m = NULL; + boolean_t pgo_draining; vm_page_t local_q; - int local_cnt; + int local_cnt; vm_page_t local_freeq = NULL; int local_freed = 0; - int local_batch_size; - int ncomps = 0; + int local_batch_size; #if DEVELOPMENT || DEBUG + int ncomps = 0; boolean_t marked_active = FALSE; #endif KERNEL_DEBUG(0xe040000c | DBG_FUNC_END, 0, 0, 0, 0, 0); q = cq->q; - local_batch_size = q->pgo_maxlaundry / (vm_compressor_thread_count * 2); + local_batch_size = q->pgo_maxlaundry / (vm_pageout_state.vm_compressor_thread_count * 2); #if RECORD_THE_COMPRESSED_DATA - if (q->pgo_laundry) + if (q->pgo_laundry) { c_compressed_record_init(); + } #endif while (TRUE) { - int pages_left_on_q = 0; + int pages_left_on_q = 0; local_cnt = 0; local_q = NULL; @@ -3721,7 +3581,7 @@ vm_pageout_iothread_internal_continue(struct cq *cq) vmct_state[cq->id] = VMCT_ACTIVE; marked_active = TRUE; if (vmct_active == 1) { - compressor_epoch_start = mach_absolute_time(); + vm_compressor_epoch_start = mach_absolute_time(); } } #endif @@ -3729,69 +3589,74 @@ vm_pageout_iothread_internal_continue(struct cq *cq) KERNEL_DEBUG(0xe0400018 | DBG_FUNC_START, q->pgo_laundry, 0, 0, 0, 0); - while ( !vm_page_queue_empty(&q->pgo_pending) && local_cnt < local_batch_size) { - - vm_page_queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq); - assert(m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q); + while (!vm_page_queue_empty(&q->pgo_pending) && local_cnt < local_batch_size) { + vm_page_queue_remove_first(&q->pgo_pending, m, vmp_pageq); + assert(m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q); VM_PAGE_CHECK(m); - - m->vm_page_q_state = VM_PAGE_NOT_ON_Q; + + m->vmp_q_state = VM_PAGE_NOT_ON_Q; VM_PAGE_ZERO_PAGEQ_ENTRY(m); - m->laundry = FALSE; + m->vmp_laundry = FALSE; - m->snext = local_q; + m->vmp_snext = local_q; local_q = m; local_cnt++; } - if (local_q == NULL) + if (local_q == NULL) { break; + } q->pgo_busy = TRUE; if ((pgo_draining = q->pgo_draining) == FALSE) { vm_pageout_throttle_up_batch(q, local_cnt); pages_left_on_q = q->pgo_laundry; - } else + } else { pages_left_on_q = q->pgo_laundry - local_cnt; + } vm_page_unlock_queues(); #if !RECORD_THE_COMPRESSED_DATA - if (pages_left_on_q >= local_batch_size && cq->id < (vm_compressor_thread_count - 1)) { + if (pages_left_on_q >= local_batch_size && cq->id < (vm_pageout_state.vm_compressor_thread_count - 1)) { thread_wakeup((event_t) ((uintptr_t)&q->pgo_pending + cq->id + 1)); } #endif KERNEL_DEBUG(0xe0400018 | DBG_FUNC_END, q->pgo_laundry, 0, 0, 0, 0); while (local_q) { - KERNEL_DEBUG(0xe0400024 | DBG_FUNC_START, local_cnt, 0, 0, 0, 0); m = local_q; - local_q = m->snext; - m->snext = NULL; + local_q = m->vmp_snext; + m->vmp_snext = NULL; - if (vm_pageout_compress_page(&cq->current_chead, cq->scratch_buf, m, FALSE) == KERN_SUCCESS) { + if (vm_pageout_compress_page(&cq->current_chead, cq->scratch_buf, m) == KERN_SUCCESS) { +#if DEVELOPMENT || DEBUG ncomps++; - m->snext = local_freeq; +#endif + KERNEL_DEBUG(0xe0400024 | DBG_FUNC_END, local_cnt, 0, 0, 0, 0); + + m->vmp_snext = local_freeq; local_freeq = m; local_freed++; if (local_freed >= MAX_FREE_BATCH) { - vm_pageout_freed_after_compression += local_freed; + OSAddAtomic64(local_freed, &vm_pageout_vminfo.vm_pageout_compressions); vm_page_free_list(local_freeq, TRUE); + local_freeq = NULL; local_freed = 0; } } #if !CONFIG_JETSAM while (vm_page_free_count < COMPRESSOR_FREE_RESERVED_LIMIT) { - kern_return_t wait_result; - int need_wakeup = 0; + kern_return_t wait_result; + int need_wakeup = 0; if (local_freeq) { - vm_pageout_freed_after_compression += local_freed; + OSAddAtomic64(local_freed, &vm_pageout_vminfo.vm_pageout_compressions); vm_page_free_list(local_freeq, TRUE); local_freeq = NULL; @@ -3802,26 +3667,28 @@ vm_pageout_iothread_internal_continue(struct cq *cq) lck_mtx_lock_spin(&vm_page_queue_free_lock); if (vm_page_free_count < COMPRESSOR_FREE_RESERVED_LIMIT) { - - if (vm_page_free_wanted_privileged++ == 0) + if (vm_page_free_wanted_privileged++ == 0) { need_wakeup = 1; + } wait_result = assert_wait((event_t)&vm_page_free_wanted_privileged, THREAD_UNINT); lck_mtx_unlock(&vm_page_queue_free_lock); - if (need_wakeup) + if (need_wakeup) { thread_wakeup((event_t)&vm_page_free_wanted); + } - if (wait_result == THREAD_WAITING) - + if (wait_result == THREAD_WAITING) { thread_block(THREAD_CONTINUE_NULL); - } else + } + } else { lck_mtx_unlock(&vm_page_queue_free_lock); + } } #endif } if (local_freeq) { - vm_pageout_freed_after_compression += local_freed; + OSAddAtomic64(local_freed, &vm_pageout_vminfo.vm_pageout_compressions); vm_page_free_list(local_freeq, TRUE); local_freeq = NULL; @@ -3848,14 +3715,15 @@ vm_pageout_iothread_internal_continue(struct cq *cq) vmct_state[cq->id] = VMCT_IDLE; if (vmct_active == 0) { - compressor_epoch_stop = mach_absolute_time(); - assert(compressor_epoch_stop > compressor_epoch_start); + vm_compressor_epoch_stop = mach_absolute_time(); + assertf(vm_compressor_epoch_stop >= vm_compressor_epoch_start, + "Compressor epoch non-monotonic: 0x%llx -> 0x%llx", + vm_compressor_epoch_start, vm_compressor_epoch_stop); /* This interval includes intervals where one or more * compressor threads were pre-empted */ - vmct_stats.vmct_cthreads_total += compressor_epoch_stop - compressor_epoch_start; + vmct_stats.vmct_cthreads_total += vm_compressor_epoch_stop - vm_compressor_epoch_start; } - } #endif vm_page_unlock_queues(); @@ -3881,25 +3749,21 @@ vm_pageout_iothread_internal_continue(struct cq *cq) kern_return_t -vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m, boolean_t object_locked_by_caller) +vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m) { - vm_object_t object; - memory_object_t pager; - int compressed_count_delta; - kern_return_t retval; + vm_object_t object; + memory_object_t pager; + int compressed_count_delta; + kern_return_t retval; object = VM_PAGE_OBJECT(m); - if (object->object_slid) { - panic("slid page %p not allowed on this path\n", m); - } - assert(!m->free_when_done); - assert(!m->laundry); + assert(!m->vmp_free_when_done); + assert(!m->vmp_laundry); pager = object->pager; - if (object_locked_by_caller == FALSE && (!object->pager_initialized || pager == MEMORY_OBJECT_NULL)) { - + if (!object->pager_initialized || pager == MEMORY_OBJECT_NULL) { KERNEL_DEBUG(0xe0400010 | DBG_FUNC_START, object, pager, 0, 0, 0); vm_object_lock(object); @@ -3909,10 +3773,12 @@ vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m, b * one and hand it to the compression pager. */ - if (!object->pager_initialized) + if (!object->pager_initialized) { vm_object_collapse(object, (vm_object_offset_t) 0, TRUE); - if (!object->pager_initialized) + } + if (!object->pager_initialized) { vm_object_compressor_pager_create(object); + } pager = object->pager; @@ -3929,9 +3795,9 @@ vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m, b vm_page_lockspin_queues(); vm_page_activate(m); - vm_pageout_dirty_no_pager++; + VM_PAGEOUT_DEBUG(vm_pageout_dirty_no_pager, 1); vm_page_unlock_queues(); - + /* * And we are done with it. */ @@ -3941,35 +3807,30 @@ vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m, b return KERN_FAILURE; } vm_object_unlock(object); - + KERNEL_DEBUG(0xe0400010 | DBG_FUNC_END, object, pager, 0, 0, 0); } assert(object->pager_initialized && pager != MEMORY_OBJECT_NULL); - - if (object_locked_by_caller == FALSE) - assert(object->activity_in_progress > 0); + assert(object->activity_in_progress > 0); retval = vm_compressor_pager_put( pager, - m->offset + object->paging_offset, + m->vmp_offset + object->paging_offset, VM_PAGE_GET_PHYS_PAGE(m), current_chead, scratch_buf, &compressed_count_delta); - if (object_locked_by_caller == FALSE) { - vm_object_lock(object); + vm_object_lock(object); - assert(object->activity_in_progress > 0); - assert(VM_PAGE_OBJECT(m) == object); - } + assert(object->activity_in_progress > 0); + assert(VM_PAGE_OBJECT(m) == object); + assert( !VM_PAGE_WIRED(m)); vm_compressor_pager_count(pager, - compressed_count_delta, - FALSE, /* shared_lock */ - object); - - assert( !VM_PAGE_WIRED(m)); + compressed_count_delta, + FALSE, /* shared_lock */ + object); if (retval == KERN_SUCCESS) { /* @@ -3979,31 +3840,31 @@ vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m, b * contributes to the owner's memory footprint, * so account for it as such. */ - if (object->purgable != VM_PURGABLE_DENY && - object->vo_purgeable_owner != NULL) { - /* one more compressed purgeable page */ - vm_purgeable_compressed_update(object, - +1); + if ((object->purgable != VM_PURGABLE_DENY || + object->vo_ledger_tag) && + object->vo_owner != NULL) { + /* one more compressed purgeable/tagged page */ + vm_object_owner_compressed_update(object, + +1); } VM_STAT_INCR(compressions); - - if (m->tabled) - vm_page_remove(m, TRUE); + if (m->vmp_tabled) { + vm_page_remove(m, TRUE); + } } else { PAGE_WAKEUP_DONE(m); vm_page_lockspin_queues(); vm_page_activate(m); - vm_compressor_failed++; + vm_pageout_vminfo.vm_compressor_failed++; vm_page_unlock_queues(); } - if (object_locked_by_caller == FALSE) { - vm_object_activity_end(object); - vm_object_unlock(object); - } + vm_object_activity_end(object); + vm_object_unlock(object); + return retval; } @@ -4011,13 +3872,13 @@ vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m, b static void vm_pageout_adjust_eq_iothrottle(struct vm_pageout_queue *eq, boolean_t req_lowpriority) { - uint32_t policy; - - if (hibernate_cleaning_in_progress == TRUE) + uint32_t policy; + + if (hibernate_cleaning_in_progress == TRUE) { req_lowpriority = FALSE; + } if (eq->pgo_inited == TRUE && eq->pgo_lowpriority != req_lowpriority) { - vm_page_unlock_queues(); if (req_lowpriority == TRUE) { @@ -4028,7 +3889,7 @@ vm_pageout_adjust_eq_iothrottle(struct vm_pageout_queue *eq, boolean_t req_lowpr DTRACE_VM(laundryunthrottle); } proc_set_thread_policy_with_tid(kernel_task, eq->pgo_tid, - TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy); + TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy); eq->pgo_lowpriority = req_lowpriority; @@ -4040,14 +3901,14 @@ vm_pageout_adjust_eq_iothrottle(struct vm_pageout_queue *eq, boolean_t req_lowpr static void vm_pageout_iothread_external(void) { - thread_t self = current_thread(); + thread_t self = current_thread(); self->options |= TH_OPT_VMPRIV; DTRACE_VM2(laundrythrottle, int, 1, (uint64_t *), NULL); proc_set_thread_policy(self, TASK_POLICY_EXTERNAL, - TASK_POLICY_IO, THROTTLE_LEVEL_PAGEOUT_THROTTLED); + TASK_POLICY_IO, THROTTLE_LEVEL_PAGEOUT_THROTTLED); vm_page_lock_queues(); @@ -4066,7 +3927,7 @@ vm_pageout_iothread_external(void) static void vm_pageout_iothread_internal(struct cq *cq) { - thread_t self = current_thread(); + thread_t self = current_thread(); self->options |= TH_OPT_VMPRIV; @@ -4078,8 +3939,9 @@ vm_pageout_iothread_internal(struct cq *cq) vm_page_unlock_queues(); - if (vm_restricted_to_single_processor == TRUE) + if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) { thread_vm_bind_group_add(); + } thread_set_thread_name(current_thread(), "VM_compressor"); @@ -4092,7 +3954,7 @@ vm_pageout_iothread_internal(struct cq *cq) } kern_return_t -vm_set_buffer_cleanup_callout(boolean_t (*func)(int)) +vm_set_buffer_cleanup_callout(boolean_t (*func)(int)) { if (OSCompareAndSwapPtr(NULL, func, (void * volatile *) &consider_buffer_cache_collect)) { return KERN_SUCCESS; @@ -4101,8 +3963,8 @@ vm_set_buffer_cleanup_callout(boolean_t (*func)(int)) } } -extern boolean_t memorystatus_manual_testing_on; -extern unsigned int memorystatus_level; +extern boolean_t memorystatus_manual_testing_on; +extern unsigned int memorystatus_level; #if VM_PRESSURE_EVENTS @@ -4112,14 +3974,14 @@ boolean_t vm_pressure_events_enabled = FALSE; void vm_pressure_response(void) { + vm_pressure_level_t old_level = kVMPressureNormal; + int new_level = -1; + unsigned int total_pages; + uint64_t available_memory = 0; - vm_pressure_level_t old_level = kVMPressureNormal; - int new_level = -1; - unsigned int total_pages; - uint64_t available_memory = 0; - - if (vm_pressure_events_enabled == FALSE) + if (vm_pressure_events_enabled == FALSE) { return; + } #if CONFIG_EMBEDDED @@ -4141,88 +4003,90 @@ vm_pressure_response(void) if (memorystatus_manual_testing_on) { return; } - + old_level = memorystatus_vm_pressure_level; switch (memorystatus_vm_pressure_level) { - - case kVMPressureNormal: - { - if (VM_PRESSURE_WARNING_TO_CRITICAL()) { - new_level = kVMPressureCritical; - } else if (VM_PRESSURE_NORMAL_TO_WARNING()) { - new_level = kVMPressureWarning; - } - break; + case kVMPressureNormal: + { + if (VM_PRESSURE_WARNING_TO_CRITICAL()) { + new_level = kVMPressureCritical; + } else if (VM_PRESSURE_NORMAL_TO_WARNING()) { + new_level = kVMPressureWarning; } + break; + } - case kVMPressureWarning: - case kVMPressureUrgent: - { - if (VM_PRESSURE_WARNING_TO_NORMAL()) { - new_level = kVMPressureNormal; - } else if (VM_PRESSURE_WARNING_TO_CRITICAL()) { - new_level = kVMPressureCritical; - } - break; + case kVMPressureWarning: + case kVMPressureUrgent: + { + if (VM_PRESSURE_WARNING_TO_NORMAL()) { + new_level = kVMPressureNormal; + } else if (VM_PRESSURE_WARNING_TO_CRITICAL()) { + new_level = kVMPressureCritical; } + break; + } - case kVMPressureCritical: - { - if (VM_PRESSURE_WARNING_TO_NORMAL()) { - new_level = kVMPressureNormal; - } else if (VM_PRESSURE_CRITICAL_TO_WARNING()) { - new_level = kVMPressureWarning; - } - break; + case kVMPressureCritical: + { + if (VM_PRESSURE_WARNING_TO_NORMAL()) { + new_level = kVMPressureNormal; + } else if (VM_PRESSURE_CRITICAL_TO_WARNING()) { + new_level = kVMPressureWarning; } + break; + } - default: - return; + default: + return; } - + if (new_level != -1) { memorystatus_vm_pressure_level = (vm_pressure_level_t) new_level; - if ((memorystatus_vm_pressure_level != kVMPressureNormal) || (old_level != new_level)) { - if (vm_pressure_thread_running == FALSE) { + if (new_level != (int) old_level) { + VM_DEBUG_CONSTANT_EVENT(vm_pressure_level_change, VM_PRESSURE_LEVEL_CHANGE, DBG_FUNC_NONE, + new_level, old_level, 0, 0); + } + + if ((memorystatus_vm_pressure_level != kVMPressureNormal) || (old_level != memorystatus_vm_pressure_level)) { + if (vm_pageout_state.vm_pressure_thread_running == FALSE) { thread_wakeup(&vm_pressure_thread); } - if (old_level != new_level) { - thread_wakeup(&vm_pressure_changed); + if (old_level != memorystatus_vm_pressure_level) { + thread_wakeup(&vm_pageout_state.vm_pressure_changed); } } } - } #endif /* VM_PRESSURE_EVENTS */ kern_return_t -mach_vm_pressure_level_monitor(__unused boolean_t wait_for_pressure, __unused unsigned int *pressure_level) { - +mach_vm_pressure_level_monitor(__unused boolean_t wait_for_pressure, __unused unsigned int *pressure_level) +{ #if CONFIG_EMBEDDED - + return KERN_FAILURE; #elif !VM_PRESSURE_EVENTS - + return KERN_FAILURE; #else /* VM_PRESSURE_EVENTS */ - kern_return_t kr = KERN_SUCCESS; + kern_return_t kr = KERN_SUCCESS; if (pressure_level != NULL) { - - vm_pressure_level_t old_level = memorystatus_vm_pressure_level; + vm_pressure_level_t old_level = memorystatus_vm_pressure_level; if (wait_for_pressure == TRUE) { - wait_result_t wr = 0; + wait_result_t wr = 0; while (old_level == *pressure_level) { - wr = assert_wait((event_t) &vm_pressure_changed, - THREAD_INTERRUPTIBLE); + wr = assert_wait((event_t) &vm_pageout_state.vm_pressure_changed, + THREAD_INTERRUPTIBLE); if (wr == THREAD_WAITING) { wr = thread_block(THREAD_CONTINUE_NULL); } @@ -4230,7 +4094,6 @@ mach_vm_pressure_level_monitor(__unused boolean_t wait_for_pressure, __unused un return KERN_ABORTED; } if (wr == THREAD_AWAKENED) { - old_level = memorystatus_vm_pressure_level; if (old_level != *pressure_level) { @@ -4252,15 +4115,17 @@ mach_vm_pressure_level_monitor(__unused boolean_t wait_for_pressure, __unused un #if VM_PRESSURE_EVENTS void -vm_pressure_thread(void) { +vm_pressure_thread(void) +{ static boolean_t thread_initialized = FALSE; if (thread_initialized == TRUE) { - vm_pressure_thread_running = TRUE; + vm_pageout_state.vm_pressure_thread_running = TRUE; consider_vm_pressure_events(); - vm_pressure_thread_running = FALSE; + vm_pageout_state.vm_pressure_thread_running = FALSE; } + thread_set_thread_name(current_thread(), "VM_pressure"); thread_initialized = TRUE; assert_wait((event_t) &vm_pressure_thread, THREAD_UNINT); thread_block((thread_continue_t)vm_pressure_thread); @@ -4268,17 +4133,14 @@ vm_pressure_thread(void) { #endif /* VM_PRESSURE_EVENTS */ -uint32_t vm_pageout_considered_page_last = 0; - /* * called once per-second via "compute_averages" */ void compute_pageout_gc_throttle(__unused void *arg) { - if (vm_pageout_considered_page != vm_pageout_considered_page_last) { - - vm_pageout_considered_page_last = vm_pageout_considered_page; + if (vm_pageout_vminfo.vm_pageout_considered_page != vm_pageout_state.vm_pageout_considered_page_last) { + vm_pageout_state.vm_pageout_considered_page_last = vm_pageout_vminfo.vm_pageout_considered_page; thread_wakeup((event_t) &vm_pageout_garbage_collect); } @@ -4330,7 +4192,6 @@ vm_pageout_garbage_collect(int collect) * up again. */ consider_zone_gc(TRUE); - } else { /* Woken up by vm_pageout_scan or compute_pageout_gc_throttle. */ boolean_t buf_large_zfree = FALSE; @@ -4339,7 +4200,7 @@ vm_pageout_garbage_collect(int collect) stack_collect(); consider_machine_collect(); - m_drain(); + mbuf_drain(FALSE); do { if (consider_buffer_cache_collect != NULL) { @@ -4353,7 +4214,6 @@ vm_pageout_garbage_collect(int collect) consider_zone_gc(FALSE); } first_try = FALSE; - } while (buf_large_zfree == TRUE && vm_page_free_count < vm_page_free_target); consider_machine_adjust(); @@ -4388,13 +4248,13 @@ vm_set_restrictions() if (hinfo.max_cpus <= 3) { /* - * on systems with a limited number of CPUS, bind the + * on systems with a limited number of CPUS, bind the * 4 major threads that can free memory and that tend to use * a fair bit of CPU under pressured conditions to a single processor. * This insures that these threads don't hog all of the available CPUs * (important for camera launch), while allowing them to run independently * w/r to locks... the 4 threads are - * vm_pageout_scan, vm_pageout_iothread_internal (compressor), + * vm_pageout_scan, vm_pageout_iothread_internal (compressor), * vm_compressor_swap_trigger_thread (minor and major compactions), * memorystatus_thread (jetsams). * @@ -4403,17 +4263,19 @@ vm_set_restrictions() * thread_bind_master... someday this should be replaced with a group * scheduling mechanism and KPI. */ - vm_restricted_to_single_processor = TRUE; + vm_pageout_state.vm_restricted_to_single_processor = TRUE; + } else { + vm_pageout_state.vm_restricted_to_single_processor = FALSE; } } void vm_pageout(void) { - thread_t self = current_thread(); - thread_t thread; - kern_return_t result; - spl_t s; + thread_t self = current_thread(); + thread_t thread; + kern_return_t result; + spl_t s; /* * Set thread privileges. @@ -4425,11 +4287,13 @@ vm_pageout(void) sched_set_thread_base_priority(self, BASEPRI_VM); thread_unlock(self); - if (!self->reserved_stack) + if (!self->reserved_stack) { self->reserved_stack = self->kernel_stack; + } - if (vm_restricted_to_single_processor == TRUE) + if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) { thread_vm_bind_group_add(); + } splx(s); @@ -4439,43 +4303,70 @@ vm_pageout(void) * Initialize some paging parameters. */ - if (vm_pageout_swap_wait == 0) - vm_pageout_swap_wait = VM_PAGEOUT_SWAP_WAIT; - - if (vm_pageout_idle_wait == 0) - vm_pageout_idle_wait = VM_PAGEOUT_IDLE_WAIT; - - if (vm_pageout_burst_wait == 0) - vm_pageout_burst_wait = VM_PAGEOUT_BURST_WAIT; - - if (vm_pageout_empty_wait == 0) - vm_pageout_empty_wait = VM_PAGEOUT_EMPTY_WAIT; + vm_pageout_state.vm_pressure_thread_running = FALSE; + vm_pageout_state.vm_pressure_changed = FALSE; + vm_pageout_state.memorystatus_purge_on_warning = 2; + vm_pageout_state.memorystatus_purge_on_urgent = 5; + vm_pageout_state.memorystatus_purge_on_critical = 8; + vm_pageout_state.vm_page_speculative_q_age_ms = VM_PAGE_SPECULATIVE_Q_AGE_MS; + vm_pageout_state.vm_page_speculative_percentage = 5; + vm_pageout_state.vm_page_speculative_target = 0; + + vm_pageout_state.vm_pageout_external_iothread = THREAD_NULL; + vm_pageout_state.vm_pageout_internal_iothread = THREAD_NULL; + + vm_pageout_state.vm_pageout_swap_wait = 0; + vm_pageout_state.vm_pageout_idle_wait = 0; + vm_pageout_state.vm_pageout_empty_wait = 0; + vm_pageout_state.vm_pageout_burst_wait = 0; + vm_pageout_state.vm_pageout_deadlock_wait = 0; + vm_pageout_state.vm_pageout_deadlock_relief = 0; + vm_pageout_state.vm_pageout_burst_inactive_throttle = 0; + + vm_pageout_state.vm_pageout_inactive = 0; + vm_pageout_state.vm_pageout_inactive_used = 0; + vm_pageout_state.vm_pageout_inactive_clean = 0; + + vm_pageout_state.vm_memory_pressure = 0; + vm_pageout_state.vm_page_filecache_min = 0; +#if CONFIG_JETSAM + vm_pageout_state.vm_page_filecache_min_divisor = 70; + vm_pageout_state.vm_page_xpmapped_min_divisor = 40; +#else + vm_pageout_state.vm_page_filecache_min_divisor = 27; + vm_pageout_state.vm_page_xpmapped_min_divisor = 36; +#endif + vm_pageout_state.vm_page_free_count_init = vm_page_free_count; - if (vm_pageout_deadlock_wait == 0) - vm_pageout_deadlock_wait = VM_PAGEOUT_DEADLOCK_WAIT; + vm_pageout_state.vm_pageout_considered_page_last = 0; - if (vm_pageout_deadlock_relief == 0) - vm_pageout_deadlock_relief = VM_PAGEOUT_DEADLOCK_RELIEF; + if (vm_pageout_state.vm_pageout_swap_wait == 0) { + vm_pageout_state.vm_pageout_swap_wait = VM_PAGEOUT_SWAP_WAIT; + } - if (vm_pageout_inactive_relief == 0) - vm_pageout_inactive_relief = VM_PAGEOUT_INACTIVE_RELIEF; + if (vm_pageout_state.vm_pageout_idle_wait == 0) { + vm_pageout_state.vm_pageout_idle_wait = VM_PAGEOUT_IDLE_WAIT; + } - if (vm_pageout_burst_active_throttle == 0) - vm_pageout_burst_active_throttle = VM_PAGEOUT_BURST_ACTIVE_THROTTLE; + if (vm_pageout_state.vm_pageout_burst_wait == 0) { + vm_pageout_state.vm_pageout_burst_wait = VM_PAGEOUT_BURST_WAIT; + } - if (vm_pageout_burst_inactive_throttle == 0) - vm_pageout_burst_inactive_throttle = VM_PAGEOUT_BURST_INACTIVE_THROTTLE; + if (vm_pageout_state.vm_pageout_empty_wait == 0) { + vm_pageout_state.vm_pageout_empty_wait = VM_PAGEOUT_EMPTY_WAIT; + } - /* - * Set kernel task to low backing store privileged - * status - */ - task_lock(kernel_task); - kernel_task->priv_flags |= VM_BACKING_STORE_PRIV; - task_unlock(kernel_task); + if (vm_pageout_state.vm_pageout_deadlock_wait == 0) { + vm_pageout_state.vm_pageout_deadlock_wait = VM_PAGEOUT_DEADLOCK_WAIT; + } - vm_page_free_count_init = vm_page_free_count; + if (vm_pageout_state.vm_pageout_deadlock_relief == 0) { + vm_pageout_state.vm_pageout_deadlock_relief = VM_PAGEOUT_DEADLOCK_RELIEF; + } + if (vm_pageout_state.vm_pageout_burst_inactive_throttle == 0) { + vm_pageout_state.vm_pageout_burst_inactive_throttle = VM_PAGEOUT_BURST_INACTIVE_THROTTLE; + } /* * even if we've already called vm_page_free_reserve * call it again here to insure that the targets are @@ -4485,8 +4376,9 @@ vm_pageout(void) */ if (vm_page_free_reserved < VM_PAGE_FREE_RESERVED(processor_count)) { vm_page_free_reserve((VM_PAGE_FREE_RESERVED(processor_count)) - vm_page_free_reserved); - } else + } else { vm_page_free_reserve(0); + } vm_page_queue_init(&vm_pageout_queue_external.pgo_pending); @@ -4514,29 +4406,32 @@ vm_pageout(void) /* internal pageout thread started when default pager registered first time */ /* external pageout and garbage collection threads started here */ - result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_external, NULL, - BASEPRI_VM, - &vm_pageout_external_iothread); - if (result != KERN_SUCCESS) + result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_external, NULL, + BASEPRI_VM, + &vm_pageout_state.vm_pageout_external_iothread); + if (result != KERN_SUCCESS) { panic("vm_pageout_iothread_external: create failed"); + } - thread_deallocate(vm_pageout_external_iothread); + thread_deallocate(vm_pageout_state.vm_pageout_external_iothread); result = kernel_thread_start_priority((thread_continue_t)vm_pageout_garbage_collect, NULL, - BASEPRI_DEFAULT, - &thread); - if (result != KERN_SUCCESS) + BASEPRI_DEFAULT, + &thread); + if (result != KERN_SUCCESS) { panic("vm_pageout_garbage_collect: create failed"); + } thread_deallocate(thread); #if VM_PRESSURE_EVENTS result = kernel_thread_start_priority((thread_continue_t)vm_pressure_thread, NULL, - BASEPRI_DEFAULT, - &thread); + BASEPRI_DEFAULT, + &thread); - if (result != KERN_SUCCESS) + if (result != KERN_SUCCESS) { panic("vm_pressure_thread: create failed"); + } thread_deallocate(thread); #endif @@ -4546,8 +4441,7 @@ vm_pageout(void) bzero(&vm_config, sizeof(vm_config)); - switch(vm_compressor_mode) { - + switch (vm_compressor_mode) { case VM_PAGER_DEFAULT: printf("mapping deprecated VM_PAGER_DEFAULT to VM_PAGER_COMPRESSOR_WITH_SWAP\n"); @@ -4586,8 +4480,9 @@ vm_pageout(void) printf("unknown compressor mode - %x\n", vm_compressor_mode); break; } - if (VM_CONFIG_COMPRESSOR_IS_PRESENT) + if (VM_CONFIG_COMPRESSOR_IS_PRESENT) { vm_compressor_pager_init(); + } #if VM_PRESSURE_EVENTS vm_pressure_events_enabled = TRUE; @@ -4599,12 +4494,12 @@ vm_pageout(void) #if VM_PAGE_BUCKETS_CHECK #if VM_PAGE_FAKE_BUCKETS printf("**** DEBUG: protecting fake buckets [0x%llx:0x%llx]\n", - (uint64_t) vm_page_fake_buckets_start, - (uint64_t) vm_page_fake_buckets_end); + (uint64_t) vm_page_fake_buckets_start, + (uint64_t) vm_page_fake_buckets_end); pmap_protect(kernel_pmap, - vm_page_fake_buckets_start, - vm_page_fake_buckets_end, - VM_PROT_READ); + vm_page_fake_buckets_start, + vm_page_fake_buckets_end, + VM_PROT_READ); // *(char *) vm_page_fake_buckets_start = 'x'; /* panic! */ #endif /* VM_PAGE_FAKE_BUCKETS */ #endif /* VM_PAGE_BUCKETS_CHECK */ @@ -4643,20 +4538,14 @@ vm_pageout(void) -#if CONFIG_EMBEDDED -int vm_compressor_thread_count = 1; -#else -int vm_compressor_thread_count = 2; -#endif - kern_return_t vm_pageout_internal_start(void) { - kern_return_t result; - int i; + kern_return_t result; + int i; host_basic_info_data_t hinfo; - assert (VM_CONFIG_COMPRESSOR_IS_PRESENT); + assert(VM_CONFIG_COMPRESSOR_IS_PRESENT); mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT; #define BSD_HOST 1 @@ -4664,30 +4553,47 @@ vm_pageout_internal_start(void) assert(hinfo.max_cpus > 0); - PE_parse_boot_argn("vmcomp_threads", &vm_compressor_thread_count, sizeof(vm_compressor_thread_count)); - if (vm_compressor_thread_count >= hinfo.max_cpus) - vm_compressor_thread_count = hinfo.max_cpus - 1; - if (vm_compressor_thread_count <= 0) - vm_compressor_thread_count = 1; - else if (vm_compressor_thread_count > MAX_COMPRESSOR_THREAD_COUNT) - vm_compressor_thread_count = MAX_COMPRESSOR_THREAD_COUNT; + lck_grp_init(&vm_pageout_lck_grp, "vm_pageout", LCK_GRP_ATTR_NULL); + +#if CONFIG_EMBEDDED + vm_pageout_state.vm_compressor_thread_count = 1; +#else + if (hinfo.max_cpus > 4) { + vm_pageout_state.vm_compressor_thread_count = 2; + } else { + vm_pageout_state.vm_compressor_thread_count = 1; + } +#endif + PE_parse_boot_argn("vmcomp_threads", &vm_pageout_state.vm_compressor_thread_count, + sizeof(vm_pageout_state.vm_compressor_thread_count)); + + if (vm_pageout_state.vm_compressor_thread_count >= hinfo.max_cpus) { + vm_pageout_state.vm_compressor_thread_count = hinfo.max_cpus - 1; + } + if (vm_pageout_state.vm_compressor_thread_count <= 0) { + vm_pageout_state.vm_compressor_thread_count = 1; + } else if (vm_pageout_state.vm_compressor_thread_count > MAX_COMPRESSOR_THREAD_COUNT) { + vm_pageout_state.vm_compressor_thread_count = MAX_COMPRESSOR_THREAD_COUNT; + } - vm_pageout_queue_internal.pgo_maxlaundry = (vm_compressor_thread_count * 4) * VM_PAGE_LAUNDRY_MAX; + vm_pageout_queue_internal.pgo_maxlaundry = (vm_pageout_state.vm_compressor_thread_count * 4) * VM_PAGE_LAUNDRY_MAX; PE_parse_boot_argn("vmpgoi_maxlaundry", &vm_pageout_queue_internal.pgo_maxlaundry, sizeof(vm_pageout_queue_internal.pgo_maxlaundry)); - for (i = 0; i < vm_compressor_thread_count; i++) { + for (i = 0; i < vm_pageout_state.vm_compressor_thread_count; i++) { ciq[i].id = i; ciq[i].q = &vm_pageout_queue_internal; ciq[i].current_chead = NULL; ciq[i].scratch_buf = kalloc(COMPRESSOR_SCRATCH_BUF_SIZE); - result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal, (void *)&ciq[i], BASEPRI_VM, &vm_pageout_internal_iothread); + result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal, (void *)&ciq[i], + BASEPRI_VM, &vm_pageout_state.vm_pageout_internal_iothread); - if (result == KERN_SUCCESS) - thread_deallocate(vm_pageout_internal_iothread); - else + if (result == KERN_SUCCESS) { + thread_deallocate(vm_pageout_state.vm_pageout_internal_iothread); + } else { break; + } } return result; } @@ -4709,41 +4615,41 @@ vm_pageout_internal_start(void) static void upl_set_decmp_info(upl_t upl, upl_t src_upl) { - assert((src_upl->flags & UPL_DECMP_REQ) != 0); - - upl_lock(src_upl); - if (src_upl->decmp_io_upl) { - /* - * If there is already an alive real I/O UPL, ignore this new UPL. - * This case should rarely happen and even if it does, it just means - * that we might issue a spurious expedite which the driver is expected - * to handle. - */ - upl_unlock(src_upl); - return; - } - src_upl->decmp_io_upl = (void *)upl; - src_upl->ref_count++; - - upl->flags |= UPL_DECMP_REAL_IO; - upl->decmp_io_upl = (void *)src_upl; + assert((src_upl->flags & UPL_DECMP_REQ) != 0); + + upl_lock(src_upl); + if (src_upl->decmp_io_upl) { + /* + * If there is already an alive real I/O UPL, ignore this new UPL. + * This case should rarely happen and even if it does, it just means + * that we might issue a spurious expedite which the driver is expected + * to handle. + */ + upl_unlock(src_upl); + return; + } + src_upl->decmp_io_upl = (void *)upl; + src_upl->ref_count++; + + upl->flags |= UPL_DECMP_REAL_IO; + upl->decmp_io_upl = (void *)src_upl; upl_unlock(src_upl); } -#endif /* CONFIG_IOSCHED */ +#endif /* CONFIG_IOSCHED */ #if UPL_DEBUG -int upl_debug_enabled = 1; +int upl_debug_enabled = 1; #else -int upl_debug_enabled = 0; +int upl_debug_enabled = 0; #endif static upl_t upl_create(int type, int flags, upl_size_t size) { - upl_t upl; - vm_size_t page_field_size = 0; - int upl_flags = 0; - vm_size_t upl_size = sizeof(struct upl); + upl_t upl; + vm_size_t page_field_size = 0; + int upl_flags = 0; + vm_size_t upl_size = sizeof(struct upl); size = round_page_32(size); @@ -4760,8 +4666,9 @@ upl_create(int type, int flags, upl_size_t size) } upl = (upl_t)kalloc(upl_size + page_field_size); - if (page_field_size) - bzero((char *)upl + upl_size, page_field_size); + if (page_field_size) { + bzero((char *)upl + upl_size, page_field_size); + } upl->flags = upl_flags | flags; upl->kaddr = (vm_offset_t)0; @@ -4773,11 +4680,12 @@ upl_create(int type, int flags, upl_size_t size) upl_lock_init(upl); upl->vector_upl = NULL; upl->associated_upl = NULL; + upl->upl_iodone = NULL; #if CONFIG_IOSCHED if (type & UPL_CREATE_IO_TRACKING) { upl->upl_priority = proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO); } - + upl->upl_reprio_info = 0; upl->decmp_io_upl = 0; if ((type & UPL_CREATE_INTERNAL) && (type & UPL_CREATE_EXPEDITE_SUP)) { @@ -4786,8 +4694,9 @@ upl_create(int type, int flags, upl_size_t size) upl->upl_reprio_info = (uint64_t *)kalloc(sizeof(uint64_t) * atop(size)); bzero(upl->upl_reprio_info, (sizeof(uint64_t) * atop(size))); upl->flags |= UPL_EXPEDITE_SUPPORTED; - if (curthread->decmp_upl != NULL) + if (curthread->decmp_upl != NULL) { upl_set_decmp_info(upl, curthread->decmp_upl); + } } #endif #if CONFIG_IOSCHED || UPL_DEBUG @@ -4810,34 +4719,34 @@ upl_create(int type, int flags, upl_size_t size) (void) OSBacktrace(&upl->upl_create_retaddr[0], UPL_DEBUG_STACK_FRAMES); #endif /* UPL_DEBUG */ - return(upl); + return upl; } static void upl_destroy(upl_t upl) { - int page_field_size; /* bit field in word size buf */ - int size; + int page_field_size; /* bit field in word size buf */ + int size; if (upl->ext_ref_count) { panic("upl(%p) ext_ref_count", upl); } #if CONFIG_IOSCHED - if ((upl->flags & UPL_DECMP_REAL_IO) && upl->decmp_io_upl) { - upl_t src_upl; - src_upl = upl->decmp_io_upl; - assert((src_upl->flags & UPL_DECMP_REQ) != 0); - upl_lock(src_upl); - src_upl->decmp_io_upl = NULL; - upl_unlock(src_upl); - upl_deallocate(src_upl); - } + if ((upl->flags & UPL_DECMP_REAL_IO) && upl->decmp_io_upl) { + upl_t src_upl; + src_upl = upl->decmp_io_upl; + assert((src_upl->flags & UPL_DECMP_REQ) != 0); + upl_lock(src_upl); + src_upl->decmp_io_upl = NULL; + upl_unlock(src_upl); + upl_deallocate(src_upl); + } #endif /* CONFIG_IOSCHED */ #if CONFIG_IOSCHED || UPL_DEBUG if ((upl->flags & UPL_TRACKED_BY_OBJECT) && !(upl->flags & UPL_VECTOR)) { - vm_object_t object; + vm_object_t object; if (upl->flags & UPL_SHADOWED) { object = upl->map_object->shadow; @@ -4856,32 +4765,35 @@ upl_destroy(upl_t upl) * drop a reference on the map_object whether or * not a pageout object is inserted */ - if (upl->flags & UPL_SHADOWED) + if (upl->flags & UPL_SHADOWED) { vm_object_deallocate(upl->map_object); + } - if (upl->flags & UPL_DEVICE_MEMORY) - size = PAGE_SIZE; - else - size = upl->size; + if (upl->flags & UPL_DEVICE_MEMORY) { + size = PAGE_SIZE; + } else { + size = upl->size; + } page_field_size = 0; if (upl->flags & UPL_LITE) { - page_field_size = ((size/PAGE_SIZE) + 7) >> 3; + page_field_size = ((size / PAGE_SIZE) + 7) >> 3; page_field_size = (page_field_size + 3) & 0xFFFFFFFC; } upl_lock_destroy(upl); upl->vector_upl = (vector_upl_t) 0xfeedbeef; #if CONFIG_IOSCHED - if (upl->flags & UPL_EXPEDITE_SUPPORTED) - kfree(upl->upl_reprio_info, sizeof(uint64_t) * (size/PAGE_SIZE)); + if (upl->flags & UPL_EXPEDITE_SUPPORTED) { + kfree(upl->upl_reprio_info, sizeof(uint64_t) * (size / PAGE_SIZE)); + } #endif if (upl->flags & UPL_INTERNAL) { kfree(upl, - sizeof(struct upl) + - (sizeof(struct upl_page_info) * (size/PAGE_SIZE)) - + page_field_size); + sizeof(struct upl) + + (sizeof(struct upl_page_info) * (size / PAGE_SIZE)) + + page_field_size); } else { kfree(upl, sizeof(struct upl) + page_field_size); } @@ -4891,14 +4803,21 @@ void upl_deallocate(upl_t upl) { upl_lock(upl); + if (--upl->ref_count == 0) { - if(vector_upl_is_valid(upl)) + if (vector_upl_is_valid(upl)) { vector_upl_deallocate(upl); - upl_unlock(upl); + } + upl_unlock(upl); + + if (upl->upl_iodone) { + upl_callout_iodone(upl); + } + upl_destroy(upl); - } - else + } else { upl_unlock(upl); + } } #if CONFIG_IOSCHED @@ -4908,21 +4827,21 @@ upl_mark_decmp(upl_t upl) if (upl->flags & UPL_TRACKED_BY_OBJECT) { upl->flags |= UPL_DECMP_REQ; upl->upl_creator->decmp_upl = (void *)upl; - } + } } void upl_unmark_decmp(upl_t upl) { - if(upl && (upl->flags & UPL_DECMP_REQ)) { + if (upl && (upl->flags & UPL_DECMP_REQ)) { upl->upl_creator->decmp_upl = NULL; } -} +} #endif /* CONFIG_IOSCHED */ -#define VM_PAGE_Q_BACKING_UP(q) \ - ((q)->pgo_laundry >= (((q)->pgo_maxlaundry * 8) / 10)) +#define VM_PAGE_Q_BACKING_UP(q) \ + ((q)->pgo_laundry >= (((q)->pgo_maxlaundry * 8) / 10)) boolean_t must_throttle_writes(void); @@ -4930,29 +4849,17 @@ boolean_t must_throttle_writes() { if (VM_PAGE_Q_BACKING_UP(&vm_pageout_queue_external) && - vm_page_pageable_external_count > (AVAILABLE_NON_COMPRESSED_MEMORY * 6) / 10) - return (TRUE); + vm_page_pageable_external_count > (AVAILABLE_NON_COMPRESSED_MEMORY * 6) / 10) { + return TRUE; + } - return (FALSE); + return FALSE; } -#if DEVELOPMENT || DEBUG -/*/* - * Statistics about UPL enforcement of copy-on-write obligations. - */ -unsigned long upl_cow = 0; -unsigned long upl_cow_again = 0; -unsigned long upl_cow_pages = 0; -unsigned long upl_cow_again_pages = 0; - -unsigned long iopl_cow = 0; -unsigned long iopl_cow_pages = 0; -#endif - -/* - * Routine: vm_object_upl_request - * Purpose: +/* + * Routine: vm_object_upl_request + * Purpose: * Cause the population of a portion of a vm_object. * Depending on the nature of the request, the pages * returned may be contain valid data or be uninitialized. @@ -4963,7 +4870,7 @@ unsigned long iopl_cow_pages = 0; * IMPORTANT NOTE: The caller must still respect the relationship * between the vm_object and its backing memory object. The * caller MUST NOT substitute changes in the backing file - * without first doing a memory_object_lock_request on the + * without first doing a memory_object_lock_request on the * target range unless it is know that the pages are not * shared with another entity at the pager level. * Copy_in_to: @@ -4981,7 +4888,7 @@ unsigned long iopl_cow_pages = 0; * all mapped pages. Where a page does not exist * map a zero filled one. Leave pages busy in * the original object. If a page list structure - * is not specified, this call is a no-op. + * is not specified, this call is a no-op. * * Note: access of default pager objects has a rather interesting * twist. The caller of this routine, presumably the file system @@ -4989,44 +4896,47 @@ unsigned long iopl_cow_pages = 0; * against a default pager backed object. Only the default * pager will make requests on backing store related vm_objects * In this way the default pager can maintain the relationship - * between backing store files (abstract memory objects) and + * between backing store files (abstract memory objects) and * the vm_objects (cache objects), they support. * */ __private_extern__ kern_return_t vm_object_upl_request( - vm_object_t object, - vm_object_offset_t offset, - upl_size_t size, - upl_t *upl_ptr, - upl_page_info_array_t user_page_list, - unsigned int *page_list_count, - upl_control_flags_t cntrl_flags, - vm_tag_t tag) + vm_object_t object, + vm_object_offset_t offset, + upl_size_t size, + upl_t *upl_ptr, + upl_page_info_array_t user_page_list, + unsigned int *page_list_count, + upl_control_flags_t cntrl_flags, + vm_tag_t tag) { - vm_page_t dst_page = VM_PAGE_NULL; - vm_object_offset_t dst_offset; - upl_size_t xfer_size; - unsigned int size_in_pages; - boolean_t dirty; - boolean_t hw_dirty; - upl_t upl = NULL; - unsigned int entry; -#if MACH_CLUSTER_STATS - boolean_t encountered_lrp = FALSE; -#endif - vm_page_t alias_page = NULL; - int refmod_state = 0; - wpl_array_t lite_list = NULL; - vm_object_t last_copy_object; - struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT]; - struct vm_page_delayed_work *dwp; - int dw_count; - int dw_limit; - int io_tracking_flag = 0; - int grab_options; - ppnum_t phys_page; + vm_page_t dst_page = VM_PAGE_NULL; + vm_object_offset_t dst_offset; + upl_size_t xfer_size; + unsigned int size_in_pages; + boolean_t dirty; + boolean_t hw_dirty; + upl_t upl = NULL; + unsigned int entry; + vm_page_t alias_page = NULL; + int refmod_state = 0; + wpl_array_t lite_list = NULL; + vm_object_t last_copy_object; + struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT]; + struct vm_page_delayed_work *dwp; + int dw_count; + int dw_limit; + int io_tracking_flag = 0; + int grab_options; + int page_grab_count = 0; + ppnum_t phys_page; + pmap_flush_context pmap_flush_context_storage; + boolean_t pmap_flushes_delayed = FALSE; +#if DEVELOPMENT || DEBUG + task_t task = current_task(); +#endif /* DEVELOPMENT || DEBUG */ if (cntrl_flags & ~UPL_VALID_FLAGS) { /* @@ -5035,42 +4945,48 @@ vm_object_upl_request( */ return KERN_INVALID_VALUE; } - if ( (!object->internal) && (object->paging_offset != 0) ) + if ((!object->internal) && (object->paging_offset != 0)) { panic("vm_object_upl_request: external object with non-zero paging offset\n"); - if (object->phys_contiguous) - panic("vm_object_upl_request: contiguous object specified\n"); + } + if (object->phys_contiguous) { + panic("vm_object_upl_request: contiguous object specified\n"); + } + VM_DEBUG_CONSTANT_EVENT(vm_object_upl_request, VM_UPL_REQUEST, DBG_FUNC_START, size, cntrl_flags, 0, 0); - if (size > MAX_UPL_SIZE_BYTES) + if (size > MAX_UPL_SIZE_BYTES) { size = MAX_UPL_SIZE_BYTES; + } - if ( (cntrl_flags & UPL_SET_INTERNAL) && page_list_count != NULL) - *page_list_count = MAX_UPL_SIZE_BYTES >> PAGE_SHIFT; + if ((cntrl_flags & UPL_SET_INTERNAL) && page_list_count != NULL) { + *page_list_count = MAX_UPL_SIZE_BYTES >> PAGE_SHIFT; + } #if CONFIG_IOSCHED || UPL_DEBUG - if (object->io_tracking || upl_debug_enabled) + if (object->io_tracking || upl_debug_enabled) { io_tracking_flag |= UPL_CREATE_IO_TRACKING; + } #endif #if CONFIG_IOSCHED - if (object->io_tracking) + if (object->io_tracking) { io_tracking_flag |= UPL_CREATE_EXPEDITE_SUP; + } #endif if (cntrl_flags & UPL_SET_INTERNAL) { - if (cntrl_flags & UPL_SET_LITE) { - + if (cntrl_flags & UPL_SET_LITE) { upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE | io_tracking_flag, 0, size); user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl)); lite_list = (wpl_array_t) - (((uintptr_t)user_page_list) + - ((size/PAGE_SIZE) * sizeof(upl_page_info_t))); + (((uintptr_t)user_page_list) + + ((size / PAGE_SIZE) * sizeof(upl_page_info_t))); if (size == 0) { user_page_list = NULL; lite_list = NULL; } } else { - upl = upl_create(UPL_CREATE_INTERNAL | io_tracking_flag, 0, size); + upl = upl_create(UPL_CREATE_INTERNAL | io_tracking_flag, 0, size); user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl)); if (size == 0) { @@ -5078,8 +4994,7 @@ vm_object_upl_request( } } } else { - if (cntrl_flags & UPL_SET_LITE) { - + if (cntrl_flags & UPL_SET_LITE) { upl = upl_create(UPL_CREATE_EXTERNAL | UPL_CREATE_LITE | io_tracking_flag, 0, size); lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl)); @@ -5087,18 +5002,19 @@ vm_object_upl_request( lite_list = NULL; } } else { - upl = upl_create(UPL_CREATE_EXTERNAL | io_tracking_flag, 0, size); + upl = upl_create(UPL_CREATE_EXTERNAL | io_tracking_flag, 0, size); } } *upl_ptr = upl; - - if (user_page_list) - user_page_list[0].device = FALSE; + + if (user_page_list) { + user_page_list[0].device = FALSE; + } if (cntrl_flags & UPL_SET_LITE) { - upl->map_object = object; + upl->map_object = object; } else { - upl->map_object = vm_object_allocate(size); + upl->map_object = vm_object_allocate(size); /* * No neeed to lock the new object: nobody else knows * about it yet, so it's all ours so far. @@ -5114,8 +5030,9 @@ vm_object_upl_request( upl->flags |= UPL_SHADOWED; } - if (cntrl_flags & UPL_FOR_PAGEOUT) + if (cntrl_flags & UPL_FOR_PAGEOUT) { upl->flags |= UPL_PAGEOUT; + } vm_object_lock(object); vm_object_activity_begin(object); @@ -5150,17 +5067,16 @@ vm_object_upl_request( * the caller modify them. */ vm_object_update(object, - offset, - size, - NULL, - NULL, - FALSE, /* should_return */ - MEMORY_OBJECT_COPY_SYNC, - VM_PROT_NO_CHANGE); -#if DEVELOPMENT || DEBUG - upl_cow++; - upl_cow_pages += size >> PAGE_SHIFT; -#endif + offset, + size, + NULL, + NULL, + FALSE, /* should_return */ + MEMORY_OBJECT_COPY_SYNC, + VM_PROT_NO_CHANGE); + + VM_PAGEOUT_DEBUG(upl_cow, 1); + VM_PAGEOUT_DEBUG(upl_cow_pages, (size >> PAGE_SHIFT)); } /* * remember which copy object we synchronized with @@ -5177,11 +5093,12 @@ vm_object_upl_request( dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT); if (vm_page_free_count > (vm_page_free_target + size_in_pages) || - object->resident_page_count < ((MAX_UPL_SIZE_BYTES * 2) >> PAGE_SHIFT)) + object->resident_page_count < ((MAX_UPL_SIZE_BYTES * 2) >> PAGE_SHIFT)) { object->scan_collisions = 0; + } if ((cntrl_flags & UPL_WILL_MODIFY) && must_throttle_writes() == TRUE) { - boolean_t isSSD = FALSE; + boolean_t isSSD = FALSE; #if CONFIG_EMBEDDED isSSD = TRUE; @@ -5189,20 +5106,20 @@ vm_object_upl_request( vnode_pager_get_isSSD(object->pager, &isSSD); #endif vm_object_unlock(object); - + OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages); - if (isSSD == TRUE) + if (isSSD == TRUE) { delay(1000 * size_in_pages); - else + } else { delay(5000 * size_in_pages); + } OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages); vm_object_lock(object); } while (xfer_size) { - dwp->dw_mask = 0; if ((alias_page == NULL) && !(cntrl_flags & UPL_SET_LITE)) { @@ -5211,17 +5128,17 @@ vm_object_upl_request( vm_object_lock(object); } if (cntrl_flags & UPL_COPYOUT_FROM) { - upl->flags |= UPL_PAGE_SYNC_DONE; - - if ( ((dst_page = vm_page_lookup(object, dst_offset)) == VM_PAGE_NULL) || - dst_page->fictitious || - dst_page->absent || - dst_page->error || - dst_page->cleaning || - (VM_PAGE_WIRED(dst_page))) { - - if (user_page_list) + upl->flags |= UPL_PAGE_SYNC_DONE; + + if (((dst_page = vm_page_lookup(object, dst_offset)) == VM_PAGE_NULL) || + dst_page->vmp_fictitious || + dst_page->vmp_absent || + dst_page->vmp_error || + dst_page->vmp_cleaning || + (VM_PAGE_WIRED(dst_page))) { + if (user_page_list) { user_page_list[entry].phys_addr = 0; + } goto try_next_page; } @@ -5234,13 +5151,14 @@ vm_object_upl_request( * anyway... so we can eliminate an extra call into * the pmap layer by grabbing it here and recording it */ - if (dst_page->pmapped) - refmod_state = pmap_get_refmod(phys_page); - else - refmod_state = 0; + if (dst_page->vmp_pmapped) { + refmod_state = pmap_get_refmod(phys_page); + } else { + refmod_state = 0; + } - if ( (refmod_state & VM_MEM_REFERENCED) && VM_PAGE_INACTIVE(dst_page)) { - /* + if ((refmod_state & VM_MEM_REFERENCED) && VM_PAGE_INACTIVE(dst_page)) { + /* * page is on inactive list and referenced... * reactivate it now... this gets it out of the * way of vm_pageout_scan which would have to @@ -5249,19 +5167,20 @@ vm_object_upl_request( dwp->dw_mask |= DW_vm_page_activate; } if (cntrl_flags & UPL_RET_ONLY_DIRTY) { - /* + /* * we're only asking for DIRTY pages to be returned */ - if (dst_page->laundry || !(cntrl_flags & UPL_FOR_PAGEOUT)) { - /* + if (dst_page->vmp_laundry || !(cntrl_flags & UPL_FOR_PAGEOUT)) { + /* * if we were the page stolen by vm_pageout_scan to be - * cleaned (as opposed to a buddy being clustered in + * cleaned (as opposed to a buddy being clustered in * or this request is not being driven by a PAGEOUT cluster * then we only need to check for the page being dirty or * precious to decide whether to return it */ - if (dst_page->dirty || dst_page->precious || (refmod_state & VM_MEM_MODIFIED)) - goto check_busy; + if (dst_page->vmp_dirty || dst_page->vmp_precious || (refmod_state & VM_MEM_MODIFIED)) { + goto check_busy; + } goto dont_return; } /* @@ -5270,18 +5189,18 @@ vm_object_upl_request( * does it have to be dirty to be returned, but it also * can't have been referenced recently... */ - if ( (hibernate_cleaning_in_progress == TRUE || - (!((refmod_state & VM_MEM_REFERENCED) || dst_page->reference) || - (dst_page->vm_page_q_state == VM_PAGE_ON_THROTTLED_Q))) && - ((refmod_state & VM_MEM_MODIFIED) || dst_page->dirty || dst_page->precious) ) { - goto check_busy; + if ((hibernate_cleaning_in_progress == TRUE || + (!((refmod_state & VM_MEM_REFERENCED) || dst_page->vmp_reference) || + (dst_page->vmp_q_state == VM_PAGE_ON_THROTTLED_Q))) && + ((refmod_state & VM_MEM_MODIFIED) || dst_page->vmp_dirty || dst_page->vmp_precious)) { + goto check_busy; } dont_return: /* * if we reach here, we're not to return * the page... go on to the next one */ - if (dst_page->laundry == TRUE) { + if (dst_page->vmp_laundry == TRUE) { /* * if we get here, the page is not 'cleaning' (filtered out above). * since it has been referenced, remove it from the laundry @@ -5292,19 +5211,21 @@ dont_return: vm_pageout_steal_laundry(dst_page, TRUE); vm_page_activate(dst_page); - + vm_page_unlock_queues(); } - if (user_page_list) - user_page_list[entry].phys_addr = 0; + if (user_page_list) { + user_page_list[entry].phys_addr = 0; + } goto try_next_page; } -check_busy: - if (dst_page->busy) { - if (cntrl_flags & UPL_NOBLOCK) { - if (user_page_list) - user_page_list[entry].phys_addr = 0; +check_busy: + if (dst_page->vmp_busy) { + if (cntrl_flags & UPL_NOBLOCK) { + if (user_page_list) { + user_page_list[entry].phys_addr = 0; + } dwp->dw_mask = 0; goto try_next_page; @@ -5317,11 +5238,10 @@ check_busy: continue; } - if (dst_page->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) { - + if (dst_page->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) { vm_page_lockspin_queues(); - if (dst_page->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) { + if (dst_page->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) { /* * we've buddied up a page for a clustered pageout * that has already been moved to the pageout @@ -5333,71 +5253,66 @@ check_busy: } vm_page_unlock_queues(); } -#if MACH_CLUSTER_STATS - /* - * pageout statistics gathering. count - * all the pages we will page out that - * were not counted in the initial - * vm_pageout_scan work - */ - if (dst_page->pageout) - encountered_lrp = TRUE; - if ((dst_page->dirty || (object->internal && dst_page->precious))) { - if (encountered_lrp) - CLUSTER_STAT(pages_at_higher_offsets++;) - else - CLUSTER_STAT(pages_at_lower_offsets++;) - } -#endif hw_dirty = refmod_state & VM_MEM_MODIFIED; - dirty = hw_dirty ? TRUE : dst_page->dirty; + dirty = hw_dirty ? TRUE : dst_page->vmp_dirty; - if (phys_page > upl->highest_page) - upl->highest_page = phys_page; + if (phys_page > upl->highest_page) { + upl->highest_page = phys_page; + } - assert (!pmap_is_noencrypt(phys_page)); + assert(!pmap_is_noencrypt(phys_page)); if (cntrl_flags & UPL_SET_LITE) { - unsigned int pg_num; + unsigned int pg_num; - pg_num = (unsigned int) ((dst_offset-offset)/PAGE_SIZE); - assert(pg_num == (dst_offset-offset)/PAGE_SIZE); - lite_list[pg_num>>5] |= 1 << (pg_num & 31); + pg_num = (unsigned int) ((dst_offset - offset) / PAGE_SIZE); + assert(pg_num == (dst_offset - offset) / PAGE_SIZE); + lite_list[pg_num >> 5] |= 1 << (pg_num & 31); - if (hw_dirty) - pmap_clear_modify(phys_page); + if (hw_dirty) { + if (pmap_flushes_delayed == FALSE) { + pmap_flush_context_init(&pmap_flush_context_storage); + pmap_flushes_delayed = TRUE; + } + pmap_clear_refmod_options(phys_page, + VM_MEM_MODIFIED, + PMAP_OPTIONS_NOFLUSH | PMAP_OPTIONS_CLEAR_WRITE, + &pmap_flush_context_storage); + } /* - * Mark original page as cleaning + * Mark original page as cleaning * in place. */ - dst_page->cleaning = TRUE; - dst_page->precious = FALSE; + dst_page->vmp_cleaning = TRUE; + dst_page->vmp_precious = FALSE; } else { - /* + /* * use pageclean setup, it is more * convenient even for the pageout * cases here */ - vm_object_lock(upl->map_object); + vm_object_lock(upl->map_object); vm_pageclean_setup(dst_page, alias_page, upl->map_object, size - xfer_size); vm_object_unlock(upl->map_object); - alias_page->absent = FALSE; + alias_page->vmp_absent = FALSE; alias_page = NULL; } if (dirty) { SET_PAGE_DIRTY(dst_page, FALSE); } else { - dst_page->dirty = FALSE; + dst_page->vmp_dirty = FALSE; } - if (!dirty) - dst_page->precious = TRUE; + if (!dirty) { + dst_page->vmp_precious = TRUE; + } - if ( !(cntrl_flags & UPL_CLEAN_IN_PLACE) ) { - if ( !VM_PAGE_WIRED(dst_page)) - dst_page->free_when_done = TRUE; + if (!(cntrl_flags & UPL_CLEAN_IN_PLACE)) { + if (!VM_PAGE_WIRED(dst_page)) { + dst_page->vmp_free_when_done = TRUE; + } } } else { if ((cntrl_flags & UPL_WILL_MODIFY) && object->copy != last_copy_object) { @@ -5429,14 +5344,12 @@ check_busy: xfer_size, /* remaining size */ NULL, NULL, - FALSE, /* should_return */ + FALSE, /* should_return */ MEMORY_OBJECT_COPY_SYNC, VM_PROT_NO_CHANGE); -#if DEVELOPMENT || DEBUG - upl_cow_again++; - upl_cow_again_pages += xfer_size >> PAGE_SHIFT; -#endif + VM_PAGEOUT_DEBUG(upl_cow_again, 1); + VM_PAGEOUT_DEBUG(upl_cow_again_pages, (xfer_size >> PAGE_SHIFT)); } /* * remember the copy object we synced with @@ -5444,23 +5357,23 @@ check_busy: last_copy_object = object->copy; } dst_page = vm_page_lookup(object, dst_offset); - - if (dst_page != VM_PAGE_NULL) { + if (dst_page != VM_PAGE_NULL) { if ((cntrl_flags & UPL_RET_ONLY_ABSENT)) { /* * skip over pages already present in the cache */ - if (user_page_list) + if (user_page_list) { user_page_list[entry].phys_addr = 0; + } goto try_next_page; } - if (dst_page->fictitious) { + if (dst_page->vmp_fictitious) { panic("need corner case for fictitious page"); } - if (dst_page->busy || dst_page->cleaning) { + if (dst_page->vmp_busy || dst_page->vmp_cleaning) { /* * someone else is playing with the * page. We will have to wait. @@ -5469,21 +5382,23 @@ check_busy: continue; } - if (dst_page->laundry) + if (dst_page->vmp_laundry) { vm_pageout_steal_laundry(dst_page, FALSE); + } } else { if (object->private) { - /* - * This is a nasty wrinkle for users - * of upl who encounter device or - * private memory however, it is + /* + * This is a nasty wrinkle for users + * of upl who encounter device or + * private memory however, it is * unavoidable, only a fault can * resolve the actual backing * physical page by asking the * backing device. */ - if (user_page_list) + if (user_page_list) { user_page_list[entry].phys_addr = 0; + } goto try_next_page; } @@ -5501,9 +5416,10 @@ check_busy: */ dst_page = vm_object_page_grab(object); - if (dst_page != VM_PAGE_NULL) + if (dst_page != VM_PAGE_NULL) { vm_page_release(dst_page, - FALSE); + FALSE); + } dst_page = vm_object_page_grab(object); } @@ -5512,26 +5428,30 @@ check_busy: * need to allocate a page */ dst_page = vm_page_grab_options(grab_options); + if (dst_page != VM_PAGE_NULL) { + page_grab_count++; + } } if (dst_page == VM_PAGE_NULL) { - if ( (cntrl_flags & (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) == (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) { - /* - * we don't want to stall waiting for pages to come onto the free list - * while we're already holding absent pages in this UPL - * the caller will deal with the empty slots - */ - if (user_page_list) - user_page_list[entry].phys_addr = 0; + if ((cntrl_flags & (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) == (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) { + /* + * we don't want to stall waiting for pages to come onto the free list + * while we're already holding absent pages in this UPL + * the caller will deal with the empty slots + */ + if (user_page_list) { + user_page_list[entry].phys_addr = 0; + } goto try_next_page; } - /* + /* * no pages available... wait * then try again for the same * offset... */ vm_object_unlock(object); - + OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages); VM_DEBUG_EVENT(vm_upl_page_wait, VM_UPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0); @@ -5547,70 +5467,74 @@ check_busy: } vm_page_insert(dst_page, object, dst_offset); - dst_page->absent = TRUE; - dst_page->busy = FALSE; + dst_page->vmp_absent = TRUE; + dst_page->vmp_busy = FALSE; if (cntrl_flags & UPL_RET_ONLY_ABSENT) { - /* + /* * if UPL_RET_ONLY_ABSENT was specified, * than we're definitely setting up a - * upl for a clustered read/pagein + * upl for a clustered read/pagein * operation... mark the pages as clustered * so upl_commit_range can put them on the * speculative list */ - dst_page->clustered = TRUE; + dst_page->vmp_clustered = TRUE; - if ( !(cntrl_flags & UPL_FILE_IO)) + if (!(cntrl_flags & UPL_FILE_IO)) { VM_STAT_INCR(pageins); + } } } phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page); - dst_page->overwriting = TRUE; + dst_page->vmp_overwriting = TRUE; - if (dst_page->pmapped) { - if ( !(cntrl_flags & UPL_FILE_IO)) - /* + if (dst_page->vmp_pmapped) { + if (!(cntrl_flags & UPL_FILE_IO)) { + /* * eliminate all mappings from the * original object and its prodigy */ - refmod_state = pmap_disconnect(phys_page); - else - refmod_state = pmap_get_refmod(phys_page); - } else - refmod_state = 0; + refmod_state = pmap_disconnect(phys_page); + } else { + refmod_state = pmap_get_refmod(phys_page); + } + } else { + refmod_state = 0; + } hw_dirty = refmod_state & VM_MEM_MODIFIED; - dirty = hw_dirty ? TRUE : dst_page->dirty; + dirty = hw_dirty ? TRUE : dst_page->vmp_dirty; if (cntrl_flags & UPL_SET_LITE) { - unsigned int pg_num; + unsigned int pg_num; - pg_num = (unsigned int) ((dst_offset-offset)/PAGE_SIZE); - assert(pg_num == (dst_offset-offset)/PAGE_SIZE); - lite_list[pg_num>>5] |= 1 << (pg_num & 31); + pg_num = (unsigned int) ((dst_offset - offset) / PAGE_SIZE); + assert(pg_num == (dst_offset - offset) / PAGE_SIZE); + lite_list[pg_num >> 5] |= 1 << (pg_num & 31); - if (hw_dirty) - pmap_clear_modify(phys_page); + if (hw_dirty) { + pmap_clear_modify(phys_page); + } /* - * Mark original page as cleaning + * Mark original page as cleaning * in place. */ - dst_page->cleaning = TRUE; - dst_page->precious = FALSE; + dst_page->vmp_cleaning = TRUE; + dst_page->vmp_precious = FALSE; } else { /* * use pageclean setup, it is more * convenient even for the pageout * cases here */ - vm_object_lock(upl->map_object); + vm_object_lock(upl->map_object); vm_pageclean_setup(dst_page, alias_page, upl->map_object, size - xfer_size); - vm_object_unlock(upl->map_object); + vm_object_unlock(upl->map_object); - alias_page->absent = FALSE; + alias_page->vmp_absent = FALSE; alias_page = NULL; } @@ -5630,27 +5554,29 @@ check_busy: */ upl->flags |= UPL_CLEAR_DIRTY; } - dst_page->dirty = dirty; + dst_page->vmp_dirty = dirty; - if (!dirty) - dst_page->precious = TRUE; + if (!dirty) { + dst_page->vmp_precious = TRUE; + } - if ( !VM_PAGE_WIRED(dst_page)) { - /* + if (!VM_PAGE_WIRED(dst_page)) { + /* * deny access to the target page while * it is being worked on */ - dst_page->busy = TRUE; - } else + dst_page->vmp_busy = TRUE; + } else { dwp->dw_mask |= DW_vm_page_wire; + } /* * We might be about to satisfy a fault which has been * requested. So no need for the "restart" bit. */ - dst_page->restart = FALSE; - if (!dst_page->absent && !(cntrl_flags & UPL_WILL_MODIFY)) { - /* + dst_page->vmp_restart = FALSE; + if (!dst_page->vmp_absent && !(cntrl_flags & UPL_WILL_MODIFY)) { + /* * expect the page to be used */ dwp->dw_mask |= DW_set_reference; @@ -5658,57 +5584,62 @@ check_busy: if (cntrl_flags & UPL_PRECIOUS) { if (object->internal) { SET_PAGE_DIRTY(dst_page, FALSE); - dst_page->precious = FALSE; + dst_page->vmp_precious = FALSE; } else { - dst_page->precious = TRUE; + dst_page->vmp_precious = TRUE; } } else { - dst_page->precious = FALSE; + dst_page->vmp_precious = FALSE; } } - if (dst_page->busy) + if (dst_page->vmp_busy) { upl->flags |= UPL_HAS_BUSY; + } - if (phys_page > upl->highest_page) - upl->highest_page = phys_page; - assert (!pmap_is_noencrypt(phys_page)); + if (phys_page > upl->highest_page) { + upl->highest_page = phys_page; + } + assert(!pmap_is_noencrypt(phys_page)); if (user_page_list) { user_page_list[entry].phys_addr = phys_page; - user_page_list[entry].free_when_done = dst_page->free_when_done; - user_page_list[entry].absent = dst_page->absent; - user_page_list[entry].dirty = dst_page->dirty; - user_page_list[entry].precious = dst_page->precious; - user_page_list[entry].device = FALSE; + user_page_list[entry].free_when_done = dst_page->vmp_free_when_done; + user_page_list[entry].absent = dst_page->vmp_absent; + user_page_list[entry].dirty = dst_page->vmp_dirty; + user_page_list[entry].precious = dst_page->vmp_precious; + user_page_list[entry].device = FALSE; user_page_list[entry].needed = FALSE; - if (dst_page->clustered == TRUE) - user_page_list[entry].speculative = (dst_page->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE; - else - user_page_list[entry].speculative = FALSE; - user_page_list[entry].cs_validated = dst_page->cs_validated; - user_page_list[entry].cs_tainted = dst_page->cs_tainted; - user_page_list[entry].cs_nx = dst_page->cs_nx; + if (dst_page->vmp_clustered == TRUE) { + user_page_list[entry].speculative = (dst_page->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE; + } else { + user_page_list[entry].speculative = FALSE; + } + user_page_list[entry].cs_validated = dst_page->vmp_cs_validated; + user_page_list[entry].cs_tainted = dst_page->vmp_cs_tainted; + user_page_list[entry].cs_nx = dst_page->vmp_cs_nx; user_page_list[entry].mark = FALSE; } - /* + /* * if UPL_RET_ONLY_ABSENT is set, then * we are working with a fresh page and we've * just set the clustered flag on it to * indicate that it was drug in as part of a * speculative cluster... so leave it alone */ - if ( !(cntrl_flags & UPL_RET_ONLY_ABSENT)) { - /* + if (!(cntrl_flags & UPL_RET_ONLY_ABSENT)) { + /* * someone is explicitly grabbing this page... * update clustered and speculative state - * + * */ - if (dst_page->clustered) + if (dst_page->vmp_clustered) { VM_PAGE_CONSUME_CLUSTERED(dst_page); + } } try_next_page: if (dwp->dw_mask) { - if (dwp->dw_mask & DW_vm_page_activate) + if (dwp->dw_mask & DW_vm_page_activate) { VM_STAT_INCR(reactivations); + } VM_PAGE_ADD_DELAYED_WORK(dwp, dst_page, dw_count); @@ -5723,30 +5654,42 @@ try_next_page: dst_offset += PAGE_SIZE_64; xfer_size -= PAGE_SIZE; } - if (dw_count) + if (dw_count) { vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count); + } if (alias_page != NULL) { VM_PAGE_FREE(alias_page); } + if (pmap_flushes_delayed == TRUE) { + pmap_flush(&pmap_flush_context_storage); + } if (page_list_count != NULL) { - if (upl->flags & UPL_INTERNAL) + if (upl->flags & UPL_INTERNAL) { *page_list_count = 0; - else if (*page_list_count > entry) + } else if (*page_list_count > entry) { *page_list_count = entry; + } } #if UPL_DEBUG upl->upl_state = 1; #endif vm_object_unlock(object); + VM_DEBUG_CONSTANT_EVENT(vm_object_upl_request, VM_UPL_REQUEST, DBG_FUNC_END, page_grab_count, 0, 0, 0); +#if DEVELOPMENT || DEBUG + if (task != NULL) { + ledger_credit(task->ledger, task_ledgers.pages_grabbed_upl, page_grab_count); + } +#endif /* DEVELOPMENT || DEBUG */ + return KERN_SUCCESS; } -/* +/* * Routine: vm_object_super_upl_request - * Purpose: + * Purpose: * Cause the population of a portion of a vm_object * in much the same way as memory_object_upl_request. * Depending on the nature of the request, the pages @@ -5758,38 +5701,38 @@ try_next_page: __private_extern__ kern_return_t vm_object_super_upl_request( vm_object_t object, - vm_object_offset_t offset, - upl_size_t size, - upl_size_t super_cluster, - upl_t *upl, - upl_page_info_t *user_page_list, - unsigned int *page_list_count, - upl_control_flags_t cntrl_flags, - vm_tag_t tag) + vm_object_offset_t offset, + upl_size_t size, + upl_size_t super_cluster, + upl_t *upl, + upl_page_info_t *user_page_list, + unsigned int *page_list_count, + upl_control_flags_t cntrl_flags, + vm_tag_t tag) { - if (object->paging_offset > offset || ((cntrl_flags & UPL_VECTOR)==UPL_VECTOR)) + if (object->paging_offset > offset || ((cntrl_flags & UPL_VECTOR) == UPL_VECTOR)) { return KERN_FAILURE; + } assert(object->paging_in_progress); offset = offset - object->paging_offset; if (super_cluster > size) { - - vm_object_offset_t base_offset; - upl_size_t super_size; - vm_object_size_t super_size_64; + vm_object_offset_t base_offset; + upl_size_t super_size; + vm_object_size_t super_size_64; base_offset = (offset & ~((vm_object_offset_t) super_cluster - 1)); - super_size = (offset + size) > (base_offset + super_cluster) ? super_cluster<<1 : super_cluster; + super_size = (offset + size) > (base_offset + super_cluster) ? super_cluster << 1 : super_cluster; super_size_64 = ((base_offset + super_size) > object->vo_size) ? (object->vo_size - base_offset) : super_size; super_size = (upl_size_t) super_size_64; assert(super_size == super_size_64); if (offset > (base_offset + super_size)) { - panic("vm_object_super_upl_request: Missed target pageout" - " %#llx,%#llx, %#x, %#x, %#x, %#llx\n", - offset, base_offset, super_size, super_cluster, - size, object->paging_offset); + panic("vm_object_super_upl_request: Missed target pageout" + " %#llx,%#llx, %#x, %#x, %#x, %#llx\n", + offset, base_offset, super_size, super_cluster, + size, object->paging_offset); } /* * apparently there is a case where the vm requests a @@ -5797,7 +5740,7 @@ vm_object_super_upl_request( * object size */ if ((offset + size) > (base_offset + super_size)) { - super_size_64 = (offset + size) - base_offset; + super_size_64 = (offset + size) - base_offset; super_size = (upl_size_t) super_size_64; assert(super_size == super_size_64); } @@ -5816,23 +5759,23 @@ extern char *proc_name_address(void *p); kern_return_t vm_map_create_upl( - vm_map_t map, - vm_map_address_t offset, - upl_size_t *upl_size, - upl_t *upl, - upl_page_info_array_t page_list, - unsigned int *count, - upl_control_flags_t *flags, - vm_tag_t tag) + vm_map_t map, + vm_map_address_t offset, + upl_size_t *upl_size, + upl_t *upl, + upl_page_info_array_t page_list, + unsigned int *count, + upl_control_flags_t *flags, + vm_tag_t tag) { - vm_map_entry_t entry; - upl_control_flags_t caller_flags; - int force_data_sync; - int sync_cow_data; - vm_object_t local_object; - vm_map_offset_t local_offset; - vm_map_offset_t local_start; - kern_return_t ret; + vm_map_entry_t entry; + upl_control_flags_t caller_flags; + int force_data_sync; + int sync_cow_data; + vm_object_t local_object; + vm_map_offset_t local_offset; + vm_map_offset_t local_start; + kern_return_t ret; assert(page_aligned(offset)); @@ -5848,8 +5791,9 @@ vm_map_create_upl( force_data_sync = (caller_flags & UPL_FORCE_DATA_SYNC); sync_cow_data = !(caller_flags & UPL_COPYOUT_FROM); - if (upl == NULL) + if (upl == NULL) { return KERN_INVALID_ARGUMENT; + } REDISCOVER_ENTRY: vm_map_lock_read(map); @@ -5869,11 +5813,13 @@ REDISCOVER_ENTRY: if (!entry->is_sub_map && VME_OBJECT(entry) != VM_OBJECT_NULL) { - if (VME_OBJECT(entry)->private) + if (VME_OBJECT(entry)->private) { *flags = UPL_DEV_MEMORY; + } - if (VME_OBJECT(entry)->phys_contiguous) + if (VME_OBJECT(entry)->phys_contiguous) { *flags |= UPL_PHYS_CONTIG; + } } vm_map_unlock_read(map); return KERN_SUCCESS; @@ -5881,22 +5827,23 @@ REDISCOVER_ENTRY: if (VME_OBJECT(entry) == VM_OBJECT_NULL || !VME_OBJECT(entry)->phys_contiguous) { - if (*upl_size > MAX_UPL_SIZE_BYTES) + if (*upl_size > MAX_UPL_SIZE_BYTES) { *upl_size = MAX_UPL_SIZE_BYTES; + } } /* * Create an object if necessary. */ if (VME_OBJECT(entry) == VM_OBJECT_NULL) { - - if (vm_map_lock_read_to_write(map)) + if (vm_map_lock_read_to_write(map)) { goto REDISCOVER_ENTRY; + } VME_OBJECT_SET(entry, - vm_object_allocate((vm_size_t) - (entry->vme_end - - entry->vme_start))); + vm_object_allocate((vm_size_t) + (entry->vme_end - + entry->vme_start))); VME_OFFSET_SET(entry, 0); assert(entry->use_pmap); @@ -5904,6 +5851,7 @@ REDISCOVER_ENTRY: } if (!(caller_flags & UPL_COPYOUT_FROM) && + !entry->is_sub_map && !(entry->protection & VM_PROT_WRITE)) { vm_map_unlock_read(map); return KERN_PROTECTION_FAILURE; @@ -5914,8 +5862,8 @@ REDISCOVER_ENTRY: (caller_flags & UPL_COPYOUT_FROM) && (entry->protection & VM_PROT_EXECUTE) && !(entry->protection & VM_PROT_WRITE)) { - vm_offset_t kaddr; - vm_size_t ksize; + vm_offset_t kaddr; + vm_size_t ksize; /* * We're about to create a read-only UPL backed by @@ -5935,9 +5883,9 @@ REDISCOVER_ENTRY: ksize = round_page(*upl_size); kaddr = 0; ret = kmem_alloc_pageable(kernel_map, - &kaddr, - ksize, - tag); + &kaddr, + ksize, + tag); if (ret == KERN_SUCCESS) { /* copyin the user data */ assert(page_aligned(offset)); @@ -5947,12 +5895,12 @@ REDISCOVER_ENTRY: if (ksize > *upl_size) { /* zero out the extra space in kernel buffer */ memset((void *)(kaddr + *upl_size), - 0, - ksize - *upl_size); + 0, + ksize - *upl_size); } /* create the UPL from the kernel buffer */ ret = vm_map_create_upl(kernel_map, kaddr, upl_size, - upl, page_list, count, flags, tag); + upl, page_list, count, flags, tag); } if (kaddr != 0) { /* free the kernel buffer */ @@ -5962,10 +5910,10 @@ REDISCOVER_ENTRY: } #if DEVELOPMENT || DEBUG DTRACE_VM4(create_upl_from_executable, - vm_map_t, map, - vm_map_address_t, offset, - upl_size_t, *upl_size, - kern_return_t, ret); + vm_map_t, map, + vm_map_address_t, offset, + upl_size_t, *upl_size, + kern_return_t, ret); #endif /* DEVELOPMENT || DEBUG */ return ret; } @@ -5981,12 +5929,12 @@ REDISCOVER_ENTRY: entry->wired_count == 0 && /* No COW for entries that are wired */ (map->pmap != kernel_pmap) && /* alias checks */ (vm_map_entry_should_cow_for_true_share(entry) /* case 1 */ - || - (/* case 2 */ - local_object->internal && - (local_object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) && - local_object->ref_count > 1))) { - vm_prot_t prot; + || + ( /* case 2 */ + local_object->internal && + (local_object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) && + local_object->ref_count > 1))) { + vm_prot_t prot; /* * Case 1: @@ -6011,30 +5959,31 @@ REDISCOVER_ENTRY: assert(VME_OBJECT(entry) == local_object); vm_map_clip_start(map, - entry, - vm_map_trunc_page(offset, - VM_MAP_PAGE_MASK(map))); + entry, + vm_map_trunc_page(offset, + VM_MAP_PAGE_MASK(map))); vm_map_clip_end(map, - entry, - vm_map_round_page(offset + *upl_size, - VM_MAP_PAGE_MASK(map))); + entry, + vm_map_round_page(offset + *upl_size, + VM_MAP_PAGE_MASK(map))); if ((entry->vme_end - offset) < *upl_size) { *upl_size = (upl_size_t) (entry->vme_end - offset); assert(*upl_size == entry->vme_end - offset); } prot = entry->protection & ~VM_PROT_WRITE; - if (override_nx(map, VME_ALIAS(entry)) && prot) + if (override_nx(map, VME_ALIAS(entry)) && prot) { prot |= VM_PROT_EXECUTE; + } vm_object_pmap_protect(local_object, - VME_OFFSET(entry), - entry->vme_end - entry->vme_start, - ((entry->is_shared || - map->mapped_in_other_pmaps) - ? PMAP_NULL - : map->pmap), - entry->vme_start, - prot); + VME_OFFSET(entry), + entry->vme_end - entry->vme_start, + ((entry->is_shared || + map->mapped_in_other_pmaps) + ? PMAP_NULL + : map->pmap), + entry->vme_start, + prot); assert(entry->wired_count == 0); @@ -6047,11 +5996,11 @@ REDISCOVER_ENTRY: if (local_object->true_share) { /* object is already in proper state: no COW needed */ assert(local_object->copy_strategy != - MEMORY_OBJECT_COPY_SYMMETRIC); + MEMORY_OBJECT_COPY_SYMMETRIC); } else { /* not true_share: ask for copy-on-write below */ assert(local_object->copy_strategy == - MEMORY_OBJECT_COPY_SYMMETRIC); + MEMORY_OBJECT_COPY_SYMMETRIC); entry->needs_copy = TRUE; } vm_object_unlock(local_object); @@ -6059,19 +6008,19 @@ REDISCOVER_ENTRY: vm_map_lock_write_to_read(map); } - if (entry->needs_copy) { + if (entry->needs_copy) { /* * Honor copy-on-write for COPY_SYMMETRIC * strategy. */ - vm_map_t local_map; - vm_object_t object; - vm_object_offset_t new_offset; - vm_prot_t prot; - boolean_t wired; - vm_map_version_t version; - vm_map_t real_map; - vm_prot_t fault_type; + vm_map_t local_map; + vm_object_t object; + vm_object_offset_t new_offset; + vm_prot_t prot; + boolean_t wired; + vm_map_version_t version; + vm_map_t real_map; + vm_prot_t fault_type; local_map = map; @@ -6079,17 +6028,17 @@ REDISCOVER_ENTRY: fault_type = VM_PROT_READ | VM_PROT_COPY; vm_counters.create_upl_extra_cow++; vm_counters.create_upl_extra_cow_pages += - (entry->vme_end - entry->vme_start) / PAGE_SIZE; + (entry->vme_end - entry->vme_start) / PAGE_SIZE; } else { fault_type = VM_PROT_WRITE; } if (vm_map_lookup_locked(&local_map, - offset, fault_type, - OBJECT_LOCK_EXCLUSIVE, - &version, &object, - &new_offset, &prot, &wired, - NULL, - &real_map) != KERN_SUCCESS) { + offset, fault_type, + OBJECT_LOCK_EXCLUSIVE, + &version, &object, + &new_offset, &prot, &wired, + NULL, + &real_map) != KERN_SUCCESS) { if (fault_type == VM_PROT_WRITE) { vm_counters.create_upl_lookup_failure_write++; } else { @@ -6098,8 +6047,9 @@ REDISCOVER_ENTRY: vm_map_unlock_read(local_map); return KERN_FAILURE; } - if (real_map != map) + if (real_map != map) { vm_map_unlock(real_map); + } vm_map_unlock_read(local_map); vm_object_unlock(object); @@ -6108,7 +6058,7 @@ REDISCOVER_ENTRY: } if (entry->is_sub_map) { - vm_map_t submap; + vm_map_t submap; submap = VME_SUBMAP(entry); local_start = entry->vme_start; @@ -6117,9 +6067,9 @@ REDISCOVER_ENTRY: vm_map_reference(submap); vm_map_unlock_read(map); - ret = vm_map_create_upl(submap, - local_offset + (offset - local_start), - upl_size, upl, page_list, count, flags, tag); + ret = vm_map_create_upl(submap, + local_offset + (offset - local_start), + upl_size, upl, page_list, count, flags, tag); vm_map_deallocate(submap); return ret; @@ -6127,7 +6077,7 @@ REDISCOVER_ENTRY: if (sync_cow_data && (VME_OBJECT(entry)->shadow || - VME_OBJECT(entry)->copy)) { + VME_OBJECT(entry)->copy)) { local_object = VME_OBJECT(entry); local_start = entry->vme_start; local_offset = VME_OFFSET(entry); @@ -6137,13 +6087,13 @@ REDISCOVER_ENTRY: if (local_object->shadow && local_object->copy) { vm_object_lock_request(local_object->shadow, - ((vm_object_offset_t) - ((offset - local_start) + - local_offset) + - local_object->vo_shadow_offset), - *upl_size, FALSE, - MEMORY_OBJECT_DATA_SYNC, - VM_PROT_NO_CHANGE); + ((vm_object_offset_t) + ((offset - local_start) + + local_offset) + + local_object->vo_shadow_offset), + *upl_size, FALSE, + MEMORY_OBJECT_DATA_SYNC, + VM_PROT_NO_CHANGE); } sync_cow_data = FALSE; vm_object_deallocate(local_object); @@ -6159,26 +6109,28 @@ REDISCOVER_ENTRY: vm_map_unlock_read(map); vm_object_lock_request(local_object, - ((vm_object_offset_t) - ((offset - local_start) + - local_offset)), - (vm_object_size_t)*upl_size, - FALSE, - MEMORY_OBJECT_DATA_SYNC, - VM_PROT_NO_CHANGE); + ((vm_object_offset_t) + ((offset - local_start) + + local_offset)), + (vm_object_size_t)*upl_size, + FALSE, + MEMORY_OBJECT_DATA_SYNC, + VM_PROT_NO_CHANGE); force_data_sync = FALSE; vm_object_deallocate(local_object); goto REDISCOVER_ENTRY; } - if (VME_OBJECT(entry)->private) + if (VME_OBJECT(entry)->private) { *flags = UPL_DEV_MEMORY; - else + } else { *flags = 0; + } - if (VME_OBJECT(entry)->phys_contiguous) + if (VME_OBJECT(entry)->phys_contiguous) { *flags |= UPL_PHYS_CONTIG; + } local_object = VME_OBJECT(entry); local_offset = VME_OFFSET(entry); @@ -6194,18 +6146,18 @@ REDISCOVER_ENTRY: if (entry->protection & VM_PROT_EXECUTE) { #if MACH_ASSERT printf("pid %d[%s] create_upl out of executable range from " - "0x%llx to 0x%llx: side effects may include " - "code-signing violations later on\n", - proc_selfpid(), - (current_task()->bsd_info - ? proc_name_address(current_task()->bsd_info) - : "?"), - (uint64_t) entry->vme_start, - (uint64_t) entry->vme_end); + "0x%llx to 0x%llx: side effects may include " + "code-signing violations later on\n", + proc_selfpid(), + (current_task()->bsd_info + ? proc_name_address(current_task()->bsd_info) + : "?"), + (uint64_t) entry->vme_start, + (uint64_t) entry->vme_end); #endif /* MACH_ASSERT */ DTRACE_VM2(cs_executable_create_upl, - uint64_t, (uint64_t)entry->vme_start, - uint64_t, (uint64_t)entry->vme_end); + uint64_t, (uint64_t)entry->vme_start, + uint64_t, (uint64_t)entry->vme_end); cs_executable_create_upl++; } #endif /* CONFIG_EMBEDDED */ @@ -6221,22 +6173,22 @@ REDISCOVER_ENTRY: */ if (local_object->true_share) { assert(local_object->copy_strategy != - MEMORY_OBJECT_COPY_SYMMETRIC); + MEMORY_OBJECT_COPY_SYMMETRIC); } else if (local_object != kernel_object && - local_object != compressor_object && - !local_object->phys_contiguous) { + local_object != compressor_object && + !local_object->phys_contiguous) { #if VM_OBJECT_TRACKING_OP_TRUESHARE if (!local_object->true_share && vm_object_tracking_inited) { void *bt[VM_OBJECT_TRACKING_BTDEPTH]; int num = 0; num = OSBacktrace(bt, - VM_OBJECT_TRACKING_BTDEPTH); + VM_OBJECT_TRACKING_BTDEPTH); btlog_add_entry(vm_object_tracking_btlog, - local_object, - VM_OBJECT_TRACKING_OP_TRUESHARE, - bt, - num); + local_object, + VM_OBJECT_TRACKING_OP_TRUESHARE, + bt, + num); } #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */ local_object->true_share = TRUE; @@ -6251,15 +6203,15 @@ REDISCOVER_ENTRY: vm_map_unlock_read(map); - ret = vm_object_iopl_request(local_object, - ((vm_object_offset_t) - ((offset - local_start) + local_offset)), - *upl_size, - upl, - page_list, - count, - caller_flags, - tag); + ret = vm_object_iopl_request(local_object, + ((vm_object_offset_t) + ((offset - local_start) + local_offset)), + *upl_size, + upl, + page_list, + count, + caller_flags, + tag); vm_object_deallocate(local_object); return ret; @@ -6267,76 +6219,81 @@ REDISCOVER_ENTRY: /* * Internal routine to enter a UPL into a VM map. - * + * * JMM - This should just be doable through the standard * vm_map_enter() API. */ kern_return_t vm_map_enter_upl( - vm_map_t map, - upl_t upl, - vm_map_offset_t *dst_addr) + vm_map_t map, + upl_t upl, + vm_map_offset_t *dst_addr) { - vm_map_size_t size; - vm_object_offset_t offset; - vm_map_offset_t addr; - vm_page_t m; - kern_return_t kr; - int isVectorUPL = 0, curr_upl=0; - upl_t vector_upl = NULL; - vm_offset_t vector_upl_dst_addr = 0; - vm_map_t vector_upl_submap = NULL; - upl_offset_t subupl_offset = 0; - upl_size_t subupl_size = 0; - - if (upl == UPL_NULL) + vm_map_size_t size; + vm_object_offset_t offset; + vm_map_offset_t addr; + vm_page_t m; + kern_return_t kr; + int isVectorUPL = 0, curr_upl = 0; + upl_t vector_upl = NULL; + vm_offset_t vector_upl_dst_addr = 0; + vm_map_t vector_upl_submap = NULL; + upl_offset_t subupl_offset = 0; + upl_size_t subupl_size = 0; + + if (upl == UPL_NULL) { return KERN_INVALID_ARGUMENT; + } - if((isVectorUPL = vector_upl_is_valid(upl))) { - int mapped=0,valid_upls=0; + if ((isVectorUPL = vector_upl_is_valid(upl))) { + int mapped = 0, valid_upls = 0; vector_upl = upl; upl_lock(vector_upl); - for(curr_upl=0; curr_upl < MAX_VECTOR_UPL_ELEMENTS; curr_upl++) { + for (curr_upl = 0; curr_upl < MAX_VECTOR_UPL_ELEMENTS; curr_upl++) { upl = vector_upl_subupl_byindex(vector_upl, curr_upl ); - if(upl == NULL) + if (upl == NULL) { continue; + } valid_upls++; - if (UPL_PAGE_LIST_MAPPED & upl->flags) + if (UPL_PAGE_LIST_MAPPED & upl->flags) { mapped++; + } } - if(mapped) { - if(mapped != valid_upls) + if (mapped) { + if (mapped != valid_upls) { panic("Only %d of the %d sub-upls within the Vector UPL are alread mapped\n", mapped, valid_upls); - else { + } else { upl_unlock(vector_upl); return KERN_FAILURE; } } kr = kmem_suballoc(map, &vector_upl_dst_addr, vector_upl->size, FALSE, - VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_NONE, - &vector_upl_submap); - if( kr != KERN_SUCCESS ) + VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_NONE, + &vector_upl_submap); + if (kr != KERN_SUCCESS) { panic("Vector UPL submap allocation failed\n"); + } map = vector_upl_submap; vector_upl_set_submap(vector_upl, vector_upl_submap, vector_upl_dst_addr); - curr_upl=0; - } - else + curr_upl = 0; + } else { upl_lock(upl); + } process_upl_to_enter: - if(isVectorUPL){ - if(curr_upl == MAX_VECTOR_UPL_ELEMENTS) { + if (isVectorUPL) { + if (curr_upl == MAX_VECTOR_UPL_ELEMENTS) { *dst_addr = vector_upl_dst_addr; upl_unlock(vector_upl); return KERN_SUCCESS; } upl = vector_upl_subupl_byindex(vector_upl, curr_upl++ ); - if(upl == NULL) + if (upl == NULL) { goto process_upl_to_enter; + } vector_upl_get_iostate(vector_upl, upl, &subupl_offset, &subupl_size); *dst_addr = (vm_map_offset_t)(vector_upl_dst_addr + (vm_map_offset_t)subupl_offset); @@ -6351,20 +6308,19 @@ process_upl_to_enter: } if ((!(upl->flags & UPL_SHADOWED)) && ((upl->flags & UPL_HAS_BUSY) || - !((upl->flags & (UPL_DEVICE_MEMORY | UPL_IO_WIRE)) || (upl->map_object->phys_contiguous)))) { - - vm_object_t object; - vm_page_t alias_page; - vm_object_offset_t new_offset; - unsigned int pg_num; - wpl_array_t lite_list; + !((upl->flags & (UPL_DEVICE_MEMORY | UPL_IO_WIRE)) || (upl->map_object->phys_contiguous)))) { + vm_object_t object; + vm_page_t alias_page; + vm_object_offset_t new_offset; + unsigned int pg_num; + wpl_array_t lite_list; if (upl->flags & UPL_INTERNAL) { - lite_list = (wpl_array_t) - ((((uintptr_t)upl) + sizeof(struct upl)) - + ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t))); + lite_list = (wpl_array_t) + ((((uintptr_t)upl) + sizeof(struct upl)) + + ((upl->size / PAGE_SIZE) * sizeof(upl_page_info_t))); } else { - lite_list = (wpl_array_t)(((uintptr_t)upl) + sizeof(struct upl)); + lite_list = (wpl_array_t)(((uintptr_t)upl) + sizeof(struct upl)); } object = upl->map_object; upl->map_object = vm_object_allocate(upl->size); @@ -6387,25 +6343,24 @@ process_upl_to_enter: pg_num = (unsigned int) (new_offset / PAGE_SIZE); assert(pg_num == new_offset / PAGE_SIZE); - if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) { - + if (lite_list[pg_num >> 5] & (1 << (pg_num & 31))) { VM_PAGE_GRAB_FICTITIOUS(alias_page); vm_object_lock(object); m = vm_page_lookup(object, offset); if (m == VM_PAGE_NULL) { - panic("vm_upl_map: page missing\n"); + panic("vm_upl_map: page missing\n"); } /* - * Convert the fictitious page to a private + * Convert the fictitious page to a private * shadow of the real page. */ - assert(alias_page->fictitious); - alias_page->fictitious = FALSE; - alias_page->private = TRUE; - alias_page->free_when_done = TRUE; + assert(alias_page->vmp_fictitious); + alias_page->vmp_fictitious = FALSE; + alias_page->vmp_private = TRUE; + alias_page->vmp_free_when_done = TRUE; /* * since m is a page in the upl it must * already be wired or BUSY, so it's @@ -6414,17 +6369,17 @@ process_upl_to_enter: */ VM_PAGE_SET_PHYS_PAGE(alias_page, VM_PAGE_GET_PHYS_PAGE(m)); - vm_object_unlock(object); + vm_object_unlock(object); vm_page_lockspin_queues(); vm_page_wire(alias_page, VM_KERN_MEMORY_NONE, TRUE); vm_page_unlock_queues(); - + vm_page_insert_wired(alias_page, upl->map_object, new_offset, VM_KERN_MEMORY_NONE); - assert(!alias_page->wanted); - alias_page->busy = FALSE; - alias_page->absent = FALSE; + assert(!alias_page->vmp_wanted); + alias_page->vmp_busy = FALSE; + alias_page->vmp_absent = FALSE; } size -= PAGE_SIZE; offset += PAGE_SIZE_64; @@ -6432,38 +6387,39 @@ process_upl_to_enter: } vm_object_unlock(upl->map_object); } - if (upl->flags & UPL_SHADOWED) - offset = 0; - else - offset = upl->offset - upl->map_object->paging_offset; + if (upl->flags & UPL_SHADOWED) { + offset = 0; + } else { + offset = upl->offset - upl->map_object->paging_offset; + } size = upl->size; - + vm_object_reference(upl->map_object); - if(!isVectorUPL) { + if (!isVectorUPL) { *dst_addr = 0; /* - * NEED A UPL_MAP ALIAS - */ + * NEED A UPL_MAP ALIAS + */ kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0, - VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_OSFMK, - upl->map_object, offset, FALSE, - VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); + VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_OSFMK, + upl->map_object, offset, FALSE, + VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); if (kr != KERN_SUCCESS) { vm_object_deallocate(upl->map_object); upl_unlock(upl); - return(kr); + return kr; } - } - else { + } else { kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0, - VM_FLAGS_FIXED, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_OSFMK, - upl->map_object, offset, FALSE, - VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); - if(kr) + VM_FLAGS_FIXED, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_OSFMK, + upl->map_object, offset, FALSE, + VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); + if (kr) { panic("vm_map_enter failed for a Vector UPL\n"); + } } vm_object_lock(upl->map_object); @@ -6471,14 +6427,14 @@ process_upl_to_enter: m = vm_page_lookup(upl->map_object, offset); if (m) { - m->pmapped = TRUE; + m->vmp_pmapped = TRUE; - /* CODE SIGNING ENFORCEMENT: page has been wpmapped, + /* CODE SIGNING ENFORCEMENT: page has been wpmapped, * but only in kernel space. If this was on a user map, * we'd have to set the wpmapped bit. */ - /* m->wpmapped = TRUE; */ + /* m->vmp_wpmapped = TRUE; */ assert(map->pmap == kernel_pmap); - + PMAP_ENTER(map->pmap, addr, m, VM_PROT_DEFAULT, VM_PROT_NONE, 0, TRUE, kr); assert(kr == KERN_SUCCESS); @@ -6497,15 +6453,16 @@ process_upl_to_enter: upl->flags |= UPL_PAGE_LIST_MAPPED; upl->kaddr = (vm_offset_t) *dst_addr; assert(upl->kaddr == *dst_addr); - - if(isVectorUPL) + + if (isVectorUPL) { goto process_upl_to_enter; + } upl_unlock(upl); return KERN_SUCCESS; } - + /* * Internal routine to remove a UPL mapping from a VM map. * @@ -6518,59 +6475,63 @@ process_upl_to_enter: */ kern_return_t vm_map_remove_upl( - vm_map_t map, - upl_t upl) + vm_map_t map, + upl_t upl) { - vm_address_t addr; - upl_size_t size; - int isVectorUPL = 0, curr_upl = 0; - upl_t vector_upl = NULL; + vm_address_t addr; + upl_size_t size; + int isVectorUPL = 0, curr_upl = 0; + upl_t vector_upl = NULL; - if (upl == UPL_NULL) + if (upl == UPL_NULL) { return KERN_INVALID_ARGUMENT; + } - if((isVectorUPL = vector_upl_is_valid(upl))) { - int unmapped=0, valid_upls=0; + if ((isVectorUPL = vector_upl_is_valid(upl))) { + int unmapped = 0, valid_upls = 0; vector_upl = upl; upl_lock(vector_upl); - for(curr_upl=0; curr_upl < MAX_VECTOR_UPL_ELEMENTS; curr_upl++) { + for (curr_upl = 0; curr_upl < MAX_VECTOR_UPL_ELEMENTS; curr_upl++) { upl = vector_upl_subupl_byindex(vector_upl, curr_upl ); - if(upl == NULL) + if (upl == NULL) { continue; + } valid_upls++; - if (!(UPL_PAGE_LIST_MAPPED & upl->flags)) + if (!(UPL_PAGE_LIST_MAPPED & upl->flags)) { unmapped++; + } } - if(unmapped) { - if(unmapped != valid_upls) + if (unmapped) { + if (unmapped != valid_upls) { panic("%d of the %d sub-upls within the Vector UPL is/are not mapped\n", unmapped, valid_upls); - else { + } else { upl_unlock(vector_upl); return KERN_FAILURE; } } - curr_upl=0; - } - else + curr_upl = 0; + } else { upl_lock(upl); + } process_upl_to_remove: - if(isVectorUPL) { - if(curr_upl == MAX_VECTOR_UPL_ELEMENTS) { + if (isVectorUPL) { + if (curr_upl == MAX_VECTOR_UPL_ELEMENTS) { vm_map_t v_upl_submap; vm_offset_t v_upl_submap_dst_addr; vector_upl_get_submap(vector_upl, &v_upl_submap, &v_upl_submap_dst_addr); - vm_map_remove(map, v_upl_submap_dst_addr, v_upl_submap_dst_addr + vector_upl->size, VM_MAP_NO_FLAGS); + vm_map_remove(map, v_upl_submap_dst_addr, v_upl_submap_dst_addr + vector_upl->size, VM_MAP_REMOVE_NO_FLAGS); vm_map_deallocate(v_upl_submap); upl_unlock(vector_upl); return KERN_SUCCESS; } upl = vector_upl_subupl_byindex(vector_upl, curr_upl++ ); - if(upl == NULL) - goto process_upl_to_remove; + if (upl == NULL) { + goto process_upl_to_remove; + } } if (upl->flags & UPL_PAGE_LIST_MAPPED) { @@ -6578,30 +6539,28 @@ process_upl_to_remove: size = upl->size; assert(upl->ref_count > 1); - upl->ref_count--; /* removing mapping ref */ + upl->ref_count--; /* removing mapping ref */ upl->flags &= ~UPL_PAGE_LIST_MAPPED; upl->kaddr = (vm_offset_t) 0; - - if(!isVectorUPL) { + + if (!isVectorUPL) { upl_unlock(upl); - + vm_map_remove( map, vm_map_trunc_page(addr, - VM_MAP_PAGE_MASK(map)), + VM_MAP_PAGE_MASK(map)), vm_map_round_page(addr + size, - VM_MAP_PAGE_MASK(map)), - VM_MAP_NO_FLAGS); - + VM_MAP_PAGE_MASK(map)), + VM_MAP_REMOVE_NO_FLAGS); return KERN_SUCCESS; - } - else { + } else { /* - * If it's a Vectored UPL, we'll be removing the entire - * submap anyways, so no need to remove individual UPL - * element mappings from within the submap - */ + * If it's a Vectored UPL, we'll be removing the entire + * submap anyways, so no need to remove individual UPL + * element mappings from within the submap + */ goto process_upl_to_remove; } } @@ -6613,67 +6572,69 @@ process_upl_to_remove: kern_return_t upl_commit_range( - upl_t upl, - upl_offset_t offset, - upl_size_t size, - int flags, - upl_page_info_t *page_list, - mach_msg_type_number_t count, - boolean_t *empty) + upl_t upl, + upl_offset_t offset, + upl_size_t size, + int flags, + upl_page_info_t *page_list, + mach_msg_type_number_t count, + boolean_t *empty) { - upl_size_t xfer_size, subupl_size = size; - vm_object_t shadow_object; - vm_object_t object; - vm_object_t m_object; - vm_object_offset_t target_offset; - upl_offset_t subupl_offset = offset; - int entry; - wpl_array_t lite_list; - int occupied; - int clear_refmod = 0; - int pgpgout_count = 0; - struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT]; - struct vm_page_delayed_work *dwp; - int dw_count; - int dw_limit; - int isVectorUPL = 0; - upl_t vector_upl = NULL; - boolean_t should_be_throttled = FALSE; - - vm_page_t nxt_page = VM_PAGE_NULL; - int fast_path_possible = 0; - int fast_path_full_commit = 0; - int throttle_page = 0; - int unwired_count = 0; - int local_queue_count = 0; - vm_page_t first_local, last_local; + upl_size_t xfer_size, subupl_size = size; + vm_object_t shadow_object; + vm_object_t object; + vm_object_t m_object; + vm_object_offset_t target_offset; + upl_offset_t subupl_offset = offset; + int entry; + wpl_array_t lite_list; + int occupied; + int clear_refmod = 0; + int pgpgout_count = 0; + struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT]; + struct vm_page_delayed_work *dwp; + int dw_count; + int dw_limit; + int isVectorUPL = 0; + upl_t vector_upl = NULL; + boolean_t should_be_throttled = FALSE; + + vm_page_t nxt_page = VM_PAGE_NULL; + int fast_path_possible = 0; + int fast_path_full_commit = 0; + int throttle_page = 0; + int unwired_count = 0; + int local_queue_count = 0; + vm_page_t first_local, last_local; *empty = FALSE; - if (upl == UPL_NULL) + if (upl == UPL_NULL) { return KERN_INVALID_ARGUMENT; + } - if (count == 0) + if (count == 0) { page_list = NULL; + } - if((isVectorUPL = vector_upl_is_valid(upl))) { + if ((isVectorUPL = vector_upl_is_valid(upl))) { vector_upl = upl; upl_lock(vector_upl); - } - else + } else { upl_lock(upl); + } process_upl_to_commit: - if(isVectorUPL) { + if (isVectorUPL) { size = subupl_size; offset = subupl_offset; - if(size == 0) { + if (size == 0) { upl_unlock(vector_upl); return KERN_SUCCESS; } upl = vector_upl_subupl_byoffset(vector_upl, &offset, &size); - if(upl == NULL) { + if (upl == NULL) { upl_unlock(vector_upl); return KERN_FAILURE; } @@ -6685,54 +6646,58 @@ process_upl_to_commit: #if UPL_DEBUG if (upl->upl_commit_index < UPL_DEBUG_COMMIT_RECORDS) { (void) OSBacktrace(&upl->upl_commit_records[upl->upl_commit_index].c_retaddr[0], UPL_DEBUG_STACK_FRAMES); - + upl->upl_commit_records[upl->upl_commit_index].c_beg = offset; upl->upl_commit_records[upl->upl_commit_index].c_end = (offset + size); upl->upl_commit_index++; } #endif - if (upl->flags & UPL_DEVICE_MEMORY) + if (upl->flags & UPL_DEVICE_MEMORY) { xfer_size = 0; - else if ((offset + size) <= upl->size) - xfer_size = size; - else { - if(!isVectorUPL) + } else if ((offset + size) <= upl->size) { + xfer_size = size; + } else { + if (!isVectorUPL) { upl_unlock(upl); - else { + } else { upl_unlock(vector_upl); } return KERN_FAILURE; } - if (upl->flags & UPL_SET_DIRTY) + if (upl->flags & UPL_SET_DIRTY) { flags |= UPL_COMMIT_SET_DIRTY; - if (upl->flags & UPL_CLEAR_DIRTY) - flags |= UPL_COMMIT_CLEAR_DIRTY; + } + if (upl->flags & UPL_CLEAR_DIRTY) { + flags |= UPL_COMMIT_CLEAR_DIRTY; + } - if (upl->flags & UPL_INTERNAL) + if (upl->flags & UPL_INTERNAL) { lite_list = (wpl_array_t) ((((uintptr_t)upl) + sizeof(struct upl)) - + ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t))); - else + + ((upl->size / PAGE_SIZE) * sizeof(upl_page_info_t))); + } else { lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl)); + } object = upl->map_object; if (upl->flags & UPL_SHADOWED) { - vm_object_lock(object); + vm_object_lock(object); shadow_object = object->shadow; } else { shadow_object = object; } - entry = offset/PAGE_SIZE; + entry = offset / PAGE_SIZE; target_offset = (vm_object_offset_t)offset; assert(!(target_offset & PAGE_MASK)); assert(!(xfer_size & PAGE_MASK)); - if (upl->flags & UPL_KERNEL_OBJECT) + if (upl->flags & UPL_KERNEL_OBJECT) { vm_object_lock_shared(shadow_object); - else + } else { vm_object_lock(shadow_object); + } VM_OBJECT_WIRED_PAGE_UPDATE_START(shadow_object); @@ -6752,14 +6717,15 @@ process_upl_to_commit: */ flags &= ~UPL_COMMIT_CS_VALIDATED; } - if (! page_list) { + if (!page_list) { /* * No page list to get the code-signing info from !? */ flags &= ~UPL_COMMIT_CS_VALIDATED; } - if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal) + if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal) { should_be_throttled = TRUE; + } dwp = &dw_array[0]; dw_count = 0; @@ -6770,9 +6736,7 @@ process_upl_to_commit: !isVectorUPL && shadow_object->purgable != VM_PURGABLE_VOLATILE && shadow_object->purgable != VM_PURGABLE_EMPTY) { - if (!vm_page_queue_empty(&shadow_object->memq)) { - if (size == shadow_object->vo_size) { nxt_page = (vm_page_t)vm_page_queue_first(&shadow_object->memq); fast_path_full_commit = 1; @@ -6781,8 +6745,8 @@ process_upl_to_commit: if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal && (shadow_object->purgable == VM_PURGABLE_DENY || - shadow_object->purgable == VM_PURGABLE_NONVOLATILE || - shadow_object->purgable == VM_PURGABLE_VOLATILE)) { + shadow_object->purgable == VM_PURGABLE_NONVOLATILE || + shadow_object->purgable == VM_PURGABLE_VOLATILE)) { throttle_page = 1; } } @@ -6791,7 +6755,7 @@ process_upl_to_commit: last_local = VM_PAGE_NULL; while (xfer_size) { - vm_page_t t, m; + vm_page_t t, m; dwp->dw_mask = 0; clear_refmod = 0; @@ -6799,42 +6763,45 @@ process_upl_to_commit: m = VM_PAGE_NULL; if (upl->flags & UPL_LITE) { - unsigned int pg_num; + unsigned int pg_num; if (nxt_page != VM_PAGE_NULL) { m = nxt_page; - nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->listq); - target_offset = m->offset; + nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->vmp_listq); + target_offset = m->vmp_offset; } - pg_num = (unsigned int) (target_offset/PAGE_SIZE); - assert(pg_num == target_offset/PAGE_SIZE); + pg_num = (unsigned int) (target_offset / PAGE_SIZE); + assert(pg_num == target_offset / PAGE_SIZE); - if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) { - lite_list[pg_num>>5] &= ~(1 << (pg_num & 31)); + if (lite_list[pg_num >> 5] & (1 << (pg_num & 31))) { + lite_list[pg_num >> 5] &= ~(1 << (pg_num & 31)); - if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL) + if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL) { m = vm_page_lookup(shadow_object, target_offset + (upl->offset - shadow_object->paging_offset)); - } else + } + } else { m = NULL; + } } if (upl->flags & UPL_SHADOWED) { - if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) { - - t->free_when_done = FALSE; + if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) { + t->vmp_free_when_done = FALSE; VM_PAGE_FREE(t); - if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL) + if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL) { m = vm_page_lookup(shadow_object, target_offset + object->vo_shadow_offset); + } } } - if (m == VM_PAGE_NULL) + if (m == VM_PAGE_NULL) { goto commit_next_page; + } m_object = VM_PAGE_OBJECT(m); - if (m->vm_page_q_state == VM_PAGE_USED_BY_COMPRESSOR) { - assert(m->busy); + if (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) { + assert(m->vmp_busy); dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); goto commit_next_page; @@ -6846,25 +6813,26 @@ process_upl_to_commit: * Set the code signing bits according to * what the UPL says they should be. */ - m->cs_validated = page_list[entry].cs_validated; - m->cs_tainted = page_list[entry].cs_tainted; - m->cs_nx = page_list[entry].cs_nx; + m->vmp_cs_validated = page_list[entry].cs_validated; + m->vmp_cs_tainted = page_list[entry].cs_tainted; + m->vmp_cs_nx = page_list[entry].cs_nx; + } + if (flags & UPL_COMMIT_WRITTEN_BY_KERNEL) { + m->vmp_written_by_kernel = TRUE; } - if (flags & UPL_COMMIT_WRITTEN_BY_KERNEL) - m->written_by_kernel = TRUE; if (upl->flags & UPL_IO_WIRE) { - - if (page_list) + if (page_list) { page_list[entry].phys_addr = 0; + } if (flags & UPL_COMMIT_SET_DIRTY) { SET_PAGE_DIRTY(m, FALSE); } else if (flags & UPL_COMMIT_CLEAR_DIRTY) { - m->dirty = FALSE; + m->vmp_dirty = FALSE; - if (! (flags & UPL_COMMIT_CS_VALIDATED) && - m->cs_validated && !m->cs_tainted) { + if (!(flags & UPL_COMMIT_CS_VALIDATED) && + m->vmp_cs_validated && !m->vmp_cs_tainted) { /* * CODE SIGNING: * This page is no longer dirty @@ -6872,15 +6840,10 @@ process_upl_to_commit: * so it will need to be * re-validated. */ - if (m->slid) { - panic("upl_commit_range(%p): page %p was slid\n", - upl, m); - } - assert(!m->slid); - m->cs_validated = FALSE; -#if DEVELOPMENT || DEBUG - vm_cs_validated_resets++; -#endif + m->vmp_cs_validated = FALSE; + + VM_PAGEOUT_DEBUG(vm_cs_validated_resets, 1); + pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)); } clear_refmod |= VM_MEM_MODIFIED; @@ -6896,33 +6859,34 @@ process_upl_to_commit: if (fast_path_possible) { assert(m_object->purgable != VM_PURGABLE_EMPTY); assert(m_object->purgable != VM_PURGABLE_VOLATILE); - if (m->absent) { - assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q); - assert(m->wire_count == 0); - assert(m->busy); + if (m->vmp_absent) { + assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q); + assert(m->vmp_wire_count == 0); + assert(m->vmp_busy); - m->absent = FALSE; + m->vmp_absent = FALSE; dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); } else { - if (m->wire_count == 0) + if (m->vmp_wire_count == 0) { panic("wire_count == 0, m = %p, obj = %p\n", m, shadow_object); - assert(m->vm_page_q_state == VM_PAGE_IS_WIRED); + } + assert(m->vmp_q_state == VM_PAGE_IS_WIRED); /* * XXX FBDP need to update some other * counters here (purgeable_wired_count) * (ledgers), ... */ - assert(m->wire_count > 0); - m->wire_count--; + assert(m->vmp_wire_count > 0); + m->vmp_wire_count--; - if (m->wire_count == 0) { - m->vm_page_q_state = VM_PAGE_NOT_ON_Q; + if (m->vmp_wire_count == 0) { + m->vmp_q_state = VM_PAGE_NOT_ON_Q; unwired_count++; } } - if (m->wire_count == 0) { - assert(m->pageq.next == 0 && m->pageq.prev == 0); + if (m->vmp_wire_count == 0) { + assert(m->vmp_pageq.next == 0 && m->vmp_pageq.prev == 0); if (last_local == VM_PAGE_NULL) { assert(first_local == VM_PAGE_NULL); @@ -6932,22 +6896,24 @@ process_upl_to_commit: } else { assert(first_local != VM_PAGE_NULL); - m->pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local); - first_local->pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(m); + m->vmp_pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local); + first_local->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(m); first_local = m; } local_queue_count++; if (throttle_page) { - m->vm_page_q_state = VM_PAGE_ON_THROTTLED_Q; + m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q; } else { if (flags & UPL_COMMIT_INACTIVATE) { - if (shadow_object->internal) - m->vm_page_q_state = VM_PAGE_ON_INACTIVE_INTERNAL_Q; - else - m->vm_page_q_state = VM_PAGE_ON_INACTIVE_EXTERNAL_Q; - } else - m->vm_page_q_state = VM_PAGE_ON_ACTIVE_Q; + if (shadow_object->internal) { + m->vmp_q_state = VM_PAGE_ON_INACTIVE_INTERNAL_Q; + } else { + m->vmp_q_state = VM_PAGE_ON_INACTIVE_EXTERNAL_Q; + } + } else { + m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q; + } } } } else { @@ -6955,25 +6921,28 @@ process_upl_to_commit: dwp->dw_mask |= DW_vm_page_deactivate_internal; clear_refmod |= VM_MEM_REFERENCED; } - if (m->absent) { - if (flags & UPL_COMMIT_FREE_ABSENT) + if (m->vmp_absent) { + if (flags & UPL_COMMIT_FREE_ABSENT) { dwp->dw_mask |= DW_vm_page_free; - else { - m->absent = FALSE; + } else { + m->vmp_absent = FALSE; dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); - if ( !(dwp->dw_mask & DW_vm_page_deactivate_internal)) + if (!(dwp->dw_mask & DW_vm_page_deactivate_internal)) { dwp->dw_mask |= DW_vm_page_activate; + } } - } else + } else { dwp->dw_mask |= DW_vm_page_unwire; + } } goto commit_next_page; } - assert(m->vm_page_q_state != VM_PAGE_USED_BY_COMPRESSOR); + assert(m->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR); - if (page_list) + if (page_list) { page_list[entry].phys_addr = 0; + } /* * make sure to clear the hardware @@ -6983,18 +6952,20 @@ process_upl_to_commit: * change of state */ if (flags & UPL_COMMIT_CLEAR_DIRTY) { - m->dirty = FALSE; + m->vmp_dirty = FALSE; clear_refmod |= VM_MEM_MODIFIED; } - if (m->laundry) + if (m->vmp_laundry) { dwp->dw_mask |= DW_vm_pageout_throttle_up; + } + + if (VM_PAGE_WIRED(m)) { + m->vmp_free_when_done = FALSE; + } - if (VM_PAGE_WIRED(m)) - m->free_when_done = FALSE; - - if (! (flags & UPL_COMMIT_CS_VALIDATED) && - m->cs_validated && !m->cs_tainted) { + if (!(flags & UPL_COMMIT_CS_VALIDATED) && + m->vmp_cs_validated && !m->vmp_cs_tainted) { /* * CODE SIGNING: * This page is no longer dirty @@ -7002,27 +6973,23 @@ process_upl_to_commit: * so it will need to be * re-validated. */ - if (m->slid) { - panic("upl_commit_range(%p): page %p was slid\n", - upl, m); - } - assert(!m->slid); - m->cs_validated = FALSE; -#if DEVELOPMENT || DEBUG - vm_cs_validated_resets++; -#endif + m->vmp_cs_validated = FALSE; + + VM_PAGEOUT_DEBUG(vm_cs_validated_resets, 1); + pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)); } - if (m->overwriting) { + if (m->vmp_overwriting) { /* * the (COPY_OUT_FROM == FALSE) request_page_list case */ - if (m->busy) { + if (m->vmp_busy) { #if CONFIG_PHANTOM_CACHE - if (m->absent && !m_object->internal) + if (m->vmp_absent && !m_object->internal) { dwp->dw_mask |= DW_vm_phantom_cache_update; + } #endif - m->absent = FALSE; + m->vmp_absent = FALSE; dwp->dw_mask |= DW_clear_busy; } else { @@ -7035,37 +7002,34 @@ process_upl_to_commit: dwp->dw_mask |= DW_vm_page_unwire; /* reactivates */ } - m->overwriting = FALSE; + m->vmp_overwriting = FALSE; } - m->cleaning = FALSE; + m->vmp_cleaning = FALSE; - if (m->free_when_done) { - /* + if (m->vmp_free_when_done) { + /* * With the clean queue enabled, UPL_PAGEOUT should - * no longer set the pageout bit. It's pages now go + * no longer set the pageout bit. It's pages now go * to the clean queue. */ assert(!(flags & UPL_PAGEOUT)); assert(!m_object->internal); - m->free_when_done = FALSE; -#if MACH_CLUSTER_STATS - if (m->wanted) vm_pageout_target_collisions++; -#endif + m->vmp_free_when_done = FALSE; + if ((flags & UPL_COMMIT_SET_DIRTY) || - (m->pmapped && (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED))) { + (m->vmp_pmapped && (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED))) { /* * page was re-dirtied after we started - * the pageout... reactivate it since + * the pageout... reactivate it since * we don't know whether the on-disk * copy matches what is now in memory */ SET_PAGE_DIRTY(m, FALSE); - + dwp->dw_mask |= DW_vm_page_activate | DW_PAGE_WAKEUP; if (upl->flags & UPL_PAGEOUT) { - CLUSTER_STAT(vm_pageout_target_page_dirtied++;) VM_STAT_INCR(reactivations); DTRACE_VM2(pgrec, int, 1, (uint64_t *), NULL); } @@ -7079,21 +7043,13 @@ process_upl_to_commit: } else { DTRACE_VM2(fspgout, int, 1, (uint64_t *), NULL); } - m->dirty = FALSE; - m->busy = TRUE; + m->vmp_dirty = FALSE; + m->vmp_busy = TRUE; dwp->dw_mask |= DW_vm_page_free; } goto commit_next_page; } -#if MACH_CLUSTER_STATS - if (m->wpmapped) - m->dirty = pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m)); - - if (m->dirty) vm_pageout_cluster_dirtied++; - else vm_pageout_cluster_cleaned++; - if (m->wanted) vm_pageout_cluster_collisions++; -#endif /* * It is a part of the semantic of COPYOUT_FROM * UPLs that a commit implies cache sync @@ -7101,25 +7057,25 @@ process_upl_to_commit: * this can be used to strip the precious bit * as well as clean */ - if ((upl->flags & UPL_PAGE_SYNC_DONE) || (flags & UPL_COMMIT_CLEAR_PRECIOUS)) - m->precious = FALSE; + if ((upl->flags & UPL_PAGE_SYNC_DONE) || (flags & UPL_COMMIT_CLEAR_PRECIOUS)) { + m->vmp_precious = FALSE; + } if (flags & UPL_COMMIT_SET_DIRTY) { SET_PAGE_DIRTY(m, FALSE); } else { - m->dirty = FALSE; + m->vmp_dirty = FALSE; } /* with the clean queue on, move *all* cleaned pages to the clean queue */ - if (hibernate_cleaning_in_progress == FALSE && !m->dirty && (upl->flags & UPL_PAGEOUT)) { + if (hibernate_cleaning_in_progress == FALSE && !m->vmp_dirty && (upl->flags & UPL_PAGEOUT)) { pgpgout_count++; VM_STAT_INCR(pageouts); DTRACE_VM2(pgout, int, 1, (uint64_t *), NULL); dwp->dw_mask |= DW_enqueue_cleaned; - vm_pageout_enqueued_cleaned_from_inactive_dirty++; - } else if (should_be_throttled == TRUE && (m->vm_page_q_state == VM_PAGE_NOT_ON_Q)) { + } else if (should_be_throttled == TRUE && (m->vmp_q_state == VM_PAGE_NOT_ON_Q)) { /* * page coming back in from being 'frozen'... * it was dirty before it was frozen, so keep it so @@ -7128,18 +7084,16 @@ process_upl_to_commit: */ SET_PAGE_DIRTY(m, FALSE); dwp->dw_mask |= DW_vm_page_activate; - } else { - if ((flags & UPL_COMMIT_INACTIVATE) && !m->clustered && (m->vm_page_q_state != VM_PAGE_ON_SPECULATIVE_Q)) { + if ((flags & UPL_COMMIT_INACTIVATE) && !m->vmp_clustered && (m->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q)) { dwp->dw_mask |= DW_vm_page_deactivate_internal; clear_refmod |= VM_MEM_REFERENCED; - } else if ( !VM_PAGE_PAGEABLE(m)) { - - if (m->clustered || (flags & UPL_COMMIT_SPECULATE)) + } else if (!VM_PAGE_PAGEABLE(m)) { + if (m->vmp_clustered || (flags & UPL_COMMIT_SPECULATE)) { dwp->dw_mask |= DW_vm_page_speculate; - else if (m->reference) + } else if (m->vmp_reference) { dwp->dw_mask |= DW_vm_page_activate; - else { + } else { dwp->dw_mask |= DW_vm_page_deactivate_internal; clear_refmod |= VM_MEM_REFERENCED; } @@ -7159,8 +7113,9 @@ process_upl_to_commit: dwp->dw_mask |= DW_PAGE_WAKEUP; commit_next_page: - if (clear_refmod) + if (clear_refmod) { pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(m), clear_refmod); + } target_offset += PAGE_SIZE_64; xfer_size -= PAGE_SIZE; @@ -7172,43 +7127,46 @@ commit_next_page: if (dw_count >= dw_limit) { vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count); - + dwp = &dw_array[0]; dw_count = 0; } } else { - if (dwp->dw_mask & DW_clear_busy) - m->busy = FALSE; + if (dwp->dw_mask & DW_clear_busy) { + m->vmp_busy = FALSE; + } - if (dwp->dw_mask & DW_PAGE_WAKEUP) + if (dwp->dw_mask & DW_PAGE_WAKEUP) { PAGE_WAKEUP(m); + } } } } - if (dw_count) + if (dw_count) { vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count); + } if (fast_path_possible) { - assert(shadow_object->purgable != VM_PURGABLE_VOLATILE); assert(shadow_object->purgable != VM_PURGABLE_EMPTY); if (local_queue_count || unwired_count) { - if (local_queue_count) { - vm_page_t first_target; - vm_page_queue_head_t *target_queue; + vm_page_t first_target; + vm_page_queue_head_t *target_queue; - if (throttle_page) + if (throttle_page) { target_queue = &vm_page_queue_throttled; - else { + } else { if (flags & UPL_COMMIT_INACTIVATE) { - if (shadow_object->internal) + if (shadow_object->internal) { target_queue = &vm_page_queue_anonymous; - else + } else { target_queue = &vm_page_queue_inactive; - } else + } + } else { target_queue = &vm_page_queue_active; + } } /* * Transfer the entire local queue to a regular LRU page queues. @@ -7217,14 +7175,15 @@ commit_next_page: first_target = (vm_page_t) vm_page_queue_first(target_queue); - if (vm_page_queue_empty(target_queue)) + if (vm_page_queue_empty(target_queue)) { target_queue->prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local); - else - first_target->pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local); + } else { + first_target->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local); + } target_queue->next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local); - first_local->pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(target_queue); - last_local->pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_target); + first_local->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(target_queue); + last_local->vmp_pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_target); /* * Adjust the global page counts. @@ -7233,23 +7192,26 @@ commit_next_page: vm_page_throttled_count += local_queue_count; } else { if (flags & UPL_COMMIT_INACTIVATE) { - if (shadow_object->internal) + if (shadow_object->internal) { vm_page_anonymous_count += local_queue_count; + } vm_page_inactive_count += local_queue_count; token_new_pagecount += local_queue_count; - } else + } else { vm_page_active_count += local_queue_count; + } - if (shadow_object->internal) + if (shadow_object->internal) { vm_page_pageable_internal_count += local_queue_count; - else + } else { vm_page_pageable_external_count += local_queue_count; + } } } else { vm_page_lockspin_queues(); } - if (unwired_count) { + if (unwired_count) { vm_page_wire_count -= unwired_count; VM_CHECK_MEMORYSTATUS; } @@ -7260,16 +7222,16 @@ commit_next_page: } occupied = 1; - if (upl->flags & UPL_DEVICE_MEMORY) { + if (upl->flags & UPL_DEVICE_MEMORY) { occupied = 0; } else if (upl->flags & UPL_LITE) { - int pg_num; - int i; + int pg_num; + int i; occupied = 0; if (!fast_path_full_commit) { - pg_num = upl->size/PAGE_SIZE; + pg_num = upl->size / PAGE_SIZE; pg_num = (pg_num + 31) >> 5; for (i = 0; i < pg_num; i++) { @@ -7280,8 +7242,9 @@ commit_next_page: } } } else { - if (vm_page_queue_empty(&upl->map_object->memq)) + if (vm_page_queue_empty(&upl->map_object->memq)) { occupied = 0; + } } if (occupied == 0) { /* @@ -7292,11 +7255,12 @@ commit_next_page: * should be considered relevant for the Vector UPL and not * the internal UPLs. */ - if ((upl->flags & UPL_COMMIT_NOTIFY_EMPTY) || isVectorUPL) + if ((upl->flags & UPL_COMMIT_NOTIFY_EMPTY) || isVectorUPL) { *empty = TRUE; + } if (object == shadow_object && !(upl->flags & UPL_KERNEL_OBJECT)) { - /* + /* * this is not a paging object * so we need to drop the paging reference * that was taken when we created the UPL @@ -7305,31 +7269,32 @@ commit_next_page: vm_object_activity_end(shadow_object); vm_object_collapse(shadow_object, 0, TRUE); } else { - /* - * we dontated the paging reference to - * the map object... vm_pageout_object_terminate - * will drop this reference - */ + /* + * we dontated the paging reference to + * the map object... vm_pageout_object_terminate + * will drop this reference + */ } } VM_OBJECT_WIRED_PAGE_UPDATE_END(shadow_object, shadow_object->wire_tag); vm_object_unlock(shadow_object); - if (object != shadow_object) - vm_object_unlock(object); - - if(!isVectorUPL) + if (object != shadow_object) { + vm_object_unlock(object); + } + + if (!isVectorUPL) { upl_unlock(upl); - else { - /* + } else { + /* * If we completed our operations on an UPL that is * part of a Vectored UPL and if empty is TRUE, then - * we should go ahead and deallocate this UPL element. + * we should go ahead and deallocate this UPL element. * Then we check if this was the last of the UPL elements * within that Vectored UPL. If so, set empty to TRUE * so that in ubc_upl_commit_range or ubc_upl_commit, we * can go ahead and deallocate the Vector UPL too. */ - if(*empty==TRUE) { + if (*empty == TRUE) { *empty = vector_upl_set_subupl(vector_upl, upl, 0); upl_deallocate(upl); } @@ -7344,53 +7309,55 @@ commit_next_page: kern_return_t upl_abort_range( - upl_t upl, - upl_offset_t offset, - upl_size_t size, - int error, - boolean_t *empty) + upl_t upl, + upl_offset_t offset, + upl_size_t size, + int error, + boolean_t *empty) { - upl_page_info_t *user_page_list = NULL; - upl_size_t xfer_size, subupl_size = size; - vm_object_t shadow_object; - vm_object_t object; - vm_object_offset_t target_offset; - upl_offset_t subupl_offset = offset; - int entry; - wpl_array_t lite_list; - int occupied; - struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT]; - struct vm_page_delayed_work *dwp; - int dw_count; - int dw_limit; - int isVectorUPL = 0; - upl_t vector_upl = NULL; + upl_page_info_t *user_page_list = NULL; + upl_size_t xfer_size, subupl_size = size; + vm_object_t shadow_object; + vm_object_t object; + vm_object_offset_t target_offset; + upl_offset_t subupl_offset = offset; + int entry; + wpl_array_t lite_list; + int occupied; + struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT]; + struct vm_page_delayed_work *dwp; + int dw_count; + int dw_limit; + int isVectorUPL = 0; + upl_t vector_upl = NULL; *empty = FALSE; - if (upl == UPL_NULL) + if (upl == UPL_NULL) { return KERN_INVALID_ARGUMENT; + } - if ( (upl->flags & UPL_IO_WIRE) && !(error & UPL_ABORT_DUMP_PAGES) ) + if ((upl->flags & UPL_IO_WIRE) && !(error & UPL_ABORT_DUMP_PAGES)) { return upl_commit_range(upl, offset, size, UPL_COMMIT_FREE_ABSENT, NULL, 0, empty); + } - if((isVectorUPL = vector_upl_is_valid(upl))) { + if ((isVectorUPL = vector_upl_is_valid(upl))) { vector_upl = upl; upl_lock(vector_upl); - } - else + } else { upl_lock(upl); + } process_upl_to_abort: - if(isVectorUPL) { + if (isVectorUPL) { size = subupl_size; offset = subupl_offset; - if(size == 0) { + if (size == 0) { upl_unlock(vector_upl); return KERN_SUCCESS; } upl = vector_upl_subupl_byoffset(vector_upl, &offset, &size); - if(upl == NULL) { + if (upl == NULL) { upl_unlock(vector_upl); return KERN_FAILURE; } @@ -7403,7 +7370,7 @@ process_upl_to_abort: #if UPL_DEBUG if (upl->upl_commit_index < UPL_DEBUG_COMMIT_RECORDS) { (void) OSBacktrace(&upl->upl_commit_records[upl->upl_commit_index].c_retaddr[0], UPL_DEBUG_STACK_FRAMES); - + upl->upl_commit_records[upl->upl_commit_index].c_beg = offset; upl->upl_commit_records[upl->upl_commit_index].c_end = (offset + size); upl->upl_commit_records[upl->upl_commit_index].c_aborted = 1; @@ -7411,47 +7378,49 @@ process_upl_to_abort: upl->upl_commit_index++; } #endif - if (upl->flags & UPL_DEVICE_MEMORY) + if (upl->flags & UPL_DEVICE_MEMORY) { xfer_size = 0; - else if ((offset + size) <= upl->size) - xfer_size = size; - else { - if(!isVectorUPL) + } else if ((offset + size) <= upl->size) { + xfer_size = size; + } else { + if (!isVectorUPL) { upl_unlock(upl); - else { + } else { upl_unlock(vector_upl); } return KERN_FAILURE; } if (upl->flags & UPL_INTERNAL) { - lite_list = (wpl_array_t) - ((((uintptr_t)upl) + sizeof(struct upl)) - + ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t))); + lite_list = (wpl_array_t) + ((((uintptr_t)upl) + sizeof(struct upl)) + + ((upl->size / PAGE_SIZE) * sizeof(upl_page_info_t))); user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl)); } else { - lite_list = (wpl_array_t) - (((uintptr_t)upl) + sizeof(struct upl)); + lite_list = (wpl_array_t) + (((uintptr_t)upl) + sizeof(struct upl)); } object = upl->map_object; if (upl->flags & UPL_SHADOWED) { - vm_object_lock(object); + vm_object_lock(object); shadow_object = object->shadow; - } else + } else { shadow_object = object; + } - entry = offset/PAGE_SIZE; + entry = offset / PAGE_SIZE; target_offset = (vm_object_offset_t)offset; assert(!(target_offset & PAGE_MASK)); assert(!(xfer_size & PAGE_MASK)); - if (upl->flags & UPL_KERNEL_OBJECT) + if (upl->flags & UPL_KERNEL_OBJECT) { vm_object_lock_shared(shadow_object); - else + } else { vm_object_lock(shadow_object); + } if (upl->flags & UPL_ACCESS_BLOCKED) { assert(shadow_object->blocked_access); @@ -7463,54 +7432,57 @@ process_upl_to_abort: dw_count = 0; dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT); - if ((error & UPL_ABORT_DUMP_PAGES) && (upl->flags & UPL_KERNEL_OBJECT)) + if ((error & UPL_ABORT_DUMP_PAGES) && (upl->flags & UPL_KERNEL_OBJECT)) { panic("upl_abort_range: kernel_object being DUMPED"); + } while (xfer_size) { - vm_page_t t, m; - unsigned int pg_num; - boolean_t needed; + vm_page_t t, m; + unsigned int pg_num; + boolean_t needed; - pg_num = (unsigned int) (target_offset/PAGE_SIZE); - assert(pg_num == target_offset/PAGE_SIZE); + pg_num = (unsigned int) (target_offset / PAGE_SIZE); + assert(pg_num == target_offset / PAGE_SIZE); needed = FALSE; - if (user_page_list) + if (user_page_list) { needed = user_page_list[pg_num].needed; + } dwp->dw_mask = 0; m = VM_PAGE_NULL; if (upl->flags & UPL_LITE) { + if (lite_list[pg_num >> 5] & (1 << (pg_num & 31))) { + lite_list[pg_num >> 5] &= ~(1 << (pg_num & 31)); - if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) { - lite_list[pg_num>>5] &= ~(1 << (pg_num & 31)); - - if ( !(upl->flags & UPL_KERNEL_OBJECT)) + if (!(upl->flags & UPL_KERNEL_OBJECT)) { m = vm_page_lookup(shadow_object, target_offset + - (upl->offset - shadow_object->paging_offset)); + (upl->offset - shadow_object->paging_offset)); + } } } if (upl->flags & UPL_SHADOWED) { - if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) { - t->free_when_done = FALSE; + if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) { + t->vmp_free_when_done = FALSE; VM_PAGE_FREE(t); - if (m == VM_PAGE_NULL) + if (m == VM_PAGE_NULL) { m = vm_page_lookup(shadow_object, target_offset + object->vo_shadow_offset); + } } } - if ((upl->flags & UPL_KERNEL_OBJECT)) + if ((upl->flags & UPL_KERNEL_OBJECT)) { goto abort_next_page; + } if (m != VM_PAGE_NULL) { + assert(m->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR); - assert(m->vm_page_q_state != VM_PAGE_USED_BY_COMPRESSOR); - - if (m->absent) { - boolean_t must_free = TRUE; + if (m->vmp_absent) { + boolean_t must_free = TRUE; /* * COPYOUT = FALSE case @@ -7518,22 +7490,22 @@ process_upl_to_abort: * be passed back to the pages customer */ if (error & UPL_ABORT_RESTART) { - m->restart = TRUE; - m->absent = FALSE; - m->unusual = TRUE; + m->vmp_restart = TRUE; + m->vmp_absent = FALSE; + m->vmp_unusual = TRUE; must_free = FALSE; } else if (error & UPL_ABORT_UNAVAILABLE) { - m->restart = FALSE; - m->unusual = TRUE; + m->vmp_restart = FALSE; + m->vmp_unusual = TRUE; must_free = FALSE; } else if (error & UPL_ABORT_ERROR) { - m->restart = FALSE; - m->absent = FALSE; - m->error = TRUE; - m->unusual = TRUE; + m->vmp_restart = FALSE; + m->vmp_absent = FALSE; + m->vmp_error = TRUE; + m->vmp_unusual = TRUE; must_free = FALSE; } - if (m->clustered && needed == FALSE) { + if (m->vmp_clustered && needed == FALSE) { /* * This page was a part of a speculative * read-ahead initiated by the kernel @@ -7545,14 +7517,14 @@ process_upl_to_abort: */ must_free = TRUE; } - m->cleaning = FALSE; + m->vmp_cleaning = FALSE; - if (m->overwriting && !m->busy) { + if (m->vmp_overwriting && !m->vmp_busy) { /* * this shouldn't happen since * this is an 'absent' page, but * it doesn't hurt to check for - * the 'alternate' method of + * the 'alternate' method of * stabilizing the page... * we will mark 'busy' to be cleared * in the following code which will @@ -7561,20 +7533,22 @@ process_upl_to_abort: */ dwp->dw_mask |= DW_vm_page_unwire; } - m->overwriting = FALSE; + m->vmp_overwriting = FALSE; dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); - if (must_free == TRUE) + if (must_free == TRUE) { dwp->dw_mask |= DW_vm_page_free; - else + } else { dwp->dw_mask |= DW_vm_page_activate; + } } else { - /* + /* * Handle the trusted pager throttle. - */ - if (m->laundry) + */ + if (m->vmp_laundry) { dwp->dw_mask |= DW_vm_pageout_throttle_up; + } if (upl->flags & UPL_ACCESS_BLOCKED) { /* @@ -7584,10 +7558,10 @@ process_upl_to_abort: */ dwp->dw_mask |= DW_clear_busy; } - if (m->overwriting) { - if (m->busy) + if (m->vmp_overwriting) { + if (m->vmp_busy) { dwp->dw_mask |= DW_clear_busy; - else { + } else { /* * deal with the 'alternate' method * of stabilizing the page... @@ -7599,10 +7573,10 @@ process_upl_to_abort: */ dwp->dw_mask |= DW_vm_page_unwire; } - m->overwriting = FALSE; + m->vmp_overwriting = FALSE; } - m->free_when_done = FALSE; - m->cleaning = FALSE; + m->vmp_free_when_done = FALSE; + m->vmp_cleaning = FALSE; if (error & UPL_ABORT_DUMP_PAGES) { pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)); @@ -7613,14 +7587,14 @@ process_upl_to_abort: if (error & UPL_ABORT_REFERENCE) { /* * we've been told to explictly - * reference this page... for + * reference this page... for * file I/O, this is done by * implementing an LRU on the inactive q */ dwp->dw_mask |= DW_vm_page_lru; - - } else if ( !VM_PAGE_PAGEABLE(m)) + } else if (!VM_PAGE_PAGEABLE(m)) { dwp->dw_mask |= DW_vm_page_deactivate_internal; + } } dwp->dw_mask |= DW_PAGE_WAKEUP; } @@ -7637,31 +7611,34 @@ abort_next_page: if (dw_count >= dw_limit) { vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count); - + dwp = &dw_array[0]; dw_count = 0; } } else { - if (dwp->dw_mask & DW_clear_busy) - m->busy = FALSE; + if (dwp->dw_mask & DW_clear_busy) { + m->vmp_busy = FALSE; + } - if (dwp->dw_mask & DW_PAGE_WAKEUP) + if (dwp->dw_mask & DW_PAGE_WAKEUP) { PAGE_WAKEUP(m); + } } } } - if (dw_count) + if (dw_count) { vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count); + } occupied = 1; - if (upl->flags & UPL_DEVICE_MEMORY) { + if (upl->flags & UPL_DEVICE_MEMORY) { occupied = 0; } else if (upl->flags & UPL_LITE) { - int pg_num; - int i; + int pg_num; + int i; - pg_num = upl->size/PAGE_SIZE; + pg_num = upl->size / PAGE_SIZE; pg_num = (pg_num + 31) >> 5; occupied = 0; @@ -7672,8 +7649,9 @@ abort_next_page: } } } else { - if (vm_page_queue_empty(&upl->map_object->memq)) + if (vm_page_queue_empty(&upl->map_object->memq)) { occupied = 0; + } } if (occupied == 0) { /* @@ -7684,11 +7662,12 @@ abort_next_page: * should be considered relevant for the Vector UPL and * not the internal UPLs. */ - if ((upl->flags & UPL_COMMIT_NOTIFY_EMPTY) || isVectorUPL) + if ((upl->flags & UPL_COMMIT_NOTIFY_EMPTY) || isVectorUPL) { *empty = TRUE; + } if (object == shadow_object && !(upl->flags & UPL_KERNEL_OBJECT)) { - /* + /* * this is not a paging object * so we need to drop the paging reference * that was taken when we created the UPL @@ -7697,31 +7676,32 @@ abort_next_page: vm_object_activity_end(shadow_object); vm_object_collapse(shadow_object, 0, TRUE); } else { - /* - * we dontated the paging reference to - * the map object... vm_pageout_object_terminate - * will drop this reference - */ + /* + * we dontated the paging reference to + * the map object... vm_pageout_object_terminate + * will drop this reference + */ } } vm_object_unlock(shadow_object); - if (object != shadow_object) - vm_object_unlock(object); - - if(!isVectorUPL) + if (object != shadow_object) { + vm_object_unlock(object); + } + + if (!isVectorUPL) { upl_unlock(upl); - else { - /* - * If we completed our operations on an UPL that is - * part of a Vectored UPL and if empty is TRUE, then - * we should go ahead and deallocate this UPL element. - * Then we check if this was the last of the UPL elements - * within that Vectored UPL. If so, set empty to TRUE - * so that in ubc_upl_abort_range or ubc_upl_abort, we - * can go ahead and deallocate the Vector UPL too. - */ - if(*empty == TRUE) { - *empty = vector_upl_set_subupl(vector_upl, upl,0); + } else { + /* + * If we completed our operations on an UPL that is + * part of a Vectored UPL and if empty is TRUE, then + * we should go ahead and deallocate this UPL element. + * Then we check if this was the last of the UPL elements + * within that Vectored UPL. If so, set empty to TRUE + * so that in ubc_upl_abort_range or ubc_upl_abort, we + * can go ahead and deallocate the Vector UPL too. + */ + if (*empty == TRUE) { + *empty = vector_upl_set_subupl(vector_upl, upl, 0); upl_deallocate(upl); } goto process_upl_to_abort; @@ -7733,13 +7713,14 @@ abort_next_page: kern_return_t upl_abort( - upl_t upl, - int error) + upl_t upl, + int error) { - boolean_t empty; + boolean_t empty; - if (upl == UPL_NULL) + if (upl == UPL_NULL) { return KERN_INVALID_ARGUMENT; + } return upl_abort_range(upl, 0, upl->size, error, &empty); } @@ -7748,14 +7729,15 @@ upl_abort( /* an option on commit should be wire */ kern_return_t upl_commit( - upl_t upl, - upl_page_info_t *page_list, - mach_msg_type_number_t count) + upl_t upl, + upl_page_info_t *page_list, + mach_msg_type_number_t count) { - boolean_t empty; + boolean_t empty; - if (upl == UPL_NULL) + if (upl == UPL_NULL) { return KERN_INVALID_ARGUMENT; + } return upl_commit_range(upl, 0, upl->size, 0, page_list, count, &empty); } @@ -7763,83 +7745,90 @@ upl_commit( void iopl_valid_data( - upl_t upl, + upl_t upl, vm_tag_t tag) { - vm_object_t object; - vm_offset_t offset; - vm_page_t m, nxt_page = VM_PAGE_NULL; - upl_size_t size; - int wired_count = 0; + vm_object_t object; + vm_offset_t offset; + vm_page_t m, nxt_page = VM_PAGE_NULL; + upl_size_t size; + int wired_count = 0; - if (upl == NULL) + if (upl == NULL) { panic("iopl_valid_data: NULL upl"); - if (vector_upl_is_valid(upl)) + } + if (vector_upl_is_valid(upl)) { panic("iopl_valid_data: vector upl"); - if ((upl->flags & (UPL_DEVICE_MEMORY|UPL_SHADOWED|UPL_ACCESS_BLOCKED|UPL_IO_WIRE|UPL_INTERNAL)) != UPL_IO_WIRE) + } + if ((upl->flags & (UPL_DEVICE_MEMORY | UPL_SHADOWED | UPL_ACCESS_BLOCKED | UPL_IO_WIRE | UPL_INTERNAL)) != UPL_IO_WIRE) { panic("iopl_valid_data: unsupported upl, flags = %x", upl->flags); + } object = upl->map_object; - if (object == kernel_object || object == compressor_object) + if (object == kernel_object || object == compressor_object) { panic("iopl_valid_data: object == kernel or compressor"); + } if (object->purgable == VM_PURGABLE_VOLATILE || - object->purgable == VM_PURGABLE_EMPTY) + object->purgable == VM_PURGABLE_EMPTY) { panic("iopl_valid_data: object %p purgable %d", - object, object->purgable); + object, object->purgable); + } size = upl->size; vm_object_lock(object); VM_OBJECT_WIRED_PAGE_UPDATE_START(object); - if (object->vo_size == size && object->resident_page_count == (size / PAGE_SIZE)) + if (object->vo_size == size && object->resident_page_count == (size / PAGE_SIZE)) { nxt_page = (vm_page_t)vm_page_queue_first(&object->memq); - else + } else { offset = 0 + upl->offset - object->paging_offset; + } while (size) { - if (nxt_page != VM_PAGE_NULL) { m = nxt_page; - nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->listq); + nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->vmp_listq); } else { m = vm_page_lookup(object, offset); offset += PAGE_SIZE; - if (m == VM_PAGE_NULL) + if (m == VM_PAGE_NULL) { panic("iopl_valid_data: missing expected page at offset %lx", (long)offset); + } } - if (m->busy) { - if (!m->absent) + if (m->vmp_busy) { + if (!m->vmp_absent) { panic("iopl_valid_data: busy page w/o absent"); + } - if (m->pageq.next || m->pageq.prev) + if (m->vmp_pageq.next || m->vmp_pageq.prev) { panic("iopl_valid_data: busy+absent page on page queue"); - if (m->reusable) { + } + if (m->vmp_reusable) { panic("iopl_valid_data: %p is reusable", m); } - m->absent = FALSE; - m->dirty = TRUE; - assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q); - assert(m->wire_count == 0); - m->wire_count++; - assert(m->wire_count); - if (m->wire_count == 1) { - m->vm_page_q_state = VM_PAGE_IS_WIRED; + m->vmp_absent = FALSE; + m->vmp_dirty = TRUE; + assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q); + assert(m->vmp_wire_count == 0); + m->vmp_wire_count++; + assert(m->vmp_wire_count); + if (m->vmp_wire_count == 1) { + m->vmp_q_state = VM_PAGE_IS_WIRED; wired_count++; } else { panic("iopl_valid_data: %p already wired\n", m); } - + PAGE_WAKEUP_DONE(m); } size -= PAGE_SIZE; } if (wired_count) { - VM_OBJECT_WIRED_PAGE_COUNT(object, wired_count); assert(object->resident_page_count >= object->wired_page_count); @@ -7858,10 +7847,10 @@ iopl_valid_data( void vm_object_set_pmap_cache_attr( - vm_object_t object, - upl_page_info_array_t user_page_list, - unsigned int num_pages, - boolean_t batch_pmap_op) + vm_object_t object, + upl_page_info_array_t user_page_list, + unsigned int num_pages, + boolean_t batch_pmap_op) { unsigned int cache_attr = 0; @@ -7873,21 +7862,21 @@ vm_object_set_pmap_cache_attr( } -boolean_t vm_object_iopl_wire_full(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t, vm_tag_t); -kern_return_t vm_object_iopl_wire_empty(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t, vm_tag_t, vm_object_offset_t *, int); +boolean_t vm_object_iopl_wire_full(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t, vm_tag_t); +kern_return_t vm_object_iopl_wire_empty(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t, vm_tag_t, vm_object_offset_t *, int, int*); boolean_t vm_object_iopl_wire_full(vm_object_t object, upl_t upl, upl_page_info_array_t user_page_list, - wpl_array_t lite_list, upl_control_flags_t cntrl_flags, vm_tag_t tag) + wpl_array_t lite_list, upl_control_flags_t cntrl_flags, vm_tag_t tag) { - vm_page_t dst_page; - unsigned int entry; - int page_count; - int delayed_unlock = 0; - boolean_t retval = TRUE; - ppnum_t phys_page; + vm_page_t dst_page; + unsigned int entry; + int page_count; + int delayed_unlock = 0; + boolean_t retval = TRUE; + ppnum_t phys_page; vm_object_lock_assert_exclusive(object); assert(object->purgable != VM_PURGABLE_VOLATILE); @@ -7902,48 +7891,48 @@ vm_object_iopl_wire_full(vm_object_t object, upl_t upl, upl_page_info_array_t us vm_page_lock_queues(); while (page_count--) { - - if (dst_page->busy || - dst_page->fictitious || - dst_page->absent || - dst_page->error || - dst_page->cleaning || - dst_page->restart || - dst_page->laundry) { + if (dst_page->vmp_busy || + dst_page->vmp_fictitious || + dst_page->vmp_absent || + dst_page->vmp_error || + dst_page->vmp_cleaning || + dst_page->vmp_restart || + dst_page->vmp_laundry) { retval = FALSE; goto done; } - if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->written_by_kernel == TRUE) { + if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->vmp_written_by_kernel == TRUE) { retval = FALSE; goto done; } - dst_page->reference = TRUE; + dst_page->vmp_reference = TRUE; vm_page_wire(dst_page, tag, FALSE); if (!(cntrl_flags & UPL_COPYOUT_FROM)) { SET_PAGE_DIRTY(dst_page, FALSE); } - entry = (unsigned int)(dst_page->offset / PAGE_SIZE); + entry = (unsigned int)(dst_page->vmp_offset / PAGE_SIZE); assert(entry >= 0 && entry < object->resident_page_count); - lite_list[entry>>5] |= 1 << (entry & 31); - + lite_list[entry >> 5] |= 1 << (entry & 31); + phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page); - if (phys_page > upl->highest_page) + if (phys_page > upl->highest_page) { upl->highest_page = phys_page; + } if (user_page_list) { user_page_list[entry].phys_addr = phys_page; - user_page_list[entry].absent = dst_page->absent; - user_page_list[entry].dirty = dst_page->dirty; - user_page_list[entry].free_when_done = dst_page->free_when_done; - user_page_list[entry].precious = dst_page->precious; + user_page_list[entry].absent = dst_page->vmp_absent; + user_page_list[entry].dirty = dst_page->vmp_dirty; + user_page_list[entry].free_when_done = dst_page->vmp_free_when_done; + user_page_list[entry].precious = dst_page->vmp_precious; user_page_list[entry].device = FALSE; user_page_list[entry].speculative = FALSE; user_page_list[entry].cs_validated = FALSE; user_page_list[entry].cs_tainted = FALSE; - user_page_list[entry].cs_nx = FALSE; + user_page_list[entry].cs_nx = FALSE; user_page_list[entry].needed = FALSE; user_page_list[entry].mark = FALSE; } @@ -7953,31 +7942,32 @@ vm_object_iopl_wire_full(vm_object_t object, upl_t upl, upl_page_info_array_t us VM_CHECK_MEMORYSTATUS; } - dst_page = (vm_page_t)vm_page_queue_next(&dst_page->listq); + dst_page = (vm_page_t)vm_page_queue_next(&dst_page->vmp_listq); } done: vm_page_unlock_queues(); VM_CHECK_MEMORYSTATUS; - return (retval); + return retval; } kern_return_t vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t user_page_list, - wpl_array_t lite_list, upl_control_flags_t cntrl_flags, vm_tag_t tag, vm_object_offset_t *dst_offset, int page_count) + wpl_array_t lite_list, upl_control_flags_t cntrl_flags, vm_tag_t tag, vm_object_offset_t *dst_offset, + int page_count, int* page_grab_count) { - vm_page_t dst_page; - boolean_t no_zero_fill = FALSE; - int interruptible; - int pages_wired = 0; - int pages_inserted = 0; - int entry = 0; - uint64_t delayed_ledger_update = 0; - kern_return_t ret = KERN_SUCCESS; - int grab_options; - ppnum_t phys_page; + vm_page_t dst_page; + boolean_t no_zero_fill = FALSE; + int interruptible; + int pages_wired = 0; + int pages_inserted = 0; + int entry = 0; + uint64_t delayed_ledger_update = 0; + kern_return_t ret = KERN_SUCCESS; + int grab_options; + ppnum_t phys_page; vm_object_lock_assert_exclusive(object); assert(object->purgable != VM_PURGABLE_VOLATILE); @@ -7986,13 +7976,15 @@ vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t u assert(object->copy == NULL); assert(object->shadow == NULL); - if (cntrl_flags & UPL_SET_INTERRUPTIBLE) + if (cntrl_flags & UPL_SET_INTERRUPTIBLE) { interruptible = THREAD_ABORTSAFE; - else + } else { interruptible = THREAD_UNINT; + } - if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO)) - no_zero_fill = TRUE; + if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO)) { + no_zero_fill = TRUE; + } grab_options = 0; #if CONFIG_SECLUDED_MEMORY @@ -8002,10 +7994,8 @@ vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t u #endif /* CONFIG_SECLUDED_MEMORY */ while (page_count--) { - while ((dst_page = vm_page_grab_options(grab_options)) - == VM_PAGE_NULL) { - + == VM_PAGE_NULL) { OSAddAtomic(page_count, &vm_upl_wait_for_pages); VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0); @@ -8017,7 +8007,7 @@ vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t u OSAddAtomic(-page_count, &vm_upl_wait_for_pages); VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1); - + ret = MACH_SEND_INTERRUPTED; goto done; } @@ -8025,22 +8015,23 @@ vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t u VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0); } - if (no_zero_fill == FALSE) + if (no_zero_fill == FALSE) { vm_page_zero_fill(dst_page); - else - dst_page->absent = TRUE; + } else { + dst_page->vmp_absent = TRUE; + } - dst_page->reference = TRUE; + dst_page->vmp_reference = TRUE; if (!(cntrl_flags & UPL_COPYOUT_FROM)) { - SET_PAGE_DIRTY(dst_page, FALSE); - } - if (dst_page->absent == FALSE) { - assert(dst_page->vm_page_q_state == VM_PAGE_NOT_ON_Q); - assert(dst_page->wire_count == 0); - dst_page->wire_count++; - dst_page->vm_page_q_state = VM_PAGE_IS_WIRED; - assert(dst_page->wire_count); + SET_PAGE_DIRTY(dst_page, FALSE); + } + if (dst_page->vmp_absent == FALSE) { + assert(dst_page->vmp_q_state == VM_PAGE_NOT_ON_Q); + assert(dst_page->vmp_wire_count == 0); + dst_page->vmp_wire_count++; + dst_page->vmp_q_state = VM_PAGE_IS_WIRED; + assert(dst_page->vmp_wire_count); pages_wired++; PAGE_WAKEUP_DONE(dst_page); } @@ -8048,20 +8039,21 @@ vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t u vm_page_insert_internal(dst_page, object, *dst_offset, tag, FALSE, TRUE, TRUE, TRUE, &delayed_ledger_update); - lite_list[entry>>5] |= 1 << (entry & 31); - + lite_list[entry >> 5] |= 1 << (entry & 31); + phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page); - if (phys_page > upl->highest_page) + if (phys_page > upl->highest_page) { upl->highest_page = phys_page; + } if (user_page_list) { - user_page_list[entry].phys_addr = phys_page; - user_page_list[entry].absent = dst_page->absent; - user_page_list[entry].dirty = dst_page->dirty; - user_page_list[entry].free_when_done = FALSE; - user_page_list[entry].precious = FALSE; - user_page_list[entry].device = FALSE; + user_page_list[entry].phys_addr = phys_page; + user_page_list[entry].absent = dst_page->vmp_absent; + user_page_list[entry].dirty = dst_page->vmp_dirty; + user_page_list[entry].free_when_done = FALSE; + user_page_list[entry].precious = FALSE; + user_page_list[entry].device = FALSE; user_page_list[entry].speculative = FALSE; user_page_list[entry].cs_validated = FALSE; user_page_list[entry].cs_tainted = FALSE; @@ -8086,64 +8078,85 @@ done: } } if (delayed_ledger_update) { - task_t owner; - - owner = object->vo_purgeable_owner; + task_t owner; + int ledger_idx_volatile; + int ledger_idx_nonvolatile; + int ledger_idx_volatile_compressed; + int ledger_idx_nonvolatile_compressed; + boolean_t do_footprint; + + owner = VM_OBJECT_OWNER(object); assert(owner); + vm_object_ledger_tag_ledgers(object, + &ledger_idx_volatile, + &ledger_idx_nonvolatile, + &ledger_idx_volatile_compressed, + &ledger_idx_nonvolatile_compressed, + &do_footprint); + /* more non-volatile bytes */ ledger_credit(owner->ledger, - task_ledgers.purgeable_nonvolatile, - delayed_ledger_update); - /* more footprint */ - ledger_credit(owner->ledger, - task_ledgers.phys_footprint, - delayed_ledger_update); + ledger_idx_nonvolatile, + delayed_ledger_update); + if (do_footprint) { + /* more footprint */ + ledger_credit(owner->ledger, + task_ledgers.phys_footprint, + delayed_ledger_update); + } } - return (ret); -} + assert(page_grab_count); + *page_grab_count = pages_inserted; + + return ret; +} -unsigned int vm_object_iopl_request_sleep_for_cleaning = 0; kern_return_t vm_object_iopl_request( - vm_object_t object, - vm_object_offset_t offset, - upl_size_t size, - upl_t *upl_ptr, - upl_page_info_array_t user_page_list, - unsigned int *page_list_count, - upl_control_flags_t cntrl_flags, - vm_tag_t tag) + vm_object_t object, + vm_object_offset_t offset, + upl_size_t size, + upl_t *upl_ptr, + upl_page_info_array_t user_page_list, + unsigned int *page_list_count, + upl_control_flags_t cntrl_flags, + vm_tag_t tag) { - vm_page_t dst_page; - vm_object_offset_t dst_offset; - upl_size_t xfer_size; - upl_t upl = NULL; - unsigned int entry; - wpl_array_t lite_list = NULL; - int no_zero_fill = FALSE; - unsigned int size_in_pages; - u_int32_t psize; - kern_return_t ret; - vm_prot_t prot; - struct vm_object_fault_info fault_info; - struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT]; - struct vm_page_delayed_work *dwp; - int dw_count; - int dw_limit; - int dw_index; - boolean_t caller_lookup; - int io_tracking_flag = 0; - int interruptible; - ppnum_t phys_page; - - boolean_t set_cache_attr_needed = FALSE; - boolean_t free_wired_pages = FALSE; - boolean_t fast_path_empty_req = FALSE; - boolean_t fast_path_full_req = FALSE; + vm_page_t dst_page; + vm_object_offset_t dst_offset; + upl_size_t xfer_size; + upl_t upl = NULL; + unsigned int entry; + wpl_array_t lite_list = NULL; + int no_zero_fill = FALSE; + unsigned int size_in_pages; + int page_grab_count = 0; + u_int32_t psize; + kern_return_t ret; + vm_prot_t prot; + struct vm_object_fault_info fault_info = {}; + struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT]; + struct vm_page_delayed_work *dwp; + int dw_count; + int dw_limit; + int dw_index; + boolean_t caller_lookup; + int io_tracking_flag = 0; + int interruptible; + ppnum_t phys_page; + + boolean_t set_cache_attr_needed = FALSE; + boolean_t free_wired_pages = FALSE; + boolean_t fast_path_empty_req = FALSE; + boolean_t fast_path_full_req = FALSE; + +#if DEVELOPMENT || DEBUG + task_t task = current_task(); +#endif /* DEVELOPMENT || DEBUG */ if (cntrl_flags & ~UPL_VALID_FLAGS) { /* @@ -8152,72 +8165,93 @@ vm_object_iopl_request( */ return KERN_INVALID_VALUE; } - if (vm_lopage_needed == FALSE) - cntrl_flags &= ~UPL_NEED_32BIT_ADDR; + if (vm_lopage_needed == FALSE) { + cntrl_flags &= ~UPL_NEED_32BIT_ADDR; + } if (cntrl_flags & UPL_NEED_32BIT_ADDR) { - if ( (cntrl_flags & (UPL_SET_IO_WIRE | UPL_SET_LITE)) != (UPL_SET_IO_WIRE | UPL_SET_LITE)) - return KERN_INVALID_VALUE; + if ((cntrl_flags & (UPL_SET_IO_WIRE | UPL_SET_LITE)) != (UPL_SET_IO_WIRE | UPL_SET_LITE)) { + return KERN_INVALID_VALUE; + } if (object->phys_contiguous) { - if ((offset + object->vo_shadow_offset) >= (vm_object_offset_t)max_valid_dma_address) - return KERN_INVALID_ADDRESS; - - if (((offset + object->vo_shadow_offset) + size) >= (vm_object_offset_t)max_valid_dma_address) - return KERN_INVALID_ADDRESS; + if ((offset + object->vo_shadow_offset) >= (vm_object_offset_t)max_valid_dma_address) { + return KERN_INVALID_ADDRESS; + } + + if (((offset + object->vo_shadow_offset) + size) >= (vm_object_offset_t)max_valid_dma_address) { + return KERN_INVALID_ADDRESS; + } } } - if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO)) - no_zero_fill = TRUE; + if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO)) { + no_zero_fill = TRUE; + } - if (cntrl_flags & UPL_COPYOUT_FROM) + if (cntrl_flags & UPL_COPYOUT_FROM) { prot = VM_PROT_READ; - else + } else { prot = VM_PROT_READ | VM_PROT_WRITE; + } - if ((!object->internal) && (object->paging_offset != 0)) + if ((!object->internal) && (object->paging_offset != 0)) { panic("vm_object_iopl_request: external object with non-zero paging offset\n"); + } + + VM_DEBUG_CONSTANT_EVENT(vm_object_iopl_request, VM_IOPL_REQUEST, DBG_FUNC_START, size, cntrl_flags, prot, 0); #if CONFIG_IOSCHED || UPL_DEBUG - if ((object->io_tracking && object != kernel_object) || upl_debug_enabled) + if ((object->io_tracking && object != kernel_object) || upl_debug_enabled) { io_tracking_flag |= UPL_CREATE_IO_TRACKING; + } #endif #if CONFIG_IOSCHED if (object->io_tracking) { /* Check if we're dealing with the kernel object. We do not support expedite on kernel object UPLs */ - if (object != kernel_object) + if (object != kernel_object) { io_tracking_flag |= UPL_CREATE_EXPEDITE_SUP; + } } #endif - if (object->phys_contiguous) - psize = PAGE_SIZE; - else - psize = size; + if (object->phys_contiguous) { + psize = PAGE_SIZE; + } else { + psize = size; + } if (cntrl_flags & UPL_SET_INTERNAL) { - upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE | io_tracking_flag, UPL_IO_WIRE, psize); + upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE | io_tracking_flag, UPL_IO_WIRE, psize); user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl)); lite_list = (wpl_array_t) (((uintptr_t)user_page_list) + - ((psize / PAGE_SIZE) * sizeof(upl_page_info_t))); + ((psize / PAGE_SIZE) * sizeof(upl_page_info_t))); if (size == 0) { user_page_list = NULL; lite_list = NULL; } } else { - upl = upl_create(UPL_CREATE_LITE | io_tracking_flag, UPL_IO_WIRE, psize); + upl = upl_create(UPL_CREATE_LITE | io_tracking_flag, UPL_IO_WIRE, psize); lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl)); if (size == 0) { lite_list = NULL; } } - if (user_page_list) - user_page_list[0].device = FALSE; + if (user_page_list) { + user_page_list[0].device = FALSE; + } *upl_ptr = upl; + if (cntrl_flags & UPL_NOZEROFILLIO) { + DTRACE_VM4(upl_nozerofillio, + vm_object_t, object, + vm_object_offset_t, offset, + upl_size_t, size, + upl_t, upl); + } + upl->map_object = object; upl->size = size; @@ -8256,7 +8290,6 @@ vm_object_iopl_request( #endif if (object->phys_contiguous) { - if (upl->flags & UPL_ACCESS_BLOCKED) { assert(!object->blocked_access); object->blocked_access = TRUE; @@ -8270,18 +8303,26 @@ vm_object_iopl_request( */ upl->flags |= UPL_DEVICE_MEMORY; - upl->highest_page = (ppnum_t) ((offset + object->vo_shadow_offset + size - 1)>>PAGE_SHIFT); + upl->highest_page = (ppnum_t) ((offset + object->vo_shadow_offset + size - 1) >> PAGE_SHIFT); if (user_page_list) { - user_page_list[0].phys_addr = (ppnum_t) ((offset + object->vo_shadow_offset)>>PAGE_SHIFT); + user_page_list[0].phys_addr = (ppnum_t) ((offset + object->vo_shadow_offset) >> PAGE_SHIFT); user_page_list[0].device = TRUE; } if (page_list_count != NULL) { - if (upl->flags & UPL_INTERNAL) - *page_list_count = 0; - else - *page_list_count = 1; + if (upl->flags & UPL_INTERNAL) { + *page_list_count = 0; + } else { + *page_list_count = 1; + } + } + + VM_DEBUG_CONSTANT_EVENT(vm_object_iopl_request, VM_IOPL_REQUEST, DBG_FUNC_END, page_grab_count, KERN_SUCCESS, 0, 0); +#if DEVELOPMENT || DEBUG + if (task != NULL) { + ledger_credit(task->ledger, task_ledgers.pages_grabbed_iopl, page_grab_count); } +#endif /* DEVELOPMENT || DEBUG */ return KERN_SUCCESS; } if (object != kernel_object && object != compressor_object) { @@ -8295,20 +8336,21 @@ vm_object_iopl_request( int num = 0; num = OSBacktrace(bt, - VM_OBJECT_TRACKING_BTDEPTH); + VM_OBJECT_TRACKING_BTDEPTH); btlog_add_entry(vm_object_tracking_btlog, - object, - VM_OBJECT_TRACKING_OP_TRUESHARE, - bt, - num); + object, + VM_OBJECT_TRACKING_OP_TRUESHARE, + bt, + num); } #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */ vm_object_lock_assert_exclusive(object); object->true_share = TRUE; - if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) + if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) { object->copy_strategy = MEMORY_OBJECT_COPY_DELAY; + } } if (!(cntrl_flags & UPL_COPYOUT_FROM) && @@ -8331,17 +8373,15 @@ vm_object_iopl_request( * code path for that... */ vm_object_update(object, - offset, - size, - NULL, - NULL, - FALSE, /* should_return */ - MEMORY_OBJECT_COPY_SYNC, - VM_PROT_NO_CHANGE); -#if DEVELOPMENT || DEBUG - iopl_cow++; - iopl_cow_pages += size >> PAGE_SHIFT; -#endif + offset, + size, + NULL, + NULL, + FALSE, /* should_return */ + MEMORY_OBJECT_COPY_SYNC, + VM_PROT_NO_CHANGE); + VM_PAGEOUT_DEBUG(iopl_cow, 1); + VM_PAGEOUT_DEBUG(iopl_cow_pages, (size >> PAGE_SHIFT)); } if (!(cntrl_flags & (UPL_NEED_32BIT_ADDR | UPL_BLOCK_ACCESS)) && object->purgable != VM_PURGABLE_VOLATILE && @@ -8350,16 +8390,12 @@ vm_object_iopl_request( size == object->vo_size && offset == 0 && object->shadow == NULL && - object->pager == NULL) - { - if (object->resident_page_count == size_in_pages) - { + object->pager == NULL) { + if (object->resident_page_count == size_in_pages) { assert(object != compressor_object); assert(object != kernel_object); fast_path_full_req = TRUE; - } - else if (object->resident_page_count == 0) - { + } else if (object->resident_page_count == 0) { assert(object != compressor_object); assert(object != kernel_object); fast_path_empty_req = TRUE; @@ -8367,10 +8403,11 @@ vm_object_iopl_request( } } - if (cntrl_flags & UPL_SET_INTERRUPTIBLE) + if (cntrl_flags & UPL_SET_INTERRUPTIBLE) { interruptible = THREAD_ABORTSAFE; - else + } else { interruptible = THREAD_UNINT; + } entry = 0; @@ -8379,22 +8416,20 @@ vm_object_iopl_request( dw_count = 0; if (fast_path_full_req) { - - if (vm_object_iopl_wire_full(object, upl, user_page_list, lite_list, cntrl_flags, tag) == TRUE) + if (vm_object_iopl_wire_full(object, upl, user_page_list, lite_list, cntrl_flags, tag) == TRUE) { goto finish; + } /* * we couldn't complete the processing of this request on the fast path * so fall through to the slow path and finish up */ - } else if (fast_path_empty_req) { - if (cntrl_flags & UPL_REQUEST_NO_FAULT) { ret = KERN_MEMORY_ERROR; goto return_err; } - ret = vm_object_iopl_wire_empty(object, upl, user_page_list, lite_list, cntrl_flags, tag, &dst_offset, size_in_pages); - + ret = vm_object_iopl_wire_empty(object, upl, user_page_list, lite_list, cntrl_flags, tag, &dst_offset, size_in_pages, &page_grab_count); + if (ret) { free_wired_pages = TRUE; goto return_err; @@ -8403,13 +8438,8 @@ vm_object_iopl_request( } fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL; - fault_info.user_tag = 0; fault_info.lo_offset = offset; fault_info.hi_offset = offset + xfer_size; - fault_info.no_cache = FALSE; - fault_info.stealth = FALSE; - fault_info.io_sync = FALSE; - fault_info.cs_bypass = FALSE; fault_info.mark_zf_absent = TRUE; fault_info.interruptible = interruptible; fault_info.batch_pmap_op = TRUE; @@ -8418,7 +8448,7 @@ vm_object_iopl_request( dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT); while (xfer_size) { - vm_fault_return_t result; + vm_fault_return_t result; dwp->dw_mask = 0; @@ -8431,157 +8461,159 @@ vm_object_iopl_request( * the following check is needed to determine whether * this page was already processed in the fast path */ - if (lite_list[entry>>5] & (1 << (entry & 31))) + if (lite_list[entry >> 5] & (1 << (entry & 31))) { goto skip_page; + } } dst_page = vm_page_lookup(object, dst_offset); if (dst_page == VM_PAGE_NULL || - dst_page->busy || - dst_page->error || - dst_page->restart || - dst_page->absent || - dst_page->fictitious) { - - if (object == kernel_object) - panic("vm_object_iopl_request: missing/bad page in kernel object\n"); - if (object == compressor_object) - panic("vm_object_iopl_request: missing/bad page in compressor object\n"); - - if (cntrl_flags & UPL_REQUEST_NO_FAULT) { - ret = KERN_MEMORY_ERROR; - goto return_err; - } - set_cache_attr_needed = TRUE; - - /* - * We just looked up the page and the result remains valid - * until the object lock is release, so send it to - * vm_fault_page() (as "dst_page"), to avoid having to - * look it up again there. - */ - caller_lookup = TRUE; - - do { - vm_page_t top_page; - kern_return_t error_code; - - fault_info.cluster_size = xfer_size; - - vm_object_paging_begin(object); - - result = vm_fault_page(object, dst_offset, - prot | VM_PROT_WRITE, FALSE, - caller_lookup, - &prot, &dst_page, &top_page, - (int *)0, - &error_code, no_zero_fill, - FALSE, &fault_info); - - /* our lookup is no longer valid at this point */ - caller_lookup = FALSE; - - switch (result) { - - case VM_FAULT_SUCCESS: - - if ( !dst_page->absent) { - PAGE_WAKEUP_DONE(dst_page); - } else { - /* - * we only get back an absent page if we - * requested that it not be zero-filled - * because we are about to fill it via I/O - * - * absent pages should be left BUSY - * to prevent them from being faulted - * into an address space before we've - * had a chance to complete the I/O on - * them since they may contain info that - * shouldn't be seen by the faulting task - */ - } - /* - * Release paging references and - * top-level placeholder page, if any. - */ - if (top_page != VM_PAGE_NULL) { - vm_object_t local_object; + dst_page->vmp_busy || + dst_page->vmp_error || + dst_page->vmp_restart || + dst_page->vmp_absent || + dst_page->vmp_fictitious) { + if (object == kernel_object) { + panic("vm_object_iopl_request: missing/bad page in kernel object\n"); + } + if (object == compressor_object) { + panic("vm_object_iopl_request: missing/bad page in compressor object\n"); + } + + if (cntrl_flags & UPL_REQUEST_NO_FAULT) { + ret = KERN_MEMORY_ERROR; + goto return_err; + } + set_cache_attr_needed = TRUE; + + /* + * We just looked up the page and the result remains valid + * until the object lock is release, so send it to + * vm_fault_page() (as "dst_page"), to avoid having to + * look it up again there. + */ + caller_lookup = TRUE; + + do { + vm_page_t top_page; + kern_return_t error_code; + + fault_info.cluster_size = xfer_size; + + vm_object_paging_begin(object); + + result = vm_fault_page(object, dst_offset, + prot | VM_PROT_WRITE, FALSE, + caller_lookup, + &prot, &dst_page, &top_page, + (int *)0, + &error_code, no_zero_fill, + FALSE, &fault_info); + + /* our lookup is no longer valid at this point */ + caller_lookup = FALSE; - local_object = VM_PAGE_OBJECT(top_page); - + switch (result) { + case VM_FAULT_SUCCESS: + page_grab_count++; + + if (!dst_page->vmp_absent) { + PAGE_WAKEUP_DONE(dst_page); + } else { + /* + * we only get back an absent page if we + * requested that it not be zero-filled + * because we are about to fill it via I/O + * + * absent pages should be left BUSY + * to prevent them from being faulted + * into an address space before we've + * had a chance to complete the I/O on + * them since they may contain info that + * shouldn't be seen by the faulting task + */ + } /* - * comparing 2 packed pointers + * Release paging references and + * top-level placeholder page, if any. */ - if (top_page->vm_page_object != dst_page->vm_page_object) { - vm_object_lock(local_object); - VM_PAGE_FREE(top_page); - vm_object_paging_end(local_object); - vm_object_unlock(local_object); - } else { - VM_PAGE_FREE(top_page); - vm_object_paging_end(local_object); + if (top_page != VM_PAGE_NULL) { + vm_object_t local_object; + + local_object = VM_PAGE_OBJECT(top_page); + + /* + * comparing 2 packed pointers + */ + if (top_page->vmp_object != dst_page->vmp_object) { + vm_object_lock(local_object); + VM_PAGE_FREE(top_page); + vm_object_paging_end(local_object); + vm_object_unlock(local_object); + } else { + VM_PAGE_FREE(top_page); + vm_object_paging_end(local_object); + } } - } - vm_object_paging_end(object); - break; - - case VM_FAULT_RETRY: - vm_object_lock(object); - break; + vm_object_paging_end(object); + break; - case VM_FAULT_MEMORY_SHORTAGE: - OSAddAtomic((size_in_pages - entry), &vm_upl_wait_for_pages); + case VM_FAULT_RETRY: + vm_object_lock(object); + break; - VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0); + case VM_FAULT_MEMORY_SHORTAGE: + OSAddAtomic((size_in_pages - entry), &vm_upl_wait_for_pages); - if (vm_page_wait(interruptible)) { - OSAddAtomic(-(size_in_pages - entry), &vm_upl_wait_for_pages); + VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0); - VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0); - vm_object_lock(object); + if (vm_page_wait(interruptible)) { + OSAddAtomic(-(size_in_pages - entry), &vm_upl_wait_for_pages); - break; - } - OSAddAtomic(-(size_in_pages - entry), &vm_upl_wait_for_pages); + VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0); + vm_object_lock(object); - VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1); + break; + } + OSAddAtomic(-(size_in_pages - entry), &vm_upl_wait_for_pages); - /* fall thru */ + VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1); - case VM_FAULT_INTERRUPTED: - error_code = MACH_SEND_INTERRUPTED; - case VM_FAULT_MEMORY_ERROR: - memory_error: - ret = (error_code ? error_code: KERN_MEMORY_ERROR); + /* fall thru */ - vm_object_lock(object); - goto return_err; + case VM_FAULT_INTERRUPTED: + error_code = MACH_SEND_INTERRUPTED; + case VM_FAULT_MEMORY_ERROR: +memory_error: + ret = (error_code ? error_code: KERN_MEMORY_ERROR); - case VM_FAULT_SUCCESS_NO_VM_PAGE: - /* success but no page: fail */ - vm_object_paging_end(object); - vm_object_unlock(object); - goto memory_error; + vm_object_lock(object); + goto return_err; - default: - panic("vm_object_iopl_request: unexpected error" - " 0x%x from vm_fault_page()\n", result); - } - } while (result != VM_FAULT_SUCCESS); + case VM_FAULT_SUCCESS_NO_VM_PAGE: + /* success but no page: fail */ + vm_object_paging_end(object); + vm_object_unlock(object); + goto memory_error; + default: + panic("vm_object_iopl_request: unexpected error" + " 0x%x from vm_fault_page()\n", result); + } + } while (result != VM_FAULT_SUCCESS); } phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page); - if (upl->flags & UPL_KERNEL_OBJECT) + if (upl->flags & UPL_KERNEL_OBJECT) { goto record_phys_addr; + } - if (dst_page->vm_page_q_state == VM_PAGE_USED_BY_COMPRESSOR) { - dst_page->busy = TRUE; + if (dst_page->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) { + dst_page->vmp_busy = TRUE; goto record_phys_addr; } - if (dst_page->cleaning) { + if (dst_page->vmp_cleaning) { /* * Someone else is cleaning this page in place. * In theory, we should be able to proceed and use this @@ -8592,17 +8624,18 @@ vm_object_iopl_request( * We'd better wait for the cleaning to complete and * then try again. */ - vm_object_iopl_request_sleep_for_cleaning++; + VM_PAGEOUT_DEBUG(vm_object_iopl_request_sleep_for_cleaning, 1); PAGE_SLEEP(object, dst_page, THREAD_UNINT); continue; } - if (dst_page->laundry) + if (dst_page->vmp_laundry) { vm_pageout_steal_laundry(dst_page, FALSE); + } - if ( (cntrl_flags & UPL_NEED_32BIT_ADDR) && - phys_page >= (max_valid_dma_address >> PAGE_SHIFT) ) { - vm_page_t low_page; - int refmod; + if ((cntrl_flags & UPL_NEED_32BIT_ADDR) && + phys_page >= (max_valid_dma_address >> PAGE_SHIFT)) { + vm_page_t low_page; + int refmod; /* * support devices that can't DMA above 32 bits @@ -8613,13 +8646,13 @@ vm_object_iopl_request( * handed out to some other 64 bit capable DMA device to use */ if (VM_PAGE_WIRED(dst_page)) { - ret = KERN_PROTECTION_FAILURE; + ret = KERN_PROTECTION_FAILURE; goto return_err; } low_page = vm_page_grablo(); if (low_page == VM_PAGE_NULL) { - ret = KERN_RESOURCE_SHORTAGE; + ret = KERN_RESOURCE_SHORTAGE; goto return_err; } /* @@ -8629,22 +8662,25 @@ vm_object_iopl_request( * it after we disconnect it... we want the fault * to find the new page being substituted. */ - if (dst_page->pmapped) - refmod = pmap_disconnect(phys_page); - else - refmod = 0; + if (dst_page->vmp_pmapped) { + refmod = pmap_disconnect(phys_page); + } else { + refmod = 0; + } - if (!dst_page->absent) + if (!dst_page->vmp_absent) { vm_page_copy(dst_page, low_page); - - low_page->reference = dst_page->reference; - low_page->dirty = dst_page->dirty; - low_page->absent = dst_page->absent; + } + + low_page->vmp_reference = dst_page->vmp_reference; + low_page->vmp_dirty = dst_page->vmp_dirty; + low_page->vmp_absent = dst_page->vmp_absent; - if (refmod & VM_MEM_REFERENCED) - low_page->reference = TRUE; + if (refmod & VM_MEM_REFERENCED) { + low_page->vmp_reference = TRUE; + } if (refmod & VM_MEM_MODIFIED) { - SET_PAGE_DIRTY(low_page, FALSE); + SET_PAGE_DIRTY(low_page, FALSE); } vm_page_replace(low_page, object, dst_offset); @@ -8655,13 +8691,15 @@ vm_object_iopl_request( * BUSY... we don't need a PAGE_WAKEUP_DONE * here, because we've never dropped the object lock */ - if ( !dst_page->absent) - dst_page->busy = FALSE; + if (!dst_page->vmp_absent) { + dst_page->vmp_busy = FALSE; + } phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page); } - if ( !dst_page->busy) + if (!dst_page->vmp_busy) { dwp->dw_mask |= DW_vm_page_wire; + } if (cntrl_flags & UPL_BLOCK_ACCESS) { /* @@ -8670,8 +8708,8 @@ vm_object_iopl_request( * We'll also remove the mapping * of all these pages before leaving this routine. */ - assert(!dst_page->fictitious); - dst_page->busy = TRUE; + assert(!dst_page->vmp_fictitious); + dst_page->vmp_busy = TRUE; } /* * expect the page to be used @@ -8679,48 +8717,52 @@ vm_object_iopl_request( */ dwp->dw_mask |= DW_set_reference; - if (!(cntrl_flags & UPL_COPYOUT_FROM)) { - SET_PAGE_DIRTY(dst_page, TRUE); + if (!(cntrl_flags & UPL_COPYOUT_FROM)) { + SET_PAGE_DIRTY(dst_page, TRUE); } - if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->written_by_kernel == TRUE) { + if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->vmp_written_by_kernel == TRUE) { pmap_sync_page_attributes_phys(phys_page); - dst_page->written_by_kernel = FALSE; + dst_page->vmp_written_by_kernel = FALSE; } record_phys_addr: - if (dst_page->busy) + if (dst_page->vmp_busy) { upl->flags |= UPL_HAS_BUSY; + } - lite_list[entry>>5] |= 1 << (entry & 31); + lite_list[entry >> 5] |= 1 << (entry & 31); - if (phys_page > upl->highest_page) - upl->highest_page = phys_page; + if (phys_page > upl->highest_page) { + upl->highest_page = phys_page; + } if (user_page_list) { - user_page_list[entry].phys_addr = phys_page; - user_page_list[entry].free_when_done = dst_page->free_when_done; - user_page_list[entry].absent = dst_page->absent; - user_page_list[entry].dirty = dst_page->dirty; - user_page_list[entry].precious = dst_page->precious; - user_page_list[entry].device = FALSE; + user_page_list[entry].phys_addr = phys_page; + user_page_list[entry].free_when_done = dst_page->vmp_free_when_done; + user_page_list[entry].absent = dst_page->vmp_absent; + user_page_list[entry].dirty = dst_page->vmp_dirty; + user_page_list[entry].precious = dst_page->vmp_precious; + user_page_list[entry].device = FALSE; user_page_list[entry].needed = FALSE; - if (dst_page->clustered == TRUE) - user_page_list[entry].speculative = (dst_page->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE; - else - user_page_list[entry].speculative = FALSE; - user_page_list[entry].cs_validated = dst_page->cs_validated; - user_page_list[entry].cs_tainted = dst_page->cs_tainted; - user_page_list[entry].cs_nx = dst_page->cs_nx; + if (dst_page->vmp_clustered == TRUE) { + user_page_list[entry].speculative = (dst_page->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE; + } else { + user_page_list[entry].speculative = FALSE; + } + user_page_list[entry].cs_validated = dst_page->vmp_cs_validated; + user_page_list[entry].cs_tainted = dst_page->vmp_cs_tainted; + user_page_list[entry].cs_nx = dst_page->vmp_cs_nx; user_page_list[entry].mark = FALSE; } if (object != kernel_object && object != compressor_object) { /* * someone is explicitly grabbing this page... * update clustered and speculative state - * + * */ - if (dst_page->clustered) + if (dst_page->vmp_clustered) { VM_PAGE_CONSUME_CLUSTERED(dst_page); + } } skip_page: entry++; @@ -8732,7 +8774,7 @@ skip_page: if (dw_count >= dw_limit) { vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count); - + dwp = &dw_array[0]; dw_count = 0; } @@ -8740,17 +8782,20 @@ skip_page: } assert(entry == size_in_pages); - if (dw_count) + if (dw_count) { vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count); + } finish: - if (user_page_list && set_cache_attr_needed == TRUE) + if (user_page_list && set_cache_attr_needed == TRUE) { vm_object_set_pmap_cache_attr(object, user_page_list, size_in_pages, TRUE); + } if (page_list_count != NULL) { - if (upl->flags & UPL_INTERNAL) + if (upl->flags & UPL_INTERNAL) { *page_list_count = 0; - else if (*page_list_count > size_in_pages) + } else if (*page_list_count > size_in_pages) { *page_list_count = size_in_pages; + } } vm_object_unlock(object); @@ -8762,11 +8807,17 @@ finish: * can't be accessed without causing a page fault. */ vm_object_pmap_protect(object, offset, (vm_object_size_t)size, - PMAP_NULL, 0, VM_PROT_NONE); + PMAP_NULL, 0, VM_PROT_NONE); assert(!object->blocked_access); object->blocked_access = TRUE; } + VM_DEBUG_CONSTANT_EVENT(vm_object_iopl_request, VM_IOPL_REQUEST, DBG_FUNC_END, page_grab_count, KERN_SUCCESS, 0, 0); +#if DEVELOPMENT || DEBUG + if (task != NULL) { + ledger_credit(task->ledger, task_ledgers.pages_grabbed_iopl, page_grab_count); + } +#endif /* DEVELOPMENT || DEBUG */ return KERN_SUCCESS; return_err: @@ -8775,13 +8826,14 @@ return_err: for (; offset < dst_offset; offset += PAGE_SIZE) { boolean_t need_unwire; - dst_page = vm_page_lookup(object, offset); + dst_page = vm_page_lookup(object, offset); - if (dst_page == VM_PAGE_NULL) - panic("vm_object_iopl_request: Wired page missing. \n"); + if (dst_page == VM_PAGE_NULL) { + panic("vm_object_iopl_request: Wired page missing. \n"); + } /* - * if we've already processed this page in an earlier + * if we've already processed this page in an earlier * dw_do_work, we need to undo the wiring... we will * leave the dirty and reference bits on if they * were set, since we don't have a good way of knowing @@ -8807,47 +8859,55 @@ return_err: } vm_page_lock_queues(); - if (dst_page->absent || free_wired_pages == TRUE) { + if (dst_page->vmp_absent || free_wired_pages == TRUE) { vm_page_free(dst_page); need_unwire = FALSE; } else { - if (need_unwire == TRUE) + if (need_unwire == TRUE) { vm_page_unwire(dst_page, TRUE); + } PAGE_WAKEUP_DONE(dst_page); } vm_page_unlock_queues(); - if (need_unwire == TRUE) + if (need_unwire == TRUE) { VM_STAT_INCR(reactivations); + } } #if UPL_DEBUG upl->upl_state = 2; #endif - if (! (upl->flags & UPL_KERNEL_OBJECT)) { + if (!(upl->flags & UPL_KERNEL_OBJECT)) { vm_object_activity_end(object); vm_object_collapse(object, 0, TRUE); } vm_object_unlock(object); upl_destroy(upl); + VM_DEBUG_CONSTANT_EVENT(vm_object_iopl_request, VM_IOPL_REQUEST, DBG_FUNC_END, page_grab_count, ret, 0, 0); +#if DEVELOPMENT || DEBUG + if (task != NULL) { + ledger_credit(task->ledger, task_ledgers.pages_grabbed_iopl, page_grab_count); + } +#endif /* DEVELOPMENT || DEBUG */ return ret; } kern_return_t upl_transpose( - upl_t upl1, - upl_t upl2) + upl_t upl1, + upl_t upl2) { - kern_return_t retval; - boolean_t upls_locked; - vm_object_t object1, object2; + kern_return_t retval; + boolean_t upls_locked; + vm_object_t object1, object2; - if (upl1 == UPL_NULL || upl2 == UPL_NULL || upl1 == upl2 || ((upl1->flags & UPL_VECTOR)==UPL_VECTOR) || ((upl2->flags & UPL_VECTOR)==UPL_VECTOR)) { + if (upl1 == UPL_NULL || upl2 == UPL_NULL || upl1 == upl2 || ((upl1->flags & UPL_VECTOR) == UPL_VECTOR) || ((upl2->flags & UPL_VECTOR) == UPL_VECTOR)) { return KERN_INVALID_ARGUMENT; } - + upls_locked = FALSE; /* @@ -8861,7 +8921,7 @@ upl_transpose( upl_lock(upl2); upl_lock(upl1); } - upls_locked = TRUE; /* the UPLs will need to be unlocked */ + upls_locked = TRUE; /* the UPLs will need to be unlocked */ object1 = upl1->map_object; object2 = upl2->map_object; @@ -8882,7 +8942,7 @@ upl_transpose( * Tranpose the VM objects' backing store. */ retval = vm_object_transpose(object1, object2, - (vm_object_size_t) upl1->size); + (vm_object_size_t) upl1->size); if (retval == KERN_SUCCESS) { /* @@ -8894,19 +8954,23 @@ upl_transpose( vm_object_lock(object1); vm_object_lock(object2); } - if (upl1->flags & UPL_TRACKED_BY_OBJECT) + if (upl1->flags & UPL_TRACKED_BY_OBJECT) { queue_remove(&object1->uplq, upl1, upl_t, uplq); - if (upl2->flags & UPL_TRACKED_BY_OBJECT) + } + if (upl2->flags & UPL_TRACKED_BY_OBJECT) { queue_remove(&object2->uplq, upl2, upl_t, uplq); + } #endif upl1->map_object = object2; upl2->map_object = object1; #if CONFIG_IOSCHED || UPL_DEBUG - if (upl1->flags & UPL_TRACKED_BY_OBJECT) + if (upl1->flags & UPL_TRACKED_BY_OBJECT) { queue_enter(&object2->uplq, upl1, upl_t, uplq); - if (upl2->flags & UPL_TRACKED_BY_OBJECT) + } + if (upl2->flags & UPL_TRACKED_BY_OBJECT) { queue_enter(&object1->uplq, upl2, upl_t, uplq); + } if ((upl1->flags & UPL_TRACKED_BY_OBJECT) || (upl2->flags & UPL_TRACKED_BY_OBJECT)) { vm_object_unlock(object2); vm_object_unlock(object1); @@ -8929,22 +8993,24 @@ done: void upl_range_needed( - upl_t upl, - int index, - int count) + upl_t upl, + int index, + int count) { - upl_page_info_t *user_page_list; - int size_in_pages; + upl_page_info_t *user_page_list; + int size_in_pages; - if ( !(upl->flags & UPL_INTERNAL) || count <= 0) + if (!(upl->flags & UPL_INTERNAL) || count <= 0) { return; + } size_in_pages = upl->size / PAGE_SIZE; user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl)); - while (count-- && index < size_in_pages) + while (count-- && index < size_in_pages) { user_page_list[index++].needed = TRUE; + } } @@ -8957,25 +9023,26 @@ upl_range_needed( * virtaul address space each time we need to work with * a physical page. */ -decl_simple_lock_data(,vm_paging_lock) -#define VM_PAGING_NUM_PAGES 64 +decl_simple_lock_data(, vm_paging_lock) +#define VM_PAGING_NUM_PAGES 64 vm_map_offset_t vm_paging_base_address = 0; -boolean_t vm_paging_page_inuse[VM_PAGING_NUM_PAGES] = { FALSE, }; -int vm_paging_max_index = 0; -int vm_paging_page_waiter = 0; -int vm_paging_page_waiter_total = 0; -unsigned long vm_paging_no_kernel_page = 0; -unsigned long vm_paging_objects_mapped = 0; -unsigned long vm_paging_pages_mapped = 0; -unsigned long vm_paging_objects_mapped_slow = 0; -unsigned long vm_paging_pages_mapped_slow = 0; +boolean_t vm_paging_page_inuse[VM_PAGING_NUM_PAGES] = { FALSE, }; +int vm_paging_max_index = 0; +int vm_paging_page_waiter = 0; +int vm_paging_page_waiter_total = 0; + +unsigned long vm_paging_no_kernel_page = 0; +unsigned long vm_paging_objects_mapped = 0; +unsigned long vm_paging_pages_mapped = 0; +unsigned long vm_paging_objects_mapped_slow = 0; +unsigned long vm_paging_pages_mapped_slow = 0; void vm_paging_map_init(void) { - kern_return_t kr; - vm_map_offset_t page_map_offset; - vm_map_entry_t map_entry; + kern_return_t kr; + vm_map_offset_t page_map_offset; + vm_map_entry_t map_entry; assert(vm_paging_base_address == 0); @@ -8985,13 +9052,13 @@ vm_paging_map_init(void) */ page_map_offset = 0; kr = vm_map_find_space(kernel_map, - &page_map_offset, - VM_PAGING_NUM_PAGES * PAGE_SIZE, - 0, - 0, - VM_MAP_KERNEL_FLAGS_NONE, - VM_KERN_MEMORY_NONE, - &map_entry); + &page_map_offset, + VM_PAGING_NUM_PAGES * PAGE_SIZE, + 0, + 0, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_NONE, + &map_entry); if (kr != KERN_SUCCESS) { panic("vm_paging_map_init: kernel_map full\n"); } @@ -9010,58 +9077,58 @@ vm_paging_map_init(void) /* * vm_paging_map_object: * Maps part of a VM object's pages in the kernel - * virtual address space, using the pre-allocated + * virtual address space, using the pre-allocated * kernel virtual addresses, if possible. * Context: - * The VM object is locked. This lock will get - * dropped and re-acquired though, so the caller - * must make sure the VM object is kept alive + * The VM object is locked. This lock will get + * dropped and re-acquired though, so the caller + * must make sure the VM object is kept alive * (by holding a VM map that has a reference - * on it, for example, or taking an extra reference). - * The page should also be kept busy to prevent + * on it, for example, or taking an extra reference). + * The page should also be kept busy to prevent * it from being reclaimed. */ kern_return_t vm_paging_map_object( - vm_page_t page, - vm_object_t object, - vm_object_offset_t offset, - vm_prot_t protection, - boolean_t can_unlock_object, - vm_map_size_t *size, /* IN/OUT */ - vm_map_offset_t *address, /* OUT */ - boolean_t *need_unmap) /* OUT */ + vm_page_t page, + vm_object_t object, + vm_object_offset_t offset, + vm_prot_t protection, + boolean_t can_unlock_object, + vm_map_size_t *size, /* IN/OUT */ + vm_map_offset_t *address, /* OUT */ + boolean_t *need_unmap) /* OUT */ { - kern_return_t kr; - vm_map_offset_t page_map_offset; - vm_map_size_t map_size; - vm_object_offset_t object_offset; - int i; + kern_return_t kr; + vm_map_offset_t page_map_offset; + vm_map_size_t map_size; + vm_object_offset_t object_offset; + int i; if (page != VM_PAGE_NULL && *size == PAGE_SIZE) { /* use permanent 1-to-1 kernel mapping of physical memory ? */ #if __x86_64__ *address = (vm_map_offset_t) - PHYSMAP_PTOV((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(page) << - PAGE_SHIFT); + PHYSMAP_PTOV((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(page) << + PAGE_SHIFT); *need_unmap = FALSE; return KERN_SUCCESS; #elif __arm__ || __arm64__ *address = (vm_map_offset_t) - phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(page) << PAGE_SHIFT); + phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(page) << PAGE_SHIFT); *need_unmap = FALSE; return KERN_SUCCESS; #else #warn "vm_paging_map_object: no 1-to-1 kernel mapping of physical memory..." #endif - assert(page->busy); + assert(page->vmp_busy); /* * Use one of the pre-allocated kernel virtual addresses * and just enter the VM page in the kernel address space * at that virtual address. */ - simple_lock(&vm_paging_lock); + simple_lock(&vm_paging_lock, &vm_pageout_lck_grp); /* * Try and find an available kernel virtual address @@ -9072,8 +9139,8 @@ vm_paging_map_object( for (i = 0; i < VM_PAGING_NUM_PAGES; i++) { if (vm_paging_page_inuse[i] == FALSE) { page_map_offset = - vm_paging_base_address + - (i * PAGE_SIZE); + vm_paging_base_address + + (i * PAGE_SIZE); break; } } @@ -9099,7 +9166,7 @@ vm_paging_map_object( if (kr == THREAD_WAITING) { simple_unlock(&vm_paging_lock); kr = thread_block(THREAD_CONTINUE_NULL); - simple_lock(&vm_paging_lock); + simple_lock(&vm_paging_lock, &vm_pageout_lck_grp); } vm_paging_page_waiter--; /* ... and try again */ @@ -9116,25 +9183,25 @@ vm_paging_map_object( vm_paging_page_inuse[i] = TRUE; simple_unlock(&vm_paging_lock); - page->pmapped = TRUE; + page->vmp_pmapped = TRUE; /* * Keep the VM object locked over the PMAP_ENTER * and the actual use of the page by the kernel, - * or this pmap mapping might get undone by a + * or this pmap mapping might get undone by a * vm_object_pmap_protect() call... */ PMAP_ENTER(kernel_pmap, - page_map_offset, - page, - protection, - VM_PROT_NONE, - 0, - TRUE, - kr); + page_map_offset, + page, + protection, + VM_PROT_NONE, + 0, + TRUE, + kr); assert(kr == KERN_SUCCESS); vm_paging_objects_mapped++; - vm_paging_pages_mapped++; + vm_paging_pages_mapped++; *address = page_map_offset; *need_unmap = TRUE; @@ -9155,7 +9222,7 @@ vm_paging_map_object( simple_unlock(&vm_paging_lock); } - if (! can_unlock_object) { + if (!can_unlock_object) { *address = 0; *size = 0; *need_unmap = FALSE; @@ -9164,34 +9231,34 @@ vm_paging_map_object( object_offset = vm_object_trunc_page(offset); map_size = vm_map_round_page(*size, - VM_MAP_PAGE_MASK(kernel_map)); + VM_MAP_PAGE_MASK(kernel_map)); /* * Try and map the required range of the object * in the kernel_map */ - vm_object_reference_locked(object); /* for the map entry */ + vm_object_reference_locked(object); /* for the map entry */ vm_object_unlock(object); kr = vm_map_enter(kernel_map, - address, - map_size, - 0, - VM_FLAGS_ANYWHERE, - VM_MAP_KERNEL_FLAGS_NONE, - VM_KERN_MEMORY_NONE, - object, - object_offset, - FALSE, - protection, - VM_PROT_ALL, - VM_INHERIT_NONE); + address, + map_size, + 0, + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_NONE, + object, + object_offset, + FALSE, + protection, + VM_PROT_ALL, + VM_INHERIT_NONE); if (kr != KERN_SUCCESS) { *address = 0; *size = 0; *need_unmap = FALSE; - vm_object_deallocate(object); /* for the map entry */ + vm_object_deallocate(object); /* for the map entry */ vm_object_lock(object); return kr; } @@ -9210,15 +9277,14 @@ vm_paging_map_object( */ for (page_map_offset = 0; - map_size != 0; - map_size -= PAGE_SIZE_64, page_map_offset += PAGE_SIZE_64) { - + map_size != 0; + map_size -= PAGE_SIZE_64, page_map_offset += PAGE_SIZE_64) { page = vm_page_lookup(object, offset + page_map_offset); if (page == VM_PAGE_NULL) { printf("vm_paging_map_object: no page !?"); vm_object_unlock(object); kr = vm_map_remove(kernel_map, *address, *size, - VM_MAP_NO_FLAGS); + VM_MAP_REMOVE_NO_FLAGS); assert(kr == KERN_SUCCESS); *address = 0; *size = 0; @@ -9226,23 +9292,23 @@ vm_paging_map_object( vm_object_lock(object); return KERN_MEMORY_ERROR; } - page->pmapped = TRUE; + page->vmp_pmapped = TRUE; //assert(pmap_verify_free(VM_PAGE_GET_PHYS_PAGE(page))); PMAP_ENTER(kernel_pmap, - *address + page_map_offset, - page, - protection, - VM_PROT_NONE, - 0, - TRUE, - kr); + *address + page_map_offset, + page, + protection, + VM_PROT_NONE, + 0, + TRUE, + kr); assert(kr == KERN_SUCCESS); #if KASAN kasan_notify_address(*address + page_map_offset, PAGE_SIZE); #endif } - + vm_paging_objects_mapped_slow++; vm_paging_pages_mapped_slow += (unsigned long) (map_size / PAGE_SIZE_64); @@ -9254,24 +9320,24 @@ vm_paging_map_object( /* * vm_paging_unmap_object: * Unmaps part of a VM object's pages from the kernel - * virtual address space. + * virtual address space. * Context: - * The VM object is locked. This lock will get - * dropped and re-acquired though. + * The VM object is locked. This lock will get + * dropped and re-acquired though. */ void vm_paging_unmap_object( - vm_object_t object, - vm_map_offset_t start, - vm_map_offset_t end) + vm_object_t object, + vm_map_offset_t start, + vm_map_offset_t end) { - kern_return_t kr; - int i; + kern_return_t kr; + int i; if ((vm_paging_base_address == 0) || (start < vm_paging_base_address) || (end > (vm_paging_base_address - + (VM_PAGING_NUM_PAGES * PAGE_SIZE)))) { + + (VM_PAGING_NUM_PAGES * PAGE_SIZE)))) { /* * We didn't use our pre-allocated pool of * kernel virtual address. Deallocate the @@ -9280,7 +9346,8 @@ vm_paging_unmap_object( if (object != VM_OBJECT_NULL) { vm_object_unlock(object); } - kr = vm_map_remove(kernel_map, start, end, VM_MAP_NO_FLAGS); + kr = vm_map_remove(kernel_map, start, end, + VM_MAP_REMOVE_NO_FLAGS); if (object != VM_OBJECT_NULL) { vm_object_lock(object); } @@ -9298,7 +9365,7 @@ vm_paging_unmap_object( /* undo the pmap mapping */ pmap_remove(kernel_pmap, start, end); - simple_lock(&vm_paging_lock); + simple_lock(&vm_paging_lock, &vm_pageout_lck_grp); vm_paging_page_inuse[i] = FALSE; if (vm_paging_page_waiter) { thread_wakeup(&vm_paging_page_waiter); @@ -9309,7 +9376,7 @@ vm_paging_unmap_object( /* - * page->object must be locked + * page->vmp_object must be locked */ void vm_pageout_steal_laundry(vm_page_t page, boolean_t queues_locked) @@ -9318,7 +9385,7 @@ vm_pageout_steal_laundry(vm_page_t page, boolean_t queues_locked) vm_page_lockspin_queues(); } - page->free_when_done = FALSE; + page->vmp_free_when_done = FALSE; /* * need to drop the laundry count... * we may also need to remove it @@ -9329,8 +9396,6 @@ vm_pageout_steal_laundry(vm_page_t page, boolean_t queues_locked) */ vm_pageout_throttle_up(page); - vm_page_steal_pageout_page++; - if (!queues_locked) { vm_page_unlock_queues(); } @@ -9339,24 +9404,23 @@ vm_pageout_steal_laundry(vm_page_t page, boolean_t queues_locked) upl_t vector_upl_create(vm_offset_t upl_offset) { - int vector_upl_size = sizeof(struct _vector_upl); - int i=0; - upl_t upl; + int vector_upl_size = sizeof(struct _vector_upl); + int i = 0; + upl_t upl; vector_upl_t vector_upl = (vector_upl_t)kalloc(vector_upl_size); - upl = upl_create(0,UPL_VECTOR,0); + upl = upl_create(0, UPL_VECTOR, 0); upl->vector_upl = vector_upl; upl->offset = upl_offset; vector_upl->size = 0; vector_upl->offset = upl_offset; - vector_upl->invalid_upls=0; - vector_upl->num_upls=0; + vector_upl->invalid_upls = 0; + vector_upl->num_upls = 0; vector_upl->pagelist = NULL; - - for(i=0; i < MAX_VECTOR_UPL_ELEMENTS ; i++) { + + for (i = 0; i < MAX_VECTOR_UPL_ELEMENTS; i++) { vector_upl->upl_iostates[i].size = 0; vector_upl->upl_iostates[i].offset = 0; - } return upl; } @@ -9364,125 +9428,131 @@ vector_upl_create(vm_offset_t upl_offset) void vector_upl_deallocate(upl_t upl) { - if(upl) { + if (upl) { vector_upl_t vector_upl = upl->vector_upl; - if(vector_upl) { - if(vector_upl->invalid_upls != vector_upl->num_upls) + if (vector_upl) { + if (vector_upl->invalid_upls != vector_upl->num_upls) { panic("Deallocating non-empty Vectored UPL\n"); - kfree(vector_upl->pagelist,(sizeof(struct upl_page_info)*(vector_upl->size/PAGE_SIZE))); - vector_upl->invalid_upls=0; + } + kfree(vector_upl->pagelist, (sizeof(struct upl_page_info) * (vector_upl->size / PAGE_SIZE))); + vector_upl->invalid_upls = 0; vector_upl->num_upls = 0; vector_upl->pagelist = NULL; vector_upl->size = 0; vector_upl->offset = 0; kfree(vector_upl, sizeof(struct _vector_upl)); vector_upl = (vector_upl_t)0xfeedfeed; - } - else + } else { panic("vector_upl_deallocate was passed a non-vectored upl\n"); - } - else + } + } else { panic("vector_upl_deallocate was passed a NULL upl\n"); + } } boolean_t vector_upl_is_valid(upl_t upl) { - if(upl && ((upl->flags & UPL_VECTOR)==UPL_VECTOR)) { + if (upl && ((upl->flags & UPL_VECTOR) == UPL_VECTOR)) { vector_upl_t vector_upl = upl->vector_upl; - if(vector_upl == NULL || vector_upl == (vector_upl_t)0xfeedfeed || vector_upl == (vector_upl_t)0xfeedbeef) + if (vector_upl == NULL || vector_upl == (vector_upl_t)0xfeedfeed || vector_upl == (vector_upl_t)0xfeedbeef) { return FALSE; - else + } else { return TRUE; + } } return FALSE; } boolean_t -vector_upl_set_subupl(upl_t upl,upl_t subupl, uint32_t io_size) +vector_upl_set_subupl(upl_t upl, upl_t subupl, uint32_t io_size) { - if(vector_upl_is_valid(upl)) { + if (vector_upl_is_valid(upl)) { vector_upl_t vector_upl = upl->vector_upl; - - if(vector_upl) { - if(subupl) { - if(io_size) { - if(io_size < PAGE_SIZE) + + if (vector_upl) { + if (subupl) { + if (io_size) { + if (io_size < PAGE_SIZE) { io_size = PAGE_SIZE; + } subupl->vector_upl = (void*)vector_upl; vector_upl->upl_elems[vector_upl->num_upls++] = subupl; vector_upl->size += io_size; upl->size += io_size; - } - else { - uint32_t i=0,invalid_upls=0; - for(i = 0; i < vector_upl->num_upls; i++) { - if(vector_upl->upl_elems[i] == subupl) + } else { + uint32_t i = 0, invalid_upls = 0; + for (i = 0; i < vector_upl->num_upls; i++) { + if (vector_upl->upl_elems[i] == subupl) { break; + } } - if(i == vector_upl->num_upls) + if (i == vector_upl->num_upls) { panic("Trying to remove sub-upl when none exists"); - + } + vector_upl->upl_elems[i] = NULL; - invalid_upls = hw_atomic_add(&(vector_upl)->invalid_upls, 1); - if(invalid_upls == vector_upl->num_upls) + invalid_upls = hw_atomic_add(&(vector_upl)->invalid_upls, 1); + if (invalid_upls == vector_upl->num_upls) { return TRUE; - else + } else { return FALSE; + } } - } - else + } else { panic("vector_upl_set_subupl was passed a NULL upl element\n"); - } - else + } + } else { panic("vector_upl_set_subupl was passed a non-vectored upl\n"); - } - else + } + } else { panic("vector_upl_set_subupl was passed a NULL upl\n"); + } return FALSE; -} +} void vector_upl_set_pagelist(upl_t upl) { - if(vector_upl_is_valid(upl)) { - uint32_t i=0; + if (vector_upl_is_valid(upl)) { + uint32_t i = 0; vector_upl_t vector_upl = upl->vector_upl; - if(vector_upl) { - vm_offset_t pagelist_size=0, cur_upl_pagelist_size=0; + if (vector_upl) { + vm_offset_t pagelist_size = 0, cur_upl_pagelist_size = 0; + + vector_upl->pagelist = (upl_page_info_array_t)kalloc(sizeof(struct upl_page_info) * (vector_upl->size / PAGE_SIZE)); - vector_upl->pagelist = (upl_page_info_array_t)kalloc(sizeof(struct upl_page_info)*(vector_upl->size/PAGE_SIZE)); - - for(i=0; i < vector_upl->num_upls; i++) { - cur_upl_pagelist_size = sizeof(struct upl_page_info) * vector_upl->upl_elems[i]->size/PAGE_SIZE; + for (i = 0; i < vector_upl->num_upls; i++) { + cur_upl_pagelist_size = sizeof(struct upl_page_info) * vector_upl->upl_elems[i]->size / PAGE_SIZE; bcopy(UPL_GET_INTERNAL_PAGE_LIST_SIMPLE(vector_upl->upl_elems[i]), (char*)vector_upl->pagelist + pagelist_size, cur_upl_pagelist_size); pagelist_size += cur_upl_pagelist_size; - if(vector_upl->upl_elems[i]->highest_page > upl->highest_page) + if (vector_upl->upl_elems[i]->highest_page > upl->highest_page) { upl->highest_page = vector_upl->upl_elems[i]->highest_page; + } } - assert( pagelist_size == (sizeof(struct upl_page_info)*(vector_upl->size/PAGE_SIZE)) ); - } - else + assert( pagelist_size == (sizeof(struct upl_page_info) * (vector_upl->size / PAGE_SIZE))); + } else { panic("vector_upl_set_pagelist was passed a non-vectored upl\n"); - } - else + } + } else { panic("vector_upl_set_pagelist was passed a NULL upl\n"); - + } } upl_t vector_upl_subupl_byindex(upl_t upl, uint32_t index) { - if(vector_upl_is_valid(upl)) { + if (vector_upl_is_valid(upl)) { vector_upl_t vector_upl = upl->vector_upl; - if(vector_upl) { - if(index < vector_upl->num_upls) + if (vector_upl) { + if (index < vector_upl->num_upls) { return vector_upl->upl_elems[index]; - } - else + } + } else { panic("vector_upl_subupl_byindex was passed a non-vectored upl\n"); + } } return NULL; } @@ -9490,39 +9560,42 @@ vector_upl_subupl_byindex(upl_t upl, uint32_t index) upl_t vector_upl_subupl_byoffset(upl_t upl, upl_offset_t *upl_offset, upl_size_t *upl_size) { - if(vector_upl_is_valid(upl)) { - uint32_t i=0; + if (vector_upl_is_valid(upl)) { + uint32_t i = 0; vector_upl_t vector_upl = upl->vector_upl; - if(vector_upl) { + if (vector_upl) { upl_t subupl = NULL; vector_upl_iostates_t subupl_state; - for(i=0; i < vector_upl->num_upls; i++) { + for (i = 0; i < vector_upl->num_upls; i++) { subupl = vector_upl->upl_elems[i]; subupl_state = vector_upl->upl_iostates[i]; - if( *upl_offset <= (subupl_state.offset + subupl_state.size - 1)) { + if (*upl_offset <= (subupl_state.offset + subupl_state.size - 1)) { /* We could have been passed an offset/size pair that belongs * to an UPL element that has already been committed/aborted. * If so, return NULL. */ - if(subupl == NULL) + if (subupl == NULL) { return NULL; - if((subupl_state.offset + subupl_state.size) < (*upl_offset + *upl_size)) { + } + if ((subupl_state.offset + subupl_state.size) < (*upl_offset + *upl_size)) { *upl_size = (subupl_state.offset + subupl_state.size) - *upl_offset; - if(*upl_size > subupl_state.size) + if (*upl_size > subupl_state.size) { *upl_size = subupl_state.size; + } } - if(*upl_offset >= subupl_state.offset) + if (*upl_offset >= subupl_state.offset) { *upl_offset -= subupl_state.offset; - else if(i) + } else if (i) { panic("Vector UPL offset miscalculation\n"); + } return subupl; - } + } } - } - else + } else { panic("vector_upl_subupl_byoffset was passed a non-vectored UPL\n"); + } } return NULL; } @@ -9532,107 +9605,112 @@ vector_upl_get_submap(upl_t upl, vm_map_t *v_upl_submap, vm_offset_t *submap_dst { *v_upl_submap = NULL; - if(vector_upl_is_valid(upl)) { + if (vector_upl_is_valid(upl)) { vector_upl_t vector_upl = upl->vector_upl; - if(vector_upl) { + if (vector_upl) { *v_upl_submap = vector_upl->submap; *submap_dst_addr = vector_upl->submap_dst_addr; - } - else + } else { panic("vector_upl_get_submap was passed a non-vectored UPL\n"); - } - else + } + } else { panic("vector_upl_get_submap was passed a null UPL\n"); + } } void vector_upl_set_submap(upl_t upl, vm_map_t submap, vm_offset_t submap_dst_addr) { - if(vector_upl_is_valid(upl)) { + if (vector_upl_is_valid(upl)) { vector_upl_t vector_upl = upl->vector_upl; - if(vector_upl) { + if (vector_upl) { vector_upl->submap = submap; vector_upl->submap_dst_addr = submap_dst_addr; - } - else + } else { panic("vector_upl_get_submap was passed a non-vectored UPL\n"); - } - else + } + } else { panic("vector_upl_get_submap was passed a NULL UPL\n"); + } } void vector_upl_set_iostate(upl_t upl, upl_t subupl, upl_offset_t offset, upl_size_t size) { - if(vector_upl_is_valid(upl)) { + if (vector_upl_is_valid(upl)) { uint32_t i = 0; vector_upl_t vector_upl = upl->vector_upl; - if(vector_upl) { - for(i = 0; i < vector_upl->num_upls; i++) { - if(vector_upl->upl_elems[i] == subupl) + if (vector_upl) { + for (i = 0; i < vector_upl->num_upls; i++) { + if (vector_upl->upl_elems[i] == subupl) { break; + } } - - if(i == vector_upl->num_upls) + + if (i == vector_upl->num_upls) { panic("setting sub-upl iostate when none exists"); + } vector_upl->upl_iostates[i].offset = offset; - if(size < PAGE_SIZE) + if (size < PAGE_SIZE) { size = PAGE_SIZE; + } vector_upl->upl_iostates[i].size = size; - } - else + } else { panic("vector_upl_set_iostate was passed a non-vectored UPL\n"); - } - else + } + } else { panic("vector_upl_set_iostate was passed a NULL UPL\n"); + } } void vector_upl_get_iostate(upl_t upl, upl_t subupl, upl_offset_t *offset, upl_size_t *size) { - if(vector_upl_is_valid(upl)) { + if (vector_upl_is_valid(upl)) { uint32_t i = 0; vector_upl_t vector_upl = upl->vector_upl; - if(vector_upl) { - for(i = 0; i < vector_upl->num_upls; i++) { - if(vector_upl->upl_elems[i] == subupl) + if (vector_upl) { + for (i = 0; i < vector_upl->num_upls; i++) { + if (vector_upl->upl_elems[i] == subupl) { break; + } } - - if(i == vector_upl->num_upls) + + if (i == vector_upl->num_upls) { panic("getting sub-upl iostate when none exists"); + } *offset = vector_upl->upl_iostates[i].offset; *size = vector_upl->upl_iostates[i].size; - } - else + } else { panic("vector_upl_get_iostate was passed a non-vectored UPL\n"); - } - else + } + } else { panic("vector_upl_get_iostate was passed a NULL UPL\n"); + } } void vector_upl_get_iostate_byindex(upl_t upl, uint32_t index, upl_offset_t *offset, upl_size_t *size) { - if(vector_upl_is_valid(upl)) { + if (vector_upl_is_valid(upl)) { vector_upl_t vector_upl = upl->vector_upl; - if(vector_upl) { - if(index < vector_upl->num_upls) { + if (vector_upl) { + if (index < vector_upl->num_upls) { *offset = vector_upl->upl_iostates[index].offset; *size = vector_upl->upl_iostates[index].size; - } - else + } else { *offset = *size = 0; - } - else + } + } else { panic("vector_upl_get_iostate_byindex was passed a non-vectored UPL\n"); - } - else + } + } else { panic("vector_upl_get_iostate_byindex was passed a NULL UPL\n"); + } } upl_page_info_t * @@ -9655,8 +9733,8 @@ upl_get_internal_pagelist_offset(void) void upl_clear_dirty( - upl_t upl, - boolean_t value) + upl_t upl, + boolean_t value) { if (value) { upl->flags |= UPL_CLEAR_DIRTY; @@ -9667,8 +9745,8 @@ upl_clear_dirty( void upl_set_referenced( - upl_t upl, - boolean_t value) + upl_t upl, + boolean_t value) { upl_lock(upl); if (value) { @@ -9685,216 +9763,78 @@ upl_set_referenced( #if CONFIG_IOSCHED void upl_set_blkno( - upl_t upl, - vm_offset_t upl_offset, - int io_size, - int64_t blkno) -{ - int i,j; - if ((upl->flags & UPL_EXPEDITE_SUPPORTED) == 0) - return; - - assert(upl->upl_reprio_info != 0); - for(i = (int)(upl_offset / PAGE_SIZE), j = 0; j < io_size; i++, j += PAGE_SIZE) { - UPL_SET_REPRIO_INFO(upl, i, blkno, io_size); - } -} -#endif - -boolean_t -vm_page_is_slideable(vm_page_t m) -{ - boolean_t result = FALSE; - vm_shared_region_slide_info_t si; - vm_object_t m_object; - - m_object = VM_PAGE_OBJECT(m); - - vm_object_lock_assert_held(m_object); - - /* make sure our page belongs to the one object allowed to do this */ - if (!m_object->object_slid) { - goto done; - } - - si = m_object->vo_slide_info; - if (si == NULL) { - goto done; - } - - if(!m->slid && (si->start <= m->offset && si->end > m->offset)) { - result = TRUE; - } - -done: - return result; -} - -int vm_page_slide_counter = 0; -int vm_page_slide_errors = 0; -kern_return_t -vm_page_slide( - vm_page_t page, - vm_map_offset_t kernel_mapping_offset) + upl_t upl, + vm_offset_t upl_offset, + int io_size, + int64_t blkno) { - kern_return_t kr; - vm_map_size_t kernel_mapping_size; - boolean_t kernel_mapping_needs_unmap; - vm_offset_t kernel_vaddr; - uint32_t pageIndex; - uint32_t slide_chunk; - vm_object_t page_object; - - page_object = VM_PAGE_OBJECT(page); - - assert(!page->slid); - assert(page_object->object_slid); - vm_object_lock_assert_exclusive(page_object); - - if (page->error) - return KERN_FAILURE; - - /* - * Take a paging-in-progress reference to keep the object - * alive even if we have to unlock it (in vm_paging_map_object() - * for example)... - */ - vm_object_paging_begin(page_object); - - if (kernel_mapping_offset == 0) { - /* - * The page hasn't already been mapped in kernel space - * by the caller. Map it now, so that we can access - * its contents and decrypt them. - */ - kernel_mapping_size = PAGE_SIZE; - kernel_mapping_needs_unmap = FALSE; - kr = vm_paging_map_object(page, - page_object, - page->offset, - VM_PROT_READ | VM_PROT_WRITE, - FALSE, - &kernel_mapping_size, - &kernel_mapping_offset, - &kernel_mapping_needs_unmap); - if (kr != KERN_SUCCESS) { - panic("vm_page_slide: " - "could not map page in kernel: 0x%x\n", - kr); - } - } else { - kernel_mapping_size = 0; - kernel_mapping_needs_unmap = FALSE; - } - kernel_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping_offset); - - /* - * Slide the pointers on the page. - */ - - /*assert that slide_file_info.start/end are page-aligned?*/ - - assert(!page->slid); - assert(page_object->object_slid); - - pageIndex = (uint32_t)((page->offset - - page_object->vo_slide_info->start) / - PAGE_SIZE_FOR_SR_SLIDE); - for (slide_chunk = 0; - slide_chunk < PAGE_SIZE / PAGE_SIZE_FOR_SR_SLIDE; - slide_chunk++) { - kr = vm_shared_region_slide_page(page_object->vo_slide_info, - (kernel_vaddr + - (slide_chunk * - PAGE_SIZE_FOR_SR_SLIDE)), - (pageIndex + slide_chunk)); - if (kr != KERN_SUCCESS) { - break; - } + int i, j; + if ((upl->flags & UPL_EXPEDITE_SUPPORTED) == 0) { + return; } - vm_page_slide_counter++; - - /* - * Unmap the page from the kernel's address space, - */ - if (kernel_mapping_needs_unmap) { - vm_paging_unmap_object(page_object, - kernel_vaddr, - kernel_vaddr + PAGE_SIZE); - } - - page->dirty = FALSE; - pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(page), VM_MEM_MODIFIED | VM_MEM_REFERENCED); - - if (kr != KERN_SUCCESS || cs_debug > 1) { - printf("vm_page_slide(%p): " - "obj %p off 0x%llx mobj %p moff 0x%llx\n", - page, - page_object, page->offset, - page_object->pager, - page->offset + page_object->paging_offset); - } - - if (kr == KERN_SUCCESS) { - page->slid = TRUE; - } else { - page->error = TRUE; - vm_page_slide_errors++; + assert(upl->upl_reprio_info != 0); + for (i = (int)(upl_offset / PAGE_SIZE), j = 0; j < io_size; i++, j += PAGE_SIZE) { + UPL_SET_REPRIO_INFO(upl, i, blkno, io_size); } - - vm_object_paging_end(page_object); - - return kr; } +#endif -void inline memoryshot(unsigned int event, unsigned int control) +void inline +memoryshot(unsigned int event, unsigned int control) { if (vm_debug_events) { KERNEL_DEBUG_CONSTANT1((MACHDBG_CODE(DBG_MACH_VM_PRESSURE, event)) | control, - vm_page_active_count, vm_page_inactive_count, - vm_page_free_count, vm_page_speculative_count, - vm_page_throttled_count); + vm_page_active_count, vm_page_inactive_count, + vm_page_free_count, vm_page_speculative_count, + vm_page_throttled_count); } else { (void) event; (void) control; } - } #ifdef MACH_BSD -boolean_t upl_device_page(upl_page_info_t *upl) +boolean_t +upl_device_page(upl_page_info_t *upl) { - return(UPL_DEVICE_PAGE(upl)); + return UPL_DEVICE_PAGE(upl); } -boolean_t upl_page_present(upl_page_info_t *upl, int index) +boolean_t +upl_page_present(upl_page_info_t *upl, int index) { - return(UPL_PAGE_PRESENT(upl, index)); + return UPL_PAGE_PRESENT(upl, index); } -boolean_t upl_speculative_page(upl_page_info_t *upl, int index) +boolean_t +upl_speculative_page(upl_page_info_t *upl, int index) { - return(UPL_SPECULATIVE_PAGE(upl, index)); + return UPL_SPECULATIVE_PAGE(upl, index); } -boolean_t upl_dirty_page(upl_page_info_t *upl, int index) +boolean_t +upl_dirty_page(upl_page_info_t *upl, int index) { - return(UPL_DIRTY_PAGE(upl, index)); + return UPL_DIRTY_PAGE(upl, index); } -boolean_t upl_valid_page(upl_page_info_t *upl, int index) +boolean_t +upl_valid_page(upl_page_info_t *upl, int index) { - return(UPL_VALID_PAGE(upl, index)); + return UPL_VALID_PAGE(upl, index); } -ppnum_t upl_phys_page(upl_page_info_t *upl, int index) +ppnum_t +upl_phys_page(upl_page_info_t *upl, int index) { - return(UPL_PHYS_PAGE(upl, index)); + return UPL_PHYS_PAGE(upl, index); } -void upl_page_set_mark(upl_page_info_t *upl, int index, boolean_t v) +void +upl_page_set_mark(upl_page_info_t *upl, int index, boolean_t v) { upl[index].mark = v; } -boolean_t upl_page_get_mark(upl_page_info_t *upl, int index) +boolean_t +upl_page_get_mark(upl_page_info_t *upl, int index) { return upl[index].mark; } @@ -9908,138 +9848,211 @@ vm_countdirtypages(void) int precpages; - dpages=0; - pgopages=0; - precpages=0; + dpages = 0; + pgopages = 0; + precpages = 0; vm_page_lock_queues(); m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive); do { - if (m ==(vm_page_t )0) break; + if (m == (vm_page_t)0) { + break; + } - if(m->dirty) dpages++; - if(m->free_when_done) pgopages++; - if(m->precious) precpages++; + if (m->vmp_dirty) { + dpages++; + } + if (m->vmp_free_when_done) { + pgopages++; + } + if (m->vmp_precious) { + precpages++; + } assert(VM_PAGE_OBJECT(m) != kernel_object); - m = (vm_page_t) vm_page_queue_next(&m->pageq); - if (m ==(vm_page_t )0) break; - + m = (vm_page_t) vm_page_queue_next(&m->vmp_pageq); + if (m == (vm_page_t)0) { + break; + } } while (!vm_page_queue_end(&vm_page_queue_inactive, (vm_page_queue_entry_t) m)); vm_page_unlock_queues(); vm_page_lock_queues(); m = (vm_page_t) vm_page_queue_first(&vm_page_queue_throttled); do { - if (m ==(vm_page_t )0) break; + if (m == (vm_page_t)0) { + break; + } dpages++; - assert(m->dirty); - assert(!m->free_when_done); + assert(m->vmp_dirty); + assert(!m->vmp_free_when_done); assert(VM_PAGE_OBJECT(m) != kernel_object); - m = (vm_page_t) vm_page_queue_next(&m->pageq); - if (m ==(vm_page_t )0) break; - + m = (vm_page_t) vm_page_queue_next(&m->vmp_pageq); + if (m == (vm_page_t)0) { + break; + } } while (!vm_page_queue_end(&vm_page_queue_throttled, (vm_page_queue_entry_t) m)); vm_page_unlock_queues(); vm_page_lock_queues(); m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous); do { - if (m ==(vm_page_t )0) break; + if (m == (vm_page_t)0) { + break; + } - if(m->dirty) dpages++; - if(m->free_when_done) pgopages++; - if(m->precious) precpages++; + if (m->vmp_dirty) { + dpages++; + } + if (m->vmp_free_when_done) { + pgopages++; + } + if (m->vmp_precious) { + precpages++; + } assert(VM_PAGE_OBJECT(m) != kernel_object); - m = (vm_page_t) vm_page_queue_next(&m->pageq); - if (m ==(vm_page_t )0) break; - + m = (vm_page_t) vm_page_queue_next(&m->vmp_pageq); + if (m == (vm_page_t)0) { + break; + } } while (!vm_page_queue_end(&vm_page_queue_anonymous, (vm_page_queue_entry_t) m)); vm_page_unlock_queues(); printf("IN Q: %d : %d : %d\n", dpages, pgopages, precpages); - dpages=0; - pgopages=0; - precpages=0; + dpages = 0; + pgopages = 0; + precpages = 0; vm_page_lock_queues(); m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active); do { - if(m == (vm_page_t )0) break; - if(m->dirty) dpages++; - if(m->free_when_done) pgopages++; - if(m->precious) precpages++; + if (m == (vm_page_t)0) { + break; + } + if (m->vmp_dirty) { + dpages++; + } + if (m->vmp_free_when_done) { + pgopages++; + } + if (m->vmp_precious) { + precpages++; + } assert(VM_PAGE_OBJECT(m) != kernel_object); - m = (vm_page_t) vm_page_queue_next(&m->pageq); - if(m == (vm_page_t )0) break; - + m = (vm_page_t) vm_page_queue_next(&m->vmp_pageq); + if (m == (vm_page_t)0) { + break; + } } while (!vm_page_queue_end(&vm_page_queue_active, (vm_page_queue_entry_t) m)); vm_page_unlock_queues(); printf("AC Q: %d : %d : %d\n", dpages, pgopages, precpages); - } #endif /* MACH_BSD */ #if CONFIG_IOSCHED -int upl_get_cached_tier(upl_t upl) +int +upl_get_cached_tier(upl_t upl) +{ + assert(upl); + if (upl->flags & UPL_TRACKED_BY_OBJECT) { + return upl->upl_priority; + } + return -1; +} +#endif /* CONFIG_IOSCHED */ + + +void +upl_callout_iodone(upl_t upl) +{ + struct upl_io_completion *upl_ctx = upl->upl_iodone; + + if (upl_ctx) { + void (*iodone_func)(void *, int) = upl_ctx->io_done; + + assert(upl_ctx->io_done); + + (*iodone_func)(upl_ctx->io_context, upl_ctx->io_error); + } +} + +void +upl_set_iodone(upl_t upl, void *upl_iodone) +{ + upl->upl_iodone = (struct upl_io_completion *)upl_iodone; +} + +void +upl_set_iodone_error(upl_t upl, int error) { - assert(upl); - if (upl->flags & UPL_TRACKED_BY_OBJECT) - return (upl->upl_priority); - return (-1); + struct upl_io_completion *upl_ctx = upl->upl_iodone; + + if (upl_ctx) { + upl_ctx->io_error = error; + } } -#endif /* CONFIG_IOSCHED */ -ppnum_t upl_get_highest_page( - upl_t upl) + +ppnum_t +upl_get_highest_page( + upl_t upl) { - return upl->highest_page; + return upl->highest_page; } -upl_size_t upl_get_size( - upl_t upl) +upl_size_t +upl_get_size( + upl_t upl) { - return upl->size; + return upl->size; } -upl_t upl_associated_upl(upl_t upl) +upl_t +upl_associated_upl(upl_t upl) { return upl->associated_upl; } -void upl_set_associated_upl(upl_t upl, upl_t associated_upl) +void +upl_set_associated_upl(upl_t upl, upl_t associated_upl) { upl->associated_upl = associated_upl; } -struct vnode * upl_lookup_vnode(upl_t upl) +struct vnode * +upl_lookup_vnode(upl_t upl) { - if (!upl->map_object->internal) + if (!upl->map_object->internal) { return vnode_pager_lookup_vnode(upl->map_object->pager); - else + } else { return NULL; -} + } +} #if UPL_DEBUG -kern_return_t upl_ubc_alias_set(upl_t upl, uintptr_t alias1, uintptr_t alias2) +kern_return_t +upl_ubc_alias_set(upl_t upl, uintptr_t alias1, uintptr_t alias2) { upl->ubc_alias1 = alias1; upl->ubc_alias2 = alias2; return KERN_SUCCESS; } -int upl_ubc_alias_get(upl_t upl, uintptr_t * al, uintptr_t * al2) +int +upl_ubc_alias_get(upl_t upl, uintptr_t * al, uintptr_t * al2) { - if(al) + if (al) { *al = upl->ubc_alias1; - if(al2) + } + if (al2) { *al2 = upl->ubc_alias2; + } return KERN_SUCCESS; } #endif /* UPL_DEBUG */ @@ -10051,10 +10064,9 @@ int upl_ubc_alias_get(upl_t upl, uintptr_t * al, uintptr_t * al2) extern boolean_t vm_compressor_low_on_space(void); boolean_t -VM_PRESSURE_NORMAL_TO_WARNING(void) { - - if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) { - +VM_PRESSURE_NORMAL_TO_WARNING(void) +{ + if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) { /* Available pages below our threshold */ if (memorystatus_available_pages < memorystatus_available_pages_pressure) { /* No frozen processes to kill */ @@ -10066,24 +10078,22 @@ VM_PRESSURE_NORMAL_TO_WARNING(void) { } } return FALSE; - } else { - return ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0); + return (AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0; } } boolean_t -VM_PRESSURE_WARNING_TO_CRITICAL(void) { - - if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) { - +VM_PRESSURE_WARNING_TO_CRITICAL(void) +{ + if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) { /* Available pages below our threshold */ if (memorystatus_available_pages < memorystatus_available_pages_critical) { return TRUE; } return FALSE; } else { - return (vm_compressor_low_on_space() || (AVAILABLE_NON_COMPRESSED_MEMORY < ((12 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0); + return vm_compressor_low_on_space() || (AVAILABLE_NON_COMPRESSED_MEMORY < ((12 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0; } } @@ -10091,10 +10101,9 @@ VM_PRESSURE_WARNING_TO_CRITICAL(void) { * Downward trajectory. */ boolean_t -VM_PRESSURE_WARNING_TO_NORMAL(void) { - - if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) { - +VM_PRESSURE_WARNING_TO_NORMAL(void) +{ + if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) { /* Available pages above our threshold */ unsigned int target_threshold = (unsigned int) (memorystatus_available_pages_pressure + ((15 * memorystatus_available_pages_pressure) / 100)); if (memorystatus_available_pages > target_threshold) { @@ -10102,15 +10111,14 @@ VM_PRESSURE_WARNING_TO_NORMAL(void) { } return FALSE; } else { - return ((AVAILABLE_NON_COMPRESSED_MEMORY > ((12 * VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) / 10)) ? 1 : 0); + return (AVAILABLE_NON_COMPRESSED_MEMORY > ((12 * VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) / 10)) ? 1 : 0; } } boolean_t -VM_PRESSURE_CRITICAL_TO_WARNING(void) { - - if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) { - +VM_PRESSURE_CRITICAL_TO_WARNING(void) +{ + if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) { /* Available pages above our threshold */ unsigned int target_threshold = (unsigned int)(memorystatus_available_pages_critical + ((15 * memorystatus_available_pages_critical) / 100)); if (memorystatus_available_pages > target_threshold) { @@ -10118,20 +10126,20 @@ VM_PRESSURE_CRITICAL_TO_WARNING(void) { } return FALSE; } else { - return ((AVAILABLE_NON_COMPRESSED_MEMORY > ((14 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0); + return (AVAILABLE_NON_COMPRESSED_MEMORY > ((14 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0; } } #endif /* VM_PRESSURE_EVENTS */ -#define VM_TEST_COLLAPSE_COMPRESSOR 0 -#define VM_TEST_WIRE_AND_EXTRACT 0 -#define VM_TEST_PAGE_WIRE_OVERFLOW_PANIC 0 +#define VM_TEST_COLLAPSE_COMPRESSOR 0 +#define VM_TEST_WIRE_AND_EXTRACT 0 +#define VM_TEST_PAGE_WIRE_OVERFLOW_PANIC 0 #if __arm64__ -#define VM_TEST_KERNEL_OBJECT_FAULT 0 +#define VM_TEST_KERNEL_OBJECT_FAULT 0 #endif /* __arm64__ */ -#define VM_TEST_DEVICE_PAGER_TRANSPOSE (DEVELOPMENT || DEBUG) +#define VM_TEST_DEVICE_PAGER_TRANSPOSE (DEVELOPMENT || DEBUG) #if VM_TEST_COLLAPSE_COMPRESSOR extern boolean_t vm_object_collapse_compressor_allowed; @@ -10139,11 +10147,11 @@ extern boolean_t vm_object_collapse_compressor_allowed; static void vm_test_collapse_compressor(void) { - vm_object_size_t backing_size, top_size; - vm_object_t backing_object, top_object; - vm_map_offset_t backing_offset, top_offset; - unsigned char *backing_address, *top_address; - kern_return_t kr; + vm_object_size_t backing_size, top_size; + vm_object_t backing_object, top_object; + vm_map_offset_t backing_offset, top_offset; + unsigned char *backing_address, *top_address; + kern_return_t kr; printf("VM_TEST_COLLAPSE_COMPRESSOR:\n"); @@ -10152,45 +10160,46 @@ vm_test_collapse_compressor(void) backing_object = vm_object_allocate(backing_size); assert(backing_object != VM_OBJECT_NULL); printf("VM_TEST_COLLAPSE_COMPRESSOR: created backing object %p\n", - backing_object); + backing_object); /* map backing object */ backing_offset = 0; kr = vm_map_enter(kernel_map, &backing_offset, backing_size, 0, - VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, - backing_object, 0, FALSE, - VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT); + VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, + backing_object, 0, FALSE, + VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT); assert(kr == KERN_SUCCESS); backing_address = (unsigned char *) backing_offset; printf("VM_TEST_COLLAPSE_COMPRESSOR: " - "mapped backing object %p at 0x%llx\n", - backing_object, (uint64_t) backing_offset); + "mapped backing object %p at 0x%llx\n", + backing_object, (uint64_t) backing_offset); /* populate with pages to be compressed in backing object */ - backing_address[0x1*PAGE_SIZE] = 0xB1; - backing_address[0x4*PAGE_SIZE] = 0xB4; - backing_address[0x7*PAGE_SIZE] = 0xB7; - backing_address[0xa*PAGE_SIZE] = 0xBA; - backing_address[0xd*PAGE_SIZE] = 0xBD; + backing_address[0x1 * PAGE_SIZE] = 0xB1; + backing_address[0x4 * PAGE_SIZE] = 0xB4; + backing_address[0x7 * PAGE_SIZE] = 0xB7; + backing_address[0xa * PAGE_SIZE] = 0xBA; + backing_address[0xd * PAGE_SIZE] = 0xBD; printf("VM_TEST_COLLAPSE_COMPRESSOR: " - "populated pages to be compressed in " - "backing_object %p\n", backing_object); + "populated pages to be compressed in " + "backing_object %p\n", backing_object); /* compress backing object */ vm_object_pageout(backing_object); printf("VM_TEST_COLLAPSE_COMPRESSOR: compressing backing_object %p\n", - backing_object); + backing_object); /* wait for all the pages to be gone */ - while (*(volatile int *)&backing_object->resident_page_count != 0) + while (*(volatile int *)&backing_object->resident_page_count != 0) { IODelay(10); + } printf("VM_TEST_COLLAPSE_COMPRESSOR: backing_object %p compressed\n", - backing_object); + backing_object); /* populate with pages to be resident in backing object */ - backing_address[0x0*PAGE_SIZE] = 0xB0; - backing_address[0x3*PAGE_SIZE] = 0xB3; - backing_address[0x6*PAGE_SIZE] = 0xB6; - backing_address[0x9*PAGE_SIZE] = 0xB9; - backing_address[0xc*PAGE_SIZE] = 0xBC; + backing_address[0x0 * PAGE_SIZE] = 0xB0; + backing_address[0x3 * PAGE_SIZE] = 0xB3; + backing_address[0x6 * PAGE_SIZE] = 0xB6; + backing_address[0x9 * PAGE_SIZE] = 0xB9; + backing_address[0xc * PAGE_SIZE] = 0xBC; printf("VM_TEST_COLLAPSE_COMPRESSOR: " - "populated pages to be resident in " - "backing_object %p\n", backing_object); + "populated pages to be resident in " + "backing_object %p\n", backing_object); /* leave the other pages absent */ /* mess with the paging_offset of the backing_object */ assert(backing_object->paging_offset == 0); @@ -10201,41 +10210,42 @@ vm_test_collapse_compressor(void) top_object = vm_object_allocate(top_size); assert(top_object != VM_OBJECT_NULL); printf("VM_TEST_COLLAPSE_COMPRESSOR: created top object %p\n", - top_object); + top_object); /* map top object */ top_offset = 0; kr = vm_map_enter(kernel_map, &top_offset, top_size, 0, - VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, - top_object, 0, FALSE, - VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT); + VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, + top_object, 0, FALSE, + VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT); assert(kr == KERN_SUCCESS); top_address = (unsigned char *) top_offset; printf("VM_TEST_COLLAPSE_COMPRESSOR: " - "mapped top object %p at 0x%llx\n", - top_object, (uint64_t) top_offset); + "mapped top object %p at 0x%llx\n", + top_object, (uint64_t) top_offset); /* populate with pages to be compressed in top object */ - top_address[0x3*PAGE_SIZE] = 0xA3; - top_address[0x4*PAGE_SIZE] = 0xA4; - top_address[0x5*PAGE_SIZE] = 0xA5; + top_address[0x3 * PAGE_SIZE] = 0xA3; + top_address[0x4 * PAGE_SIZE] = 0xA4; + top_address[0x5 * PAGE_SIZE] = 0xA5; printf("VM_TEST_COLLAPSE_COMPRESSOR: " - "populated pages to be compressed in " - "top_object %p\n", top_object); + "populated pages to be compressed in " + "top_object %p\n", top_object); /* compress top object */ vm_object_pageout(top_object); printf("VM_TEST_COLLAPSE_COMPRESSOR: compressing top_object %p\n", - top_object); + top_object); /* wait for all the pages to be gone */ - while (top_object->resident_page_count != 0) + while (top_object->resident_page_count != 0) { IODelay(10); + } printf("VM_TEST_COLLAPSE_COMPRESSOR: top_object %p compressed\n", - top_object); + top_object); /* populate with pages to be resident in top object */ - top_address[0x0*PAGE_SIZE] = 0xA0; - top_address[0x1*PAGE_SIZE] = 0xA1; - top_address[0x2*PAGE_SIZE] = 0xA2; + top_address[0x0 * PAGE_SIZE] = 0xA0; + top_address[0x1 * PAGE_SIZE] = 0xA1; + top_address[0x2 * PAGE_SIZE] = 0xA2; printf("VM_TEST_COLLAPSE_COMPRESSOR: " - "populated pages to be resident in " - "top_object %p\n", top_object); + "populated pages to be resident in " + "top_object %p\n", top_object); /* leave the other pages absent */ /* link the 2 objects */ @@ -10243,18 +10253,18 @@ vm_test_collapse_compressor(void) top_object->shadow = backing_object; top_object->vo_shadow_offset = 0x3000; printf("VM_TEST_COLLAPSE_COMPRESSOR: linked %p and %p\n", - top_object, backing_object); + top_object, backing_object); /* unmap backing object */ vm_map_remove(kernel_map, - backing_offset, - backing_offset + backing_size, - 0); + backing_offset, + backing_offset + backing_size, + VM_MAP_REMOVE_NO_FLAGS); printf("VM_TEST_COLLAPSE_COMPRESSOR: " - "unmapped backing_object %p [0x%llx:0x%llx]\n", - backing_object, - (uint64_t) backing_offset, - (uint64_t) (backing_offset + backing_size)); + "unmapped backing_object %p [0x%llx:0x%llx]\n", + backing_object, + (uint64_t) backing_offset, + (uint64_t) (backing_offset + backing_size)); /* collapse */ printf("VM_TEST_COLLAPSE_COMPRESSOR: collapsing %p\n", top_object); @@ -10273,32 +10283,32 @@ vm_test_collapse_compressor(void) } else { /* check the contents of the mapping */ unsigned char expect[9] = - { 0xA0, 0xA1, 0xA2, /* resident in top */ - 0xA3, 0xA4, 0xA5, /* compressed in top */ - 0xB9, /* resident in backing + shadow_offset */ - 0xBD, /* compressed in backing + shadow_offset + paging_offset */ - 0x00 }; /* absent in both */ + { 0xA0, 0xA1, 0xA2, /* resident in top */ + 0xA3, 0xA4, 0xA5, /* compressed in top */ + 0xB9, /* resident in backing + shadow_offset */ + 0xBD, /* compressed in backing + shadow_offset + paging_offset */ + 0x00 }; /* absent in both */ unsigned char actual[9]; unsigned int i, errors; errors = 0; - for (i = 0; i < sizeof (actual); i++) { - actual[i] = (unsigned char) top_address[i*PAGE_SIZE]; + for (i = 0; i < sizeof(actual); i++) { + actual[i] = (unsigned char) top_address[i * PAGE_SIZE]; if (actual[i] != expect[i]) { errors++; } } printf("VM_TEST_COLLAPSE_COMPRESSOR: " - "actual [%x %x %x %x %x %x %x %x %x] " - "expect [%x %x %x %x %x %x %x %x %x] " - "%d errors\n", - actual[0], actual[1], actual[2], actual[3], - actual[4], actual[5], actual[6], actual[7], - actual[8], - expect[0], expect[1], expect[2], expect[3], - expect[4], expect[5], expect[6], expect[7], - expect[8], - errors); + "actual [%x %x %x %x %x %x %x %x %x] " + "expect [%x %x %x %x %x %x %x %x %x] " + "%d errors\n", + actual[0], actual[1], actual[2], actual[3], + actual[4], actual[5], actual[6], actual[7], + actual[8], + expect[0], expect[1], expect[2], expect[3], + expect[4], expect[5], expect[6], expect[7], + expect[8], + errors); if (errors) { panic("VM_TEST_COLLAPSE_COMPRESSOR: FAIL\n"); } else { @@ -10311,69 +10321,69 @@ vm_test_collapse_compressor(void) #endif /* VM_TEST_COLLAPSE_COMPRESSOR */ #if VM_TEST_WIRE_AND_EXTRACT -extern ledger_template_t task_ledger_template; +extern ledger_template_t task_ledger_template; #include extern ppnum_t vm_map_get_phys_page(vm_map_t map, - vm_offset_t offset); + vm_offset_t offset); static void vm_test_wire_and_extract(void) { - ledger_t ledger; - vm_map_t user_map, wire_map; - mach_vm_address_t user_addr, wire_addr; - mach_vm_size_t user_size, wire_size; - mach_vm_offset_t cur_offset; - vm_prot_t cur_prot, max_prot; - ppnum_t user_ppnum, wire_ppnum; - kern_return_t kr; + ledger_t ledger; + vm_map_t user_map, wire_map; + mach_vm_address_t user_addr, wire_addr; + mach_vm_size_t user_size, wire_size; + mach_vm_offset_t cur_offset; + vm_prot_t cur_prot, max_prot; + ppnum_t user_ppnum, wire_ppnum; + kern_return_t kr; ledger = ledger_instantiate(task_ledger_template, - LEDGER_CREATE_ACTIVE_ENTRIES); + LEDGER_CREATE_ACTIVE_ENTRIES); user_map = vm_map_create(pmap_create(ledger, 0, PMAP_CREATE_64BIT), - 0x100000000ULL, - 0x200000000ULL, - TRUE); + 0x100000000ULL, + 0x200000000ULL, + TRUE); wire_map = vm_map_create(NULL, - 0x100000000ULL, - 0x200000000ULL, - TRUE); + 0x100000000ULL, + 0x200000000ULL, + TRUE); user_addr = 0; user_size = 0x10000; kr = mach_vm_allocate(user_map, - &user_addr, - user_size, - VM_FLAGS_ANYWHERE); + &user_addr, + user_size, + VM_FLAGS_ANYWHERE); assert(kr == KERN_SUCCESS); wire_addr = 0; wire_size = user_size; kr = mach_vm_remap(wire_map, - &wire_addr, - wire_size, - 0, - VM_FLAGS_ANYWHERE, - user_map, - user_addr, - FALSE, - &cur_prot, - &max_prot, - VM_INHERIT_NONE); + &wire_addr, + wire_size, + 0, + VM_FLAGS_ANYWHERE, + user_map, + user_addr, + FALSE, + &cur_prot, + &max_prot, + VM_INHERIT_NONE); assert(kr == KERN_SUCCESS); for (cur_offset = 0; - cur_offset < wire_size; - cur_offset += PAGE_SIZE) { + cur_offset < wire_size; + cur_offset += PAGE_SIZE) { kr = vm_map_wire_and_extract(wire_map, - wire_addr + cur_offset, - VM_PROT_DEFAULT | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK), - TRUE, - &wire_ppnum); + wire_addr + cur_offset, + VM_PROT_DEFAULT | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK), + TRUE, + &wire_ppnum); assert(kr == KERN_SUCCESS); user_ppnum = vm_map_get_phys_page(user_map, - user_addr + cur_offset); + user_addr + cur_offset); printf("VM_TEST_WIRE_AND_EXTRACT: kr=0x%x " - "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n", - kr, - user_map, user_addr + cur_offset, user_ppnum, - wire_map, wire_addr + cur_offset, wire_ppnum); + "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n", + kr, + user_map, user_addr + cur_offset, user_ppnum, + wire_map, wire_addr + cur_offset, wire_ppnum); if (kr != KERN_SUCCESS || wire_ppnum == 0 || wire_ppnum != user_ppnum) { @@ -10382,16 +10392,16 @@ vm_test_wire_and_extract(void) } cur_offset -= PAGE_SIZE; kr = vm_map_wire_and_extract(wire_map, - wire_addr + cur_offset, - VM_PROT_DEFAULT, - TRUE, - &wire_ppnum); + wire_addr + cur_offset, + VM_PROT_DEFAULT, + TRUE, + &wire_ppnum); assert(kr == KERN_SUCCESS); printf("VM_TEST_WIRE_AND_EXTRACT: re-wire kr=0x%x " - "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n", - kr, - user_map, user_addr + cur_offset, user_ppnum, - wire_map, wire_addr + cur_offset, wire_ppnum); + "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n", + kr, + user_map, user_addr + cur_offset, user_ppnum, + wire_map, wire_addr + cur_offset, wire_ppnum); if (kr != KERN_SUCCESS || wire_ppnum == 0 || wire_ppnum != user_ppnum) { @@ -10423,7 +10433,7 @@ vm_test_page_wire_overflow_panic(void) vm_page_unlock_queues(); vm_object_unlock(object); panic("FBDP(%p,%p): wire_count overflow not detected\n", - object, page); + object, page); } #else /* VM_TEST_PAGE_WIRE_OVERFLOW_PANIC */ #define vm_test_page_wire_overflow_panic() @@ -10440,11 +10450,11 @@ vm_test_kernel_object_fault(void) int ret; kr = kernel_memory_allocate(kernel_map, &stack, - kernel_stack_size + (2*PAGE_SIZE), - 0, - (KMA_KSTACK | KMA_KOBJECT | - KMA_GUARD_FIRST | KMA_GUARD_LAST), - VM_KERN_MEMORY_STACK); + kernel_stack_size + (2 * PAGE_SIZE), + 0, + (KMA_KSTACK | KMA_KOBJECT | + KMA_GUARD_FIRST | KMA_GUARD_LAST), + VM_KERN_MEMORY_STACK); if (kr != KERN_SUCCESS) { panic("VM_TEST_KERNEL_OBJECT_FAULT: kernel_memory_allocate kr 0x%x\n", kr); } @@ -10455,9 +10465,9 @@ vm_test_kernel_object_fault(void) printf("VM_TEST_KERNEL_OBJECT_FAULT: FAIL\n"); } vm_map_remove(kernel_map, - stack, - stack + kernel_stack_size + (2*PAGE_SIZE), - VM_MAP_REMOVE_KUNWIRE); + stack, + stack + kernel_stack_size + (2 * PAGE_SIZE), + VM_MAP_REMOVE_KUNWIRE); stack = 0; } #else /* __arm64__ && VM_TEST_KERNEL_OBJECT_FAULT */ @@ -10468,11 +10478,11 @@ vm_test_kernel_object_fault(void) static void vm_test_device_pager_transpose(void) { - memory_object_t device_pager; - vm_object_t anon_object, device_object; - vm_size_t size; - vm_map_offset_t anon_mapping, device_mapping; - kern_return_t kr; + memory_object_t device_pager; + vm_object_t anon_object, device_object; + vm_size_t size; + vm_map_offset_t device_mapping; + kern_return_t kr; size = 3 * PAGE_SIZE; anon_object = vm_object_allocate(size); @@ -10481,20 +10491,27 @@ vm_test_device_pager_transpose(void) assert(device_pager != NULL); device_object = memory_object_to_vm_object(device_pager); assert(device_object != VM_OBJECT_NULL); - anon_mapping = 0; +#if 0 + /* + * Can't actually map this, since another thread might do a + * vm_map_enter() that gets coalesced into this object, which + * would cause the test to fail. + */ + vm_map_offset_t anon_mapping = 0; kr = vm_map_enter(kernel_map, &anon_mapping, size, 0, - VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_NONE, - anon_object, 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, - VM_INHERIT_DEFAULT); + VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_NONE, + anon_object, 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, + VM_INHERIT_DEFAULT); assert(kr == KERN_SUCCESS); +#endif device_mapping = 0; kr = vm_map_enter_mem_object(kernel_map, &device_mapping, size, 0, - VM_FLAGS_ANYWHERE, - VM_MAP_KERNEL_FLAGS_NONE, - VM_KERN_MEMORY_NONE, - (void *)device_pager, 0, FALSE, - VM_PROT_DEFAULT, VM_PROT_ALL, - VM_INHERIT_DEFAULT); + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_NONE, + (void *)device_pager, 0, FALSE, + VM_PROT_DEFAULT, VM_PROT_ALL, + VM_INHERIT_DEFAULT); assert(kr == KERN_SUCCESS); memory_object_deallocate(device_pager); @@ -10526,8 +10543,10 @@ vm_test_device_pager_transpose(void) assert(anon_object->ref_count == 2); assert(anon_object->named); +#if 0 kr = vm_deallocate(kernel_map, anon_mapping, size); assert(kr == KERN_SUCCESS); +#endif assert(device_object->ref_count == 1); assert(!device_object->named); kr = vm_deallocate(kernel_map, device_mapping, size);